Merge branch 'ibi-group:dev' into dev

src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java

@@ -198,10 +198,15 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st
             Field[] fieldsFoundInZip = gtfsTable.getFieldsFromFieldHeaders(headers, null);
             int transformFieldIndex = getFieldIndex(fieldsFoundInZip, fieldName);
 
+            // If the index is -1, this is a new column, and we need to add it accordingly.
+            if (transformFieldIndex == -1) {
+                writer.write(expandArray(headers, fieldName));
+            } else {
+                writer.write(headers);
+            }
+
             int modifiedRowCount = 0;
 
-            // Write headers and processed CSV rows.
-            writer.write(headers);
             while (csvReader.readRecord()) {
                 String originalValue = csvReader.get(transformFieldIndex);
                 String transformedValue = originalValue;
@@ -219,10 +224,16 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st
 
                 // Re-assemble the CSV line and place in buffer.
                 String[] csvValues = csvReader.getValues();
-                csvValues[transformFieldIndex] = transformedValue;
 
-                // Write line to table (plus new line char).
-                writer.write(csvValues);
+                // If the index is -1, this is a new column, and we need to add it accordingly.
+                if (transformFieldIndex == -1) {
+                    writer.write(expandArray(csvValues, transformedValue));
+                } else {
+                    csvValues[transformFieldIndex] = transformedValue;
+
+                    // Write line to table (plus new line char).
+                    writer.write(csvValues);
+                }
 
                 // Count number of CSV rows changed.
                 if (!originalValue.equals(transformedValue)) {
@@ -287,4 +298,14 @@ public String performSubstitutions(String inputString) {
         }
         return result;
     }
+
+    /**
+     * Copies a fixed length array, and appends a new element at the end.
+     */
+    private String[] expandArray(String[] array, String value) {
+        String[] expanded = new String[array.length + 1];
+        System.arraycopy(array, 0, expanded, 0, array.length);
+        expanded[array.length] = value;
+        return expanded;
+    }
 }

src/test/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformationTest.java

@@ -106,6 +106,24 @@ private static Stream<Arguments> createSubstitutionCasesWithOwnExceptions() {
         );
     }
 
+
+    @ParameterizedTest
+    @MethodSource("createCreateNewField")
+    void testCreateNewField(String input, String expected) {
+        NormalizeFieldTransformation transform = createTransformation(
+                "table", "new_field",  null, Lists.newArrayList(
+                        new Substitution("", "new_value")
+                ));
+        assertEquals(expected, transform.performSubstitutions(input));
+    }
+
+    private static Stream<Arguments> createCreateNewField() {
+        return Stream.of(
+                Arguments.of("", "new_value")
+        );
+    }
+
+
     /**
      * Proxy to create a transformation
      * (called by {@link com.conveyal.datatools.manager.jobs.NormalizeFieldTransformJobTest}).