diff --git a/src/versions/1.1/csv.ts b/src/versions/1.1/csv.ts index b215a99..d2a2c78 100644 --- a/src/versions/1.1/csv.ts +++ b/src/versions/1.1/csv.ts @@ -57,15 +57,13 @@ const ERRORS = { `Header column is "${actual}", it should be "${expected}"`, HEADER_COLUMN_MISSING: (column: string) => `Header column should be "${column}", but it is not present`, - HEADER_COLUMN_COUNT: (actual: number) => - `${HEADER_COLUMNS.length} header fields are required and only ${actual} are present`, HEADER_COLUMN_BLANK: (column: string) => `"${column}" is blank`, HEADER_STATE_CODE: (column: string, stateCode: string) => `Header column "${column}" includes an invalid state code "${stateCode}"`, - COLUMN_COUNT: (actual: number, expected: number) => - `Received ${actual} columns, less than the required number ${expected}`, COLUMN_NAME: (actual: string, expected: string, format: string) => `Column is "${actual}" and should be "${expected}" for ${format} format`, + COLUMN_MISSING: (column: string, format: string) => + `Column ${column} is missing, but it is required for ${format} format`, NOTES_COLUMN: (column: string) => `The last column should be "additional_generic_notes", is "${column}"`, ALLOWED_VALUES: (column: string, value: string, allowedValues: string[]) => @@ -89,106 +87,76 @@ export function validateHeader( columns: string[], row: string[] ): CsvValidationError[] { - return [...validateHeaderColumns(columns), ...validateHeaderRow(row)] + const { errors: headerErrors, columns: headerColumns } = + validateHeaderColumns(columns) + const rowErrors = validateHeaderRow(headerColumns, row) + return [...headerErrors, ...rowErrors] } /** @private */ -export function validateHeaderColumns(columns: string[]): CsvValidationError[] { +export function validateHeaderColumns(columns: string[]): { + errors: CsvValidationError[] + columns: (string | undefined)[] +} { const rowIndex = 0 - const errors: CsvValidationError[] = [] - HEADER_COLUMNS.forEach((headerColumn, index) => { - if (index < columns.length) { - if (headerColumn === "license_number | state") { - errors.push( - ...validateLicenseStateColumn(columns[index], rowIndex, index) - ) - return - } - if (!sepColumnsEqual(columns[index], headerColumn)) { - errors.push( - csvErr( - rowIndex, - index, - headerColumn, - ERRORS.HEADER_COLUMN_NAME(columns[index], headerColumn), - false - ) - ) - } - } else { - errors.push( - csvErr( + const remainingColumns = [...HEADER_COLUMNS] + const discoveredColumns: string[] = [] + columns.forEach((column, index) => { + const matchingColumnIndex = remainingColumns.findIndex((requiredColumn) => { + if (requiredColumn === "license_number | state") { + // see if it works + const licenseStateErrors = validateLicenseStateColumn( + column, rowIndex, - index, - headerColumn, - ERRORS.HEADER_COLUMN_MISSING(headerColumn) + index ) - ) + return licenseStateErrors.length === 0 + } else { + return sepColumnsEqual(column, requiredColumn) + } + }) + if (matchingColumnIndex > -1) { + discoveredColumns[index] = column + remainingColumns.splice(matchingColumnIndex, 1) } }) - return errors + return { + errors: remainingColumns.map((requiredColumn) => { + return csvErr( + rowIndex, + columns.length, + requiredColumn, + ERRORS.HEADER_COLUMN_MISSING(requiredColumn) + ) + }), + columns: discoveredColumns, + } } /** @private */ -export function validateHeaderRow(row: string[]): CsvValidationError[] { +export function validateHeaderRow( + headers: (string | undefined)[], + row: string[] +): CsvValidationError[] { const errors: CsvValidationError[] = [] const rowIndex = 1 - if (row.length < HEADER_COLUMNS.length) { - return [ - { - row: rowIndex, - column: 0, - message: ERRORS.HEADER_COLUMN_COUNT(row.length), - }, - ] - } - - const checkBlankColumns = [ - "hospital_name", - "version", - "hospital_location", - "financial_aid_policy", - "last_updated_on", - ] - const requiredColumns = ["last_updated_on"] - checkBlankColumns.forEach((checkBlankColumn) => { - const headerIndex = HEADER_COLUMNS.indexOf(checkBlankColumn) - if (!row[headerIndex].trim()) { - errors.push( - csvErr( - rowIndex, - headerIndex, - checkBlankColumn, - ERRORS.HEADER_COLUMN_BLANK(checkBlankColumn), - !requiredColumns.includes(row[headerIndex].trim()) + headers.forEach((header, index) => { + if (header != null) { + if (!row[index]?.trim()) { + errors.push( + csvErr(rowIndex, index, header, ERRORS.HEADER_COLUMN_BLANK(header)) ) - ) + } } }) - const licenseStateIndex = HEADER_COLUMNS.findIndex((c) => - c.includes("license_number") - ) - if (!row[licenseStateIndex].trim()) { - errors.push( - csvErr( - rowIndex, - licenseStateIndex, - HEADER_COLUMNS[licenseStateIndex], - ERRORS.HEADER_COLUMN_BLANK(HEADER_COLUMNS[licenseStateIndex]), - true - ) - ) - } - return errors } /** @private */ export function validateColumns(columns: string[]): CsvValidationError[] { const rowIndex = 2 - const errors: CsvValidationError[] = [] const tall = isTall(columns) @@ -196,56 +164,25 @@ export function validateColumns(columns: string[]): CsvValidationError[] { const wideColumns = getWideColumns(columns) const tallColumns = getTallColumns(columns) const schemaFormat = tall ? "tall" : "wide" - const totalColumns = baseColumns.concat(tall ? tallColumns : wideColumns) + const remainingColumns = baseColumns.concat(tall ? tallColumns : wideColumns) - if (columns.length < totalColumns.length) { - return [ - csvErr( - rowIndex, - 0, - undefined, - ERRORS.COLUMN_COUNT(columns.length, baseColumns.length) - ), - ] - } - - totalColumns.forEach((column, index) => { - if (!sepColumnsEqual(columns[index], column)) { - errors.push( - csvErr( - rowIndex, - index, - column, - ERRORS.COLUMN_NAME(columns[index], column, schemaFormat) - ) - ) + columns.forEach((column) => { + const matchingColumnIndex = remainingColumns.findIndex((requiredColumn) => + sepColumnsEqual(column, requiredColumn) + ) + if (matchingColumnIndex > -1) { + remainingColumns.splice(matchingColumnIndex, 1) } }) - if (!tall) { - errors.push(...validateWideColumns(columns)) - } - - return errors -} - -/** @private */ -export function validateWideColumns(columns: string[]): CsvValidationError[] { - const rowIndex = 2 - const errors: CsvValidationError[] = [] - - if (columns[columns.length - 1] !== "additional_generic_notes") { - errors.push( - csvErr( - rowIndex, - columns.length - 1, - "additional_generic_notes", - ERRORS.NOTES_COLUMN(columns[columns.length - 1]) - ) + return remainingColumns.map((requiredColumn) => { + return csvErr( + rowIndex, + columns.length, + requiredColumn, + ERRORS.COLUMN_MISSING(requiredColumn, schemaFormat) ) - } - - return errors + }) } /** @private */ @@ -555,6 +492,7 @@ export function getWideColumns(columns: string[]): string[] { ...payersPlansColumns.slice(0, 2), ...MIN_MAX_COLUMNS, ...payersPlansColumns.slice(2), + "additional_generic_notes", ] } diff --git a/src/versions/2.0/csv.ts b/src/versions/2.0/csv.ts index 924050b..362eb37 100644 --- a/src/versions/2.0/csv.ts +++ b/src/versions/2.0/csv.ts @@ -57,15 +57,13 @@ const ERRORS = { `Header column is "${actual}", it should be "${expected}"`, HEADER_COLUMN_MISSING: (column: string) => `Header column should be "${column}", but it is not present`, - HEADER_COLUMN_COUNT: (actual: number) => - `${HEADER_COLUMNS.length} header fields are required and only ${actual} are present`, HEADER_COLUMN_BLANK: (column: string) => `"${column}" is blank`, HEADER_STATE_CODE: (column: string, stateCode: string) => `Header column "${column}" includes an invalid state code "${stateCode}"`, - COLUMN_COUNT: (actual: number, expected: number) => - `Received ${actual} columns, less than the required number ${expected}`, COLUMN_NAME: (actual: string, expected: string, format: string) => `Column is "${actual}" and should be "${expected}" for ${format} format`, + COLUMN_MISSING: (column: string, format: string) => + `Column ${column} is missing, but it is required for ${format} format`, NOTES_COLUMN: (column: string) => `The last column should be "additional_generic_notes", is "${column}"`, ALLOWED_VALUES: (column: string, value: string, allowedValues: string[]) => @@ -89,117 +87,76 @@ export function validateHeader( columns: string[], row: string[] ): CsvValidationError[] { - return [...validateHeaderColumns(columns), ...validateHeaderRow(row)] + const { errors: headerErrors, columns: headerColumns } = + validateHeaderColumns(columns) + const rowErrors = validateHeaderRow(headerColumns, row) + return [...headerErrors, ...rowErrors] } /** @private */ -export function validateHeaderColumns(columns: string[]): CsvValidationError[] { +export function validateHeaderColumns(columns: string[]): { + errors: CsvValidationError[] + columns: (string | undefined)[] +} { const rowIndex = 0 - const errors: CsvValidationError[] = [] - HEADER_COLUMNS.forEach((headerColumn, index) => { - if (index < columns.length) { - if (headerColumn === "license_number | state") { - errors.push( - ...validateLicenseStateColumn(columns[index], rowIndex, index) - ) - return - } - if (!sepColumnsEqual(columns[index], headerColumn)) { - errors.push( - csvErr( - rowIndex, - index, - headerColumn, - ERRORS.HEADER_COLUMN_NAME(columns[index], headerColumn), - false - ) - ) - } - if (!sepColumnsEqual(columns[index], headerColumn)) { - errors.push( - csvErr( - rowIndex, - index, - headerColumn, - ERRORS.HEADER_COLUMN_NAME(columns[index], headerColumn), - false - ) - ) - } - } else { - errors.push( - csvErr( + const remainingColumns = [...HEADER_COLUMNS] + const discoveredColumns: string[] = [] + columns.forEach((column, index) => { + const matchingColumnIndex = remainingColumns.findIndex((requiredColumn) => { + if (requiredColumn === "license_number | state") { + // see if it works + const licenseStateErrors = validateLicenseStateColumn( + column, rowIndex, - index, - headerColumn, - ERRORS.HEADER_COLUMN_MISSING(headerColumn) + index ) - ) + return licenseStateErrors.length === 0 + } else { + return sepColumnsEqual(column, requiredColumn) + } + }) + if (matchingColumnIndex > -1) { + discoveredColumns[index] = column + remainingColumns.splice(matchingColumnIndex, 1) } }) - return errors + return { + errors: remainingColumns.map((requiredColumn) => { + return csvErr( + rowIndex, + columns.length, + requiredColumn, + ERRORS.HEADER_COLUMN_MISSING(requiredColumn) + ) + }), + columns: discoveredColumns, + } } /** @private */ -export function validateHeaderRow(row: string[]): CsvValidationError[] { +export function validateHeaderRow( + headers: (string | undefined)[], + row: string[] +): CsvValidationError[] { const errors: CsvValidationError[] = [] const rowIndex = 1 - if (row.length < HEADER_COLUMNS.length) { - return [ - { - row: rowIndex, - column: 0, - message: ERRORS.HEADER_COLUMN_COUNT(row.length), - }, - ] - } - - const checkBlankColumns = [ - "hospital_name", - "version", - "hospital_location", - "financial_aid_policy", - "last_updated_on", - ] - const requiredColumns = ["last_updated_on"] - checkBlankColumns.forEach((checkBlankColumn) => { - const headerIndex = HEADER_COLUMNS.indexOf(checkBlankColumn) - if (!row[headerIndex].trim()) { - errors.push( - csvErr( - rowIndex, - headerIndex, - checkBlankColumn, - ERRORS.HEADER_COLUMN_BLANK(checkBlankColumn), - !requiredColumns.includes(row[headerIndex].trim()) + headers.forEach((header, index) => { + if (header != null) { + if (!row[index]?.trim()) { + errors.push( + csvErr(rowIndex, index, header, ERRORS.HEADER_COLUMN_BLANK(header)) ) - ) + } } }) - const licenseStateIndex = HEADER_COLUMNS.findIndex((c) => - c.includes("license_number") - ) - if (!row[licenseStateIndex].trim()) { - errors.push( - csvErr( - rowIndex, - licenseStateIndex, - HEADER_COLUMNS[licenseStateIndex], - ERRORS.HEADER_COLUMN_BLANK(HEADER_COLUMNS[licenseStateIndex]), - true - ) - ) - } - return errors } /** @private */ export function validateColumns(columns: string[]): CsvValidationError[] { const rowIndex = 2 - const errors: CsvValidationError[] = [] const tall = isTall(columns) @@ -207,56 +164,25 @@ export function validateColumns(columns: string[]): CsvValidationError[] { const wideColumns = getWideColumns(columns) const tallColumns = getTallColumns(columns) const schemaFormat = tall ? "tall" : "wide" - const totalColumns = baseColumns.concat(tall ? tallColumns : wideColumns) + const remainingColumns = baseColumns.concat(tall ? tallColumns : wideColumns) - if (columns.length < totalColumns.length) { - return [ - csvErr( - rowIndex, - 0, - undefined, - ERRORS.COLUMN_COUNT(columns.length, baseColumns.length) - ), - ] - } - - totalColumns.forEach((column, index) => { - if (!sepColumnsEqual(columns[index], column)) { - errors.push( - csvErr( - rowIndex, - index, - column, - ERRORS.COLUMN_NAME(columns[index], column, schemaFormat) - ) - ) + columns.forEach((column) => { + const matchingColumnIndex = remainingColumns.findIndex((requiredColumn) => + sepColumnsEqual(column, requiredColumn) + ) + if (matchingColumnIndex > -1) { + remainingColumns.splice(matchingColumnIndex, 1) } }) - if (!tall) { - errors.push(...validateWideColumns(columns)) - } - - return errors -} - -/** @private */ -export function validateWideColumns(columns: string[]): CsvValidationError[] { - const rowIndex = 2 - const errors: CsvValidationError[] = [] - - if (columns[columns.length - 1] !== "additional_generic_notes") { - errors.push( - csvErr( - rowIndex, - columns.length - 1, - "additional_generic_notes", - ERRORS.NOTES_COLUMN(columns[columns.length - 1]) - ) + return remainingColumns.map((requiredColumn) => { + return csvErr( + rowIndex, + columns.length, + requiredColumn, + ERRORS.COLUMN_MISSING(requiredColumn, schemaFormat) ) - } - - return errors + }) } /** @private */ @@ -566,6 +492,7 @@ export function getWideColumns(columns: string[]): string[] { ...payersPlansColumns.slice(0, 2), ...MIN_MAX_COLUMNS, ...payersPlansColumns.slice(2), + "additional_generic_notes", ] } diff --git a/test/csv.spec.ts b/test/csv.spec.ts index 0cd07eb..968af4a 100644 --- a/test/csv.spec.ts +++ b/test/csv.spec.ts @@ -2,7 +2,6 @@ import test from "ava" import { validateHeaderColumns, validateHeaderRow, - validateWideColumns, validateColumns, validateTallFields, validateWideFields, @@ -21,20 +20,60 @@ const VALID_HEADER_COLUMNS = HEADER_COLUMNS.map((c) => ) test("validateHeaderColumns", (t) => { - t.is(validateHeaderColumns([]).length, HEADER_COLUMNS.length) - t.deepEqual(validateHeaderColumns(VALID_HEADER_COLUMNS), []) - t.is( - validateHeaderColumns(VALID_HEADER_COLUMNS.slice(0, -1))[0].column, - VALID_HEADER_COLUMNS.length - 1 - ) + const emptyResult = validateHeaderColumns([]) + t.is(emptyResult.errors.length, HEADER_COLUMNS.length) + t.is(emptyResult.columns.length, 0) + const basicResult = validateHeaderColumns(VALID_HEADER_COLUMNS) + t.is(basicResult.errors.length, 0) + t.deepEqual(basicResult.columns, VALID_HEADER_COLUMNS) + const reversedColumns = [...VALID_HEADER_COLUMNS].reverse() + const reverseResult = validateHeaderColumns(reversedColumns) + t.is(reverseResult.errors.length, 0) + t.deepEqual(reverseResult.columns, reversedColumns) + const extraColumns = [ + "extra1", + ...VALID_HEADER_COLUMNS.slice(0, 2), + "extra2", + ...VALID_HEADER_COLUMNS.slice(2), + ] + const extraResult = validateHeaderColumns(extraColumns) + t.is(extraResult.errors.length, 0) + t.deepEqual(extraResult.columns, [ + undefined, + ...VALID_HEADER_COLUMNS.slice(0, 2), + undefined, + ...VALID_HEADER_COLUMNS.slice(2), + ]) }) test("validateHeaderRow", (t) => { - t.is(validateHeaderRow([]).length, 1) t.is( - validateHeaderRow([ + validateHeaderRow(VALID_HEADER_COLUMNS, []).length, + VALID_HEADER_COLUMNS.length + ) + t.is( + validateHeaderRow(VALID_HEADER_COLUMNS, [ + "name", + "2022-01-01", + "1.0.0", + "Woodlawn", + "Aid", + "001 | MD", + ]).length, + 0 + ) + const extraColumns = [ + undefined, + ...VALID_HEADER_COLUMNS.slice(0, 2), + undefined, + ...VALID_HEADER_COLUMNS.slice(2), + ] + t.is( + validateHeaderRow(extraColumns, [ + "", "name", "2022-01-01", + "", "1.0.0", "Woodlawn", "Aid", @@ -43,7 +82,7 @@ test("validateHeaderRow", (t) => { 0 ) t.assert( - validateHeaderRow([ + validateHeaderRow(VALID_HEADER_COLUMNS, [ "", "2022-01-01", "1.0.0", @@ -93,19 +132,68 @@ test("validateColumns tall", (t) => { ...getTallColumns([]), "test", ]).length, - 9 + 1 ) }) -test("validateWideColumns", (t) => { - // Currently only checking for the order of additional_generic_notes +test("validateColumns wide", (t) => { + const columns = [ + ...BASE_COLUMNS, + ...MIN_MAX_COLUMNS, + "code | 1", + "code | 1 | type", + "standard_charge | Payer | Plan", + "standard_charge | Payer | Plan | percent", + "standard_charge | Payer | Plan | contracting_method", + "additional_payer_notes | Payer | Plan", + "additional_generic_notes", + ] + t.is(validateColumns(columns).length, 0) + // any order is fine + const reverseColumns = [...columns].reverse() + t.is(validateColumns(reverseColumns).length, 0) + // extra payer and plan are fine t.is( - validateWideColumns([ - ...BASE_COLUMNS, - ...MIN_MAX_COLUMNS, - "standard_charge | Payer | Plan", - "additional_generic_notes", - "standard_charge | Payer | Plan | pct", + validateColumns([ + ...columns, + "standard_charge | Payer | Plan 2", + "standard_charge | Payer | Plan 2 | percent", + "standard_charge | Payer | Plan 2 | contracting_method", + "additional_payer_notes | Payer | Plan 2", + "standard_charge | Another Payer | Plan", + "standard_charge | Another Payer | Plan | percent", + "standard_charge | Another Payer | Plan | contracting_method", + "additional_payer_notes | Another Payer | Plan", + ]).length, + 0 + ) + // missing percent is an error + t.is( + validateColumns([ + ...columns, + "standard_charge | Payer | Plan 2", + "standard_charge | Payer | Plan 2 | contracting_method", + "additional_payer_notes | Payer | Plan 2", + ]).length, + 1 + ) + // missing contracting_method is an error + t.is( + validateColumns([ + ...columns, + "standard_charge | Payer | Plan 2", + "standard_charge | Payer | Plan 2 | percent", + "additional_payer_notes | Payer | Plan 2", + ]).length, + 1 + ) + // missing additional_payer_notes is an error + t.is( + validateColumns([ + ...columns, + "standard_charge | Payer | Plan 2", + "standard_charge | Payer | Plan 2 | percent", + "standard_charge | Payer | Plan 2 | contracting_method", ]).length, 1 )