diff --git a/src/versions/2.0/csv.ts b/src/versions/2.0/csv.ts index f2234b0..fd060ea 100644 --- a/src/versions/2.0/csv.ts +++ b/src/versions/2.0/csv.ts @@ -9,15 +9,11 @@ import { } from "../common/csv.js" import { BILLING_CODE_TYPES, - BillingCodeType, CHARGE_BILLING_CLASSES, CHARGE_SETTINGS, STANDARD_CHARGE_METHODOLOGY, - ChargeBillingClass, - ChargeSetting, StandardChargeMethod, DRUG_UNITS, - DrugUnit, } from "./types" const ATTESTATION = @@ -70,24 +66,33 @@ const ERRORS = { HEADER_COLUMN_BLANK: (column: string) => `"${column}" is blank`, HEADER_STATE_CODE: (column: string, stateCode: string) => `Header column "${column}" includes an invalid state code "${stateCode}"`, + DUPLICATE_HEADER_COLUMN: (column: string) => + `Column ${column} duplicated in header`, COLUMN_NAME: (actual: string, expected: string, format: string) => `Column is "${actual}" and should be "${expected}" for ${format} format`, COLUMN_MISSING: (column: string, format: string) => `Column ${column} is missing, but it is required for ${format} format`, - ALLOWED_VALUES: (column: string, value: string, allowedValues: string[]) => + ALLOWED_VALUES: ( + column: string, + value: string, + allowedValues: readonly string[] + ) => `"${column}" value "${value}" is not one of the allowed values: ${allowedValues .map((t) => `"${t}"`) .join(", ")}`, INVALID_DATE: (column: string, value: string) => `"${column}" value "${value}" is not a valid YYYY-MM-DD date`, INVALID_NUMBER: (column: string, value: string) => - `"${column}" value "${value}" is not a valid number`, + `"${column}" value "${value}" is not a valid positive number`, POSITIVE_NUMBER: (column: string, suffix = ``) => `"${column}" is required to be a positive number${suffix}`, CHARGE_ONE_REQUIRED: (column: string) => { const fieldName = column.replace(" | percent", "") return `One of "${fieldName}" or "${fieldName} | percent" is required` }, + CODE_ONE_REQUIRED: () => { + return "At least one code and code type must be specified" + }, REQUIRED: (column: string, suffix = ``) => `"${column}" is required${suffix}`, CHARGE_PERCENT_REQUIRED_SUFFIX: " (one of charge or percent is required)", } @@ -111,6 +116,7 @@ export function validateHeaderColumns(columns: string[]): { const rowIndex = 0 const remainingColumns = [...HEADER_COLUMNS] const discoveredColumns: string[] = [] + const duplicateErrors: CsvValidationError[] = [] columns.forEach((column, index) => { const matchingColumnIndex = remainingColumns.findIndex((requiredColumn) => { if (requiredColumn === "license_number | state") { @@ -128,17 +134,35 @@ export function validateHeaderColumns(columns: string[]): { if (matchingColumnIndex > -1) { discoveredColumns[index] = column remainingColumns.splice(matchingColumnIndex, 1) + } else { + // if we already found this column, it's a duplicate + const existingColumn = discoveredColumns.find((discovered) => { + return discovered != null && sepColumnsEqual(discovered, column) + }) + if (existingColumn) { + duplicateErrors.push( + csvErr( + rowIndex, + index, + "column", + ERRORS.DUPLICATE_HEADER_COLUMN(column) + ) + ) + } } }) return { - errors: remainingColumns.map((requiredColumn) => { - return csvErr( - rowIndex, - columns.length, - requiredColumn, - ERRORS.HEADER_COLUMN_MISSING(requiredColumn) - ) - }), + errors: [ + ...duplicateErrors, + ...remainingColumns.map((requiredColumn) => { + return csvErr( + rowIndex, + columns.length, + requiredColumn, + ERRORS.HEADER_COLUMN_MISSING(requiredColumn) + ) + }), + ], columns: discoveredColumns, } } @@ -183,12 +207,11 @@ export function validateColumns(columns: string[]): CsvValidationError[] { const rowIndex = 2 const tall = isTall(columns) - const baseColumns = getBaseColumns(columns) - const wideColumns = getWideColumns(columns) - const tallColumns = getTallColumns(columns) const schemaFormat = tall ? "tall" : "wide" - const remainingColumns = baseColumns.concat(tall ? tallColumns : wideColumns) + const remainingColumns = baseColumns.concat( + tall ? getTallColumns() : getWideColumns(columns) + ) columns.forEach((column) => { const matchingColumnIndex = remainingColumns.findIndex((requiredColumn) => @@ -218,108 +241,117 @@ export function validateRow( ): CsvValidationError[] { const errors: CsvValidationError[] = [] - const requiredFields = ["description", "code | 1"] + const requiredFields = ["description"] requiredFields.forEach((field) => errors.push( ...validateRequiredField(row, field, index, columns.indexOf(field)) ) ) - if (!BILLING_CODE_TYPES.includes(row["code | 1 | type"] as BillingCodeType)) { - errors.push( - csvErr( - index, - columns.indexOf("code | 1 | type"), - "code | 1 | type", - ERRORS.ALLOWED_VALUES( - "code | 1 | type", - row["code | 1 | type"], - BILLING_CODE_TYPES as unknown as string[] - ), - true + // check code and code-type columns + const codeColumns = columns.filter((column) => { + return /^code \| \d+$/.test(column) + }) + let foundCode = false + codeColumns.forEach((codeColumn) => { + const codeTypeColumn = `${codeColumn} | type` + // if the type column is missing, we already created an error when checking the columns + // if both columns exist, we can check the values + if (row[codeTypeColumn] != null) { + const trimCode = row[codeColumn].trim() + const trimType = row[codeTypeColumn].trim() + if (trimCode.length === 0 && trimType.length > 0) { + foundCode = true + errors.push( + csvErr( + index, + columns.indexOf(codeColumn), + codeColumn, + ERRORS.REQUIRED(codeColumn) + ) + ) + } else if (trimCode.length > 0 && trimType.length === 0) { + foundCode = true + errors.push( + csvErr( + index, + columns.indexOf(codeTypeColumn), + codeTypeColumn, + ERRORS.REQUIRED(codeTypeColumn) + ) + ) + } else if (trimCode.length > 0 && trimType.length > 0) { + foundCode = true + } + errors.push( + ...validateOptionalEnumField( + row, + codeTypeColumn, + index, + columns.indexOf(codeTypeColumn), + BILLING_CODE_TYPES + ) ) + } + }) + if (!foundCode) { + errors.push( + csvErr(index, columns.length, "code | 1", ERRORS.CODE_ONE_REQUIRED()) ) } - // TODO: Code itself is required, need to check all of those, not all checked here - if ( - row["code | 2"] && - !BILLING_CODE_TYPES.includes(row["code | 2 | type"] as BillingCodeType) - ) { - errors.push( - csvErr( - index, - columns.indexOf("code | 2 | type"), - "code | 2 | type", - ERRORS.ALLOWED_VALUES( - "code | 2 | type", - row["code | 2 | type"], - BILLING_CODE_TYPES as unknown as string[] - ) - ) + errors.push( + ...validateOptionalEnumField( + row, + "billing_class", + index, + columns.indexOf("billing_class"), + CHARGE_BILLING_CLASSES ) - } + ) - if ( - !CHARGE_BILLING_CLASSES.includes(row["billing_class"] as ChargeBillingClass) - ) { + errors.push( + ...validateRequiredEnumField( + row, + "setting", + index, + columns.indexOf("setting"), + CHARGE_SETTINGS + ) + ) + + if ((row["drug_unit_of_measurement"] || "").trim()) { errors.push( - csvErr( + ...validateOptionalFloatField( + row, + "drug_unit_of_measurement", index, - columns.indexOf("billing_class"), - "billing_class", - ERRORS.ALLOWED_VALUES( - "billing_class", - row["billing_class"], - CHARGE_BILLING_CLASSES as unknown as string[] - ) + columns.indexOf("drug_unit_of_measurement") ) ) - } - - if (!CHARGE_SETTINGS.includes(row["setting"] as ChargeSetting)) { errors.push( - csvErr( + ...validateRequiredEnumField( + row, + "drug_type_of_measurement", index, - columns.indexOf("setting"), - "setting", - ERRORS.ALLOWED_VALUES( - "setting", - row["setting"], - CHARGE_SETTINGS as unknown as string[] - ) + columns.indexOf("drug_type_of_measurement"), + DRUG_UNITS ) ) - } - - if (row["drug_unit_of_measurement"]) { - if (!/\d+(\.\d+)?/g.test(row["drug_unit_of_measurement"])) { - errors.push( - csvErr( - index, - columns.indexOf("drug_unit_of_measurement"), - "drug_unit_of_measurement", - ERRORS.INVALID_NUMBER( - "drug_unit_of_measurement", - row["drug_unit_of_measurement"] - ) - ) - ) - } - if (!DRUG_UNITS.includes(row["drug_type_of_measurement"] as DrugUnit)) { - errors.push( - csvErr( - index, - columns.indexOf("drug_type_of_measurement"), - "drug_type_of_measurement", - ERRORS.ALLOWED_VALUES( - "drug_type_of_measurement", - row["drug_type_of_measurement"], - DRUG_UNITS as unknown as string[] - ) - ) - ) - } + // if (!DRUG_UNITS.includes(row["drug_type_of_measurement"] as DrugUnit)) { + // errors.push( + // csvErr( + // index, + // columns.indexOf("drug_type_of_measurement"), + // "drug_type_of_measurement", + // ERRORS.ALLOWED_VALUES( + // "drug_type_of_measurement", + // row["drug_type_of_measurement"], + // DRUG_UNITS as unknown as string[] + // ) + // ) + // ) + // } } const chargeFields = [ @@ -330,14 +362,14 @@ export function validateRow( ] chargeFields.forEach((field) => errors.push( - ...validateRequiredFloatField(row, field, index, columns.indexOf(field)) + ...validateOptionalFloatField(row, field, index, columns.indexOf(field)) ) ) if (wide) { errors.push(...validateWideFields(row, index)) } else { - errors.push(...validateTallFields(row, index)) + errors.push(...validateTallFields(row, index, columns)) } return errors @@ -421,7 +453,8 @@ function validateLicenseStateColumn( /** @private */ export function validateTallFields( row: { [key: string]: string }, - index: number + index: number, + columns: string[] ): CsvValidationError[] { const errors: CsvValidationError[] = [] @@ -457,25 +490,15 @@ export function validateTallFields( errors.push(...floatErrors) } - if ( - !STANDARD_CHARGE_METHODOLOGY.includes( - row["standard_charge | methodology"] as StandardChargeMethod - ) - ) { - errors.push( - csvErr( - index, - BASE_COLUMNS.indexOf("standard_charge | methodology"), - // TODO: Change to constants - "standard_charge | methodology", - ERRORS.ALLOWED_VALUES( - "standard_charge | methodology", - row["standard_charge | methodology"], - STANDARD_CHARGE_METHODOLOGY as unknown as string[] - ) - ) + errors.push( + ...validateRequiredEnumField( + row, + "standard_charge | methodology", + index, + columns.indexOf("standard_charge | methodology"), + STANDARD_CHARGE_METHODOLOGY ) - } + ) return errors } @@ -506,8 +529,7 @@ export function getWideColumns(columns: string[]): string[] { } /** @private */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -export function getTallColumns(columns: string[]): string[] { +export function getTallColumns(): string[] { return TALL_COLUMNS } @@ -555,7 +577,7 @@ function validateRequiredFloatField( columnIndex: number, suffix = `` ): CsvValidationError[] { - if (!/\d+(\.\d+)?/g.test(row[field] || "")) { + if (!/^\d+(\.\d+)?$/g.test((row[field] || "").trim())) { return [ csvErr( rowIndex, @@ -568,6 +590,71 @@ function validateRequiredFloatField( return [] } +function validateOptionalFloatField( + row: { [key: string]: string }, + field: string, + rowIndex: number, + columnIndex: number +): CsvValidationError[] { + if (!(row[field] || "").trim()) { + return [] + } else if (!/^\d+(\.\d+)?$/g.test(row[field].trim())) { + return [ + csvErr( + rowIndex, + columnIndex, + field, + ERRORS.INVALID_NUMBER(field, row[field]) + ), + ] + } + return [] +} + +function validateRequiredEnumField( + row: { [key: string]: string }, + field: string, + rowIndex: number, + columnIndex: number, + allowedValues: readonly string[] +) { + if (!(row[field] || "").trim()) { + return [csvErr(rowIndex, columnIndex, field, ERRORS.REQUIRED(field))] + } else if (!allowedValues.includes(row[field])) { + return [ + csvErr( + rowIndex, + columnIndex, + field, + ERRORS.ALLOWED_VALUES(field, row[field], allowedValues) + ), + ] + } + return [] +} + +function validateOptionalEnumField( + row: { [key: string]: string }, + field: string, + rowIndex: number, + columnIndex: number, + allowedValues: readonly string[] +) { + if (!(row[field] || "").trim()) { + return [] + } else if (!allowedValues.includes(row[field])) { + return [ + csvErr( + rowIndex, + columnIndex, + field, + ERRORS.ALLOWED_VALUES(field, row[field], allowedValues) + ), + ] + } + return [] +} + // TODO: Better way of getting this? /** @private */ export function isTall(columns: string[]): boolean { diff --git a/src/versions/common/csv.ts b/src/versions/common/csv.ts index 74cf56a..ca295c4 100644 --- a/src/versions/common/csv.ts +++ b/src/versions/common/csv.ts @@ -77,7 +77,11 @@ export function getCodeCount(columns: string[]): number { .map((v) => v.trim()) .filter((v) => !!v) ) - .filter((c) => c[0] === "code" && c.length === 2) + .filter( + (c) => + c[0] === "code" && + (c.length === 2 || (c.length === 3 && c[2] === "type")) + ) .map((c) => +c[1].replace(/\D/g, "")) .filter((v) => !isNaN(v)) ) diff --git a/test/2.0/csv.spec.ts b/test/2.0/csv.spec.ts index 6de1e6f..c9ac5db 100644 --- a/test/2.0/csv.spec.ts +++ b/test/2.0/csv.spec.ts @@ -2,7 +2,11 @@ import test from "ava" import { validateHeaderColumns, validateHeaderRow, + validateColumns, + validateRow, HEADER_COLUMNS, + BASE_COLUMNS, + TALL_COLUMNS, } from "../../src/versions/2.0/csv.js" const VALID_HEADER_COLUMNS = HEADER_COLUMNS.map((c) => @@ -34,6 +38,14 @@ test("validateHeaderColumns", (t) => { undefined, ...VALID_HEADER_COLUMNS.slice(2), ]) + const duplicateColumns = [...VALID_HEADER_COLUMNS, "hospital_location"] + const duplicateResult = validateHeaderColumns(duplicateColumns) + t.is(duplicateResult.errors.length, 1) + t.is( + duplicateResult.errors[0].message, + "Column hospital_location duplicated in header" + ) + t.deepEqual(duplicateResult.columns, VALID_HEADER_COLUMNS) }) test("validateHeaderRow", (t) => { @@ -78,3 +90,232 @@ test("validateHeaderRow", (t) => { t.is(wrongAffirmationResult.length, 1) t.assert(wrongAffirmationResult[0].message.includes("allowed value")) }) + +test("validateColumns tall", (t) => { + const columns = [ + ...BASE_COLUMNS, + ...TALL_COLUMNS, + "code | 1", + "code | 1 | type", + ] + t.is(validateColumns(columns).length, 0) + // any order is okay + const reverseColumns = [...columns].reverse() + t.is(validateColumns(reverseColumns).length, 0) + // extra code columns may appear + const extraCodes = [ + ...columns, + "code|2", + "code|2|type", + "code|3", + "code|3|type", + ] + t.is(validateColumns(extraCodes).length, 0) + // if a column is missing, that's an error + const missingBase = columns.slice(1) + const missingBaseResult = validateColumns(missingBase) + t.is(missingBaseResult.length, 1) + t.assert(missingBaseResult[0].message.includes("description is missing")) + // this also applies to code columns, where code|i means that code|i|type must appear + const missingCode = [...columns, "code | 2"] + const missingCodeResult = validateColumns(missingCode) + t.is(missingCodeResult.length, 1) + t.assert(missingCodeResult[0].message.includes("code | 2 | type is missing")) + // code|i|type means that code|i must be present + const missingType = [...columns, "code | 2 | type"] + const missingTypeResult = validateColumns(missingType) + t.is(missingTypeResult.length, 1) + t.assert(missingTypeResult[0].message.includes("code | 2 is missing")) +}) + +test("validateRow tall", (t) => { + const columns = [ + ...BASE_COLUMNS, + ...TALL_COLUMNS, + "code | 1", + "code | 1 | type", + "code | 2", + "code | 2 | type", + ] + const basicRow = { + description: "basic description", + setting: "inpatient", + "code | 1": "12345", + "code | 1 | type": "DRG", + "code | 2": "", + "code | 2 | type": "", + drug_unit_of_measurement: "8.5", + drug_type_of_measurement: "ML", + modifiers: "", + "standard_charge | gross": "100", + "standard_charge | discounted_cash": "200.50", + "standard_charge | min": "50", + "standard_charge | max": "500", + additional_generic_notes: "some notes", + payer_name: "Acme Payer", + plan_name: "Acme Basic Coverage", + "standard_charge | negotiated_dollar": "300", + "standard_charge | negotiated_percentage": "", + "standard_charge | negotiated_algorithm": "", + "standard_charge | methodology": "fee schedule", + estimated_amount: "", + } + const basicResult = validateRow(basicRow, 5, columns, false) + t.is(basicResult.length, 0) + // description must not be empty + const noDescriptionRow = { ...basicRow, description: "" } + const noDescriptionResult = validateRow(noDescriptionRow, 6, columns, false) + t.is(noDescriptionResult.length, 1) + t.assert(noDescriptionResult[0].message.includes('"description" is required')) + // setting must not be empty + const noSettingRow = { ...basicRow, setting: "" } + const noSettingResult = validateRow(noSettingRow, 7, columns, false) + t.is(noSettingResult.length, 1) + t.assert(noSettingResult[0].message.includes('"setting" is required')) + // setting must be one of CHARGE_SETTINGS + const wrongSettingRow = { ...basicRow, setting: "everywhere" } + const wrongSettingResult = validateRow(wrongSettingRow, 8, columns, false) + t.is(wrongSettingResult.length, 1) + t.assert( + wrongSettingResult[0].message.includes( + '"setting" value "everywhere" is not one of the allowed values' + ) + ) + // drug_unit_of_measurement must be positive number if present + const emptyDrugUnitRow = { ...basicRow, drug_unit_of_measurement: "" } + const emptyDrugUnitResult = validateRow(emptyDrugUnitRow, 9, columns, false) + t.is(emptyDrugUnitResult.length, 0) + const wrongDrugUnitRow = { ...basicRow, drug_unit_of_measurement: "-4" } + const wrongDrugUnitResult = validateRow(wrongDrugUnitRow, 10, columns, false) + t.is(wrongDrugUnitResult.length, 1) + t.assert( + wrongDrugUnitResult[0].message.includes( + '"drug_unit_of_measurement" value "-4" is not a valid positive number' + ) + ) + // drug_type_of_measurement must be one of DRUG_UNITS if present + const wrongDrugTypeRow = { ...basicRow, drug_type_of_measurement: "KG" } + const wrongDrugTypeResult = validateRow(wrongDrugTypeRow, 12, columns, false) + t.is(wrongDrugTypeResult.length, 1) + t.assert( + wrongDrugTypeResult[0].message.includes( + '"drug_type_of_measurement" value "KG" is not one of the allowed values' + ) + ) + // standard_charge | gross must be positive number if present + const emptyGrossRow = { ...basicRow, "standard_charge | gross": "" } + const emptyGrossResult = validateRow(emptyGrossRow, 13, columns, false) + t.is(emptyGrossResult.length, 0) + const wrongGrossRow = { ...basicRow, "standard_charge | gross": "3,000" } + const wrongGrossResult = validateRow(wrongGrossRow, 14, columns, false) + t.is(wrongGrossResult.length, 1) + t.assert( + wrongGrossResult[0].message.includes( + '"standard_charge | gross" value "3,000" is not a valid positive number' + ) + ) + // standard_charge | discounted_cash must be positive number if present + const emptyDiscountedRow = { + ...basicRow, + "standard_charge | discounted_cash": "", + } + const emptyDiscountedResult = validateRow( + emptyDiscountedRow, + 15, + columns, + false + ) + t.is(emptyDiscountedResult.length, 0) + const wrongDiscountedRow = { + ...basicRow, + "standard_charge | discounted_cash": "300.25.1", + } + const wrongDiscountedResult = validateRow( + wrongDiscountedRow, + 16, + columns, + false + ) + t.is(wrongDiscountedResult.length, 1) + t.assert( + wrongDiscountedResult[0].message.includes( + '"standard_charge | discounted_cash" value "300.25.1" is not a valid positive number' + ) + ) + // standard_charge | min must be positive number if present + const emptyMinRow = { ...basicRow, "standard_charge | min": "" } + const emptyMinResult = validateRow(emptyMinRow, 17, columns, false) + t.is(emptyMinResult.length, 0) + const wrongMinRow = { + ...basicRow, + "standard_charge | min": "-5", + } + const wrongMinResult = validateRow(wrongMinRow, 18, columns, false) + t.is(wrongMinResult.length, 1) + t.assert( + wrongMinResult[0].message.includes( + '"standard_charge | min" value "-5" is not a valid positive number' + ) + ) + // standard_charge | max must be positive number if present + const emptyMaxRow = { ...basicRow, "standard_charge | max": "" } + const emptyMaxResult = validateRow(emptyMaxRow, 19, columns, false) + t.is(emptyMaxResult.length, 0) + const wrongMaxRow = { + ...basicRow, + "standard_charge | max": "-2", + } + const wrongMaxResult = validateRow(wrongMaxRow, 20, columns, false) + t.is(wrongMaxResult.length, 1) + t.assert( + wrongMaxResult[0].message.includes( + '"standard_charge | max" value "-2" is not a valid positive number' + ) + ) + // no code pairs is invalid + const noCodesRow = { ...basicRow, "code | 1": "", "code | 1 | type": "" } + const noCodesResult = validateRow(noCodesRow, 21, columns, false) + t.is(noCodesResult.length, 1) + t.assert( + noCodesResult[0].message.includes( + "At least one code and code type must be specified" + ) + ) + // a code pair not in the first column is valid + const secondCodeRow = { + ...basicRow, + "code | 1": "", + "code | 1 | type": "", + "code | 2": "234", + "code | 2 | type": "LOCAL", + } + const secondCodeResult = validateRow(secondCodeRow, 22, columns, false) + t.is(secondCodeResult.length, 0) + // a code without a code type is invalid + const noTypeRow = { ...basicRow, "code | 1 | type": "" } + const noTypeResult = validateRow(noTypeRow, 23, columns, false) + t.is(noTypeResult.length, 1) + t.assert( + noTypeResult[0].message.includes( + '"code | 1 | type" is required' // should this be a unique message instead? + ) + ) + // a code type without a code is invalid + const onlyTypeRow = { ...basicRow, "code | 1": "" } + const onlyTypeResult = validateRow(onlyTypeRow, 24, columns, false) + t.is(onlyTypeResult.length, 1) + t.assert( + onlyTypeResult[0].message.includes( + '"code | 1" is required' // should this be a unique message instead? + ) + ) + // a code type must be one of BILLING_CODE_TYPES + const wrongTypeRow = { ...basicRow, "code | 1 | type": "GUS" } + const wrongTypeResult = validateRow(wrongTypeRow, 25, columns, false) + t.is(wrongTypeResult.length, 1) + t.assert( + wrongTypeResult[0].message.includes( + '"code | 1 | type" value "GUS" is not one of the allowed values' + ) + ) +})