Add header columns for 2.0 CSV format

Update enumerations for 2.0 CSV format. Refactor cleanRowFields to cleanColumnNames so that it only needs to be called once per parse.
CMSgov · Jan 11, 2024 · 986c859 · 986c859
1 parent 1318c7f
commit 986c859
Show file tree

Hide file tree

Showing 6 changed files with 191 additions and 80 deletions.
diff --git a/src/csv.ts b/src/csv.ts
@@ -9,8 +9,8 @@ import {
   csvErrorToValidationError,
   rowIsEmpty,
   csvCellName,
-  cleanRowFields,
   objectFromKeysValues,
+  cleanColumnNames,
 } from "./versions/common/csv.js"
 import { CsvValidatorOneOne } from "./versions/1.1/csv.js"
 
@@ -82,7 +82,7 @@ export async function validateCsv(
     } else if (index === 1) {
       errors.push(...validator.validateHeader(headerColumns, row))
     } else if (index === 2) {
-      dataColumns = row
+      dataColumns = cleanColumnNames(row)
       errors.push(...validator.validateColumns(dataColumns))
       if (errors.length > 0) {
         resolve({
@@ -97,7 +97,7 @@ export async function validateCsv(
         tall = validator.isTall(dataColumns)
       }
     } else {
-      const cleanRow = cleanRowFields(objectFromKeysValues(dataColumns, row))
+      const cleanRow = objectFromKeysValues(dataColumns, row)
       errors.push(...validator.validateRow(cleanRow, index, dataColumns, !tall))
 
       if (options.onValueCallback) {

diff --git a/src/versions/2.0/csv.ts b/src/versions/2.0/csv.ts
@@ -4,6 +4,8 @@ import {
   sepColumnsEqual,
   parseSepField,
   getCodeCount,
+  isValidDate,
+  matchesString,
 } from "../common/csv.js"
 import {
   BILLING_CODE_TYPES,
@@ -18,38 +20,46 @@ import {
   DrugUnit,
 } from "./types"
 
+const ATTESTATION =
+  "To the best of its knowledge and belief, the hospital has included all applicable standard charge information in accordance with the requirements of 45 CFR 180.50, and the information encoded is true, accurate, and complete as of the date indicated."
+
+// headers must all be non-empty
 export const HEADER_COLUMNS = [
-  "hospital_name",
-  "last_updated_on",
-  "version",
-  "hospital_location",
-  "financial_aid_policy",
-  "license_number | state",
-]
+  "hospital_name", // string
+  "last_updated_on", // date
+  "version", // string - maybe one of the known versions?
+  "hospital_location", // string
+  "hospital_address", // string
+  "license_number | state", // string, check for valid postal code in header
+  ATTESTATION, // "true"
+] as const
 
 export const BASE_COLUMNS = [
-  "description",
-  "billing_class",
-  "setting",
-  "drug_unit_of_measurement",
-  "drug_type_of_measurement",
-  "modifiers",
-  "standard_charge | gross",
-  "standard_charge | discounted_cash",
+  "description", // non-empty string
+  "setting", // one of CHARGE_SETTINGS
+  "drug_unit_of_measurement", // positive number or blank
+  "drug_type_of_measurement", // one of DRUG_UNITS or blank
+  "modifiers", // string
+  "standard_charge | gross", // positive number or blank
+  "standard_charge | discounted_cash", // positive number or blank
+  "standard_charge | min", // positive number or blank
+  "standard_charge | max", // positive number or blank
+  "additional_generic_notes", // string
 ]
 
-export const MIN_MAX_COLUMNS = [
-  "standard_charge | min",
-  "standard_charge | max",
+export const OPTIONAL_COLUMNS = [
+  "financial_aid_policy", // string
+  "billing_class", // CHARGE_BILLING_CLASSES or blank
 ]
 
 export const TALL_COLUMNS = [
-  "payer_name",
-  "plan_name",
-  "standard_charge | negotiated_dollar",
-  "standard_charge | negotiated_percent",
-  "standard_charge | contracting_method",
-  "additional_generic_notes",
+  "payer_name", // string
+  "plan_name", // string
+  "standard_charge | negotiated_dollar", // positive number or blank
+  "standard_charge | negotiated_percentage", // positive number or blank
+  "standard_charge | negotiated_algorithm", // string
+  "standard_charge | methodology", // one of CONTRACTING_METHODS or blank
+  "estimated_amount", // positive number or blank
 ]
 
 const ERRORS = {
@@ -64,12 +74,12 @@ const ERRORS = {
     `Column is "${actual}" and should be "${expected}" for ${format} format`,
   COLUMN_MISSING: (column: string, format: string) =>
     `Column ${column} is missing, but it is required for ${format} format`,
-  NOTES_COLUMN: (column: string) =>
-    `The last column should be "additional_generic_notes", is "${column}"`,
   ALLOWED_VALUES: (column: string, value: string, allowedValues: string[]) =>
     `"${column}" value "${value}" is not one of the allowed values: ${allowedValues
       .map((t) => `"${t}"`)
       .join(", ")}`,
+  INVALID_DATE: (column: string, value: string) =>
+    `"${column}" value "${value}" is not a valid YYYY-MM-DD date`,
   INVALID_NUMBER: (column: string, value: string) =>
     `"${column}" value "${value}" is not a valid number`,
   POSITIVE_NUMBER: (column: string, suffix = ``) =>
@@ -143,10 +153,24 @@ export function validateHeaderRow(
 
   headers.forEach((header, index) => {
     if (header != null) {
-      if (!row[index]?.trim()) {
+      const value = row[index]?.trim() ?? ""
+      if (!value) {
         errors.push(
           csvErr(rowIndex, index, header, ERRORS.HEADER_COLUMN_BLANK(header))
         )
+      } else if (header === "last_updated_on" && !isValidDate(value)) {
+        errors.push(
+          csvErr(rowIndex, index, header, ERRORS.INVALID_DATE(header, value))
+        )
+      } else if (header === ATTESTATION && !matchesString(value, "true")) {
+        errors.push(
+          csvErr(
+            rowIndex,
+            index,
+            "ATTESTATION",
+            ERRORS.ALLOWED_VALUES("ATTESTATION", value, ["true"])
+          )
+        )
       }
     }
   })
@@ -435,18 +459,18 @@ export function validateTallFields(
 
   if (
     !CONTRACTING_METHODS.includes(
-      row["standard_charge | contracting_method"] as ContractingMethod
+      row["standard_charge | methodology"] as ContractingMethod
     )
   ) {
     errors.push(
       csvErr(
         index,
-        BASE_COLUMNS.indexOf("standard_charge | contracting_method"),
+        BASE_COLUMNS.indexOf("standard_charge | methodology"),
         // TODO: Change to constants
-        "standard_charge | contracting_method",
+        "standard_charge | methodology",
         ERRORS.ALLOWED_VALUES(
-          "standard_charge | contracting_method",
-          row["standard_charge | contracting_method"],
+          "standard_charge | methodology",
+          row["standard_charge | methodology"],
           CONTRACTING_METHODS as unknown as string[]
         )
       )
@@ -463,17 +487,7 @@ export function getBaseColumns(columns: string[]): string[] {
     .fill(0)
     .flatMap((_, i) => [`code | ${i + 1}`, `code | ${i + 1} | type`])
 
-  return [
-    "description",
-    ...codeColumns,
-    "billing_class",
-    "setting",
-    "drug_unit_of_measurement",
-    "drug_type_of_measurement",
-    "modifiers",
-    "standard_charge | gross",
-    "standard_charge | discounted_cash",
-  ]
+  return [...BASE_COLUMNS, ...codeColumns]
 }
 
 /** @private */
@@ -488,27 +502,13 @@ export function getWideColumns(columns: string[]): string[] {
     ])
     .map((c) => c.join(" | "))
 
-  return [
-    ...payersPlansColumns.slice(0, 2),
-    ...MIN_MAX_COLUMNS,
-    ...payersPlansColumns.slice(2),
-    "additional_generic_notes",
-  ]
+  return payersPlansColumns
 }
 
 /** @private */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 export function getTallColumns(columns: string[]): string[] {
-  return [
-    "payer_name",
-    "plan_name",
-    "standard_charge | negotiated_dollar",
-    "standard_charge | negotiated_percent",
-    "standard_charge | min",
-    "standard_charge | max",
-    "standard_charge | contracting_method",
-    "additional_generic_notes",
-  ]
+  return TALL_COLUMNS
 }
 
 function getPayersPlans(columns: string[]): string[][] {
@@ -518,7 +518,6 @@ function getPayersPlans(columns: string[]): string[][] {
     "max",
     "gross",
     "discounted_cash",
-    "contracting_method",
   ]
   return Array.from(
     new Set(

diff --git a/src/versions/2.0/types.ts b/src/versions/2.0/types.ts
@@ -2,6 +2,7 @@ export const BILLING_CODE_TYPES = [
   "CPT",
   "HCPCS",
   "ICD",
+  "DRG",
   "MS-DRG",
   "R-DRG",
   "S-DRG",
@@ -16,19 +17,24 @@ export const BILLING_CODE_TYPES = [
   "CDT",
   "RC",
   "CDM",
+  "TRIS-DRG",
 ] as const
 type BillingCodeTypeTuple = typeof BILLING_CODE_TYPES
 export type BillingCodeType = BillingCodeTypeTuple[number]
 
-export const DRUG_UNITS = ["GR", "ME", "ML", "UN"]
+export const DRUG_UNITS = ["GR", "ME", "ML", "UN", "F2", "EA", "GM"]
 type DrugUnitTuple = typeof DRUG_UNITS
 export type DrugUnit = DrugUnitTuple[number]
 
 export const CHARGE_SETTINGS = ["inpatient", "outpatient", "both"] as const
 type ChargeSettingTuple = typeof CHARGE_SETTINGS
 export type ChargeSetting = ChargeSettingTuple[number]
 
-export const CHARGE_BILLING_CLASSES = ["professional", "facility"] as const
+export const CHARGE_BILLING_CLASSES = [
+  "professional",
+  "facility",
+  "both",
+] as const
 type ChargeBillingClassTuple = typeof CHARGE_BILLING_CLASSES
 export type ChargeBillingClass = ChargeBillingClassTuple[number]
 

diff --git a/src/versions/common/csv.ts b/src/versions/common/csv.ts
@@ -22,19 +22,13 @@ export function csvErr(
   return { row, column, field, message, warning }
 }
 
-export function cleanRowFields(row: { [key: string]: string }): {
-  [key: string]: string
-} {
-  const newRow: { [key: string]: string } = {}
-  Object.entries(row).forEach(([key, value]: string[]) => {
-    newRow[
-      key
-        .split("|")
-        .map((v) => v.trim())
-        .join(" | ")
-    ] = value
-  })
-  return newRow
+export function cleanColumnNames(columns: string[]) {
+  return columns.map((col) =>
+    col
+      .split("|")
+      .map((v) => v.trim())
+      .join(" | ")
+  )
 }
 
 export function sepColumnsEqual(colA: string, colB: string) {
@@ -88,3 +82,35 @@ export function getCodeCount(columns: string[]): number {
       .filter((v) => !isNaN(v))
   )
 }
+
+export function isEmptyString(value: string) {
+  return value.trim().length === 0
+}
+
+export function isNonEmptyString(value: string) {
+  return value.trim().length > 0
+}
+
+export function isValidDate(value: string) {
+  // required format is YYYY-MM-DD
+  const match = value.match(/^(\d{4})-(\d{2})-(\d{2})$/)
+  if (match != null) {
+    // UTC methods are used because "date-only forms are interpreted as a UTC time",
+    // as per https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date#date_time_string_format
+    // check that the parsed date matches the input, to guard against e.g. February 31
+    const expectedYear = parseInt(match[1])
+    const expectedMonth = parseInt(match[2]) - 1
+    const expectedDate = parseInt(match[3])
+    const parsedDate = new Date(value)
+    return (
+      expectedYear === parsedDate.getUTCFullYear() &&
+      expectedMonth === parsedDate.getUTCMonth() &&
+      expectedDate === parsedDate.getUTCDate()
+    )
+  }
+  return false
+}
+
+export function matchesString(value: string, target: string) {
+  return value.trim().toLocaleUpperCase() === target.trim().toLocaleUpperCase()
+}
diff --git a/test/csv.spec.ts → test/1.1/csv.spec.ts b/test/csv.spec.ts → test/1.1/csv.spec.ts
@@ -12,8 +12,8 @@ import {
   MIN_MAX_COLUMNS,
   HEADER_COLUMNS,
   TALL_COLUMNS,
-} from "../src/versions/1.1/csv.js"
-import { CONTRACTING_METHODS } from "../src/versions/1.1/types.js"
+} from "../../src/versions/1.1/csv.js"
+import { CONTRACTING_METHODS } from "../../src/versions/1.1/types.js"
 
 const VALID_HEADER_COLUMNS = HEADER_COLUMNS.map((c) =>
   c === "license_number | state" ? "license_number | MD" : c