Improve logger colors and usability

Squashed from the following commits from ngfreiter:

Differentiate colors for messages

Switch default to showing duplicate errors

Match total count colors to warning/error colors

Suppressing error output after 10 repeats.

Adding strip-ansi

Clarifying error counting

PrettyPrintDuplexStreamJson.js

@@ -3,20 +3,23 @@ const Transform = require('stream').Transform;
 const fs = require('fs');
 const path = require('path');
 const chalk = require('chalk');   //library for colorizing Strings
+const stripAnsi = require('strip-ansi');
 const { nameFromLevel } = require('bunyan');
 // color palette for messages -- can also do rgb; e.g.  chalk.rgb(123, 45, 67)
 const originalErrorColor = chalk.bold.greenBright;
 const errorDetailColor = chalk.bold.cyan;
-const errorCodeColor = chalk.bold.redBright;
+let codeColor = chalk.bold.redBright;
 const noErrorCode = '-1';
+const deduplicationNumber = 10;
 
 //https://stackoverflow.com/questions/48507828/pipe-issue-with-node-js-duplex-stream-example
 
 // make a class that implements a Duplex stream; this means you can use it to pipe into and out of
 class PrettyPrintDuplexStreamJson extends Transform {
-  constructor(options, errorFiles, showDuplicateErrors) {
+  constructor(options, errorFiles, showDuplicateErrors, outFile) {
     super(options);
     this.showDuplicateErrors = showDuplicateErrors;
+    this.outFile = outFile;
     this.solutionMap = {};
     this.ruleMap = {};
     this.templateStrings = {};
@@ -29,6 +32,9 @@ class PrettyPrintDuplexStreamJson extends Transform {
     // populate a set with the key as the ERROR_CODE number and the value is the suggested solution
     this.idSet = new Set();
     this.idSet.add( noErrorCode );
+    fs.writeFileSync(this.outFile, '');
+    // Need a way to track how many times an error has appeared
+    this.errorCount = {};
   }
 
 
@@ -176,8 +182,27 @@ class PrettyPrintDuplexStreamJson extends Transform {
     const mappingRulePart = this.getAttributeOrEmptyString( myJson.mappingRule ); //grab mappingRule
     const targetPart = this.getAttributeOrEmptyString( myJson.target );          //grab targetPart
     const targetSpecPart = this.getAttributeOrEmptyString( myJson.targetSpec );  //grab targetSpec
+
+    switch (level) {
+      case 'FATAL':
+      case 'ERROR':
+        codeColor = chalk.bold.redBright;
+        break;
+      case 'WARN':
+        codeColor = chalk.bold.yellowBright;
+        break;
+      case 'INFO':
+        codeColor = chalk.bold.whiteBright;
+        break;
+      case 'DEBUG':
+        codeColor = chalk.bold.greenBright;
+        break;
+      default:
+        codeColor = chalk.bold.whiteBright;
+    }
+
     // now we have pieces; assemble the pieces into a formatted, colorized, multi-line message
-    let outline =  errorCodeColor('\n' + level + ' ' + eCode + ': ' + detailMsg );  // first part of new message is ERROR xxxxx: <<detail>>
+    let outline =  codeColor('\n' + level + ' ' + eCode + ': ' + detailMsg );  // first part of new message is ERROR xxxxx: <<detail>>
     outline +=   errorDetailColor ( '\n    During:         ' + this.translateNames( modulePart));
     // if parts are optional/missing, then only print them if they are found
     if (shrIdPart !== '') {
@@ -198,28 +223,41 @@ class PrettyPrintDuplexStreamJson extends Transform {
       suggestedFix = suggestedFix.replace(/'/g, '').trim() ;
       // only print suggested fix if it's available from the resource file (i.e. in the solutionMap)
       if (suggestedFix !== 'Unknown' && suggestedFix !== '') {
-        outline += errorDetailColor(   '\n    Suggested Fix:  ' + suggestedFix.trim() + '\n'  );
+        outline += errorDetailColor(   '\n    Suggested Fix:  ' + suggestedFix.trim());
       }
     }
     const myDedupHashKey = this.buildHashKey( eCode, this.ruleMap,  myJson ) ;
 
-    if (myDedupHashKey === '' || this.showDuplicateErrors) { // if you have no keys for deduplication in errorMessages.txt for thisd, print everything
-      return outline ;
+    if (myDedupHashKey === '' || this.showDuplicateErrors || printAllErrors) { // if you have no keys for deduplication in errorMessages.txt for this, print everything
+      return [outline, outline];
     }
     else if (this.idSet.has(myDedupHashKey)) {
-      return '';
+      let consoleOutput;
+      // If more than deduplicationNumber errors of the same code are logged, start suppressing
+      if (this.errorCount[myDedupHashKey] === deduplicationNumber) {
+        consoleOutput = '\nOver ' + deduplicationNumber + ' logs with code ' + eCode + '. Suppressing further output of this code. Full logs can be found in out.log.';
+        consoleOutput = chalk.bold.magentaBright(consoleOutput);
+      } else if (this.errorCount[myDedupHashKey] > deduplicationNumber) {
+        consoleOutput = '';
+      } else {
+        consoleOutput = outline;
+      }
+      this.errorCount[myDedupHashKey] += 1;
+      return [consoleOutput, outline];
+
     }
     else {
       this.idSet.add(myDedupHashKey);
-
-      return outline ;
+      this.errorCount[myDedupHashKey] = 1;
+      return [outline, outline];
     }
   }
 
   _transform(chunk, encoding, callback) {
-    const ans = this.processLine(chunk, false);
-    if (ans.length > 0) {
-      console.log( ans );
+    const [consoleOut, fileOut] = this.processLine(chunk, false);
+    fs.appendFileSync(this.outFile, stripAnsi(fileOut));
+    if (consoleOut.length > 0) {
+      console.log(consoleOut);
     }
     callback();
   }

app.js

@@ -3,6 +3,7 @@ const path = require('path');
 const mkdirp = require('mkdirp');
 const bunyan = require('bunyan');
 const program = require('commander');
+const chalk = require('chalk');
 const { sanityCheckModules } = require('shr-models');
 const shrTI = require('shr-text-import');
 const shrEx = require('shr-expand');
@@ -34,7 +35,7 @@ program
   .option('-s, --skip <feature>', 'skip an export feature <fhir,cimcore,json-schema,model-doc,data-dict,all>', collect, [])
   .option('-o, --out <out>', `the path to the output folder`, path.join('.', 'out'))
   .option('-c, --config <config>', 'the name of the config file', 'config.json')
-  .option('-d, --duplicate', 'show duplicate error messages (default: false)')
+  .option('-d, --deduplicate', 'do not show duplicate error messages (default: false)')
   .option('-j, --export-es6', 'export ES6 JavaScript classes (experimental, default: false)')
   .option('-i, --import-cimcore', 'import CIMCORE files instead of CIMPL (default: false)')
   .option('-n, --clean', 'Save archive of old output directory and perform clean build (default: false)')
@@ -58,7 +59,7 @@ const doDD = program.skip.every(a => a.toLowerCase() != 'data-dict' && a.toLower
 
 // Process the de-duplicate error flag
 
-const showDuplicateErrors = program.duplicate;
+const showDuplicateErrors = !program.deduplicate;
 const importCimcore = program.importCimcore;
 const doES6 = program.exportEs6;
 const clean = program.clean;
@@ -94,7 +95,7 @@ const errorFiles = [shrTI.errorFilePath(), shrEx.errorFilePath(), shrFE.errorFil
   shrEE.errorFilePath(), shrJSE.errorFilePath(), path.join(__dirname, "errorMessages.txt")]
 
 const PrettyPrintDuplexStreamJson = require('./PrettyPrintDuplexStreamJson');
-const mdpStream = new PrettyPrintDuplexStreamJson(null, errorFiles, showDuplicateErrors);
+const mdpStream = new PrettyPrintDuplexStreamJson(null, errorFiles, showDuplicateErrors, path.join(program.out, 'out.log'));
 
 // Set up the logger streams
 const [ll, lm] = [program.logLevel.toLowerCase(), program.logMode.toLowerCase()];
@@ -109,7 +110,6 @@ if (lm == 'normal') {
 const logCounter = new LogCounter();
 streams.push({ level: 'warn', type: 'raw', stream: logCounter});
 // Always do a full JSON log
-streams.push({ level: 'trace', path: path.join(program.out, 'out.log') });
 const logger = bunyan.createLogger({
   name: 'shr',
   module: 'shr-cli',
@@ -142,15 +142,13 @@ let configSpecifications = shrTI.importConfigFromFilePath(input, program.config)
 if (!configSpecifications) {
   process.exit(1);
 }
-configSpecifications.showDuplicateErrors = showDuplicateErrors;
 let specifications;
 let expSpecifications;
 if (!importCimcore) {
   specifications = shrTI.importFromFilePath(input, configSpecifications);
   expSpecifications = shrEx.expand(specifications, shrFE);
 } else {
   [configSpecifications, expSpecifications] = shrTI.importCIMCOREFromFilePath(input);
-  configSpecifications.showDuplicateErrors = showDuplicateErrors;
 }
 
 
@@ -440,24 +438,22 @@ logger.info('05002');
 const ftlCounter = logCounter.fatal;
 const errCounter = logCounter.error;
 const wrnCounter = logCounter.warn;
-let [errColor, errLabel, wrnColor, wrnLabel, resetColor, ftlColor, ftlLabel] = ['\x1b[32m', 'errors', '\x1b[32m', 'warnings', '\x1b[0m', '\x1b[31m', 'fatal errors'];
+let [errLabel, wrnLabel, ftlLabel] = ['errors', 'warnings', 'fatal errors'];
 if (ftlCounter.count > 0) {
   ftlLabel = `fatal errors (${failedExports.join(', ')})`;
 }
 if (errCounter.count > 0) {
-  errColor = '\x1b[31m'; // red
   errLabel = `errors (${errCounter.modules.join(', ')})`;
 }
 if (wrnCounter.count > 0) {
-  wrnColor = '\x1b[35m'; // magenta
   wrnLabel = `warnings (${wrnCounter.modules.join(', ')})`;
 }
 
 // Get the elapsed time
 const hrend = process.hrtime(hrstart);
 console.log('------------------------------------------------------------');
 console.log('Elapsed time: %d.%ds', hrend[0], Math.floor(hrend[1]/1000000));
-if (ftlCounter.count > 0) console.log('%s%d %s%s', ftlColor, ftlCounter.count, ftlLabel, resetColor);
-console.log('%s%d %s%s', errColor, errCounter.count, errLabel, resetColor);
-console.log('%s%d %s%s', wrnColor, wrnCounter.count, wrnLabel, resetColor);
+if (ftlCounter.count > 0) console.log(chalk.redBright('%d %s'), ftlCounter.count, ftlLabel);
+console.log(chalk.bold.redBright('%d %s'), errCounter.count, errLabel);
+console.log(chalk.bold.yellowBright('%d %s'), wrnCounter.count, wrnLabel);
 console.log('------------------------------------------------------------');

package.json

@@ -28,7 +28,8 @@
     "shr-json-javadoc": "^6.2.0",
     "shr-json-schema-export": "^6.1.0",
     "shr-models": "^6.5.0",
-    "shr-text-import": "^6.5.0"
+    "shr-text-import": "^6.5.0",
+    "strip-ansi": "^5.2.0"
   },
   "devDependencies": {
     "eslint": "^4.6.1",

yarn.lock

@@ -46,6 +46,11 @@ ansi-regex@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998"
 
+ansi-regex@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997"
+  integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==
+
 ansi-styles@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
@@ -1374,6 +1379,13 @@ strip-ansi@^4.0.0:
   dependencies:
     ansi-regex "^3.0.0"
 
+strip-ansi@^5.2.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae"
+  integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==
+  dependencies:
+    ansi-regex "^4.1.0"
+
 strip-eof@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf"