Merge branch 'master' of https://github.com/medialab/toflit18

medialab · Nov 12, 2019 · 4323f9c · 4323f9c
2 parents 0549c4e + 768ba0d
commit 4323f9c
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 4 deletions.
diff --git a/client/docker-nginx-vhost.template b/client/docker-nginx-vhost.template
@@ -15,6 +15,7 @@ server {
     location /api/ {
         rewrite ^/api/(.*) /$1  break;
         proxy_pass $api$uri$is_args$args;
+        proxy_read_timeout 190s;
     }
 
 ### Static HTML5/JS

diff --git a/scripts/import.js b/scripts/import.js
@@ -316,7 +316,11 @@ function importer(csvLine) {
       nodeData.year = normalizeYear(csvLine.year);
     }
   }
-
+  if (isNaN(nodeData.year)) {
+    console.error(`${csvLine.year} is not a year`);
+    console.error(csvLine);
+    return;
+  }
   // Unit
   if (csvLine.quantity_unit) {
     nodeData.rawUnit = csvLine.quantity_unit;
@@ -531,8 +535,9 @@ function makeClassificationConsumer(groupIndex, classificationNode, parentNode,
           item = line[itemKey];
 
     // Dropping empty values
-    if (!group || !item)
+    if (!group || !item || group.trim().toLowerCase() === '[vide]' || item.trim().toLowerCase() === '[vide]') {
       return;
+    }
 
     const itemNode = itemIndex[item];
 
@@ -591,6 +596,8 @@ async.series({
     parseCsv(csvDirections, {delimiter: ','}, function(err, data) {
 
       data.forEach(row => {
+        if (_.some(row, e => e.trim().toLowerCase() === '[vide]'))
+          return;
         const sourceDirection = cleanText(row[0]),
               targetDirection = cleanText(row[1]);
 

diff --git a/scripts/quantities.js b/scripts/quantities.js
@@ -15,6 +15,7 @@ import FuzzyMap from 'mnemonist/fuzzy-map'
 import {parse as parseCSV} from 'csv';
 import database from '../api/connection';
 import {cleanText, cleanNumber} from '../lib/clean';
+import {some, values} from 'lodash';
 
 /**
  * Reading arguments.
@@ -123,6 +124,10 @@ async.series({
             return callback(err);
 
           lines.forEach(line => {
+
+            if (some(values(line), e => e.trim().toLowerCase() === '[vide]'))
+              return;
+
             const data = {
               name: cleanText(line.quantity_unit),
               ortho: cleanText(line.quantity_unit_ortho)
@@ -147,7 +152,7 @@ async.series({
           lines.forEach(line => {
 
             // Filtering empty lines
-            if (!line.u_conv || !line.q_conv)
+            if (!line.u_conv || !line.q_conv || some(values(line), e => e.trim().toLowerCase() === '[vide]'))
               return;
 
             const data = {
@@ -179,7 +184,7 @@ async.series({
           lines.forEach(line => {
 
             // Filtering empty lines
-            if (!line.u_conv || !line.q_conv)
+            if (!line.u_conv || !line.q_conv || some(values(line), e => e.trim().toLowerCase() === '[vide]'))
               return;
 
             const data = {