From 16ca8db8c75a7f31806202b103dbfe4eab726fc9 Mon Sep 17 00:00:00 2001
From: Paul Bienkowski <pb@opatut.de>
Date: Sat, 21 Nov 2020 13:26:52 +0100
Subject: [PATCH] feat: properly parse data types in OBSVER1

---
 logic/tracks.js | 55 +++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/logic/tracks.js b/logic/tracks.js
index 43a4b88..d77cf1e 100644
--- a/logic/tracks.js
+++ b/logic/tracks.js
@@ -1,16 +1,57 @@
 const csvParse = require('csv-parse/lib/sync');
 
 function _parseFloat(token) {
+  if (typeof token !== 'string') {
+    return null
+  }
+
+  token = token.trim()
+
+  if (token === '') {
+    return null
+  }
+
+  if (/^nan$/i.test(token)) {
+    return null
+  }
+
   let f = parseFloat(token);
+
   if (isNaN(f)) {
     f = parseFloat(token.substring(0, 10));
   }
+
   if (isNaN(f)) {
     f = 0.0;
   }
+
   return f;
 }
 
+function _parseInt(token) {
+  const asFloat = parseFloat(token)
+  if (asFloat !== null) {
+    return Math.floor(asFloat)
+  } else{
+    return asFloat
+  }
+}
+
+function _parseString(token) {
+  if (typeof token !== 'string') {
+    return null
+  }
+  // This time we do not trim -- because we assume that the quoting mechanism
+  // from CSV might have kicked in and we actually want the spacing around the
+  // token.
+
+  if (token === '') {
+    return null
+  }
+
+  return token
+}
+
 function replaceDollarNewlinesHack(body) {
   // see if we are using the hack with $ as newlines, replace them for the csv parser
   if (body.endsWith('$')) {
@@ -87,13 +128,23 @@ function* parseObsver1(body) {
   for (const record of csvParse(body, {
     delimiter: ';',
     encoding: 'utf8',
+    // We specify  different column names here, as the order of columns was
+    // always the same, but their naming was different. By enforicing these
+    // column names we don't have to translate between them. Then we just
+    // ignore the first line (or any line that starts with "Date;").
+    // Original header usually is:
+    // Date;Time;Latitude;Longitude;Course;Speed;Right;Left;Confirmed;insidePrivacyArea
     columns: ['date', 'time', 'latitude', 'longitude', 'course', 'speed', 'd1', 'd2', 'flag', 'private'],
     relax_column_count: true,
     cast(value, { column }) {
-      if (/latitude|longitude|course|speed/.test(column)) {
+      if (['latitude', 'longitude', 'course', 'speed'].includes(column)) {
         return _parseFloat(value);
+      } else if (['d1', 'd2', 'flag'].includes(column)) {
+        return _parseInt(value);
+      } else if (column === 'private') {
+        return Boolean(_parseInt(value));
       } else {
-        return value;
+        return _parseString(value);
       }
     },
   })) {