feat: add migrations, sort script for reconstructing body as one, and build recordedAt and numEvents in migrations and track routes, too

.migrations.js

@@ -0,0 +1,8 @@
+const isProduction = process.env.NODE_ENV === 'production';
+const mongodbUrl =
+  process.env.MONGODB_URL || (isProduction ? 'mongodb://localhost/obs' : 'mongodb://localhost/obsTest');
+
+module.exports = {
+    mongoose: 'src/db',
+    db: mongodbUrl,
+}

migrations/2020-12-01-1945-reconstruct-track-body.js

@@ -0,0 +1,61 @@
+const Track = require('../src/models/Track');
+const { replaceDollarNewlinesHack, detectFormat, buildObsver1 } = require('../src/logic/tracks');
+
+function shouldRebuildBody(track) {
+  if (!track.trackData || !track.trackData.points.length) {
+    return false;
+  }
+
+  if (!track.body) {
+    return true;
+  }
+  const body = track.body.trim();
+  if (!body) {
+    return true;
+  }
+
+  const actualBody = replaceDollarNewlinesHack(body).trim();
+  if (body !== actualBody) {
+    return true;
+  }
+
+  const lineCount = (actualBody.match(/\n/g) || []).length + 1;
+
+  const format = detectFormat(body);
+  if (format === 'invalid') {
+    return true;
+  }
+
+  // never reconstruct body of version 2
+  if (format > 1) {
+    return false;
+  }
+
+  // not enough data in the file
+  if (lineCount < track.trackData.points.length + 1) {
+    return true;
+  }
+
+  return false;
+}
+
+async function up(next) {
+  const query = Track.find().populate('trackData');
+  for await (const track of query) {
+    const rebuild = shouldRebuildBody(track);
+    if (rebuild) {
+      track.body = buildObsver1(track.trackData.points);
+    }
+
+    await track.save();
+  }
+
+  next();
+}
+
+async function down(next) {
+  // nothing to do
+  next();
+}
+
+module.exports = { up, down };

migrations/2020-12-01-1948-set-recorded-at.js

@@ -0,0 +1,26 @@
+const Track = require('../src/models/Track');
+
+module.exports = {
+  async up(next) {
+    const query = Track.find().populate('trackData');
+    for await (const track of query) {
+      if (!track.recordedAt) {
+        track.recordedAt = track.trackData.getRecoredAt();
+      }
+
+      await track.save();
+    }
+
+    next();
+  },
+
+  async down(next) {
+    const query = Track.find();
+    for await (const track of query) {
+      track.recordedAt = null;
+      await track.save();
+    }
+
+    next();
+  },
+};

migrations/2020-12-01-1950-set-num-events.js

@@ -0,0 +1,26 @@
+const Track = require('../src/models/Track');
+
+module.exports = {
+  async up(next) {
+    const query = Track.find().populate('trackData');
+    for await (const track of query) {
+      if (!track.numEvents) {
+        track.numEvents = track.trackData.countEvents();
+      }
+
+      await track.save();
+    }
+
+    next();
+  },
+
+  async down(next) {
+    const query = Track.find();
+    for await (const track of query) {
+      track.numEvents = null;
+      await track.save();
+    }
+
+    next();
+  },
+};

package-lock.json

@@ -1934,6 +1934,35 @@
         }
       }
     },
+    "cli": {
+      "version": "0.6.6",
+      "resolved": "https://registry.npmjs.org/cli/-/cli-0.6.6.tgz",
+      "integrity": "sha1-Aq1Eo4Cr8nraxebwzdewQ9dMU+M=",
+      "requires": {
+        "exit": "0.1.2",
+        "glob": "~ 3.2.1"
+      },
+      "dependencies": {
+        "glob": {
+          "version": "3.2.11",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.11.tgz",
+          "integrity": "sha1-Spc/Y1uRkPcV0QmH1cAP0oFevj0=",
+          "requires": {
+            "inherits": "2",
+            "minimatch": "0.3"
+          }
+        },
+        "minimatch": {
+          "version": "0.3.0",
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.3.0.tgz",
+          "integrity": "sha1-J12O2qxPG7MyZHIInnlJyDlGmd0=",
+          "requires": {
+            "lru-cache": "2",
+            "sigmund": "~1.0.0"
+          }
+        }
+      }
+    },
     "cli-boxes": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.0.tgz",
@@ -6013,6 +6042,11 @@
       "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==",
       "dev": true
     },
+    "lru-cache": {
+      "version": "2.7.3",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz",
+      "integrity": "sha1-bUUk6LlV+V1PW1iFHOId1y+06VI="
+    },
     "make-dir": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
@@ -6236,6 +6270,33 @@
         }
       }
     },
+    "mongoose-data-migrate": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/mongoose-data-migrate/-/mongoose-data-migrate-0.1.2.tgz",
+      "integrity": "sha1-0fyCVnY5r786Fuz0J3kzsAokwxY=",
+      "requires": {
+        "cli": "^0.6.5",
+        "debug": "^2.1.1",
+        "lodash": "^3.1.0",
+        "mkdirp": "^0.5.0",
+        "q": "^1.1.2"
+      },
+      "dependencies": {
+        "lodash": {
+          "version": "3.10.1",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz",
+          "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y="
+        },
+        "mkdirp": {
+          "version": "0.5.5",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+          "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+          "requires": {
+            "minimist": "^1.2.5"
+          }
+        }
+      }
+    },
     "mongoose-legacy-pluralize": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/mongoose-legacy-pluralize/-/mongoose-legacy-pluralize-1.0.2.tgz",
@@ -8697,6 +8758,11 @@
         "escape-goat": "^2.0.0"
       }
     },
+    "q": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
+      "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
+    },
     "qs": {
       "version": "6.7.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
@@ -9313,6 +9379,11 @@
       "resolved": "https://registry.npmjs.org/sift/-/sift-7.0.1.tgz",
       "integrity": "sha512-oqD7PMJ+uO6jV9EQCl0LrRw1OwsiPsiFQR5AR30heR+4Dl7jBBbDLnNvWiak20tzZlSE1H7RB30SX/1j/YYT7g=="
     },
+    "sigmund": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz",
+      "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA="
+    },
     "signal-exit": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",

package.json

@@ -10,7 +10,10 @@
     "mongo:stop": "docker stop realworld-mongo && docker rm realworld-mongo",
     "autoformat": "eslint --fix .",
     "lint": "eslint .",
-    "test": "jest"
+    "test": "jest",
+    "migrate": "mongoose-data-migrate -c .migrations.js",
+    "migrate:up": "npm run migrate -- up",
+    "migrate:down": "npm run migrate -- down"
   },
   "repository": {
     "type": "git",
@@ -35,6 +38,7 @@
     "method-override": "3.0.0",
     "methods": "1.1.2",
     "mongoose": "^5.10.7",
+    "mongoose-data-migrate": "^0.1.2",
     "mongoose-unique-validator": "2.0.3",
     "morgan": "1.10.0",
     "nodemailer": "^6.4.14",

scripts/reconstruct-obsver1-body.js

@@ -67,7 +67,7 @@ async function main() {
     }
 
     if (!track.numEvents) {
-      track.numEvents = track.trackData.points.filter((p) => p.flag).length;
+      track.numEvents = track.trackData.countEvents();
     }
 
     await track.save();

src/db.js

@@ -11,3 +11,5 @@ require('./models/User');
 require('./models/Track');
 require('./models/Comment');
 require('./config/passport');
+
+module.exports = mongoose;

src/models/TrackData.js

@@ -35,6 +35,26 @@ class TrackData extends mongoose.Model {
   slugify() {
     this.slug = 'td-' + String((Math.random() * Math.pow(36, 6)) | 0).toString(36);
   }
+
+  countEvents() {
+    return this.points.filter((p) => p.flag).length;
+  }
+
+  getRecoredAt() {
+    const firstPointWithDate = this.points.find((p) => p.date && p.time);
+    if (!firstPointWithDate) {
+      return null;
+    }
+
+    const [day, month, year] = firstPointWithDate.date.split('.');
+    const combinedString = `${year}-${month}-${day} ${firstPointWithDate.time}.000+2000`;
+    const parsedDate = new Date(combinedString);
+    if (isNaN(parsedDate.getDate())) {
+      return null;
+    }
+
+    return parsedDate;
+  }
 }
 
 mongoose.model(TrackData, schema);

src/routes/api/tracks.js

@@ -189,6 +189,8 @@ router.post(
 
     if (track.body) {
       trackData.points = Array.from(parseTrackPoints(track.body));
+      track.numEvents = trackData.countEvents();
+      track.recordedAt = trackData.getRecoredAt();
       track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']);
     }
 
@@ -273,6 +275,8 @@ router.post(
     }
 
     trackData.points = Array.from(parseTrackPoints(track.body));
+    track.numEvents = trackData.countEvents();
+    track.recordedAt = trackData.getRecoredAt();
 
     await track.save();
     await trackData.save();
@@ -330,6 +334,8 @@ router.put(
         req.track.trackData = trackData._id;
       }
       trackData.points = Array.from(parseTrackPoints(req.track.body));
+      req.track.numEvents = trackData.countEvents();
+      req.track.recordedAt = trackData.getRecoredAt();
       req.track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']);
       await trackData.save();
     }