diff --git a/migrations/2020-12-01-1948-set-recorded-at.js b/migrations/2020-12-01-1948-set-recorded-at.js deleted file mode 100644 index 2b06f5f..0000000 --- a/migrations/2020-12-01-1948-set-recorded-at.js +++ /dev/null @@ -1,26 +0,0 @@ -const Track = require('../src/models/Track'); - -module.exports = { - async up(next) { - const query = Track.find().populate('trackData'); - for await (const track of query) { - if (!track.recordedAt) { - track.recordedAt = track.trackData.getRecoredAt(); - } - - await track.save(); - } - - next(); - }, - - async down(next) { - const query = Track.find(); - for await (const track of query) { - track.recordedAt = null; - await track.save(); - } - - next(); - }, -}; diff --git a/migrations/2020-12-01-1950-rebuild-track-data.js b/migrations/2020-12-01-1950-rebuild-track-data.js new file mode 100644 index 0000000..f1779f7 --- /dev/null +++ b/migrations/2020-12-01-1950-rebuild-track-data.js @@ -0,0 +1,15 @@ +const Track = require('../src/models/Track'); + +module.exports = { + async up(next) { + for await (const track of Track.find()) { + await track.rebuildTrackDataAndSave(); + } + + next(); + }, + + async down(next) { + next(); + }, +}; diff --git a/migrations/2020-12-01-1950-set-num-events.js b/migrations/2020-12-01-1950-set-num-events.js deleted file mode 100644 index 25eee15..0000000 --- a/migrations/2020-12-01-1950-set-num-events.js +++ /dev/null @@ -1,26 +0,0 @@ -const Track = require('../src/models/Track'); - -module.exports = { - async up(next) { - const query = Track.find().populate('trackData'); - for await (const track of query) { - if (!track.numEvents) { - track.numEvents = track.trackData.countEvents(); - } - - await track.save(); - } - - next(); - }, - - async down(next) { - const query = Track.find(); - for await (const track of query) { - track.numEvents = null; - await track.save(); - } - - next(); - }, -}; diff --git a/scripts/reconstruct-obsver1-body.js b/scripts/reconstruct-obsver1-body.js deleted file mode 100644 index 4d58ed1..0000000 --- a/scripts/reconstruct-obsver1-body.js +++ /dev/null @@ -1,79 +0,0 @@ -const mongoose = require('mongoose'); -const Track = require('../src/models/Track'); - -const { replaceDollarNewlinesHack, detectFormat, buildObsver1 } = require('../src/logic/tracks'); - -// connect to database -require('../src/db'); - -function shouldRebuildBody(track) { - if (!track.trackData || !track.trackData.points.length) { - return false; - } - - if (!track.body) { - return true; - } - const body = track.body.trim(); - if (!body) { - return true; - } - - const actualBody = replaceDollarNewlinesHack(body).trim(); - if (body !== actualBody) { - return true; - } - - const lineCount = (actualBody.match(/\n/g) || []).length + 1; - - const format = detectFormat(body); - if (format === 'invalid') { - return true; - } - - // never reconstruct body of version 2 - if (format > 1) { - return false; - } - - // not enough data in the file - if (lineCount < track.trackData.points.length + 1) { - return true; - } - - return false; -} - -async function main() { - const query = Track.find().populate('trackData'); - for await (const track of query) { - const rebuild = shouldRebuildBody(track); - if (rebuild) { - console.log('Rebuilding', track.title, 'with', track.trackData.points.length, 'data points.'); - - track.body = buildObsver1(track.trackData.points); - } - - if (!track.recordedAt) { - const firstPointWithDate = track.trackData.points.find((p) => p.date && p.time); - if (firstPointWithDate) { - const [day, month, year] = firstPointWithDate.date.split('.'); - const combinedString = `${year}-${month}-${day} ${firstPointWithDate.time}.000+2000`; - const parsedDate = new Date(combinedString); - if (!isNaN(parsedDate.getDate())) { - track.recordedAt = parsedDate; - } - } - } - - if (!track.numEvents) { - track.numEvents = track.trackData.countEvents(); - } - - await track.save(); - } -} - -main() - .catch((err) => console.error(err)) - .finally(() => mongoose.connection.close()); diff --git a/src/models/Track.js b/src/models/Track.js index 4f00222..e4e0131 100644 --- a/src/models/Track.js +++ b/src/models/Track.js @@ -2,6 +2,10 @@ const mongoose = require('mongoose'); const uniqueValidator = require('mongoose-unique-validator'); const slug = require('slug'); +const { parseTrackPoints } = require('../logic/tracks'); + +const TrackData = require('./TrackData'); + const schema = new mongoose.Schema( { slug: { type: String, lowercase: true, unique: true }, @@ -10,11 +14,10 @@ const schema = new mongoose.Schema( body: String, visible: Boolean, uploadedByUserAgent: String, - recordedAt: Date, - numEvents: { type: Number, default: 0 }, comments: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Comment' }], author: { type: mongoose.Schema.Types.ObjectId, ref: 'User' }, trackData: { type: mongoose.Schema.Types.ObjectId, ref: 'TrackData' }, + publicTrackData: { type: mongoose.Schema.Types.ObjectId, ref: 'TrackData' }, }, { timestamps: true }, ); @@ -50,25 +53,52 @@ class Track extends mongoose.Model { return false; } - toJSONFor(user, include) { - const seePrivateFields = user && user._id.equals(this.author._id); + /** + * Fills the trackData and publicTrackData with references to correct + * TrackData objects. For now, this is either the same, or publicTrackData + * is set to null, depending on the visibility of the track. At some point, + * this will include the anonymisation step, and produce a distinct TrackData + * object for the publicTrackData reference. + * + * Existing TrackData objects will be deleted by this function. + */ + async rebuildTrackDataAndSave() { + // clean up existing track data, we want to actually fully delete it + if (this.trackData) { + await TrackData.findByIdAndDelete(this.trackData); + } + + if (this.publicTrackData && this.publicTrackData.equals(this.trackData)) { + await TrackData.findByIdAndDelete(this.publicTrackData); + } + + // parse the points from the body + const points = Array.from(parseTrackPoints(this.body)); + const trackData = TrackData.createFromPoints(points); + await trackData.save(); + + this.trackData = trackData._id; + + if (this.visible) { + // TODO: create a distinct object with filtered data + this.publicTrackData = trackData._id; + } + + await this.save(); + } + + toJSONFor(user) { + const includePrivateFields = user && user._id.equals(this.author._id); + return { slug: this.slug, title: this.title, description: this.description, createdAt: this.createdAt, updatedAt: this.updatedAt, - visibleForAll: this.author ? this.author.areTracksVisibleForAll : false, visible: this.visible, author: this.author.toProfileJSONFor(user), - ...(include && include.body ? { body: this.body } : {}), - ...(seePrivateFields - ? { - uploadedByUserAgent: this.uploadedByUserAgent, - recordedAt: this.recordedAt, - numEvents: this.numEvents, - } - : {}), + ...(includePrivateFields ? { uploadedByUserAgent: this.uploadedByUserAgent } : {}), }; } } diff --git a/src/models/TrackData.js b/src/models/TrackData.js index a27fbb3..2120e8f 100644 --- a/src/models/TrackData.js +++ b/src/models/TrackData.js @@ -4,6 +4,8 @@ const uniqueValidator = require('mongoose-unique-validator'); const schema = new mongoose.Schema( { slug: { type: String, lowercase: true, unique: true }, + numEvents: { type: Number, default: 0 }, + recordedAt: { type: Date }, points: [ { date: String, @@ -55,6 +57,14 @@ class TrackData extends mongoose.Model { return parsedDate; } + + static createFromPoints(points) { + const trackData = new TrackData(); + trackData.points = points; + trackData.numEvents = trackData.countEvents(); + trackData.recordedAt = trackData.getRecoredAt(); + return trackData; + } } mongoose.model(TrackData, schema); diff --git a/src/routes/api/tracks.js b/src/routes/api/tracks.js index bf0d4dd..caea62d 100644 --- a/src/routes/api/tracks.js +++ b/src/routes/api/tracks.js @@ -7,7 +7,7 @@ const User = mongoose.model('User'); const busboy = require('connect-busboy'); const auth = require('../auth'); const currentTracks = new Map(); -const { parseTrackPoints, normalizeUserAgent } = require('../../logic/tracks'); +const { normalizeUserAgent } = require('../../logic/tracks'); const wrapRoute = require('../../_helpers/wrapRoute'); function preloadByParam(target, getValueFromParam) { @@ -179,8 +179,6 @@ router.post( const { body } = await getMultipartOrJsonBody(req, (body) => body.track); const track = new Track(body); - const trackData = new TrackData(); - track.trackData = trackData._id; track.author = req.user; if (track.body) { @@ -188,10 +186,10 @@ router.post( } if (track.body) { - trackData.points = Array.from(parseTrackPoints(track.body)); - track.numEvents = trackData.countEvents(); - track.recordedAt = trackData.getRecoredAt(); - track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']); + // delete existing + if (track.trackData) { + await TrackData.findByIdAndDelete(track.trackData); + } } if (body.visible != null) { @@ -200,7 +198,6 @@ router.post( track.visible = track.author.areTracksVisibleForAll; } - await trackData.save(); await track.save(); // console.log(track.author); @@ -213,13 +210,10 @@ router.post( auth.required, wrapRoute(async (req, res) => { const track = new Track(req.body.track); - const trackData = new TrackData(); - track.trackData = trackData._id; track.author = req.user; track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']); await track.save(); - await trackData.save(); // remember which is the actively building track for this user currentTracks.set(req.user.id, track._id); @@ -255,7 +249,6 @@ router.post( auth.required, wrapRoute(async (req, res) => { let track; - let trackData; if (currentTracks.has(req.user.id)) { // the file is less than 100 lines @@ -266,20 +259,11 @@ router.post( } track.body += req.body.track.body; - trackData = await TrackData.findById(track.trackData); } else { track = new Track(req.body.track); - trackData = new TrackData(); - track.trackData = trackData._id; - track.author = req.user; } - trackData.points = Array.from(parseTrackPoints(track.body)); - track.numEvents = trackData.countEvents(); - track.recordedAt = trackData.getRecoredAt(); - - await track.save(); - await trackData.save(); + await track.rebuildTrackDataAndSave(); // We are done with this track, it is complete. currentTracks.delete(req.user.id); @@ -297,7 +281,7 @@ router.get( return res.sendStatus(403); } - return res.json({ track: req.track.toJSONFor(req.user, { body: true }) }); + return res.json({ track: req.track.toJSONFor(req.user) }); }), ); @@ -307,44 +291,44 @@ router.put( busboy(), auth.required, wrapRoute(async (req, res) => { - if (!req.track.author._id.equals(req.user.id)) { + const track = req.track; + + if (!track.author._id.equals(req.user.id)) { return res.sendStatus(403); } const { body } = await getMultipartOrJsonBody(req, (body) => body.track); if (typeof body.title !== 'undefined') { - req.track.title = (body.title || '').trim() || null; + track.title = (body.title || '').trim() || null; } if (typeof body.description !== 'undefined') { - req.track.description = (body.description || '').trim() || null; + track.description = (body.description || '').trim() || null; } if (body.visible != null) { - req.track.visible = Boolean(body.visible); + track.visible = Boolean(body.visible); } - if (body.body && body.body.trim()) { - req.track.body = body.body.trim(); + let bodyChanged = false; - let trackData = await TrackData.findById(req.track.trackData); - if (!trackData) { - trackData = new TrackData(); - req.track.trackData = trackData._id; - } - trackData.points = Array.from(parseTrackPoints(req.track.body)); - req.track.numEvents = trackData.countEvents(); - req.track.recordedAt = trackData.getRecoredAt(); - req.track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']); - await trackData.save(); + if (body.body && body.body.trim()) { + track.body = body.body.trim(); + track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']); + bodyChanged = true; } if (typeof body.tagList !== 'undefined') { - req.track.tagList = body.tagList; + track.tagList = body.tagList; + } + + if (bodyChanged) { + await track.rebuildTrackDataAndSave(); + } else { + await track.save(); } - const track = await req.track.save(); return res.json({ track: track.toJSONFor(req.user) }); }), ); @@ -437,11 +421,9 @@ router.get( return res.sendStatus(403); } - // console.log("requestTrackData"+req.track); const trackData = await TrackData.findById(req.track.trackData); - // console.log({trackData: trackData}); - return res.json({ trackData: trackData }); + return res.json({ trackData }); }), );