pub-solar/obs-portal
5
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat: move recordedAt and numEvents to trackData, add publicTrackData property

Browse source
This commit is contained in:
Paul Bienkowski 2020-12-02 18:24:43 +01:00
parent 709b1a44cb
commit a198cb620b
7 changed files with 94 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
migrations/2020-12-01-1948-set-recorded-at.js
View file

@ -1,26 +0,0 @@
const Track = require('../src/models/Track');
module.exports = {
async up(next) {
const query = Track.find().populate('trackData');
for await (const track of query) {
if (!track.recordedAt) {
track.recordedAt = track.trackData.getRecoredAt();
}
await track.save();
}
next();
},
async down(next) {
const query = Track.find();
for await (const track of query) {
track.recordedAt = null;
await track.save();
}
next();
},
};

15
migrations/2020-12-01-1950-rebuild-track-data.js Normal file
View file

@ -0,0 +1,15 @@
const Track = require('../src/models/Track');
module.exports = {
async up(next) {
for await (const track of Track.find()) {
await track.rebuildTrackDataAndSave();
}
next();
},
async down(next) {
next();
},
};

26
migrations/2020-12-01-1950-set-num-events.js
View file

@ -1,26 +0,0 @@
const Track = require('../src/models/Track');
module.exports = {
async up(next) {
const query = Track.find().populate('trackData');
for await (const track of query) {
if (!track.numEvents) {
track.numEvents = track.trackData.countEvents();
}
await track.save();
}
next();
},
async down(next) {
const query = Track.find();
for await (const track of query) {
track.numEvents = null;
await track.save();
}
next();
},
};

79
scripts/reconstruct-obsver1-body.js
View file

@ -1,79 +0,0 @@
const mongoose = require('mongoose');
const Track = require('../src/models/Track');
const { replaceDollarNewlinesHack, detectFormat, buildObsver1 } = require('../src/logic/tracks');
// connect to database
require('../src/db');
function shouldRebuildBody(track) {
if (!track.trackData || !track.trackData.points.length) {
return false;
}
if (!track.body) {
return true;
}
const body = track.body.trim();
if (!body) {
return true;
}
const actualBody = replaceDollarNewlinesHack(body).trim();
if (body !== actualBody) {
return true;
}
const lineCount = (actualBody.match(/\n/g) || []).length + 1;
const format = detectFormat(body);
if (format === 'invalid') {
return true;
}
// never reconstruct body of version 2
if (format > 1) {
return false;
}
// not enough data in the file
if (lineCount < track.trackData.points.length + 1) {
return true;
}
return false;
}
async function main() {
const query = Track.find().populate('trackData');
for await (const track of query) {
const rebuild = shouldRebuildBody(track);
if (rebuild) {
console.log('Rebuilding', track.title, 'with', track.trackData.points.length, 'data points.');
track.body = buildObsver1(track.trackData.points);
}
if (!track.recordedAt) {
const firstPointWithDate = track.trackData.points.find((p) => p.date && p.time);
if (firstPointWithDate) {
const [day, month, year] = firstPointWithDate.date.split('.');
const combinedString = `${year}-${month}-${day} ${firstPointWithDate.time}.000+2000`;
const parsedDate = new Date(combinedString);
if (!isNaN(parsedDate.getDate())) {
track.recordedAt = parsedDate;
}
}
}
if (!track.numEvents) {
track.numEvents = track.trackData.countEvents();
}
await track.save();
}
}
main()
.catch((err) => console.error(err))
.finally(() => mongoose.connection.close());

56
src/models/Track.js
View file

@ -2,6 +2,10 @@ const mongoose = require('mongoose');
const uniqueValidator = require('mongoose-unique-validator'); const uniqueValidator = require('mongoose-unique-validator');
const slug = require('slug'); const slug = require('slug');
const { parseTrackPoints } = require('../logic/tracks');
const TrackData = require('./TrackData');
const schema = new mongoose.Schema( const schema = new mongoose.Schema(
{ {
slug: { type: String, lowercase: true, unique: true }, slug: { type: String, lowercase: true, unique: true },
@ -10,11 +14,10 @@ const schema = new mongoose.Schema(
body: String, body: String,
visible: Boolean, visible: Boolean,
uploadedByUserAgent: String, uploadedByUserAgent: String,
recordedAt: Date,
numEvents: { type: Number, default: 0 },
comments: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Comment' }], comments: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Comment' }],
author: { type: mongoose.Schema.Types.ObjectId, ref: 'User' }, author: { type: mongoose.Schema.Types.ObjectId, ref: 'User' },
trackData: { type: mongoose.Schema.Types.ObjectId, ref: 'TrackData' }, trackData: { type: mongoose.Schema.Types.ObjectId, ref: 'TrackData' },
publicTrackData: { type: mongoose.Schema.Types.ObjectId, ref: 'TrackData' },
}, },
{ timestamps: true }, { timestamps: true },
); );
@ -50,25 +53,52 @@ class Track extends mongoose.Model {
return false; return false;
} }
toJSONFor(user, include) { /**
const seePrivateFields = user && user._id.equals(this.author._id); * Fills the trackData and publicTrackData with references to correct
* TrackData objects. For now, this is either the same, or publicTrackData
* is set to null, depending on the visibility of the track. At some point,
* this will include the anonymisation step, and produce a distinct TrackData
* object for the publicTrackData reference.
*
* Existing TrackData objects will be deleted by this function.
*/
async rebuildTrackDataAndSave() {
// clean up existing track data, we want to actually fully delete it
if (this.trackData) {
await TrackData.findByIdAndDelete(this.trackData);
}
if (this.publicTrackData && this.publicTrackData.equals(this.trackData)) {
await TrackData.findByIdAndDelete(this.publicTrackData);
}
// parse the points from the body
const points = Array.from(parseTrackPoints(this.body));
const trackData = TrackData.createFromPoints(points);
await trackData.save();
this.trackData = trackData._id;
if (this.visible) {
// TODO: create a distinct object with filtered data
this.publicTrackData = trackData._id;
}
await this.save();
}
toJSONFor(user) {
const includePrivateFields = user && user._id.equals(this.author._id);
return { return {
slug: this.slug, slug: this.slug,
title: this.title, title: this.title,
description: this.description, description: this.description,
createdAt: this.createdAt, createdAt: this.createdAt,
updatedAt: this.updatedAt, updatedAt: this.updatedAt,
visibleForAll: this.author ? this.author.areTracksVisibleForAll : false,
visible: this.visible, visible: this.visible,
author: this.author.toProfileJSONFor(user), author: this.author.toProfileJSONFor(user),
...(include && include.body ? { body: this.body } : {}), ...(includePrivateFields ? { uploadedByUserAgent: this.uploadedByUserAgent } : {}),
...(seePrivateFields
? {
uploadedByUserAgent: this.uploadedByUserAgent,
recordedAt: this.recordedAt,
numEvents: this.numEvents,
}
: {}),
}; };
} }
} }

10
src/models/TrackData.js
View file

@ -4,6 +4,8 @@ const uniqueValidator = require('mongoose-unique-validator');
const schema = new mongoose.Schema( const schema = new mongoose.Schema(
{ {
slug: { type: String, lowercase: true, unique: true }, slug: { type: String, lowercase: true, unique: true },
numEvents: { type: Number, default: 0 },
recordedAt: { type: Date },
points: [ points: [
{ {
date: String, date: String,
@ -55,6 +57,14 @@ class TrackData extends mongoose.Model {
return parsedDate; return parsedDate;
} }
static createFromPoints(points) {
const trackData = new TrackData();
trackData.points = points;
trackData.numEvents = trackData.countEvents();
trackData.recordedAt = trackData.getRecoredAt();
return trackData;
}
} }
mongoose.model(TrackData, schema); mongoose.model(TrackData, schema);

70
src/routes/api/tracks.js
View file

@ -7,7 +7,7 @@ const User = mongoose.model('User');
const busboy = require('connect-busboy'); const busboy = require('connect-busboy');
const auth = require('../auth'); const auth = require('../auth');
const currentTracks = new Map(); const currentTracks = new Map();
const { parseTrackPoints, normalizeUserAgent } = require('../../logic/tracks'); const { normalizeUserAgent } = require('../../logic/tracks');
const wrapRoute = require('../../_helpers/wrapRoute'); const wrapRoute = require('../../_helpers/wrapRoute');
function preloadByParam(target, getValueFromParam) { function preloadByParam(target, getValueFromParam) {
@ -179,8 +179,6 @@ router.post(
const { body } = await getMultipartOrJsonBody(req, (body) => body.track); const { body } = await getMultipartOrJsonBody(req, (body) => body.track);
const track = new Track(body); const track = new Track(body);
const trackData = new TrackData();
track.trackData = trackData._id;
track.author = req.user; track.author = req.user;
if (track.body) { if (track.body) {
@ -188,10 +186,10 @@ router.post(
} }
if (track.body) { if (track.body) {
trackData.points = Array.from(parseTrackPoints(track.body)); // delete existing
track.numEvents = trackData.countEvents(); if (track.trackData) {
track.recordedAt = trackData.getRecoredAt(); await TrackData.findByIdAndDelete(track.trackData);
track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']); }
} }
if (body.visible != null) { if (body.visible != null) {
@ -200,7 +198,6 @@ router.post(
track.visible = track.author.areTracksVisibleForAll; track.visible = track.author.areTracksVisibleForAll;
} }
await trackData.save();
await track.save(); await track.save();
// console.log(track.author); // console.log(track.author);
@ -213,13 +210,10 @@ router.post(
auth.required, auth.required,
wrapRoute(async (req, res) => { wrapRoute(async (req, res) => {
const track = new Track(req.body.track); const track = new Track(req.body.track);
const trackData = new TrackData();
track.trackData = trackData._id;
track.author = req.user; track.author = req.user;
track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']); track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']);
await track.save(); await track.save();
await trackData.save();
// remember which is the actively building track for this user // remember which is the actively building track for this user
currentTracks.set(req.user.id, track._id); currentTracks.set(req.user.id, track._id);
@ -255,7 +249,6 @@ router.post(
auth.required, auth.required,
wrapRoute(async (req, res) => { wrapRoute(async (req, res) => {
let track; let track;
let trackData;
if (currentTracks.has(req.user.id)) { if (currentTracks.has(req.user.id)) {
// the file is less than 100 lines // the file is less than 100 lines
@ -266,20 +259,11 @@ router.post(
} }
track.body += req.body.track.body; track.body += req.body.track.body;
trackData = await TrackData.findById(track.trackData);
} else { } else {
track = new Track(req.body.track); track = new Track(req.body.track);
trackData = new TrackData();
track.trackData = trackData._id;
track.author = req.user;
} }
trackData.points = Array.from(parseTrackPoints(track.body)); await track.rebuildTrackDataAndSave();
track.numEvents = trackData.countEvents();
track.recordedAt = trackData.getRecoredAt();
await track.save();
await trackData.save();
// We are done with this track, it is complete. // We are done with this track, it is complete.
currentTracks.delete(req.user.id); currentTracks.delete(req.user.id);
@ -297,7 +281,7 @@ router.get(
return res.sendStatus(403); return res.sendStatus(403);
} }
return res.json({ track: req.track.toJSONFor(req.user, { body: true }) }); return res.json({ track: req.track.toJSONFor(req.user) });
}), }),
); );
@ -307,44 +291,44 @@ router.put(
busboy(), busboy(),
auth.required, auth.required,
wrapRoute(async (req, res) => { wrapRoute(async (req, res) => {
if (!req.track.author._id.equals(req.user.id)) { const track = req.track;
if (!track.author._id.equals(req.user.id)) {
return res.sendStatus(403); return res.sendStatus(403);
} }
const { body } = await getMultipartOrJsonBody(req, (body) => body.track); const { body } = await getMultipartOrJsonBody(req, (body) => body.track);
if (typeof body.title !== 'undefined') { if (typeof body.title !== 'undefined') {
req.track.title = (body.title || '').trim() || null; track.title = (body.title || '').trim() || null;
} }
if (typeof body.description !== 'undefined') { if (typeof body.description !== 'undefined') {
req.track.description = (body.description || '').trim() || null; track.description = (body.description || '').trim() || null;
} }
if (body.visible != null) { if (body.visible != null) {
req.track.visible = Boolean(body.visible); track.visible = Boolean(body.visible);
} }
if (body.body && body.body.trim()) { let bodyChanged = false;
req.track.body = body.body.trim();
let trackData = await TrackData.findById(req.track.trackData); if (body.body && body.body.trim()) {
if (!trackData) { track.body = body.body.trim();
trackData = new TrackData(); track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']);
req.track.trackData = trackData._id; bodyChanged = true;
}
trackData.points = Array.from(parseTrackPoints(req.track.body));
req.track.numEvents = trackData.countEvents();
req.track.recordedAt = trackData.getRecoredAt();
req.track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']);
await trackData.save();
} }
if (typeof body.tagList !== 'undefined') { if (typeof body.tagList !== 'undefined') {
req.track.tagList = body.tagList; track.tagList = body.tagList;
}
if (bodyChanged) {
await track.rebuildTrackDataAndSave();
} else {
await track.save();
} }
const track = await req.track.save();
return res.json({ track: track.toJSONFor(req.user) }); return res.json({ track: track.toJSONFor(req.user) });
}), }),
); );
@ -437,11 +421,9 @@ router.get(
return res.sendStatus(403); return res.sendStatus(403);
} }
// console.log("requestTrackData"+req.track);
const trackData = await TrackData.findById(req.track.trackData); const trackData = await TrackData.findById(req.track.trackData);
// console.log({trackData: trackData}); return res.json({ trackData });
return res.json({ trackData: trackData });
}), }),
); );