This commit is contained in:
Paul Bienkowski 2020-11-18 20:53:35 +01:00
parent d8975db15a
commit 22f8c46d6a
6 changed files with 927 additions and 327 deletions

File diff suppressed because one or more lines are too long

View file

@ -1,3 +1,5 @@
const csvParse = require('csv-parse/lib/sync');
function _parseFloat(token) {
let f = parseFloat(token);
if (isNaN(f)) {
@ -9,13 +11,55 @@ function _parseFloat(token) {
return f;
}
module.exports.addPointsToTrack = function addPointsToTrack(track, body) {
function addPointsToTrack(trackInfo, body, format = null) {
const detectedFormat = format != null ? format : detectFormat(body);
let parser;
switch (detectedFormat) {
case 'invalid':
throw new Error('track format cannot be detected');
case 1:
parser = parseObsver1;
break;
case 2:
parser = parseObsver2;
break;
}
const points = trackInfo.trackData.points;
for (const newPoint of parser(body)) {
points.push(newPoint);
}
}
function detectFormat(body) {
if (!body.length) {
return 'invalid';
}
const firstLinebreakIndex = body.indexOf('\n');
if (firstLinebreakIndex === -1) {
return 1;
}
const firstLine = body.substring(0, firstLinebreakIndex);
const match = firstLine.match(/(^|&)OBSDataFormat=([\d]+)($|&)/);
if (match) {
return Number(match[2]);
}
return 'invalid';
}
function* parseObsver1(body) {
let num = 0;
let start = 0;
let end = 0;
// reference to the array we will mutate
const points = track.trackData.points;
let currentPoint;
while (end < body.length) {
@ -24,6 +68,9 @@ module.exports.addPointsToTrack = function addPointsToTrack(track, body) {
end++;
}
if (body[end] === '$') {
if (currentPoint) {
yield currentPoint;
}
// $ is replacing \n as newlines are not allowed in json strings
num = 0;
}
@ -46,7 +93,7 @@ module.exports.addPointsToTrack = function addPointsToTrack(track, body) {
switch (num) {
case 0:
currentPoint = {
date: 'dummy',
date: token,
time: '',
latitude: '',
longitude: '',
@ -57,10 +104,6 @@ module.exports.addPointsToTrack = function addPointsToTrack(track, body) {
flag: '',
private: '',
};
points.push(currentPoint);
currentPoint.date = token;
break;
case 1:
@ -104,4 +147,88 @@ module.exports.addPointsToTrack = function addPointsToTrack(track, body) {
}
}
}
if (currentPoint) {
yield currentPoint;
}
}
function* parseObsver2(body) {
for (const record of csvParse(body, {
from_line: 2,
trim: true,
columns: true,
skip_empty_lines: true,
delimiter: ';',
encoding: 'utf8',
relax_column_count: true,
cast(value, context) {
if (value === '') {
return null;
}
let type;
switch (context.column) {
case 'Millis':
case 'Left':
case 'Right':
case 'Confirmed':
case 'Invalid':
case 'InsidePrivacyArea':
case 'Measurements':
case 'Satellites':
type = 'int';
break;
case 'Date':
case 'Time':
case 'Comment':
case 'Marked':
type = 'string';
break;
case 'Latitude':
case 'Longitude':
case 'Altitude':
case 'Course':
case 'Speed':
case 'HDOP':
case 'BatteryLevel':
case 'Factor':
type = 'float';
break;
default:
type = /^(Tms|Lus|Rus)/.test(context.column) ? 'int' : 'string';
}
switch (type) {
case 'int':
return parseInt(value);
case 'float':
return parseFloat(value);
case 'string':
return value;
}
},
})) {
// We convert the new format back to the old format for storage here, until
// we upgrade the storage format as well to include all data. But we'll
// have to upgrade the obsApp first.
yield {
date: record.Date,
time: record.Time,
latitude: record.Latitude,
longitude: record.Longitude,
course: record.Course,
speed: record.Speed,
d1: record.Left,
d2: record.Right,
flag: Boolean(record.Confirmed),
private: Boolean(record.InsidePrivacyArea),
};
}
}
module.exports = { addPointsToTrack, detectFormat, parseObsver1, parseObsver2 };

View file

@ -1,7 +1,7 @@
const { addPointsToTrack } = require('./tracks');
const { addPointsToTrack, parseObsver1, detectFormat, parseObsver2 } = require('./tracks');
const TrackInfo = require('./TrackInfo');
const { test1 } = require('./_tracks_testdata');
const { test1, test2 } = require('./_tracks_testdata');
describe('addPointsToTrack', () => {
it('is a function', () => {
@ -27,3 +27,70 @@ describe('addPointsToTrack', () => {
});
});
});
describe('parseObsver1', () => {
it('can parse sample data', () => {
const points = Array.from(parseObsver1(test1));
expect(points).toHaveLength(324);
expect(points[0]).toEqual({
date: '12.07.2020',
time: '09:02:59',
latitude: 0,
longitude: 0,
course: 0,
speed: 0,
d1: '255',
d2: '255',
flag: '0',
private: '0',
});
});
});
describe('parseObsver2', () => {
it('can parse sample data', () => {
const points = Array.from(parseObsver2(test2));
expect(points).toHaveLength(18);
expect(points[0]).toEqual({
date: '18.11.2020',
time: '16:05:59',
latitude: 48.723224,
longitude: 9.094103,
course: 189.86,
speed: 3.2,
d1: 770,
d2: null,
flag: false,
private: true,
});
// this is a non-private, flagged point (i.e. "Confirmed" overtaking)
expect(points[17]).toEqual({
date: '18.11.2020',
time: '16:06:16',
latitude: 48.723109,
longitude: 9.093963,
course: 247.62,
speed: 0,
d1: 5,
d2: 89,
flag: true,
private: false,
});
});
});
describe('detectFormat', () => {
it('detects format 1', () => {
expect(detectFormat(test1)).toBe(1);
});
it('detects format 2', () => {
expect(detectFormat(test2)).toBe(2);
});
it('detects invalid format', () => {
expect(detectFormat('foobar\nbaz')).toBe('invalid');
expect(detectFormat('')).toBe('invalid');
});
});

13
package-lock.json generated
View file

@ -2143,10 +2143,9 @@
}
},
"csv-parse": {
"version": "4.12.0",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.12.0.tgz",
"integrity": "sha512-wPQl3H79vWLPI8cgKFcQXl0NBgYYEqVnT1i6/So7OjMpsI540oD7p93r3w6fDSyPvwkTepG05F69/7AViX2lXg==",
"dev": true
"version": "4.14.1",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.14.1.tgz",
"integrity": "sha512-4wmcO7QbWtDAncGFaBwlWFPhEN4Akr64IbM4zvDwEOFekI8blLc04Nw7XjQjtSNy+3AUAgBgtUa9nWo5Cq89Xg=="
},
"dashdash": {
"version": "1.14.1",
@ -6347,6 +6346,12 @@
"integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==",
"dev": true
},
"csv-parse": {
"version": "4.12.0",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.12.0.tgz",
"integrity": "sha512-wPQl3H79vWLPI8cgKFcQXl0NBgYYEqVnT1i6/So7OjMpsI540oD7p93r3w6fDSyPvwkTepG05F69/7AViX2lXg==",
"dev": true
},
"semver": {
"version": "7.3.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",

View file

@ -19,6 +19,7 @@
"dependencies": {
"body-parser": "1.19.0",
"cors": "2.8.5",
"csv-parse": "^4.14.1",
"ejs": "^3.1.5",
"errorhandler": "1.5.1",
"express": "4.17.1",

View file

@ -1,41 +1,48 @@
var router = require('express').Router();
var mongoose = require('mongoose');
var TrackData = mongoose.model('TrackData');
var Track = mongoose.model('Track');
var Comment = mongoose.model('Comment');
var User = mongoose.model('User');
var auth = require('../auth');
var currentTracks = new Map();
var TrackInfo = require('../../logic/TrackInfo');
var {addPointsToTrack} = require('../../logic/tracks');
const router = require('express').Router();
const mongoose = require('mongoose');
const TrackData = mongoose.model('TrackData');
const Track = mongoose.model('Track');
const Comment = mongoose.model('Comment');
const User = mongoose.model('User');
const auth = require('../auth');
const currentTracks = new Map();
const TrackInfo = require('../../logic/TrackInfo');
const { addPointsToTrack } = require('../../logic/tracks');
// Preload track objects on routes with ':track'
router.param('track', function (req, res, next, slug) {
Track.findOne({ slug: slug })
.populate('author')
.then(function (track) {
if (!track) { return res.sendStatus(404); }
if (!track) {
return res.sendStatus(404);
}
req.track = track;
return next();
}).catch(next);
})
.catch(next);
});
router.param('comment', function (req, res, next, id) {
Comment.findById(id).then(function(comment){
if(!comment) { return res.sendStatus(404); }
Comment.findById(id)
.then(function (comment) {
if (!comment) {
return res.sendStatus(404);
}
req.comment = comment;
return next();
}).catch(next);
})
.catch(next);
});
router.get('/', auth.optional, function (req, res, next) {
var query = {};
var limit = 20;
var offset = 0;
const query = {};
let limit = 20;
let offset = 0;
if (typeof req.query.limit !== 'undefined') {
limit = req.query.limit;
@ -46,15 +53,16 @@ router.get('/', auth.optional, function(req, res, next) {
}
if (typeof req.query.tag !== 'undefined') {
query.tagList = {"$in" : [req.query.tag]};
query.tagList = { $in: [req.query.tag] };
}
Promise.all([
req.query.author ? User.findOne({ username: req.query.author }) : null,
req.query.favorited ? User.findOne({username: req.query.favorited}) : null
]).then(function(results){
var author = results[0];
var favoriter = results[1];
req.query.favorited ? User.findOne({ username: req.query.favorited }) : null,
])
.then(function (results) {
const author = results[0];
const favoriter = results[1];
if (author) {
query.author = author._id;
@ -72,16 +80,17 @@ router.get('/', auth.optional, function(req, res, next) {
.skip(Number(offset))
.sort({ createdAt: 'desc' })
.populate('author')
.where('visible').equals(true)
.where('visible')
.equals(true)
.exec(),
Track.countDocuments(query).exec(),
req.payload ? User.findById(req.payload.id) : null,
]).then(function (results) {
var tracks = results[0];
var tracksCount = results[1];
var user = results[2];
const tracks = results[0];
const tracksCount = results[1];
const user = results[2];
// console.log(tracks);
var retTracks = [];
const retTracks = [];
for (t of tracks) {
// console.log(t);
// if (t.author.areTracksVisibleForAll || t.author == user) {
@ -92,15 +101,16 @@ router.get('/', auth.optional, function(req, res, next) {
tracks: retTracks.map(function (track) {
return track.toJSONFor(user);
}),
tracksCount: retTracks.length
tracksCount: retTracks.length,
});
});
}).catch(next);
})
.catch(next);
});
router.get('/feed', auth.required, function (req, res, next) {
var limit = 20;
var offset = 0;
let limit = 20;
let offset = 0;
if (typeof req.query.limit !== 'undefined') {
limit = req.query.limit;
@ -111,68 +121,71 @@ router.get('/feed', auth.required, function(req, res, next) {
}
User.findById(req.payload.id).then(function (user) {
if (!user) { return res.sendStatus(401); }
if(user.following != '')
{
if (!user) {
return res.sendStatus(401);
}
if (user.following != '') {
Promise.all([
Track.find({ author: { $in: user.following } })
.limit(Number(limit))
.skip(Number(offset))
.populate('author')
.exec(),
Track.countDocuments({ author: {$in: user.following}})
]).then(function(results){
var tracks = results[0];
var tracksCount = results[1];
Track.countDocuments({ author: { $in: user.following } }),
])
.then(function (results) {
const tracks = results[0];
const tracksCount = results[1];
return res.json({
tracks: tracks.map(function (track) {
return track.toJSONFor(user);
}),
tracksCount: tracksCount
tracksCount: tracksCount,
});
}).catch(next);
}
else
{
})
.catch(next);
} else {
Promise.all([
Track.find({ author: { $in: req.payload.id } })
.limit(Number(limit))
.skip(Number(offset))
.populate('author')
.exec(),
Track.countDocuments({ author: {$in: req.payload.id}})
]).then(function(results){
var tracks = results[0];
var tracksCount = results[1];
Track.countDocuments({ author: { $in: req.payload.id } }),
])
.then(function (results) {
const tracks = results[0];
const tracksCount = results[1];
return res.json({
tracks: tracks.map(function (track) {
return track.toJSONFor(user);
}),
tracksCount: tracksCount
tracksCount: tracksCount,
});
}).catch(next);
})
.catch(next);
}
});
});
router.post('/', auth.required, function (req, res, next) {
User.findById(req.payload.id).then(function(user){
if (!user) { return res.sendStatus(401); }
User.findById(req.payload.id)
.then(function (user) {
if (!user) {
return res.sendStatus(401);
}
var track = new Track(req.body.track);
var trackData = new TrackData();
const track = new Track(req.body.track);
const trackData = new TrackData();
track.trackData = trackData._id;
track.author = user;
track.visible = track.author.areTracksVisibleForAll;
trackData.save(function (err) {
if (err) {
console.log("failed to save trackData");
console.log('failed to save trackData');
}
});
@ -181,19 +194,22 @@ router.post('/', auth.required, function(req, res, next) {
return res.json({ track: track.toJSONFor(user) });
});
return res.json({ track: track.toJSONFor(user) });
}).catch(next);
})
.catch(next);
});
router.post('/add', auth.optional, function (req, res, next) {
// console.log("Add");
// console.log(req.payload);
User.findById(req.body.id).then(function (user) {
if (!user) { return res.sendStatus(401); }
User.findById(req.body.id)
.then(function (user) {
if (!user) {
return res.sendStatus(401);
}
var ti = null;
if (currentTracks.has(req.body.id))
ti = currentTracks.get(req.body.id);
let ti = null;
if (currentTracks.has(req.body.id)) ti = currentTracks.get(req.body.id);
// console.log("TI" + ti);
// console.log("TILen" + ti.trackData.points.length);
@ -210,18 +226,21 @@ router.post('/add', auth.optional, function(req, res, next) {
// return res.json({ track: track.toJSONFor(user) });
return res.sendStatus(200);
// });
}).catch(next);
})
.catch(next);
});
router.post('/begin', auth.optional, function (req, res, next) {
// console.log("Begin");
// console.log(req.payload);
User.findById(req.body.id).then(function (user) {
if (!user) { return res.sendStatus(401); }
User.findById(req.body.id)
.then(function (user) {
if (!user) {
return res.sendStatus(401);
}
if(currentTracks.has(req.body.id))
currentTracks.delete(req.body.id); // delete old parts if there are leftovers
var ti = new TrackInfo(new Track(req.body.track),new TrackData());
if (currentTracks.has(req.body.id)) currentTracks.delete(req.body.id); // delete old parts if there are leftovers
const ti = new TrackInfo(new Track(req.body.track), new TrackData());
ti.track.trackData = ti.trackData._id;
currentTracks.set(req.body.id, ti);
@ -240,23 +259,24 @@ router.post('/begin', auth.optional, function (req, res, next) {
// console.log(track.author);
return res.sendStatus(200);
// });
}).catch(next);
})
.catch(next);
});
router.post('/end', auth.optional, function (req, res, next) {
// console.log("End");
// console.log(req.payload);
User.findById(req.body.id).then(function (user) {
if (!user) { return res.sendStatus(401); }
User.findById(req.body.id)
.then(function (user) {
if (!user) {
return res.sendStatus(401);
}
var track = null;
if (currentTracks.has(req.body.id))
{
const track = null;
if (currentTracks.has(req.body.id)) {
ti = currentTracks.get(req.body.id);
addPointsToTrack(ti, req.body.track.body);
}
else
{
} else {
var ti = new TrackInfo(new Track(req.body.track), new TrackData());
ti.track.trackData = ti.trackData._id;
addPointsToTrack(ti, ti.track.body);
@ -276,26 +296,26 @@ router.post('/end', auth.optional, function (req, res, next) {
// console.log("TLen" + ti.trackData.points.length);
ti.trackData.save(function (err) {
if (err) {
console.log("failed to save trackData"+err.toString());
console.log('failed to save trackData' + err.toString());
}
});
console.log("successfulSave:");
console.log('successfulSave:');
return res.sendStatus(200);
});
}).catch(next);
})
.catch(next);
});
// return a track
router.get('/:track', auth.optional, function (req, res, next) {
Promise.all([
req.payload ? User.findById(req.payload.id) : null,
req.track.populate('author').execPopulate()
]).then(function(results){
var user = results[0];
Promise.all([req.payload ? User.findById(req.payload.id) : null, req.track.populate('author').execPopulate()])
.then(function (results) {
const user = results[0];
return res.json({ track: req.track.toJSONFor(user) });
}).catch(next);
})
.catch(next);
});
// update track
@ -318,11 +338,14 @@ router.put('/:track', auth.required, function(req, res, next) {
req.track.tagList = req.body.track.tagList;
}
req.track.visible = req.body.track.visible;
console.log("saving track");
console.log('saving track');
req.track.save().then(function(track){
req.track
.save()
.then(function (track) {
return res.json({ track: track.toJSONFor(user) });
}).catch(next);
})
.catch(next);
} else {
return res.sendStatus(403);
}
@ -331,76 +354,100 @@ router.put('/:track', auth.required, function(req, res, next) {
// delete track
router.delete('/:track', auth.required, function (req, res, next) {
User.findById(req.payload.id).then(function(user){
if (!user) { return res.sendStatus(401); }
User.findById(req.payload.id)
.then(function (user) {
if (!user) {
return res.sendStatus(401);
}
if (req.track.author._id.toString() === req.payload.id.toString()) {
TrackData.findByIdAndDelete(req.track.trackData, function (err, td) {console.log("doneDelete");}); // delet our track data
TrackData.findByIdAndDelete(req.track.trackData, function (err, td) {
console.log('doneDelete');
}); // delet our track data
return req.track.remove().then(function () {
return res.sendStatus(204);
});
} else {
return res.sendStatus(403);
}
}).catch(next);
})
.catch(next);
});
// Favorite an track
router.post('/:track/favorite', auth.required, function (req, res, next) {
var trackId = req.track._id;
const trackId = req.track._id;
User.findById(req.payload.id).then(function(user){
if (!user) { return res.sendStatus(401); }
User.findById(req.payload.id)
.then(function (user) {
if (!user) {
return res.sendStatus(401);
}
return user.favorite(trackId).then(function () {
return req.track.updateFavoriteCount().then(function (track) {
return res.json({ track: track.toJSONFor(user) });
});
});
}).catch(next);
})
.catch(next);
});
// Unfavorite an track
router.delete('/:track/favorite', auth.required, function (req, res, next) {
var trackId = req.track._id;
const trackId = req.track._id;
User.findById(req.payload.id).then(function (user){
if (!user) { return res.sendStatus(401); }
User.findById(req.payload.id)
.then(function (user) {
if (!user) {
return res.sendStatus(401);
}
return user.unfavorite(trackId).then(function () {
return req.track.updateFavoriteCount().then(function (track) {
return res.json({ track: track.toJSONFor(user) });
});
});
}).catch(next);
})
.catch(next);
});
// return an track's comments
router.get('/:track/comments', auth.optional, function (req, res, next) {
Promise.resolve(req.payload ? User.findById(req.payload.id) : null).then(function(user){
return req.track.populate({
Promise.resolve(req.payload ? User.findById(req.payload.id) : null)
.then(function (user) {
return req.track
.populate({
path: 'comments',
populate: {
path: 'author'
path: 'author',
},
options: {
sort: {
createdAt: 'desc'
}
}
}).execPopulate().then(function(track) {
return res.json({comments: req.track.comments.map(function(comment){
createdAt: 'desc',
},
},
})
.execPopulate()
.then(function (track) {
return res.json({
comments: req.track.comments.map(function (comment) {
return comment.toJSONFor(user);
})});
}),
});
}).catch(next);
});
})
.catch(next);
});
// create a new comment
router.post('/:track/comments', auth.required, function (req, res, next) {
User.findById(req.payload.id).then(function(user){
if(!user){ return res.sendStatus(401); }
User.findById(req.payload.id)
.then(function (user) {
if (!user) {
return res.sendStatus(401);
}
var comment = new Comment(req.body.comment);
const comment = new Comment(req.body.comment);
comment.track = req.track;
comment.author = user;
@ -411,13 +458,15 @@ router.post('/:track/comments', auth.required, function(req, res, next) {
res.json({ comment: comment.toJSONFor(user) });
});
});
}).catch(next);
})
.catch(next);
});
router.delete('/:track/comments/:comment', auth.required, function (req, res, next) {
if (req.comment.author.toString() === req.payload.id.toString()) {
req.track.comments.remove(req.comment._id);
req.track.save()
req.track
.save()
.then(Comment.find({ _id: req.comment._id }).remove().exec())
.then(function () {
res.sendStatus(204);
@ -429,13 +478,15 @@ router.delete('/:track/comments/:comment', auth.required, function(req, res, nex
// return an track's trackData
router.get('/:track/TrackData', auth.optional, function (req, res, next) {
Promise.resolve(req.payload ? User.findById(req.payload.id) : null).then(function(user){
Promise.resolve(req.payload ? User.findById(req.payload.id) : null)
.then(function (user) {
// console.log("requestTrackData"+req.track);
TrackData.findById(req.track.trackData, function (err, trackData) {
// console.log({trackData: trackData});
return res.json({ trackData: trackData });
});
}).catch(next);
})
.catch(next);
});
module.exports = router;