pub-solar/obs-portal
5
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #19 from Friends-of-OpenBikeSensor/streaming

Browse source 
More body parsers for uploading tracks
This commit is contained in:
Paul Bienkowski 2020-11-24 20:22:25 +01:00 committed by GitHub
parent 8cf01ba5f5 cdadbb5df4
commit 857ec7c3c6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 239 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app.js
View file

@ -5,6 +5,7 @@ const session = require('express-session');
const cors = require('cors');
const errorhandler = require('errorhandler');
const mongoose = require('mongoose');
const auth = require('./routes/auth');
const isProduction = process.env.NODE_ENV === 'production';
@ -12,6 +13,7 @@ const isProduction = process.env.NODE_ENV === 'production';
const app = express();
app.use(cors());
app.use(auth.getUserIdMiddleware);
// Normal express config defaults
app.use(require('morgan')('dev'));

29
logic/tracks.js
View file

@ -254,4 +254,31 @@ function* parseObsver2(body) {
}
}
module.exports = { parseTrackPoints, detectFormat, parseObsver1, parseObsver2, replaceDollarNewlinesHack };
/**
* This function normalizes a User-Agent header for storage in the database. It
* make sure that we only store the user-agent if it matches the pattern
* `OBS/*`, and extracts that part of the user agent, if it contains more
* information. This is the only part we are interested in, the
* remainder is too privacy sensitive to keep.
*/
function normalizeUserAgent(userAgent) {
if (!userAgent) {
return null;
}
const match = userAgent.match(/\bOBS\/[^\s]+/);
if (match) {
return match[0];
}
return null;
}
module.exports = {
detectFormat,
normalizeUserAgent,
parseObsver1,
parseObsver2,
parseTrackPoints,
replaceDollarNewlinesHack,
};

50
logic/tracks.test.js
View file

@ -1,4 +1,11 @@
const { parseTrackPoints, parseObsver1, detectFormat, parseObsver2, replaceDollarNewlinesHack } = require('./tracks');
const {
detectFormat,
normalizeUserAgent,
parseObsver1,
parseObsver2,
parseTrackPoints,
replaceDollarNewlinesHack,
} = require('./tracks');
const { test1, test2, test3 } = require('./_tracks_testdata');
@ -92,3 +99,44 @@ describe('detectFormat', () => {
expect(detectFormat('')).toBe('invalid');
});
});
describe('normalizeUserAgent', () => {
it('is a function', () => {
expect(typeof normalizeUserAgent).toBe('function');
});
it('ignores falsy values', () => {
expect(normalizeUserAgent(null)).toBe(null);
expect(normalizeUserAgent('')).toBe(null);
});
it('ignores normal browser agents', () => {
const browserAgents = [
'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 6P Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.3',
'Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36',
'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5370a Safari/604.1',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1',
];
for (const browserAgent of browserAgents) {
expect(normalizeUserAgent(browserAgent)).toBe(null);
}
});
it('detects OBS versions', () => {
const agents = ['OBS/123', 'OBS/2', 'OBS/1.2.3.4.5-rc123'];
for (const agent of agents) {
expect(normalizeUserAgent(agent)).toBe(agent);
}
});
it('extracts OBS versions from extended formats', () => {
const agents = ['foo OBS/123', 'OBS/123 bar', 'foo OBS/123 bar'];
for (const agent of agents) {
expect(normalizeUserAgent(agent)).toBe('OBS/123');
}
});
});

1
models/Track.js
View file

@ -9,6 +9,7 @@ const TrackSchema = new mongoose.Schema(
description: String,
body: String,
visible: Boolean,
uploadedByUserAgent: String,
numEvents: { type: Number, default: 0 },
comments: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Comment' }],
author: { type: mongoose.Schema.Types.ObjectId, ref: 'User' },

29
package-lock.json generated
View file

@ -1766,6 +1766,14 @@
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
},
"busboy": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz",
"integrity": "sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw==",
"requires": {
"dicer": "0.3.0"
}
},
"bytes": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@ -2048,6 +2056,14 @@
"xdg-basedir": "^4.0.0"
}
},
"connect-busboy": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/connect-busboy/-/connect-busboy-0.0.2.tgz",
"integrity": "sha1-rFyclmchcYheV2xmsr/ZXTuxEJc=",
"requires": {
"busboy": "*"
}
},
"contains-path": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz",
@ -2296,6 +2312,14 @@
"resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz",
"integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA=="
},
"dicer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz",
"integrity": "sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==",
"requires": {
"streamsearch": "0.1.2"
}
},
"diff-sequences": {
"version": "26.6.2",
"resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-26.6.2.tgz",
@ -9596,6 +9620,11 @@
"bluebird": "^2.6.2"
}
},
"streamsearch": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
"integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo="
},
"strftime": {
"version": "0.10.0",
"resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.0.tgz",

1
package.json
View file

@ -19,6 +19,7 @@
"license": "LGPLv3",
"dependencies": {
"body-parser": "1.19.0",
"connect-busboy": "0.0.2",
"cors": "2.8.5",
"csv-parse": "^4.14.1",
"ejs": "^3.1.5",

102
routes/api/tracks.js
View file

@ -4,9 +4,10 @@ const TrackData = mongoose.model('TrackData');
const Track = mongoose.model('Track');
const Comment = mongoose.model('Comment');
const User = mongoose.model('User');
const busboy = require('connect-busboy');
const auth = require('../auth');
const currentTracks = new Map();
const { parseTrackPoints } = require('../../logic/tracks');
const { parseTrackPoints, normalizeUserAgent } = require('../../logic/tracks');
const wrapRoute = require('../../_helpers/wrapRoute');
// Preload track objects on routes with ':track'
@ -140,9 +141,61 @@ router.get(
}),
);
async function readFile(file) {
let fileContent = '';
file.on('data', function (data) {
fileContent += data;
});
await new Promise((resolve, reject) => {
file.on('end', resolve);
file.on('error', reject);
});
return fileContent;
}
async function getMultipartOrJsonBody(req, mapJsonBody = (x) => x) {
const fileInfo = {};
let body;
if (req.busboy) {
body = {};
req.busboy.on('file', async function (fieldname, file, filename, encoding, mimetype) {
body[fieldname] = await readFile(file);
fileInfo[fieldname] = { filename, encoding, mimetype };
});
req.busboy.on('field', (key, value) => {
body[key] = value;
});
req.pipe(req.busboy);
await new Promise((resolve, reject) => {
req.busboy.on('finish', resolve);
req.busboy.on('error', reject);
});
} else if (req.headers['content-type'] === 'application/json') {
body = mapJsonBody(req.body);
} else {
body = { body: await readFile(req), ...req.query };
fileInfo.body = {
mimetype: req.headers['content-type'],
filename: req.headers['content-disposition'],
encoding: req.headers['content-encoding'],
};
}
return { body, fileInfo };
}
router.post(
'/',
auth.required,
busboy(), // parse multipart body
wrapRoute(async (req, res) => {
const user = await User.findById(req.payload.id);
@ -150,18 +203,25 @@ router.post(
return res.sendStatus(401);
}
const track = new Track(req.body.track);
const { body } = await getMultipartOrJsonBody(req, (body) => body.track);
const track = new Track(body);
const trackData = new TrackData();
track.trackData = trackData._id;
track.author = user;
if (req.body.track.body && req.body.track.body.trim()) {
trackData.points = Array.from(parseTrackPoints(track.body));
if (track.body) {
track.body = track.body.trim();
}
track.author = user;
track.visible = track.author.areTracksVisibleForAll;
await trackData.save();
if (track.body) {
trackData.points = Array.from(parseTrackPoints(track.body));
track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']);
}
track.visible = track.author.areTracksVisibleForAll;
await trackData.save();
await track.save();
// console.log(track.author);
@ -183,6 +243,7 @@ router.post(
const trackData = new TrackData();
track.trackData = trackData._id;
track.author = user;
track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']);
await track.save();
await trackData.save();
@ -273,6 +334,11 @@ router.get(
req.payload ? User.findById(req.payload.id) : null,
req.track.populate('author').execPopulate(),
]);
if (!req.track.visible && (!req.payload || req.track.author._id.toString() !== req.payload.id.toString())) {
return res.sendStatus(403);
}
return res.json({ track: req.track.toJSONFor(user, { body: true }) });
}),
);
@ -280,6 +346,7 @@ router.get(
// update track
router.put(
'/:track',
busboy(),
auth.required,
wrapRoute(async (req, res) => {
const user = await User.findById(req.payload.id);
@ -288,16 +355,18 @@ router.put(
return res.sendStatus(403);
}
if (typeof req.body.track.title !== 'undefined') {
req.track.title = req.body.track.title;
const { body } = await getMultipartOrJsonBody(req, (body) => body.track);
if (typeof body.title !== 'undefined') {
req.track.title = body.title;
}
if (typeof req.body.track.description !== 'undefined') {
req.track.description = req.body.track.description;
if (typeof body.description !== 'undefined') {
req.track.description = body.description;
}
if (req.body.track.body && req.body.track.body.trim()) {
req.track.body = req.body.track.body.trim();
if (body.body && body.body.trim()) {
req.track.body = body.body.trim();
let trackData = await TrackData.findById(req.track.trackData);
if (!trackData) {
@ -305,13 +374,14 @@ router.put(
req.track.trackData = trackData._id;
}
trackData.points = Array.from(parseTrackPoints(req.track.body));
req.track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']);
await trackData.save();
}
if (typeof req.body.track.tagList !== 'undefined') {
req.track.tagList = req.body.track.tagList;
if (typeof body.tagList !== 'undefined') {
req.track.tagList = body.tagList;
}
req.track.visible = req.body.track.visible;
req.track.visible = body.visible;
const track = await req.track.save();
return res.json({ track: track.toJSONFor(user) });

64
routes/auth.js
View file

@ -2,30 +2,52 @@ const jwt = require('express-jwt');
const secret = require('../config').secret;
function getTokenFromHeader(req) {
if (
(req.headers.authorization && req.headers.authorization.split(' ')[0] === 'Token') ||
(req.headers.authorization && req.headers.authorization.split(' ')[0] === 'Bearer')
) {
return req.headers.authorization.split(' ')[1];
const authorization = req.headers.authorization;
const [tokenType, token] = (authorization && authorization.split(' ')) || [];
if (tokenType === 'Token' || tokenType === 'Bearer') {
return token;
}
return null;
}
const auth = {
required: jwt({
secret: secret,
userProperty: 'payload',
getToken: getTokenFromHeader,
algorithms: ['HS256'],
}),
optional: jwt({
secret: secret,
userProperty: 'payload',
credentialsRequired: false,
getToken: getTokenFromHeader,
algorithms: ['HS256'],
}),
};
const jwtOptional = jwt({
secret: secret,
userProperty: 'payload',
credentialsRequired: false,
getToken: getTokenFromHeader,
algorithms: ['HS256'],
});
module.exports = auth;
function getUserIdMiddleware(req, res, next) {
try {
const [tokenType, token] = req.headers.authorization.split(' ') || [];
if (tokenType === 'Token' || tokenType === 'Bearer') {
return jwtOptional(req, res, next);
} else if (tokenType === 'OBSUserId') {
req.payload = { id: token.trim() };
next();
} else {
req.payload = null;
next();
}
} catch (err) {
next(err);
}
}
module.exports = {
required(req, res, next) {
if (!req.payload) {
return res.sendStatus(403);
} else {
return next();
}
},
optional(req, res, next) {
return next();
},
getUserIdMiddleware,
};