From 0edb1cc8eb3c8536f6b0723f5d4ee4c7bb9f1eca Mon Sep 17 00:00:00 2001 From: Paul Bienkowski Date: Sun, 13 Dec 2020 20:45:26 +0100 Subject: [PATCH] Upload tracks to files --- docker-compose.yaml | 2 + .../2020-12-12-1823-original-filename.js | 21 ++++++ .../2020-12-13-2025-move-to-upload-file.js | 25 +++++++ package-lock.json | 21 ++++++ package.json | 1 + src/logic/tracks.js | 3 + src/models/Track.js | 74 ++++++++++++++++--- src/routes/api/tracks.js | 56 +++++++------- 8 files changed, 165 insertions(+), 38 deletions(-) create mode 100644 migrations/2020-12-12-1823-original-filename.js create mode 100644 migrations/2020-12-13-2025-move-to-upload-file.js diff --git a/docker-compose.yaml b/docker-compose.yaml index 75c0c48..a5f77fa 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -17,9 +17,11 @@ services: dockerfile: ./Dockerfile volumes: - ./src:/opt/obsAPI/src + - ./local/api-data:/data environment: - PORT=3000 - MONGODB_URL=mongodb://mongo/obsTest + - UPLOADS_DIR=/data links: - mongo ports: diff --git a/migrations/2020-12-12-1823-original-filename.js b/migrations/2020-12-12-1823-original-filename.js new file mode 100644 index 0000000..cdf342e --- /dev/null +++ b/migrations/2020-12-12-1823-original-filename.js @@ -0,0 +1,21 @@ +const Track = require('../src/models/Track'); + +module.exports = { + async up(next) { + try { + for await (const track of Track.find()) { + track.originalFileName = track.slug + '.csv' + await track.generateOriginalFilePath(); + await track.save() + } + next(); + } catch(err) { + next(err) + } + }, + + async down(next) { + next(); + }, +}; + diff --git a/migrations/2020-12-13-2025-move-to-upload-file.js b/migrations/2020-12-13-2025-move-to-upload-file.js new file mode 100644 index 0000000..6b7be92 --- /dev/null +++ b/migrations/2020-12-13-2025-move-to-upload-file.js @@ -0,0 +1,25 @@ + +const Track = require('../src/models/Track'); + +module.exports = { + async up(next) { + try { + for await (const track of Track.find()) { + if (!track.body) { + continue + } + + await track.writeToOriginalFile(track.body) + delete track.body; + await track.save() + } + next(); + } catch(err) { + next(err) + } + }, + + async down(next) { + next(); + }, +}; diff --git a/package-lock.json b/package-lock.json index 8185a82..37b4f34 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9308,6 +9308,14 @@ } } }, + "sanitize-filename": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.3.tgz", + "integrity": "sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==", + "requires": { + "truncate-utf8-bytes": "^1.0.0" + } + }, "sanitize-html": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-1.20.1.tgz", @@ -10167,6 +10175,14 @@ "punycode": "^2.1.1" } }, + "truncate-utf8-bytes": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", + "integrity": "sha1-QFkjkJWS1W94pYGENLC3hInKXys=", + "requires": { + "utf8-byte-length": "^1.0.1" + } + }, "tsconfig-paths": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz", @@ -10957,6 +10973,11 @@ "integrity": "sha1-Qw/VEKt/yVtdWRDJAteYgMIIQ2s=", "dev": true }, + "utf8-byte-length": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz", + "integrity": "sha1-9F8VDExm7uloGGUFq5P8u4rWv2E=" + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index 35f20c8..50247fe 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "passport": "0.4.1", "passport-local": "1.0.0", "request": "2.88.2", + "sanitize-filename": "^1.6.3", "slug": "^3.3.5", "turf": "^3.0.14", "underscore": "^1.11.0" diff --git a/src/logic/tracks.js b/src/logic/tracks.js index 18e20f9..44923ed 100644 --- a/src/logic/tracks.js +++ b/src/logic/tracks.js @@ -64,6 +64,9 @@ function replaceDollarNewlinesHack(body) { } function* parseTrackPoints(body, format = null) { + if (body instanceof Buffer) { + body = body.toString('utf-8') + } body = replaceDollarNewlinesHack(body); const detectedFormat = format != null ? format : detectFormat(body); diff --git a/src/models/Track.js b/src/models/Track.js index cadd197..b57339d 100644 --- a/src/models/Track.js +++ b/src/models/Track.js @@ -1,35 +1,60 @@ const mongoose = require('mongoose'); const uniqueValidator = require('mongoose-unique-validator'); const slug = require('slug'); +const path = require('path'); +const sanitize = require('sanitize-filename'); +const fs = require('fs') const { parseTrackPoints } = require('../logic/tracks'); const TrackData = require('./TrackData'); +const DATA_DIR = process.env.DATA_DIR || path.resolve(__dirname, '../../data/') + const schema = new mongoose.Schema( { slug: { type: String, lowercase: true, unique: true }, title: String, description: String, - body: String, visible: Boolean, uploadedByUserAgent: String, + body: String, // deprecated, remove after migration has read it comments: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Comment' }], author: { type: mongoose.Schema.Types.ObjectId, ref: 'User' }, trackData: { type: mongoose.Schema.Types.ObjectId, ref: 'TrackData' }, publicTrackData: { type: mongoose.Schema.Types.ObjectId, ref: 'TrackData' }, + originalFileName: { + type: String, + required: true, + validate: { + validator: function (v) { + // Must be a sane filename, i.e. not change when being sanitized + return sanitize(v) === v && v.length > 0 && /.+\.csv$/i.test(v); + }, + message: (props) => `${props.value} is not a valid filename`, + }, + }, + originalFilePath: String, }, { timestamps: true }, ); schema.plugin(uniqueValidator, { message: 'is already taken' }); -schema.pre('validate', function (next) { - if (!this.slug) { - this.slugify(); - } +schema.pre('validate', async function (next) { + try { + if (!this.slug) { + this.slugify(); + } - next(); + if (!this.originalFilePath) { + await this.generateOriginalFilePath(); + } + + next(); + } catch (err) { + next(err); + } }); class Track extends mongoose.Model { @@ -37,6 +62,11 @@ class Track extends mongoose.Model { this.slug = slug(this.title) + '-' + ((Math.random() * Math.pow(36, 6)) | 0).toString(36); } + async generateOriginalFilePath() { + await this.populate('author').execPopulate(); + this.originalFilePath = path.join('uploads', 'originals', this.author.username, this.slug, this.originalFileName); + } + isVisibleTo(user) { if (this.visible) { return true; @@ -57,6 +87,24 @@ class Track extends mongoose.Model { return user && user._id.equals(this.author._id); } + async _ensureDirectoryExists() { + if (!this.originalFilePath) { + await this.generateOriginalFilePath() + } + + const dir = path.join(DATA_DIR, path.dirname(this.originalFilePath)) + await fs.promises.mkdir(dir, {recursive: true}) + } + + get fullOriginalFilePath() { + return path.join(DATA_DIR, this.originalFilePath) + } + + async writeToOriginalFile(fileBody) { + await this._ensureDirectoryExists() + await fs.promises.writeFile(this.fullOriginalFilePath, fileBody) + } + /** * Fills the trackData and publicTrackData with references to correct * TrackData objects. For now, this is either the same, or publicTrackData @@ -76,8 +124,11 @@ class Track extends mongoose.Model { await TrackData.findByIdAndDelete(this.publicTrackData); } - // parse the points from the body - const points = Array.from(parseTrackPoints(this.body)); + // Parse the points from the body. + // TODO: Stream file contents, if possible + const body = await fs.promises.readFile(this.fullOriginalFilePath) + const points = Array.from(parseTrackPoints(body)); + const trackData = TrackData.createFromPoints(points); await trackData.save(); @@ -102,7 +153,12 @@ class Track extends mongoose.Model { updatedAt: this.updatedAt, visible: this.visible, author: this.author.toProfileJSONFor(user), - ...(includePrivateFields ? { uploadedByUserAgent: this.uploadedByUserAgent } : {}), + ...(includePrivateFields + ? { + uploadedByUserAgent: this.uploadedByUserAgent, + originalFileName: this.originalFileName, + } + : {}), }; } } diff --git a/src/routes/api/tracks.js b/src/routes/api/tracks.js index 29fe1ab..40bc717 100644 --- a/src/routes/api/tracks.js +++ b/src/routes/api/tracks.js @@ -175,23 +175,28 @@ router.post( auth.required, busboy(), // parse multipart body wrapRoute(async (req, res) => { - const { body } = await getMultipartOrJsonBody(req, (body) => body.track); + // Read the whole file into memory. This is not optimal, instead, we should + // write the file data directly to the target file. However, we first have + // to parse the rest of the track data to know where to place the file. + // TODO: Stream into temporary file, then move it later. + const { body, fileInfo } = await getMultipartOrJsonBody(req, (body) => body.track); - const track = new Track(body); - track.author = req.user; + const {body: fileBody, visible, ...trackBody} = body - if (body.visible != null) { - track.visible = Boolean(body.visible); - } else { - track.visible = track.author.areTracksVisibleForAll; - } + const track = new Track({ + ...trackBody, + author: req.user, + visible: visible == null ? req.user.areTracksVisibleForAll : Boolean(trackBody.visible) + }) + track.slugify(); - if (track.body) { - track.body = track.body.trim(); + if (fileBody) { track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']); + track.originalFileName = fileInfo.body ? fileInfo.body.filename : track.slug + '.csv'; + await track.writeToOriginalFile(fileBody) await track.rebuildTrackDataAndSave(); } else { - await track.save(); + await track.save() } // console.log(track.author); @@ -224,32 +229,25 @@ router.put( return res.sendStatus(403); } - const { body } = await getMultipartOrJsonBody(req, (body) => body.track); + const { body: {body: fileBody, ...trackBody}, fileInfo } = await getMultipartOrJsonBody(req, (body) => body.track); - if (typeof body.title !== 'undefined') { - track.title = (body.title || '').trim() || null; + if (typeof trackBody.title !== 'undefined') { + track.title = (trackBody.title || '').trim() || null; } - if (typeof body.description !== 'undefined') { - track.description = (body.description || '').trim() || null; + if (typeof trackBody.description !== 'undefined') { + track.description = (trackBody.description || '').trim() || null; } - if (body.visible != null) { - track.visible = Boolean(body.visible); + if (trackBody.visible != null) { + track.visible = Boolean(trackBody.visible); } - if (typeof body.tagList !== 'undefined') { - track.tagList = body.tagList; - } - - if (body.body && body.body.trim()) { - // delete existing - if (track.trackData) { - await TrackData.findByIdAndDelete(track.trackData); - } - - track.body = body.body.trim(); + if (fileBody) { + track.originalFileName = fileInfo.body ? fileInfo.body.filename : track.slug + '.csv'; track.uploadedByUserAgent = normalizeUserAgent(req.headers['user-agent']); + await track.writeToOriginalFile(fileBody) + await track.rebuildTrackDataAndSave(); } else { await track.save();