From 56905fdf751c7991612c963f275d99c7c924ab27 Mon Sep 17 00:00:00 2001 From: Paul Bienkowski Date: Mon, 26 Sep 2022 11:45:49 +0200 Subject: [PATCH 1/6] Install stream-zip --- api/requirements.txt | 1 + api/setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/api/requirements.txt b/api/requirements.txt index 48277a6..2682253 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -11,3 +11,4 @@ sqlalchemy[asyncio]~=1.4.39 <2.0 asyncpg~=0.24.0 pyshp~=2.3.1 alembic~=1.7.7 +stream-zip~=0.0.50 diff --git a/api/setup.py b/api/setup.py index 2b8ce1e..3a5f09f 100644 --- a/api/setup.py +++ b/api/setup.py @@ -23,6 +23,7 @@ setup( "sqlalchemy[asyncio]~=1.4.25", "asyncpg~=0.24.0", "alembic~=1.7.7", + "stream-zip~=0.0.50", ], entry_points={ "console_scripts": [ From 5a78d7eb38437bf52ccad5235ddc0ad13dfcfcc6 Mon Sep 17 00:00:00 2001 From: Paul Bienkowski Date: Fri, 16 Sep 2022 10:23:25 +0200 Subject: [PATCH 2/6] Parse device identifiers and create UserDevice entries in database --- .../versions/f7b21148126a_add_user_device.py | 41 +++++++++++++++++++ api/obs/api/db.py | 36 ++++++++++++++++ api/obs/api/process.py | 37 +++++++++++++++-- api/scripts | 2 +- 4 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 api/migrations/versions/f7b21148126a_add_user_device.py diff --git a/api/migrations/versions/f7b21148126a_add_user_device.py b/api/migrations/versions/f7b21148126a_add_user_device.py new file mode 100644 index 0000000..2e65451 --- /dev/null +++ b/api/migrations/versions/f7b21148126a_add_user_device.py @@ -0,0 +1,41 @@ +"""add user_device + +Revision ID: f7b21148126a +Revises: a9627f63fbed +Create Date: 2022-09-15 17:48:06.764342 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "f7b21148126a" +down_revision = "a049e5eb24dd" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "user_device", + sa.Column("id", sa.Integer, autoincrement=True, primary_key=True), + sa.Column("user_id", sa.Integer, sa.ForeignKey("user.id", ondelete="CASCADE")), + sa.Column("identifier", sa.String, nullable=False), + sa.Column("display_name", sa.String, nullable=True), + sa.Index("user_id_identifier", "user_id", "identifier", unique=True), + ) + op.add_column( + "track", + sa.Column( + "user_device_id", + sa.Integer, + sa.ForeignKey("user_device.id", ondelete="RESTRICT"), + nullable=True, + ), + ) + + +def downgrade(): + op.drop_column("track", "user_device_id") + op.drop_table("user_device") diff --git a/api/obs/api/db.py b/api/obs/api/db.py index f7716a6..74b5b20 100644 --- a/api/obs/api/db.py +++ b/api/obs/api/db.py @@ -221,6 +221,12 @@ class Track(Base): Integer, ForeignKey("user.id", ondelete="CASCADE"), nullable=False ) + user_device_id = Column( + Integer, + ForeignKey("user_device.id", ondelete="RESTRICT"), + nullable=True, + ) + # Statistics... maybe we'll drop some of this if we can easily compute them from SQL recorded_at = Column(DateTime) recorded_until = Column(DateTime) @@ -409,6 +415,28 @@ class User(Base): self.username = new_name +class UserDevice(Base): + __tablename__ = "user_device" + id = Column(Integer, autoincrement=True, primary_key=True) + user_id = Column(Integer, ForeignKey("user.id", ondelete="CASCADE")) + identifier = Column(String, nullable=False) + display_name = Column(String, nullable=True) + + __table_args__ = ( + Index("user_id_identifier", "user_id", "identifier", unique=True), + ) + + def to_dict(self, for_user_id=None): + if for_user_id != self.user_id: + return {} + + return { + "id": self.id, + "identifier": self.identifier, + "displayName": self.display_name, + } + + class Comment(Base): __tablename__ = "comment" id = Column(Integer, autoincrement=True, primary_key=True) @@ -468,6 +496,14 @@ Track.overtaking_events = relationship( passive_deletes=True, ) +Track.user_device = relationship("UserDevice", back_populates="tracks") +UserDevice.tracks = relationship( + "Track", + order_by=Track.created_at, + back_populates="user_device", + passive_deletes=False, +) + # 0..4 Night, 4..10 Morning, 10..14 Noon, 14..18 Afternoon, 18..22 Evening, 22..00 Night # Two hour intervals diff --git a/api/obs/api/process.py b/api/obs/api/process.py index 6fc2c5e..79a39d8 100644 --- a/api/obs/api/process.py +++ b/api/obs/api/process.py @@ -8,7 +8,7 @@ import pytz from os.path import join from datetime import datetime -from sqlalchemy import delete, select +from sqlalchemy import delete, select, and_ from sqlalchemy.orm import joinedload from obs.face.importer import ImportMeasurementsCsv @@ -27,7 +27,7 @@ from obs.face.filter import ( from obs.face.osm import DataSource, DatabaseTileSource, OverpassTileSource -from obs.api.db import OvertakingEvent, RoadUsage, Track, make_session +from obs.api.db import OvertakingEvent, RoadUsage, Track, UserDevice, make_session from obs.api.app import app log = logging.getLogger(__name__) @@ -144,10 +144,11 @@ async def process_track(session, track, data_source): os.makedirs(output_dir, exist_ok=True) log.info("Annotating and filtering CSV file") - imported_data, statistics = ImportMeasurementsCsv().read( + imported_data, statistics, track_metadata = ImportMeasurementsCsv().read( original_file_path, user_id="dummy", # TODO: user username or id or nothing? dataset_id=Track.slug, # TODO: use track id or slug or nothing? + return_metadata=True, ) annotator = AnnotateMeasurements( @@ -217,6 +218,36 @@ async def process_track(session, track, data_source): await clear_track_data(session, track) await session.commit() + device_identifier = track_metadata.get("DeviceId") + if device_identifier: + if isinstance(device_identifier, list): + device_identifier = device_identifier[0] + + log.info("Finding or creating device %s", device_identifier) + user_device = ( + await session.execute( + select(UserDevice).where( + and_( + UserDevice.user_id == track.author_id, + UserDevice.identifier == device_identifier, + ) + ) + ) + ).scalar() + + log.debug("user_device is %s", user_device) + + if not user_device: + user_device = UserDevice( + user_id=track.author_id, identifier=device_identifier + ) + log.debug("Create new device for this user") + session.add(user_device) + + track.user_device = user_device + else: + log.info("No DeviceId in track metadata.") + log.info("Import events into database...") await import_overtaking_events(session, track, overtaking_events) diff --git a/api/scripts b/api/scripts index 8e9395f..bbc6fec 160000 --- a/api/scripts +++ b/api/scripts @@ -1 +1 @@ -Subproject commit 8e9395fd3cd0f1e83b4413546bc2d3cb0c726738 +Subproject commit bbc6feca08aee9ea4f4263bb7c07e199d9c989ee From cbab83e6e3f5d6cc30b4ccbb2b9e6679f48fb550 Mon Sep 17 00:00:00 2001 From: Paul Bienkowski Date: Thu, 22 Sep 2022 17:35:12 +0200 Subject: [PATCH 3/6] Build awesome "My Tracks" table with filters and sorting --- api/obs/api/db.py | 1 + api/obs/api/routes/tracks.py | 48 ++- api/obs/api/routes/users.py | 20 +- frontend/src/App.tsx | 219 ++++++++---- frontend/src/pages/MyTracksPage.tsx | 326 ++++++++++++++++++ frontend/src/pages/TrackPage/TrackDetails.tsx | 35 +- frontend/src/pages/index.js | 23 +- frontend/src/types.ts | 89 +++-- frontend/src/utils.js | 43 ++- 9 files changed, 645 insertions(+), 159 deletions(-) create mode 100644 frontend/src/pages/MyTracksPage.tsx diff --git a/api/obs/api/db.py b/api/obs/api/db.py index 74b5b20..4e870ca 100644 --- a/api/obs/api/db.py +++ b/api/obs/api/db.py @@ -259,6 +259,7 @@ class Track(Base): if for_user_id is not None and for_user_id == self.author_id: result["uploadedByUserAgent"] = self.uploaded_by_user_agent result["originalFileName"] = self.original_file_name + result["userDeviceId"] = self.user_device_id if self.author: result["author"] = self.author.to_dict(for_user_id=for_user_id) diff --git a/api/obs/api/routes/tracks.py b/api/obs/api/routes/tracks.py index 28f4097..e9df89c 100644 --- a/api/obs/api/routes/tracks.py +++ b/api/obs/api/routes/tracks.py @@ -23,7 +23,7 @@ def normalize_user_agent(user_agent): return m[0] if m else None -async def _return_tracks(req, extend_query, limit, offset): +async def _return_tracks(req, extend_query, limit, offset, order_by=None): if limit <= 0 or limit > 1000: raise InvalidUsage("invalid limit") @@ -39,7 +39,7 @@ async def _return_tracks(req, extend_query, limit, offset): extend_query(select(Track).options(joinedload(Track.author))) .limit(limit) .offset(offset) - .order_by(Track.created_at.desc()) + .order_by(order_by if order_by is not None else Track.created_at) ) tracks = (await req.ctx.db.execute(query)).scalars() @@ -76,16 +76,56 @@ async def get_tracks(req): return await _return_tracks(req, extend_query, limit, offset) +def parse_boolean(s): + if s is None: + return None + + s = s.lower() + if s in ("true", "1", "yes", "y", "t"): + return True + if s in ("false", "0", "no", "n", "f"): + return False + + raise ValueError("invalid value for boolean") + + @api.get("/tracks/feed") @require_auth async def get_feed(req): limit = req.ctx.get_single_arg("limit", default=20, convert=int) offset = req.ctx.get_single_arg("offset", default=0, convert=int) + user_device_id = req.ctx.get_single_arg("user_device_id", default=None, convert=int) + + order_by_columns = { + "recordedAt": Track.recorded_at, + "title": Track.title, + "visibility": Track.public, + "length": Track.length, + "duration": Track.duration, + "user_device_id": Track.user_device_id, + } + order_by = req.ctx.get_single_arg( + "order_by", default=None, convert=order_by_columns.get + ) + + reversed_ = req.ctx.get_single_arg("reversed", convert=parse_boolean, default=False) + if reversed_: + order_by = order_by.desc() + + public = req.ctx.get_single_arg("public", convert=parse_boolean, default=None) def extend_query(q): - return q.where(Track.author_id == req.ctx.user.id) + q = q.where(Track.author_id == req.ctx.user.id) - return await _return_tracks(req, extend_query, limit, offset) + if user_device_id is not None: + q = q.where(Track.user_device_id == user_device_id) + + if public is not None: + q = q.where(Track.public == public) + + return q + + return await _return_tracks(req, extend_query, limit, offset, order_by) @api.post("/tracks") diff --git a/api/obs/api/routes/users.py b/api/obs/api/routes/users.py index 60b0c19..38ffaf1 100644 --- a/api/obs/api/routes/users.py +++ b/api/obs/api/routes/users.py @@ -1,9 +1,11 @@ import logging from sanic.response import json -from sanic.exceptions import InvalidUsage +from sanic.exceptions import InvalidUsage, Forbidden +from sqlalchemy import select from obs.api.app import api, require_auth +from obs.api.db import UserDevice log = logging.getLogger(__name__) @@ -28,6 +30,22 @@ async def get_user(req): return json(user_to_json(req.ctx.user) if req.ctx.user else None) +@api.get("/user/devices") +async def get_user_devices(req): + if not req.ctx.user: + raise Forbidden() + + query = ( + select(UserDevice) + .where(UserDevice.user_id == req.ctx.user.id) + .order_by(UserDevice.id) + ) + + devices = (await req.ctx.db.execute(query)).scalars() + + return json([device.to_dict(req.ctx.user.id) for device in devices]) + + @api.put("/user") @require_auth async def put_user(req): diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 9c66e8e..c3d7cfd 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,17 +1,24 @@ -import React from 'react' -import classnames from 'classnames' -import {connect} from 'react-redux' -import {List, Grid, Container, Menu, Header, Dropdown} from 'semantic-ui-react' -import {BrowserRouter as Router, Switch, Route, Link} from 'react-router-dom' -import {useObservable} from 'rxjs-hooks' -import {from} from 'rxjs' -import {pluck} from 'rxjs/operators' -import {Helmet} from "react-helmet"; -import {useTranslation} from 'react-i18next' +import React from "react"; +import classnames from "classnames"; +import { connect } from "react-redux"; +import { + List, + Grid, + Container, + Menu, + Header, + Dropdown, +} from "semantic-ui-react"; +import { BrowserRouter as Router, Switch, Route, Link } from "react-router-dom"; +import { useObservable } from "rxjs-hooks"; +import { from } from "rxjs"; +import { pluck } from "rxjs/operators"; +import { Helmet } from "react-helmet"; +import { useTranslation } from "react-i18next"; -import {useConfig} from 'config' -import styles from './App.module.less' -import {AVAILABLE_LOCALES, setLocale} from 'i18n' +import { useConfig } from "config"; +import styles from "./App.module.less"; +import { AVAILABLE_LOCALES, setLocale } from "i18n"; import { ExportPage, @@ -25,50 +32,61 @@ import { TrackPage, TracksPage, UploadPage, -} from 'pages' -import {Avatar, LoginButton} from 'components' -import api from 'api' + MyTracksPage, +} from "pages"; +import { Avatar, LoginButton } from "components"; +import api from "api"; // This component removes the "navigate" prop before rendering a Menu.Item, // which is a workaround for an annoying warning that is somehow caused by the // and combination. -function MenuItemForLink({navigate, ...props}) { +function MenuItemForLink({ navigate, ...props }) { return ( { - e.preventDefault() - navigate() + e.preventDefault(); + navigate(); }} /> - ) + ); } -function DropdownItemForLink({navigate, ...props}) { +function DropdownItemForLink({ navigate, ...props }) { return ( { - e.preventDefault() - navigate() + e.preventDefault(); + navigate(); }} /> - ) + ); } -function Banner({text, style = 'warning'}: {text: string; style: 'warning' | 'info'}) { - return
{text}
+function Banner({ + text, + style = "warning", +}: { + text: string; + style: "warning" | "info"; +}) { + return
{text}
; } -const App = connect((state) => ({login: state.login}))(function App({login}) { - const {t} = useTranslation() - const config = useConfig() - const apiVersion = useObservable(() => from(api.get('/info')).pipe(pluck('version'))) +const App = connect((state) => ({ login: state.login }))(function App({ + login, +}) { + const { t } = useTranslation(); + const config = useConfig(); + const apiVersion = useObservable(() => + from(api.get("/info")).pipe(pluck("version")) + ); - const hasMap = Boolean(config?.obsMapSource) + const hasMap = Boolean(config?.obsMapSource); React.useEffect(() => { - api.loadUser() - }, []) + api.loadUser(); + }, []); return config ? ( @@ -79,36 +97,59 @@ const App = connect((state) => ({login: state.login}))(function App({login}) { {config?.banner && } - + OpenBikeSensor {hasMap && ( - - {t('App.menu.map')} - + + {t("App.menu.map")} + )} - {t('App.menu.tracks')} + {t("App.menu.tracks")} - {t('App.menu.export')} + {t("App.menu.export")} {login ? ( <> - {t('App.menu.myTracks')} + {t("App.menu.myTracks")} - }> + } + > - - + + - + @@ -125,14 +166,16 @@ const App = connect((state) => ({login: state.login}))(function App({login}) { - {hasMap && - - } + {hasMap && ( + + + + )} - + @@ -169,12 +212,14 @@ const App = connect((state) => ({login: state.login}))(function App({login}) { -
- {t('App.footer.aboutTheProject')} -
+
{t("App.footer.aboutTheProject")}
- + openbikesensor.org @@ -182,41 +227,57 @@ const App = connect((state) => ({login: state.login}))(function App({login}) { -
- {t('App.footer.getInvolved')} -
+
{t("App.footer.getInvolved")}
- - {t('App.footer.getHelpInForum')} + + {t("App.footer.getHelpInForum")} - - {t('App.footer.reportAnIssue')} + + {t("App.footer.reportAnIssue")} - - {t('App.footer.development')} + + {t("App.footer.development")} -
- {t('App.footer.thisInstallation')} -
+
{t("App.footer.thisInstallation")}
- - {t('App.footer.privacyPolicy')} + + {t("App.footer.privacyPolicy")} - - {t('App.footer.imprint')} + + {t("App.footer.imprint")} { config?.termsUrl && @@ -229,21 +290,29 @@ const App = connect((state) => ({login: state.login}))(function App({login}) { - {apiVersion ? t('App.footer.version', {apiVersion}) : t('App.footer.versionLoading')} + {apiVersion + ? t("App.footer.version", { apiVersion }) + : t("App.footer.versionLoading")} -
{t('App.footer.changeLanguage')}
+
{t("App.footer.changeLanguage")}
- {AVAILABLE_LOCALES.map(locale => setLocale(locale)}>{t(`locales.${locale}`)})} + {AVAILABLE_LOCALES.map((locale) => ( + + setLocale(locale)}> + {t(`locales.${locale}`)} + + + ))} @@ -251,7 +320,7 @@ const App = connect((state) => ({login: state.login}))(function App({login}) { - ) : null -}) + ) : null; +}); -export default App +export default App; diff --git a/frontend/src/pages/MyTracksPage.tsx b/frontend/src/pages/MyTracksPage.tsx new file mode 100644 index 0000000..f8615f3 --- /dev/null +++ b/frontend/src/pages/MyTracksPage.tsx @@ -0,0 +1,326 @@ +import React, { useCallback, useState } from "react"; +import { connect } from "react-redux"; +import { + Accordion, + Button, + Header, + Icon, + Item, + List, + Loader, + Dropdown, + SemanticCOLORS, + SemanticICONS, + Table, +} from "semantic-ui-react"; +import { useObservable } from "rxjs-hooks"; +import { Link } from "react-router-dom"; +import { of, from, concat } from "rxjs"; +import { map, switchMap, distinctUntilChanged } from "rxjs/operators"; +import _ from "lodash"; +import { useTranslation } from "react-i18next"; + +import type { ProcessingStatus, Track, UserDevice } from "types"; +import { Page, FormattedDate, Visibility } from "components"; +import api from "api"; +import { formatDistance, formatDuration } from "utils"; + +const COLOR_BY_STATUS: Record = { + error: "red", + complete: "green", + created: "grey", + queued: "orange", + processing: "orange", +}; + +const ICON_BY_STATUS: Record = { + error: "warning sign", + complete: "check circle outline", + created: "bolt", + queued: "bolt", + processing: "bolt", +}; + +function ProcessingStatusLabel({ status }: { status: ProcessingStatus }) { + const { t } = useTranslation(); + return ( + + + + ); +} + +function SortableHeader({ + children, + setOrderBy, + orderBy, + reversed, + setReversed, + name, + ...props +}) { + const toggleSort = (e) => { + e.preventDefault(); + e.stopPropagation(); + + if (orderBy === name) { + if (!reversed) { + setReversed(true); + } else { + setReversed(false); + setOrderBy(null); + } + } else { + setReversed(false); + setOrderBy(name); + } + }; + + let icon = + orderBy === name ? (reversed ? "sort descending" : "sort ascending") : null; + + return ( + +
+ {children} + +
+ + ); +} + +type Filters = { + userDeviceId?: null | number; + visibility?: null | boolean; +}; + +function TrackFilters({ + filters, + setFilters, + deviceNames, +}: { + filters: Filters; + setFilters: (f: Filters) => void; + deviceNames: null | Record; +}) { + return ( + + + Device + ({ + value: Number(deviceId), + key: deviceId, + text: deviceName, + }) + ), + ]} + value={filters?.userDeviceId ?? 0} + onChange={(_e, { value }) => + setFilters({ ...filters, userDeviceId: (value as number) || null }) + } + /> + + + + Visibility + + setFilters({ + ...filters, + visibility: value === "none" ? null : (value as boolean), + }) + } + /> + + + ); +} + +function TracksTable() { + const [orderBy, setOrderBy] = useState("recordedAt"); + const [reversed, setReversed] = useState(false); + const [showFilters, setShowFilters] = useState(false); + const [filters, setFilters] = useState({}); + + const query = _.pickBy( + { + limit: 1000, + offset: 0, + order_by: orderBy, + reversed: reversed ? "true" : "false", + user_device_id: filters?.userDeviceId, + public: filters?.visibility, + }, + (x) => x != null + ); + + const tracks: Track[] | null = useObservable( + (_$, inputs$) => + inputs$.pipe( + map(([query]) => query), + distinctUntilChanged(_.isEqual), + switchMap((query) => + concat( + of(null), + from(api.get("/tracks/feed", { query }).then((r) => r.tracks)) + ) + ) + ), + null, + [query] + ); + + const deviceNames: null | Record = useObservable(() => + from(api.get("/user/devices")).pipe( + map((response: UserDevice[]) => + Object.fromEntries( + response.map((device) => [ + device.id, + device.displayName || device.identifier, + ]) + ) + ) + ) + ); + + const { t } = useTranslation(); + + const p = { orderBy, setOrderBy, reversed, setReversed }; + + return ( +
+ + + + setShowFilters(!showFilters)} + > + + Filters + + + + + + + + + + + Title + + + Recorded at + + + Visibility + + + Length + + + Duration + + + Device + + + + + + {tracks?.map((track: Track) => ( + + + {track.processingStatus == null ? null : ( + + )} + + {track.title || t("general.unnamedTrack")} + + + + + + + + + {track.public == null ? null : ( + + )} + + + + {formatDistance(track.length)} + + + + {formatDuration(track.duration)} + + + + {track.userDeviceId + ? deviceNames?.[track.userDeviceId] ?? "..." + : null} + + + ))} + +
+
+ ); +} + +function UploadButton({ navigate, ...props }) { + const { t } = useTranslation(); + const onClick = useCallback( + (e) => { + e.preventDefault(); + navigate(); + }, + [navigate] + ); + return ( + + ); +} + +const MyTracksPage = connect((state) => ({ login: (state as any).login }))( + function MyTracksPage({ login }) { + const { t } = useTranslation(); + + const title = t("TracksPage.titleUser"); + + return ( + + +
{title}
+ + + ); + } +); + +export default MyTracksPage; diff --git a/frontend/src/pages/TrackPage/TrackDetails.tsx b/frontend/src/pages/TrackPage/TrackDetails.tsx index d5002b2..c94af57 100644 --- a/frontend/src/pages/TrackPage/TrackDetails.tsx +++ b/frontend/src/pages/TrackPage/TrackDetails.tsx @@ -5,10 +5,7 @@ import { Duration } from "luxon"; import { useTranslation } from "react-i18next"; import { FormattedDate, Visibility } from "components"; - -function formatDuration(seconds) { - return Duration.fromMillis((seconds ?? 0) * 1000).toFormat("h'h' mm'm'"); -} +import { formatDistance, formatDuration } from "utils"; export default function TrackDetails({ track, isAuthor }) { const { t } = useTranslation(); @@ -47,7 +44,7 @@ export default function TrackDetails({ track, isAuthor }) { track?.length != null && [ t("TrackPage.details.length"), - `${(track?.length / 1000).toFixed(2)} km`, + formatDistance(track?.length), ], track?.processingStatus != null && @@ -63,23 +60,23 @@ export default function TrackDetails({ track, isAuthor }) { ].filter(Boolean); const COLUMNS = 4; - const chunkSize = Math.ceil(items.length / COLUMNS) + const chunkSize = Math.ceil(items.length / COLUMNS); return ( - {_.chunk(items, chunkSize).map((chunkItems, idx) => ( - - - - {chunkItems.map(([title, value]) => ( - - {title} - {value} - ))} - - - ))} - + {_.chunk(items, chunkSize).map((chunkItems, idx) => ( + + + {chunkItems.map(([title, value]) => ( + + {title} + {value} + + ))} + + + ))} + ); } diff --git a/frontend/src/pages/index.js b/frontend/src/pages/index.js index 07233ed..6d0a958 100644 --- a/frontend/src/pages/index.js +++ b/frontend/src/pages/index.js @@ -1,11 +1,12 @@ -export {default as ExportPage} from './ExportPage' -export {default as HomePage} from './HomePage' -export {default as LoginRedirectPage} from './LoginRedirectPage' -export {default as LogoutPage} from './LogoutPage' -export {default as MapPage} from './MapPage' -export {default as NotFoundPage} from './NotFoundPage' -export {default as SettingsPage} from './SettingsPage' -export {default as TrackEditor} from './TrackEditor' -export {default as TrackPage} from './TrackPage' -export {default as TracksPage} from './TracksPage' -export {default as UploadPage} from './UploadPage' +export { default as ExportPage } from "./ExportPage"; +export { default as HomePage } from "./HomePage"; +export { default as LoginRedirectPage } from "./LoginRedirectPage"; +export { default as LogoutPage } from "./LogoutPage"; +export { default as MapPage } from "./MapPage"; +export { default as NotFoundPage } from "./NotFoundPage"; +export { default as SettingsPage } from "./SettingsPage"; +export { default as TrackEditor } from "./TrackEditor"; +export { default as TrackPage } from "./TrackPage"; +export { default as TracksPage } from "./TracksPage"; +export { default as MyTracksPage } from "./MyTracksPage"; +export { default as UploadPage } from "./UploadPage"; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index e3f5c74..36f5cfd 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1,52 +1,67 @@ -import type {FeatureCollection, Feature, LineString, Point} from 'geojson' +import type { FeatureCollection, Feature, LineString, Point } from "geojson"; -export type UserProfile = { - id: number | string - displayName: string - image?: string | null - bio?: string | null +export interface UserProfile { + username: string; + displayName: string; + image?: string | null; + bio?: string | null; } -export type TrackData = { - track: Feature - measurements: FeatureCollection - overtakingEvents: FeatureCollection +export interface TrackData { + track: Feature; + measurements: FeatureCollection; + overtakingEvents: FeatureCollection; } -export type Track = { - slug: string - author: UserProfile - title: string - description?: string - createdAt: string - public?: boolean - recordedAt?: Date - recordedUntil?: Date - duration?: number - length?: number - segments?: number - numEvents?: number - numMeasurements?: number - numValid?: number +export type ProcessingStatus = + | "error" + | "complete" + | "created" + | "queued" + | "processing"; + +export interface Track { + slug: string; + author: UserProfile; + title: string; + description?: string; + createdAt: string; + processingStatus?: ProcessingStatus; + public?: boolean; + recordedAt?: Date; + recordedUntil?: Date; + duration?: number; + length?: number; + segments?: number; + numEvents?: number; + numMeasurements?: number; + numValid?: number; + userDeviceId?: number; } -export type TrackPoint = { - type: 'Feature' - geometry: Point +export interface TrackPoint { + type: "Feature"; + geometry: Point; properties: { - distanceOvertaker: null | number - distanceStationary: null | number - } + distanceOvertaker: null | number; + distanceStationary: null | number; + }; } -export type TrackComment = { - id: string - body: string - createdAt: string - author: UserProfile +export interface TrackComment { + id: string; + body: string; + createdAt: string; + author: UserProfile; } -export type Location { +export interface Location { longitude: number; latitude: number; } + +export interface UserDevice { + id: number; + identifier: string; + displayName?: string; +} diff --git a/frontend/src/utils.js b/frontend/src/utils.js index 2fdb1a7..f4f3dc7 100644 --- a/frontend/src/utils.js +++ b/frontend/src/utils.js @@ -1,32 +1,51 @@ -import {useRef, useCallback} from 'react' +import { useRef, useCallback } from "react"; +import { Duration } from "luxon"; // Wraps the register callback from useForm into a new ref function, such that // any child of the provided element that is an input component will be // registered. export function findInput(register) { return (element) => { - const found = element ? element.querySelector('input, textarea, select, checkbox') : null - register(found) - } + const found = element + ? element.querySelector("input, textarea, select, checkbox") + : null; + register(found); + }; } // Generates pairs from the input iterable export function* pairwise(it) { - let lastValue - let firstRound = true + let lastValue; + let firstRound = true; for (const i of it) { if (firstRound) { - firstRound = false + firstRound = false; } else { - yield [lastValue, i] + yield [lastValue, i]; } - lastValue = i + lastValue = i; } } export function useCallbackRef(fn) { - const fnRef = useRef() - fnRef.current = fn - return useCallback(((...args) => fnRef.current(...args)), []) + const fnRef = useRef(); + fnRef.current = fn; + return useCallback((...args) => fnRef.current(...args), []); +} + +export function formatDuration(seconds) { + return Duration.fromMillis((seconds ?? 0) * 1000).toFormat("h'h' mm'm'"); +} + +export function formatDistance(meters) { + if (meters == null) return null; + + if (meters < 0) return "-" + formatDistance(meters); + + if (meters < 1000) { + return `${meters.toFixed(0)} m`; + } else { + return `${(meters / 1000).toFixed(2)} km`; + } } From 4fe7d45dec2ee28256d7d75bb789fbc3b7d9cdf0 Mon Sep 17 00:00:00 2001 From: Paul Bienkowski Date: Thu, 22 Sep 2022 20:09:54 +0200 Subject: [PATCH 4/6] Bulk update operations on tracks --- api/obs/api/routes/tracks.py | 35 +++- frontend/src/pages/MyTracksPage.tsx | 271 ++++++++++++++++++---------- 2 files changed, 213 insertions(+), 93 deletions(-) diff --git a/api/obs/api/routes/tracks.py b/api/obs/api/routes/tracks.py index e9df89c..b004c3e 100644 --- a/api/obs/api/routes/tracks.py +++ b/api/obs/api/routes/tracks.py @@ -3,7 +3,7 @@ import re from json import load as jsonload from os.path import join, exists, isfile -from sqlalchemy import select, func +from sqlalchemy import select, func, and_ from sqlalchemy.orm import joinedload from obs.api.db import Track, User, Comment, DuplicateTrackFileError @@ -128,6 +128,39 @@ async def get_feed(req): return await _return_tracks(req, extend_query, limit, offset, order_by) +@api.post("/tracks/bulk") +@require_auth +async def tracks_bulk_action(req): + body = req.json + action = body["action"] + track_slugs = body["tracks"] + + if action not in ("delete", "makePublic", "makePrivate", "reprocess"): + raise InvalidUsage("invalid action") + + query = select(Track).where( + and_(Track.author_id == req.ctx.user.id, Track.slug.in_(track_slugs)) + ) + + for track in (await req.ctx.db.execute(query)).scalars(): + if action == "delete": + await req.ctx.db.delete(track) + elif action == "makePublic": + if not track.public: + track.queue_processing() + track.public = True + elif action == "makePrivate": + if track.public: + track.queue_processing() + track.public = False + elif action == "reprocess": + track.queue_processing() + + await req.ctx.db.commit() + + return empty() + + @api.post("/tracks") @read_api_key @require_auth diff --git a/frontend/src/pages/MyTracksPage.tsx b/frontend/src/pages/MyTracksPage.tsx index f8615f3..104abde 100644 --- a/frontend/src/pages/MyTracksPage.tsx +++ b/frontend/src/pages/MyTracksPage.tsx @@ -1,8 +1,10 @@ -import React, { useCallback, useState } from "react"; +import React, { useCallback, useMemo, useState } from "react"; import { connect } from "react-redux"; import { Accordion, Button, + Checkbox, + Confirm, Header, Icon, Item, @@ -15,7 +17,7 @@ import { } from "semantic-ui-react"; import { useObservable } from "rxjs-hooks"; import { Link } from "react-router-dom"; -import { of, from, concat } from "rxjs"; +import { of, from, concat, BehaviorSubject, combineLatest } from "rxjs"; import { map, switchMap, distinctUntilChanged } from "rxjs/operators"; import _ from "lodash"; import { useTranslation } from "react-i18next"; @@ -23,7 +25,7 @@ import { useTranslation } from "react-i18next"; import type { ProcessingStatus, Track, UserDevice } from "types"; import { Page, FormattedDate, Visibility } from "components"; import api from "api"; -import { formatDistance, formatDuration } from "utils"; +import { useCallbackRef, formatDistance, formatDuration } from "utils"; const COLOR_BY_STATUS: Record = { error: "red", @@ -150,11 +152,23 @@ function TrackFilters({ ); } -function TracksTable() { +function TracksTable({ title }) { const [orderBy, setOrderBy] = useState("recordedAt"); const [reversed, setReversed] = useState(false); const [showFilters, setShowFilters] = useState(false); const [filters, setFilters] = useState({}); + const [selectedTracks, setSelectedTracks] = useState>( + {} + ); + + const toggleTrackSelection = useCallbackRef( + (slug: string, selected?: boolean) => { + const newSelected = selected ?? !selectedTracks[slug]; + setSelectedTracks( + _.pickBy({ ...selectedTracks, [slug]: newSelected }, _.identity) + ); + } + ); const query = _.pickBy( { @@ -168,12 +182,17 @@ function TracksTable() { (x) => x != null ); + const forceUpdate$ = useMemo(() => new BehaviorSubject(null), []); const tracks: Track[] | null = useObservable( (_$, inputs$) => - inputs$.pipe( - map(([query]) => query), - distinctUntilChanged(_.isEqual), - switchMap((query) => + combineLatest([ + inputs$.pipe( + map(([query]) => query), + distinctUntilChanged(_.isEqual) + ), + forceUpdate$, + ]).pipe( + switchMap(([query]) => concat( of(null), from(api.get("/tracks/feed", { query }).then((r) => r.tracks)) @@ -201,88 +220,163 @@ function TracksTable() { const p = { orderBy, setOrderBy, reversed, setReversed }; + const selectedCount = Object.keys(selectedTracks).length; + const noneSelected = selectedCount === 0; + const allSelected = selectedCount === tracks?.length; + const selectAll = () => { + setSelectedTracks( + Object.fromEntries(tracks?.map((t) => [t.slug, true]) ?? []) + ); + }; + const selectNone = () => { + setSelectedTracks({}); + }; + + const bulkAction = async (action: string) => { + await api.post("/tracks/bulk", { + body: { + action, + tracks: Object.keys(selectedTracks), + }, + }); + + setShowBulkDelete(false); + setSelectedTracks({}); + forceUpdate$.next(null); + }; + const [showBulkDelete, setShowBulkDelete] = useState(false); + return ( -
- + <> +
+ + + + Selection of {selectedCount} tracks + + bulkAction("makePrivate")}> + Make private + + bulkAction("makePublic")}> + Make public + + bulkAction("reprocess")}> + Reprocess + + setShowBulkDelete(true)}> + Delete + + + + +
- - setShowFilters(!showFilters)} - > - - Filters - - - - - +
{title}
+
+ - - - - - Title - - - Recorded at - - - Visibility - - - Length - - - Duration - - - Device - - - + + setShowFilters(!showFilters)} + > + + Filters + + + + + - - {tracks?.map((track: Track) => ( - - - {track.processingStatus == null ? null : ( - - )} - - {track.title || t("general.unnamedTrack")} - - + setShowBulkDelete(false)} + onConfirm={() => bulkAction("delete")} + content={`Are you sure you want to delete ${selectedCount} tracks?`} + confirmButton={t("general.delete")} + cancelButton={t("general.cancel")} + /> - - - +
+ + + + (noneSelected ? selectAll() : selectNone())} + /> + - - {track.public == null ? null : ( - - )} - - - - {formatDistance(track.length)} - - - - {formatDuration(track.duration)} - - - - {track.userDeviceId - ? deviceNames?.[track.userDeviceId] ?? "..." - : null} - + + Title + + + Recorded at + + + Visibility + + + Length + + + Duration + + + Device + - ))} - -
-
+ + + + {tracks?.map((track: Track) => ( + + + toggleTrackSelection(track.slug)} + checked={selectedTracks[track.slug] ?? false} + /> + + + {track.processingStatus == null ? null : ( + + )} + + {track.title || t("general.unnamedTrack")} + + + + + + + + + {track.public == null ? null : ( + + )} + + + + {formatDistance(track.length)} + + + + {formatDuration(track.duration)} + + + + {track.userDeviceId + ? deviceNames?.[track.userDeviceId] ?? "..." + : null} + + + ))} + + +
+ ); } @@ -296,12 +390,7 @@ function UploadButton({ navigate, ...props }) { [navigate] ); return ( - ); @@ -315,9 +404,7 @@ const MyTracksPage = connect((state) => ({ login: (state as any).login }))( return ( - -
{title}
- + ); } From 141460c79fc570cfb3f610f17b1db519ed0e1205 Mon Sep 17 00:00:00 2001 From: Paul Bienkowski Date: Sun, 27 Nov 2022 16:47:12 +0100 Subject: [PATCH 5/6] Split settings page --- frontend/src/components/Stats/index.tsx | 131 ++++++---- frontend/src/pages/MyTracksPage.tsx | 2 +- frontend/src/pages/SettingsPage.tsx | 227 ------------------ .../src/pages/SettingsPage/ApiKeySettings.tsx | 125 ++++++++++ .../pages/SettingsPage/UserSettingsForm.tsx | 89 +++++++ frontend/src/pages/SettingsPage/index.tsx | 64 +++++ frontend/src/pages/UploadPage.tsx | 6 +- frontend/src/translations/de.yaml | 13 +- frontend/src/translations/en.yaml | 12 +- 9 files changed, 377 insertions(+), 292 deletions(-) delete mode 100644 frontend/src/pages/SettingsPage.tsx create mode 100644 frontend/src/pages/SettingsPage/ApiKeySettings.tsx create mode 100644 frontend/src/pages/SettingsPage/UserSettingsForm.tsx create mode 100644 frontend/src/pages/SettingsPage/index.tsx diff --git a/frontend/src/components/Stats/index.tsx b/frontend/src/components/Stats/index.tsx index 2c7abd3..ef0b3c2 100644 --- a/frontend/src/components/Stats/index.tsx +++ b/frontend/src/components/Stats/index.tsx @@ -1,118 +1,145 @@ -import React, {useState, useCallback} from 'react' -import {pickBy} from 'lodash' -import {Loader, Statistic, Segment, Header, Menu} from 'semantic-ui-react' -import {useObservable} from 'rxjs-hooks' -import {of, from, concat, combineLatest} from 'rxjs' -import {map, switchMap, distinctUntilChanged} from 'rxjs/operators' -import {Duration, DateTime} from 'luxon' -import {useTranslation} from 'react-i18next' +import React, { useState, useCallback } from "react"; +import { pickBy } from "lodash"; +import { Loader, Statistic, Segment, Header, Menu } from "semantic-ui-react"; +import { useObservable } from "rxjs-hooks"; +import { of, from, concat, combineLatest } from "rxjs"; +import { map, switchMap, distinctUntilChanged } from "rxjs/operators"; +import { Duration, DateTime } from "luxon"; +import { useTranslation } from "react-i18next"; -import api from 'api' +import api from "api"; function formatDuration(seconds) { return ( Duration.fromMillis((seconds ?? 0) * 1000) - .as('hours') - .toFixed(1) + ' h' - ) + .as("hours") + .toFixed(1) + " h" + ); } -export default function Stats({user = null}: {user?: null | string}) { - const {t} = useTranslation() - const [timeframe, setTimeframe] = useState('all_time') - const onClick = useCallback((_e, {name}) => setTimeframe(name), [setTimeframe]) +export default function Stats({ user = null }: { user?: null | string }) { + const { t } = useTranslation(); + const [timeframe, setTimeframe] = useState("all_time"); + const onClick = useCallback( + (_e, { name }) => setTimeframe(name), + [setTimeframe] + ); const stats = useObservable( (_$, inputs$) => { const timeframe$ = inputs$.pipe( map((inputs) => inputs[0]), distinctUntilChanged() - ) + ); const user$ = inputs$.pipe( map((inputs) => inputs[1]), distinctUntilChanged() - ) + ); return combineLatest(timeframe$, user$).pipe( map(([timeframe_, user_]) => { - const now = DateTime.now() + const now = DateTime.now(); - let start, end + let start, end; switch (timeframe_) { - case 'this_month': - start = now.startOf('month') - end = now.endOf('month') - break + case "this_month": + start = now.startOf("month"); + end = now.endOf("month"); + break; - case 'this_year': - start = now.startOf('year') - end = now.endOf('year') - break + case "this_year": + start = now.startOf("year"); + end = now.endOf("year"); + break; } return pickBy({ start: start?.toISODate(), end: end?.toISODate(), user: user_, - }) + }); }), - switchMap((query) => concat(of(null), from(api.get('/stats', {query})))) - ) + switchMap((query) => + concat(of(null), from(api.get("/stats", { query }))) + ) + ); }, null, [timeframe, user] - ) + ); - const placeholder = t('Stats.placeholder') + const placeholder = t("Stats.placeholder"); return ( <> -
{user ? t('Stats.titleUser') : t('Stats.title')}
-
- {stats ? `${Number(stats?.trackLength / 1000).toFixed(1)} km` : placeholder} - {t('Stats.totalTrackLength')} + + {stats + ? `${Number(stats?.trackLength / 1000).toFixed(1)} km` + : placeholder} + + {t("Stats.totalTrackLength")} - {stats ? formatDuration(stats?.trackDuration) : placeholder} - {t('Stats.timeRecorded')} + + {stats ? formatDuration(stats?.trackDuration) : placeholder} + + {t("Stats.timeRecorded")} - {stats?.numEvents ?? placeholder} - {t('Stats.eventsConfirmed')} + + {stats?.numEvents ?? placeholder} + + {t("Stats.eventsConfirmed")} {user ? ( - {stats?.trackCount ?? placeholder} - {t('Stats.tracksRecorded')} + + {stats?.trackCount ?? placeholder} + + {t("Stats.tracksRecorded")} ) : ( - {stats?.userCount ?? placeholder} - {t('Stats.membersJoined')} + + {stats?.userCount ?? placeholder} + + {t("Stats.membersJoined")} )} - - {t('Stats.thisMonth')} + + {t("Stats.thisMonth")} - - {t('Stats.thisYear')} + + {t("Stats.thisYear")} - - {t('Stats.allTime')} + + {t("Stats.allTime")}
- ) + ); } diff --git a/frontend/src/pages/MyTracksPage.tsx b/frontend/src/pages/MyTracksPage.tsx index 104abde..48b9f4e 100644 --- a/frontend/src/pages/MyTracksPage.tsx +++ b/frontend/src/pages/MyTracksPage.tsx @@ -271,7 +271,7 @@ function TracksTable({ title }) { -
{title}
+
{title}
diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx deleted file mode 100644 index 11be6e5..0000000 --- a/frontend/src/pages/SettingsPage.tsx +++ /dev/null @@ -1,227 +0,0 @@ -import React from "react"; -import { connect } from "react-redux"; -import { - Message, - Icon, - Grid, - Form, - Button, - TextArea, - Ref, - Input, - Header, - Divider, - Popup, -} from "semantic-ui-react"; -import { useForm } from "react-hook-form"; -import Markdown from "react-markdown"; -import { useTranslation } from "react-i18next"; - -import { setLogin } from "reducers/login"; -import { Page, Stats } from "components"; -import api from "api"; -import { findInput } from "utils"; -import { useConfig } from "config"; - -const SettingsPage = connect((state) => ({ login: state.login }), { setLogin })( - function SettingsPage({ login, setLogin }) { - const { t } = useTranslation(); - const { register, handleSubmit } = useForm(); - const [loading, setLoading] = React.useState(false); - const [errors, setErrors] = React.useState(null); - - const onSave = React.useCallback( - async (changes) => { - setLoading(true); - setErrors(null); - try { - const response = await api.put("/user", { body: changes }); - setLogin(response); - } catch (err) { - setErrors(err.errors); - } finally { - setLoading(false); - } - }, - [setLoading, setLogin, setErrors] - ); - - const onGenerateNewKey = React.useCallback(async () => { - setLoading(true); - setErrors(null); - try { - const response = await api.put("/user", { - body: { updateApiKey: true }, - }); - setLogin(response); - } catch (err) { - setErrors(err.errors); - } finally { - setLoading(false); - } - }, [setLoading, setLogin, setErrors]); - - return ( - - - - -
{t("SettingsPage.profile.title")}
- -
- - - - - - {t("SettingsPage.profile.username.hint")} - - - - {t("SettingsPage.profile.publicNotice")} - - - - - - - - - {t("SettingsPage.profile.displayName.fallbackNotice")} - - - - - - -