From 497e1b739af0ebd71204b69253831e1f92e9c6e1 Mon Sep 17 00:00:00 2001 From: gluap Date: Thu, 25 May 2023 22:24:05 +0200 Subject: [PATCH 1/6] Add a first shot at upgrade documentation (WIP) --- UPGRADING.md | 24 +++++++++++++++++++++++- api/tools/reimport_tracks.py | 30 ++++++++++++++++++++++++++++++ api/tools/upgrade.py | 15 ++++++++++----- 3 files changed, 63 insertions(+), 6 deletions(-) create mode 100755 api/tools/reimport_tracks.py diff --git a/UPGRADING.md b/UPGRADING.md index e22bf19..e8e981e 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,10 +1,32 @@ # Upgrading - This document describes the general steps to upgrade between major changes. Simple migrations, e.g. for adding schema changes, are not documented explicitly. Their general usage is described in the [README](./README.md) (for development) and [docs/production-deployment.md](docs/production-deployment.md) (for production). + +## 0.8.0 +Upgrade to `0.7.x` first. See below for details. Then follow these steps: + +> **Warning** The update includes a reprocessing of tracks after import. Depending on the number of tracks this can take a few hours. The portal is reachable during that time but events disappear and incrementally reappear during reimport. + +> **Info** With this version the Import process for OpenStreetMap data has changed: the [new process](docs/osm-import.md) is easier on resources and finally permits to import a full country on a low-end VM. + +- Do your [usual backup](docs/production-deployment.md) +- Rebuild images +- Stop your portal and worker services +- run upgrade + ```bash + docker-compose run --rm portal tools/upgrade.py + ``` + this automatically does the following + - Migration of database schema using alembic. + - Upgrade of SQL tile schema to new schema. + - Import the nuts-regions from the web into the database. + - Trigger a re-import of all tracks. +- Start your portal and worker services. + + ## 0.7.0 Upgrade to `0.6.x` first. See below for details. Then follow these steps: diff --git a/api/tools/reimport_tracks.py b/api/tools/reimport_tracks.py new file mode 100755 index 0000000..4f201db --- /dev/null +++ b/api/tools/reimport_tracks.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +import logging +import asyncio + +from sqlalchemy import text + +from obs.api.app import app +from obs.api.db import connect_db, make_session + +log = logging.getLogger(__name__) + +async def main(): + logging.basicConfig(level=logging.DEBUG, format="%(levelname)s: %(message)s") + await reimport_tracks() + + +async def reimport_tracks(): + + async with connect_db( + app.config.POSTGRES_URL, + app.config.POSTGRES_POOL_SIZE, + app.config.POSTGRES_MAX_OVERFLOW, + ): + async with make_session() as session: + await session.execute(text("UPDATE track SET processing_status = 'queued';")) + await session.commit() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/api/tools/upgrade.py b/api/tools/upgrade.py index 86ff1d7..8b1c83d 100755 --- a/api/tools/upgrade.py +++ b/api/tools/upgrade.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 -import logging import asyncio -from alembic.config import Config -from alembic import command -from os.path import join, dirname +import logging log = logging.getLogger(__name__) from prepare_sql_tiles import prepare_sql_tiles, _run +from import_regions import main as import_nuts + +from reimport_tracks import main as reimport_tracks + async def _migrate(): await _run("alembic upgrade head") @@ -20,7 +21,11 @@ async def main(): await _migrate() log.info("Preparing SQL tiles...") await prepare_sql_tiles() - log.info("Upgraded") + log.info("Importing nuts regions...") + await import_nuts() + log.info("Nuts regions imported, scheduling reimport of tracks") + await reimport_tracks() + if __name__ == "__main__": From 0233045959b16207ff7dd6109eda1fc9360ff1bc Mon Sep 17 00:00:00 2001 From: gluap Date: Sat, 10 Jun 2023 12:56:44 +0200 Subject: [PATCH 2/6] import in chunks to avoid smaller systems chocking --- api/tools/import_osm.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/api/tools/import_osm.py b/api/tools/import_osm.py index 667ed87..84be8af 100755 --- a/api/tools/import_osm.py +++ b/api/tools/import_osm.py @@ -71,21 +71,25 @@ async def import_osm(connection, filename, import_group=None): # Pass 2: Import log.info("Pass 2: Import roads") - async with cursor.copy( - "COPY road (way_id, name, zone, directionality, oneway, geometry, import_group) FROM STDIN" - ) as copy: - for item in read_file(filename): - await copy.write_row( - ( - item.way_id, - item.name, - item.zone, - item.directionality, - item.oneway, - bytes.hex(item.geometry), - import_group, + amount = 0 + for items in chunk(read_file(filename), 10000): + amount += 10000 + log.info(f"...{amount}/{len(ids)} ({100*amount/len(ids)}%)") + async with cursor.copy( + "COPY road (way_id, name, zone, directionality, oneway, geometry, import_group) FROM STDIN" + ) as copy: + for item in items: + await copy.write_row( + ( + item.way_id, + item.name, + item.zone, + item.directionality, + item.oneway, + bytes.hex(item.geometry), + import_group, + ) ) - ) async def main(): From 7dd6b68da83fd691e8c13d2045ba508cf714bae4 Mon Sep 17 00:00:00 2001 From: gluap Date: Sat, 10 Jun 2023 13:00:45 +0200 Subject: [PATCH 3/6] fix percentage logging --- api/tools/import_osm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/tools/import_osm.py b/api/tools/import_osm.py index 84be8af..68808ce 100755 --- a/api/tools/import_osm.py +++ b/api/tools/import_osm.py @@ -74,7 +74,7 @@ async def import_osm(connection, filename, import_group=None): amount = 0 for items in chunk(read_file(filename), 10000): amount += 10000 - log.info(f"...{amount}/{len(ids)} ({100*amount/len(ids)}%)") + log.info(f"...{amount}/{len(road_ids)} ({100*amount/len(road_ids)}%)") async with cursor.copy( "COPY road (way_id, name, zone, directionality, oneway, geometry, import_group) FROM STDIN" ) as copy: From 4a87489b3f0b375d4d33ab8267fec003131d4f39 Mon Sep 17 00:00:00 2001 From: gluap Date: Sat, 10 Jun 2023 17:45:56 +0200 Subject: [PATCH 4/6] documentation update after dancing the dance on obs.adfc-hessen.de --- UPGRADING.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index e8e981e..482d109 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -10,11 +10,12 @@ Upgrade to `0.7.x` first. See below for details. Then follow these steps: > **Warning** The update includes a reprocessing of tracks after import. Depending on the number of tracks this can take a few hours. The portal is reachable during that time but events disappear and incrementally reappear during reimport. -> **Info** With this version the Import process for OpenStreetMap data has changed: the [new process](docs/osm-import.md) is easier on resources and finally permits to import a full country on a low-end VM. +> **Info** With this version the import process for OpenStreetMap data has changed: the [new process](docs/osm-import.md) is easier on resources and finally permits to import a full country on a low-end VM. - Do your [usual backup](docs/production-deployment.md) -- Rebuild images -- Stop your portal and worker services +- get the release in your source folder (``git pull; git checkout 0.8.0`` and update submodules ``git submodule update --recursive``) +- Rebuild images ``docker-compose build`` +- Stop your portal and worker services ``docker-compose stop worker portal`` - run upgrade ```bash docker-compose run --rm portal tools/upgrade.py @@ -24,7 +25,7 @@ Upgrade to `0.7.x` first. See below for details. Then follow these steps: - Upgrade of SQL tile schema to new schema. - Import the nuts-regions from the web into the database. - Trigger a re-import of all tracks. -- Start your portal and worker services. +- Start your portal and worker services. ``docker-compose up -d worker portal`` ## 0.7.0 From 1a1232f2a720c8833b7cd30d7b371927c50e0840 Mon Sep 17 00:00:00 2001 From: gluap Date: Sat, 10 Jun 2023 19:33:14 +0200 Subject: [PATCH 5/6] up openmaptiles version (7.0 has been running on adfc-hessen for ages), add changelog. --- CHANGELOG.md | 23 +++++++++++++++++++++++ deployment/docker-compose.yaml | 4 ++-- docker-compose.yaml | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab435d1..187af5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## 0.8.0 + +### Features + +* Bulk actions on users owned tracks (reprocess, download, make private, make public, delete) (#269, #38) +* Easy sorting by device for "multi-device users" (e.g. group lending out OBSes) +* Region display at higher zoom levels to easily find interesting areas (#112) +* Export of road statistics on top of the already-existing event statistics (#341) + +### Improvements + +* Refactored database access to hopefully combat portal crashes (#337) +* New infrastructure for map imports that makes import of larger maps possible on small VMs (#334) +* Reference current postgres and postgis versions in docker-compose.yaml files (#286) +* Configurable terms-and-conditions link (#320) +* French translation by @cbiteau (#303) + +### Bug Fixes + +* Logout not working (#285) +* Duplicate road usage hashes (#335, #253) +* cannot import name .... (#338) + ## 0.7.0 ### Features diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index 1f888d4..3d1487f 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -14,7 +14,7 @@ services: ############################################################ postgres: - image: "openmaptiles/postgis:6.0" + image: "openmaptiles/postgis:7.0" environment: - POSTGRES_DB=${OBS_POSTGRES_DB} - POSTGRES_USER=${OBS_POSTGRES_USER} @@ -136,7 +136,7 @@ services: - "traefik.docker.network=gateway" postgres-keycloak: - image: postgres:13.3 + image: postgres:15 restart: always networks: - backend diff --git a/docker-compose.yaml b/docker-compose.yaml index 7e2db47..3bba9a4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,7 +8,7 @@ version: '3' services: postgres: - image: "openmaptiles/postgis:6.0" + image: "openmaptiles/postgis:7.0" environment: POSTGRES_USER: obs POSTGRES_PASSWORD: obs From 43765092c3bbde7f156fc1c5c1b4038ba7bb4583 Mon Sep 17 00:00:00 2001 From: gluap Date: Sat, 10 Jun 2023 19:37:34 +0200 Subject: [PATCH 6/6] fix issue when displaying mapdetails (newer numpy deprecates numpy.bool, it was always an alias for bool apparently) --- api/obs/api/routes/mapdetails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/obs/api/routes/mapdetails.py b/api/obs/api/routes/mapdetails.py index ef82c74..1565fc1 100644 --- a/api/obs/api/routes/mapdetails.py +++ b/api/obs/api/routes/mapdetails.py @@ -88,7 +88,7 @@ async def mapdetails_road(req): data, mask = arrays[:-1], arrays[-1] data = data.astype(numpy.float64) - mask = mask.astype(numpy.bool) + mask = mask.astype(bool) def partition(arr, cond): return arr[:, cond], arr[:, ~cond]