From 0c256d8923c09ff94ce4dcf8e247921c647b442a Mon Sep 17 00:00:00 2001 From: Paul Bienkowski Date: Wed, 19 Jan 2022 09:11:06 +0100 Subject: [PATCH] Add export routes --- api/obs/api/app.py | 1 + api/obs/api/routes/exports.py | 137 +++++++++++++++++++++++++++++++ api/obs/api/routes/mapdetails.py | 7 +- api/obs/api/routes/tiles.py | 1 + api/requirements.txt | 1 + api/setup.py | 1 + 6 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 api/obs/api/routes/exports.py diff --git a/api/obs/api/app.py b/api/obs/api/app.py index 100cafa..219f9f7 100644 --- a/api/obs/api/app.py +++ b/api/obs/api/app.py @@ -254,6 +254,7 @@ from .routes import ( tracks, users, mapdetails, + exports, ) from .routes import frontend diff --git a/api/obs/api/routes/exports.py b/api/obs/api/routes/exports.py new file mode 100644 index 0000000..50d6a40 --- /dev/null +++ b/api/obs/api/routes/exports.py @@ -0,0 +1,137 @@ +import json +from gzip import decompress +from enum import Enum +from contextlib import contextmanager +from sqlite3 import connect +from obs.api.db import OvertakingEvent +from sanic.response import raw +from sanic.exceptions import InvalidUsage + +from sqlalchemy import select, text, func +from sqlalchemy.sql.expression import table, column + +from obs.api.app import app, json as json_response + +from .mapdetails import get_single_arg + +@app.get(r"/export/events.json") +async def tiles(req): + x = get_single_arg(req, "x", convert=int) + y = get_single_arg(req, "y", convert=int) + zoom = get_single_arg(req, "zoom", convert=int) + + features = [] + async for event in get_events(req.ctx.db, zoom, x, y): + features.append({ + "type": "Feature", + "geometry": json.loads(event.geometry), + "properties": { + "distance_overtaker": event.distance_overtaker, + "distance_stationary": event.distance_stationary, + "direction": -1 if event.direction_reversed else 1, + "way_id": event.way_id, + "course": event.course, + "speed": event.speed, + "time": event.time, + } + }) + + geojson = {"type": "FeatureCollection", "features": features} + return json_response(geojson) + +class ExportFormat(str, Enum): + SHAPEFILE = "shapefile" + GEOJSON = "geojson" + + +def parse_bounding_box(s): + left, bottom, right, top = map(float, s.split(",")) + return func.ST_SetSRID( + func.ST_MakeBox2D( + func.ST_Point(left, bottom), + func.ST_Point(right, top), + ), + 3857, + ) + +@contextmanager +def shapefile_zip(): + import io, shapefile + zip_buffer = io.BytesIO() + shp, shx, dbf = (io.BytesIO() for _ in range(3)) + writer = shapefile.Writer(shp=shp, shx=shx, dbf=dbf, shapeType=shapefile.POINT, encoding="utf8") + + yield writer, zip_buffer + + writer.balance() + writer.close() + + PRJ = ( + 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],' + 'AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],' + 'UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]' + ) + + import zipfile + zf = zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED, False) + zf.writestr('events.shp', shp.getbuffer()) + zf.writestr('events.shx', shx.getbuffer()) + zf.writestr('events.dbf', dbf.getbuffer()) + zf.writestr('events.prj', PRJ) + zf.close() + +@app.get(r"/export/events") +async def export_events(req): + bbox = get_single_arg(req, "bbox", default="-180,-90,180,90", convert=parse_bounding_box) + fmt = get_single_arg(req, "fmt", convert=ExportFormat) + + events = await req.ctx.db.stream_scalars( + select(OvertakingEvent) + .where(OvertakingEvent.geometry.bool_op("&&")(bbox)) + ) + + if fmt == ExportFormat.SHAPEFILE: + with shapefile_zip() as (writer, zip_buffer): + writer.field("distance_overtaker", "N", decimal=4) + writer.field("distance_stationary", "N", decimal=4) + writer.field("way_id", "N", decimal=0) + writer.field("direction", "N", decimal=0) + writer.field("course", "N", decimal=4) + writer.field("speed", "N", decimal=4) + + async for event in events: + writer.point(event.longitude, event.latitude) + writer.record( + distance_overtaker=event.distance_overtaker, + distance_stationary=event.distance_stationary, + direction=-1 if event.direction_reversed else 1, + way_id=event.way_id, + course=event.course, + speed=event.speed, + #"time"=event.time, + ) + + return raw(zip_buffer.getbuffer()) + + elif fmt == ExportFormat.GEOJSON: + features = [] + async for event in events: + features.append({ + "type": "Feature", + "geometry": json.loads(event.geometry), + "properties": { + "distance_overtaker": event.distance_overtaker, + "distance_stationary": event.distance_stationary, + "direction": -1 if event.direction_reversed else 1, + "way_id": event.way_id, + "course": event.course, + "speed": event.speed, + "time": event.time, + } + }) + + geojson = {"type": "FeatureCollection", "features": features} + return json_response(geojson) + + else: + raise InvalidUsage("unknown export format") diff --git a/api/obs/api/routes/mapdetails.py b/api/obs/api/routes/mapdetails.py index fc475d7..f05fb43 100644 --- a/api/obs/api/routes/mapdetails.py +++ b/api/obs/api/routes/mapdetails.py @@ -24,9 +24,10 @@ def get_single_arg(req, name, default=RAISE, convert=None): try: value = req.args[name][0] except LookupError as e: - if default is not RAISE: - return default - raise InvalidUsage("missing `{name}`") from e + if default is RAISE: + raise InvalidUsage("missing `{name}`") from e + + value = default if convert is not None: try: diff --git a/api/obs/api/routes/tiles.py b/api/obs/api/routes/tiles.py index 43bdce7..abaf70b 100644 --- a/api/obs/api/routes/tiles.py +++ b/api/obs/api/routes/tiles.py @@ -66,3 +66,4 @@ async def tiles(req, zoom: int, x: int, y: str): tile = decompress(tile) return raw(tile, content_type="application/x-protobuf", headers=headers) + diff --git a/api/requirements.txt b/api/requirements.txt index 10fdf03..5bf00c3 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -10,3 +10,4 @@ pyyaml<6 sqlparse~=0.4.2 sqlalchemy[asyncio]~=1.4.25 asyncpg~=0.24.0 +pyshp~=2.1.3 diff --git a/api/setup.py b/api/setup.py index ff46f83..e12ca3a 100644 --- a/api/setup.py +++ b/api/setup.py @@ -19,6 +19,7 @@ setup( "motor~=2.5.1", "sqlparse~=0.4.2", "openmaptiles-tools", # install from git + "pyshp~=2.1.3", ], entry_points={ "console_scripts": [