improve sql formatting and parameter passing

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.8.1
+
+### Improvements
+
+* The zone (urban/rural) is now also exported with the events GeoJson export.
+
+### Bug Fixes
+
+* Update to a current version of gpstime (python dependency) fix portal startup.
+
 ## 0.8.0
 
 ### Features

UPGRADING.md

@@ -5,6 +5,17 @@ explicitly. Their general usage is described in the [README](./README.md) (for
 development) and [docs/production-deployment.md](docs/production-deployment.md) (for production).
 
 
+## 0.8.1
+ 
+- Get the release in your source folder (``git pull; git checkout 0.8.0`` and update submodules ``git submodule update --recursive``)
+- Rebuild images ``docker-compose build``
+- No database upgrade is required, but tile functions need an update:
+  ```bash
+  docker-compose run --rm portal tools/prepare_sql_tiles.py
+  ```
+- Start your portal and worker services. ``docker-compose up -d worker portal``
+
+
 ## 0.8.0
 Upgrade to `0.7.x` first. See below for details. Then follow these steps:
 

api/obs/api/__init__.py

@@ -1 +1 @@
-__version__ = "0.8.0"
+__version__ = "0.8.1"

api/obs/api/routes/exports.py

@@ -4,6 +4,7 @@ from contextlib import contextmanager
 import zipfile
 import io
 import re
+import math
 from sqlite3 import connect
 
 import shapefile
@@ -15,6 +16,10 @@ from sanic.exceptions import InvalidUsage
 from obs.api.app import api, json as json_response
 from obs.api.utils import use_request_semaphore
 
+import logging
+
+log = logging.getLogger(__name__)
+
 
 class ExportFormat(str, Enum):
     SHAPEFILE = "shapefile"
@@ -63,15 +68,35 @@ def shapefile_zip(shape_type=shapefile.POINT, basename="events"):
 @api.get(r"/export/events")
 async def export_events(req):
     async with use_request_semaphore(req, "export_semaphore", timeout=30):
-        bbox = req.ctx.get_single_arg(
-            "bbox", default="-180,-90,180,90", convert=parse_bounding_box
-        )
+        bbox = req.ctx.get_single_arg("bbox", default="-180,-90,180,90")
+        assert re.match(r"(-?\d+\.?\d+,?){4}", bbox)
+        bbox = list(map(float, bbox.split(",")))
+
         fmt = req.ctx.get_single_arg("fmt", convert=ExportFormat)
 
-        events = await req.ctx.db.stream_scalars(
-            select(OvertakingEvent).where(
-                OvertakingEvent.geometry.bool_op("&&")(func.ST_Transform(bbox, 3857))
-            )
+        events = await req.ctx.db.stream(
+            text(
+                """
+                SELECT
+                    ST_AsGeoJSON(ST_Transform(geometry, 4326)) AS geometry,
+                    distance_overtaker,
+                    distance_stationary,
+                    way_id,
+                    direction,
+                    speed,
+                    time_stamp,
+                    course,
+                    zone
+                FROM
+                    layer_obs_events(
+                        ST_Transform(ST_MakeEnvelope(:bbox0, :bbox1, :bbox2, :bbox3, 4326), 3857),
+                        19,
+                        NULL,
+                        '1900-01-01'::timestamp,
+                        '2100-01-01'::timestamp
+                    )
+                """
+            ).bindparams(bbox0=bbox[0], bbox1=bbox[1], bbox2=bbox[2], bbox3=bbox[3])
         )
 
         if fmt == ExportFormat.SHAPEFILE:
@@ -82,16 +107,19 @@ async def export_events(req):
                 writer.field("direction", "N", decimal=0)
                 writer.field("course", "N", decimal=4)
                 writer.field("speed", "N", decimal=4)
+                writer.field("zone", "C")
 
                 async for event in events:
-                    writer.point(event.longitude, event.latitude)
+                    coords = json.loads(event.geometry)["coordinates"]
+                    writer.point(*coords)
                     writer.record(
                         distance_overtaker=event.distance_overtaker,
                         distance_stationary=event.distance_stationary,
-                        direction=-1 if event.direction_reversed else 1,
+                        direction=event.direction,
                         way_id=event.way_id,
                         course=event.course,
                         speed=event.speed,
+                        zone=event.zone
                         # "time"=event.time,
                     )
 
@@ -100,18 +128,33 @@ async def export_events(req):
         if fmt == ExportFormat.GEOJSON:
             features = []
             async for event in events:
+                geom = json.loads(event.geometry)
                 features.append(
                     {
                         "type": "Feature",
-                        "geometry": json.loads(event.geometry),
+                        "geometry": geom,
                         "properties": {
-                            "distance_overtaker": event.distance_overtaker,
-                            "distance_stationary": event.distance_stationary,
-                            "direction": -1 if event.direction_reversed else 1,
+                            "distance_overtaker": event.distance_overtaker
+                            if event.distance_overtaker is not None
+                            and not math.isnan(event.distance_overtaker)
+                            else None,
+                            "distance_stationary": event.distance_stationary
+                            if event.distance_stationary is not None
+                            and not math.isnan(event.distance_stationary)
+                            else None,
+                            "direction": event.direction
+                            if event.direction is not None
+                            and not math.isnan(event.direction)
+                            else None,
                             "way_id": event.way_id,
-                            "course": event.course,
-                            "speed": event.speed,
-                            "time": event.time,
+                            "course": event.course
+                            if event.course is not None and not math.isnan(event.course)
+                            else None,
+                            "speed": event.speed
+                            if event.speed is not None and not math.isnan(event.speed)
+                            else None,
+                            "time": event.time_stamp,
+                            "zone": event.zone,
                         },
                     }
                 )
@@ -125,19 +168,45 @@ async def export_events(req):
 @api.get(r"/export/segments")
 async def export_segments(req):
     async with use_request_semaphore(req, "export_semaphore", timeout=30):
-        bbox = req.ctx.get_single_arg(
-            "bbox", default="-180,-90,180,90"
-        )
+        bbox = req.ctx.get_single_arg("bbox", default="-180,-90,180,90")
         assert re.match(r"(-?\d+\.?\d+,?){4}", bbox)
+        bbox = list(map(float, bbox.split(",")))
+
         fmt = req.ctx.get_single_arg("fmt", convert=ExportFormat)
         segments = await req.ctx.db.stream(
             text(
-                f"select ST_AsGeoJSON(ST_Transform(geometry,4326)) AS geometry, way_id, distance_overtaker_mean, distance_overtaker_min,distance_overtaker_max,distance_overtaker_median,overtaking_event_count,usage_count,direction,zone,offset_direction,distance_overtaker_array from layer_obs_roads(ST_Transform(ST_MakeEnvelope({bbox},4326),3857),11,NULL,'1900-01-01'::timestamp,'2100-01-01'::timestamp) WHERE usage_count>0"
-            )
+                """
+                SELECT
+                    ST_AsGeoJSON(ST_Transform(geometry, 4326)) AS geometry,
+                    way_id,
+                    distance_overtaker_mean,
+                    distance_overtaker_min,
+                    distance_overtaker_max,
+                    distance_overtaker_median,
+                    overtaking_event_count,
+                    usage_count,
+                    direction,
+                    zone,
+                    offset_direction,
+                    distance_overtaker_array
+                FROM
+                    layer_obs_roads(
+                        ST_Transform(ST_MakeEnvelope(:bbox0, :bbox1, :bbox2, :bbox3, 4326), 3857),
+                        11,
+                        NULL,
+                        '1900-01-01'::timestamp,
+                        '2100-01-01'::timestamp
+                    )
+                WHERE usage_count > 0
+                """
+            ).bindparams(bbox0=bbox[0], bbox1=bbox[1], bbox2=bbox[2], bbox3=bbox[3])
         )
 
         if fmt == ExportFormat.SHAPEFILE:
-            with shapefile_zip(shape_type=3, basename="segments") as (writer, zip_buffer):
+            with shapefile_zip(shape_type=3, basename="segments") as (
+                writer,
+                zip_buffer,
+            ):
                 writer.field("distance_overtaker_mean", "N", decimal=4)
                 writer.field("distance_overtaker_max", "N", decimal=4)
                 writer.field("distance_overtaker_min", "N", decimal=4)

api/obs/api/routes/tiles.py

@@ -96,7 +96,7 @@ async def tiles(req, zoom: int, x: int, y: str):
 
             tile = await req.ctx.db.scalar(
                 text(
-                    f"select data from getmvt(:zoom, :x, :y, :user_id, :min_time, :max_time) as b(data, key);"
+                    "select data from getmvt(:zoom, :x, :y, :user_id, :min_time, :max_time) as b(data, key);"
                 ).bindparams(
                     zoom=int(zoom),
                     x=int(x),

api/obs/bin/openbikesensor_transform_osm.py

@@ -163,6 +163,11 @@ class OSMHandler(osmium.SimpleHandler):
         if not highway or highway not in HIGHWAY_TYPES:
             return
 
+        access = tags.get("access", None)
+        bicycle = tags.get("bicycle", None)
+        if access == "no" and bicycle not in ["designated", "yes", "permissive", "destination"]:
+            return
+
         zone = determine_zone(tags)
         directionality, oneway = determine_direction(tags, zone)
         name = tags.get("name")

api/scripts

@@ -1 +1 @@
-Subproject commit f513117e275be20008afa1e1fd2499698313a81d
+Subproject commit 664e4d606416417c0651ea1748d32dd36209be6a

tile-generator/layers/obs_events/layer.sql

@@ -1,5 +1,7 @@
+DROP FUNCTION IF EXISTS layer_obs_events(bbox geometry, zoom_level int, user_id integer, min_time timestamp, max_time timestamp);
+
 CREATE OR REPLACE FUNCTION layer_obs_events(bbox geometry, zoom_level int, user_id integer, min_time timestamp, max_time timestamp)
-RETURNS TABLE(event_id bigint, geometry geometry, distance_overtaker float, distance_stationary float, direction int, course float, speed float, zone zone_type, way_id bigint) AS $$
+RETURNS TABLE(event_id bigint, geometry geometry, distance_overtaker float, distance_stationary float, direction int, course float, speed float, time_stamp timestamp, zone zone_type, way_id bigint) AS $$
 
     SELECT
       overtaking_event.id::bigint as event_id,
@@ -9,6 +11,7 @@ RETURNS TABLE(event_id bigint, geometry geometry, distance_overtaker float, dist
       (case when direction_reversed then -1 else 1 end)::int as direction,
       course,
       speed,
+      time as time_stamp,
       CASE WHEN road.zone IS NULL THEN 'urban' else road.zone END as zone,
       overtaking_event.way_id::bigint as way_id
     FROM overtaking_event

tile-generator/layers/obs_events/obs_events.yaml

@@ -16,6 +16,8 @@ layer:
       Direction of travel, as reported by GPS, in degree from North.
     speed: |
       Speed of travel, as reported by GPS, in meters per second (?).
+    time_stamp: |
+      the time of the overtaking
     zone: |
       rural or urban
   defaults:

tile-generator/openbikesensor.yaml

@@ -4,7 +4,7 @@ tileset:
     - layers/obs_events/obs_events.yaml
     - layers/obs_roads/obs_roads.yaml
     - layers/obs_regions/obs_regions.yaml
-  version: 0.8.0
+  version: 0.8.1
   id: openbikesensor
   description: >
     A tileset for OpenBikeSensor data -- https://openbikesensor.org.