feat: publish tiles from API directly

api/Dockerfile

@@ -5,7 +5,8 @@ WORKDIR /opt/obs/api
 ADD scripts /opt/obs/scripts
 RUN pip install -e /opt/obs/scripts
 
-ADD requirements.txt setup.py obs /opt/obs/api/
+ADD requirements.txt setup.py /opt/obs/api/
+ADD obs /opt/obs/api/obs/
 RUN pip install -e .
 
 EXPOSE 8000

api/config.dev.py

@@ -1,44 +1,21 @@
-# Bind address of the server
 HOST = "0.0.0.0"
 PORT = 3000
-
+DEBUG = False
-# Extended log output, but slower
+AUTO_RESTART = True
-DEBUG = True
+SECRET = "!!!!!!!!!!!!CHANGE ME!!!!!!!!!!!!"
-
-# Required to encrypt or sign sessions, cookies, tokens, etc.
-SECRET = "CHANGEME!!!!!!!!!!@##@!!$$$$$$$$$$$$$!!"
-
-# Connection to the database
 POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs"
-
-# URL to the keycloak realm, as reachable by the API service. This is not
-# necessarily its publicly reachable URL, keycloak advertises that iself.
 KEYCLOAK_URL = "http://keycloak:8080/auth/realms/OBS%20Dev/"
-
-# Auth client credentials
 KEYCLOAK_CLIENT_ID = "portal"
 KEYCLOAK_CLIENT_SECRET = "76b84224-dc24-4824-bb98-9e1ba15bd58f"
-
-# Whether the API should run the worker loop, or a dedicated worker is used
 DEDICATED_WORKER = True
-
+FRONTEND_URL = "http://localhost:3001/"
-# The root of the frontend. Needed for redirecting after login, and for CORS.
-# Set to None if frontend is served by the API.
-FRONTEND_URL = "http://localhost:3000/"
-
-# Where to find the compiled frontend assets (must include index.html), or None
-# to disable serving the frontend.
 FRONTEND_DIR = None
-
-# Can be an object or a JSON string
 FRONTEND_CONFIG = None
-
+TILES_FILE = "/tiles/tiles.mbtiles"
-# Path overrides:
-# API_ROOT_DIR = "??" # default: api/ inside repository
 DATA_DIR = "/data"
-# PROCESSING_DIR = "??" # default: DATA_DIR/processing
+ADDITIONAL_CORS_ORIGINS = [
-# PROCESSING_OUTPUT_DIR = "??"  # default: DATA_DIR/processing-output
+    "http://localhost:8880/",  # for maputnik on 8880
-# TRACKS_DIR = "??" # default: DATA_DIR/tracks
+    "http://localhost:8888/",  # for maputnik on 8888
-# OBS_FACE_CACHE_DIR = "??" # default: DATA_DIR/obs-face-cache
+]
 
 # vim: set ft=python :

api/config.prod-test.py

@@ -1,54 +0,0 @@
-# Bind address of the server
-HOST = "0.0.0.0"
-PORT = 3000
-
-# Extended log output, but slower
-DEBUG = True
-
-# Required to encrypt or sign sessions, cookies, tokens, etc.
-SECRET = "CHANGEME!!!!!!!!!!@##@!!$$$$$$$$$$$$$!!"
-
-# Connection to the database
-POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs"
-
-# URL to the keycloak realm, as reachable by the API service. This is not
-# necessarily its publicly reachable URL, keycloak advertises that iself.
-KEYCLOAK_URL = "http://keycloak:8080/auth/realms/OBS%20Dev/"
-
-# Auth client credentials
-KEYCLOAK_CLIENT_ID = "portal"
-KEYCLOAK_CLIENT_SECRET = "76b84224-dc24-4824-bb98-9e1ba15bd58f"
-
-# Whether the API should run the worker loop, or a dedicated worker is used
-DEDICATED_WORKER = False
-
-# The root of the frontend. Needed for redirecting after login, and for CORS.
-# Set to None if frontend is served by the API.
-FRONTEND_URL = None
-
-# Where to find the compiled frontend assets (must include index.html), or None
-# to disable serving the frontend.
-FRONTEND_DIR = "../frontend/build/"
-
-# Can be an object or a JSON string
-FRONTEND_CONFIG = {
-    "imprintUrl": "https://example.com/imprint",
-    "privacyPolicyUrl": "https://example.com/privacy",
-    "mapTileset": {
-        "url": "https://tiles.wmflabs.org/bw-mapnik/{z}/{x}/{y}.png",
-        "minZoom": 0,
-        "maxZoom": 18,
-    },
-    "mapHome": {"zoom": 15, "longitude": 7.8302, "latitude": 47.9755},
-    "obsMapSource": "http://localhost:3002/data/v3.json",
-}
-
-# Path overrides:
-# API_ROOT_DIR = "??" # default: api/ inside repository
-DATA_DIR = "/data"
-# PROCESSING_DIR = "??" # default: DATA_DIR/processing
-# PROCESSING_OUTPUT_DIR = "??"  # default: DATA_DIR/processing-output
-# TRACKS_DIR = "??" # default: DATA_DIR/tracks
-# OBS_FACE_CACHE_DIR = "??" # default: DATA_DIR/obs-face-cache
-
-# vim: set ft=python :

api/config.py.example

@@ -4,6 +4,7 @@ PORT = 3000
 
 # Extended log output, but slower
 DEBUG = False
+AUTO_RESTART = DEBUG
 
 # Required to encrypt or sign sessions, cookies, tokens, etc.
 SECRET = "!!!<<<CHANGEME>>>!!!"
@@ -40,9 +41,17 @@ FRONTEND_CONFIG = {
         "maxZoom": 18,
     },
     "mapHome": {"zoom": 15, "longitude": 7.8302, "latitude": 47.9755},
-    "obsMapSource": "http://localhost:3002/data/v3.json",
+    "obsMapSource": {
+        "type": "vector",
+        "url": "http://localhost:3002/data/v3.json",
+    },
 }
 
+# If the API should serve generated tiles, this is the path where the tiles are
+# built. This is an experimental option and probably very inefficient, a proper
+# tileserver should be prefered. Set to None to disable.
+TILES_FILE = None
+
 # Path overrides:
 # API_ROOT_DIR = "??" # default: api/ inside repository
 # DATA_DIR = "??" # default: $API_ROOT_DIR/..
@@ -51,4 +60,8 @@ FRONTEND_CONFIG = {
 # TRACKS_DIR = "??" # default: DATA_DIR/tracks
 # OBS_FACE_CACHE_DIR = "??" # default: DATA_DIR/obs-face-cache
 
+# Additional allowed origins for CORS headers. The FRONTEND_URL is included by
+# default. Python list, or whitespace separated string.
+ADDITIONAL_CORS_ORIGINS = None
+
 # vim: set ft=python :

api/obs/api/app.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import re
 from json import JSONEncoder, dumps
 from functools import wraps, partial
 from urllib.parse import urlparse
@@ -17,9 +18,6 @@ from sqlalchemy.orm import sessionmaker
 
 from obs.api.db import User, make_session, connect_db
 
-from sanic_session.base import BaseSessionInterface
-from sanic_session.utils import ExpiringDict
-
 log = logging.getLogger(__name__)
 
 app = Sanic("OpenBikeSensor Portal API")
@@ -30,26 +28,57 @@ api = Blueprint("api", url_prefix="/api")
 auth = Blueprint("auth", url_prefix="")
 
 # Configure paths
-c.API_ROOT_DIR = c.get("API_ROOT_DIR") or abspath(join(dirname(__file__), "..", ".."))
+def configure_paths(c):
-c.DATA_DIR = c.get("DATA_DIR") or normpath(join(c.API_ROOT_DIR, "../data"))
+    c.API_ROOT_DIR = c.get("API_ROOT_DIR") or abspath(
-c.PROCESSING_DIR = c.get("PROCESSING_DIR") or join(c.DATA_DIR, "processing")
+        join(dirname(__file__), "..", "..")
-c.PROCESSING_OUTPUT_DIR = c.get("PROCESSING_OUTPUT_DIR") or join(
+    )
-    c.DATA_DIR, "processing-output"
+    c.DATA_DIR = c.get("DATA_DIR") or normpath(join(c.API_ROOT_DIR, "../data"))
-)
+    c.PROCESSING_DIR = c.get("PROCESSING_DIR") or join(c.DATA_DIR, "processing")
-c.TRACKS_DIR = c.get("TRACKS_DIR") or join(c.DATA_DIR, "tracks")
+    c.PROCESSING_OUTPUT_DIR = c.get("PROCESSING_OUTPUT_DIR") or join(
-c.OBS_FACE_CACHE_DIR = c.get("OBS_FACE_CACHE_DIR") or join(c.DATA_DIR, "obs-face-cache")
+        c.DATA_DIR, "processing-output"
-c.FRONTEND_DIR = c.get("FRONTEND_DIR")
+    )
+    c.TRACKS_DIR = c.get("TRACKS_DIR") or join(c.DATA_DIR, "tracks")
+    c.OBS_FACE_CACHE_DIR = c.get("OBS_FACE_CACHE_DIR") or join(
+        c.DATA_DIR, "obs-face-cache"
+    )
+    c.FRONTEND_DIR = c.get("FRONTEND_DIR")
+
+
+configure_paths(app.config)
+
+
+def setup_cors(app):
+    frontend_url = app.config.get("FRONTEND_URL")
+    additional_origins = app.config.get("ADDITIONAL_CORS_ORIGINS")
+    if not frontend_url and not additional_origins:
+        # No CORS configured
+        return
+
+    origins = []
+    if frontend_url:
+        u = urlparse(frontend_url)
+        origins.append(f"{u.scheme}://{u.netloc}")
+
+    if isinstance(additional_origins, str):
+        origins += re.split(r"\s+", additional_origins)
+    elif isinstance(additional_origins, list):
+        origins += additional_origins
+    elif additional_origins is not None:
+        raise ValueError(
+            "invalid option type for ADDITIONAL_CORS_ORIGINS, must be list or space separated str"
+        )
 
-if c.FRONTEND_URL:
     from sanic_cors import CORS
 
-    frontend_url = urlparse(c.FRONTEND_URL)
     CORS(
         app,
-        origins=[f"{frontend_url.scheme}://{frontend_url.netloc}"],
+        origins=origins,
         supports_credentials=True,
     )
 
+
+setup_cors(app)
+
 # TODO: use a different interface, maybe backed by the PostgreSQL, to allow
 # scaling the API
 Session(app, interface=InMemorySessionInterface())
@@ -57,7 +86,7 @@ Session(app, interface=InMemorySessionInterface())
 
 @app.before_server_start
 async def app_connect_db(app, loop):
-    app.ctx._db_engine_ctx = connect_db(c.POSTGRES_URL)
+    app.ctx._db_engine_ctx = connect_db(app.config.POSTGRES_URL)
     app.ctx._db_engine = await app.ctx._db_engine_ctx.__aenter__()
 
 
@@ -118,26 +147,37 @@ def json(*args, **kwargs):
 
 from . import routes
 
-INDEX_HTML = join(c.FRONTEND_DIR, "index.html")
+INDEX_HTML = (
-if exists(INDEX_HTML):
+    join(app.config.FRONTEND_DIR, "index.html")
+    if app.config.get("FRONTEND_DIR")
+    else None
+)
+if INDEX_HTML and exists(INDEX_HTML):
 
     @app.get("/config.json")
     def get_frontend_config(req):
         base_path = req.server_path.replace("config.json", "")
-        return json_response(
+        result = {
-            {
+            **req.app.config.FRONTEND_CONFIG,
-                **req.app.config.FRONTEND_CONFIG,
+            "apiUrl": f"{req.scheme}://{req.host}{base_path}api",
-                "apiUrl": f"{req.scheme}://{req.host}{base_path}api",
+            "loginUrl": f"{req.scheme}://{req.host}{base_path}login",
-                "loginUrl": f"{req.scheme}://{req.host}{base_path}login",
+        }
+        print(req.app.config)
+        if req.app.config.get("TILES_FILE"):
+            result["obsMapSource"] = {
+                "type": "vector",
+                "tiles": [req.app.url_for("tiles", zoom="{zoom}", x="{x}", y="{y}")],
+                "minzoom": 12,
+                "maxzoom": 14,
             }
-        )
+        return json_response()
 
     @app.get("/<path:path>")
     def get_frontend_static(req, path):
         if path.startswith("api/"):
             raise NotFound()
 
-        file = join(c.FRONTEND_DIR, path)
+        file = join(app.config.FRONTEND_DIR, path)
         if not exists(file) or not path or not isfile(file):
             file = INDEX_HTML
         return file_response(file)

api/obs/api/routes/__init__.py

@@ -1,7 +1,8 @@
 from . import (
+    info,
     login,
     stats,
+    tiles,
     tracks,
-    info,
     users,
 )

api/obs/api/routes/tiles.py

@@ -0,0 +1,57 @@
+import gzip
+from sqlite3 import connect
+from sanic.response import raw
+
+from obs.api.app import app
+
+
+def get_tile(filename, zoom, x, y):
+    """
+    Inspired by:
+    https://github.com/TileStache/TileStache/blob/master/TileStache/MBTiles.py
+    """
+
+    print(filename)
+    db = connect(filename)
+    db.text_factory = bytes
+
+    fmt = db.execute("SELECT value FROM metadata WHERE name='format'").fetchone()[0]
+    if fmt != b"pbf":
+        print(repr(b"pbf"), " versus ", repr(fmt))
+        raise ValueError("mbtiles file is in wrong format: %s" % fmt)
+
+    content = db.execute(
+        "SELECT tile_data FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?",
+        (zoom, x, (2 ** zoom - 1) - y),
+    ).fetchone()
+    return content and content[0] or None
+
+
+# regenerate approx. once each day
+TILE_CACHE_MAX_AGE = 3600 * 24
+
+if app.config.get("TILES_FILE"):
+
+    @app.route(r"/tiles/<zoom:int>/<x:int>/<y:(\d+)\.pbf>")
+    async def tiles(req, zoom: int, x: int, y: str):
+        tile = get_tile(req.app.config.TILES_FILE, int(zoom), int(x), int(y))
+
+        gzip = "gzip" in req.headers["accept-encoding"]
+
+        headers = {}
+        headers["Vary"] = "Accept-Encoding"
+
+        if req.app.config.DEBUG:
+            headers["Cache-Control"] = "no-cache"
+        else:
+            headers["Cache-Control"] = f"public, max-age={TILE_CACHE_MAX_AGE}"
+
+        # The tiles in the mbtiles file are gzip-compressed already, so we
+        # serve them actually as-is, and only decompress them if the browser
+        # doesn't accept gzip
+        if gzip:
+            headers["Content-Encoding"] = "gzip"
+        else:
+            tile = gzip.decompress(tile)
+
+        return raw(tile, content_type="application/x-protobuf", headers=headers)

api/obs/api/routes/users.py

@@ -21,9 +21,8 @@ def user_to_json(user):
 
 
 @api.get("/user")
-@require_auth
 async def get_user(req):
-    return json(user_to_json(req.ctx.user))
+    return json(user_to_json(req.ctx.user) if req.ctx.user else None)
 
 
 @api.put("/user")

api/obs/bin/openbikesensor_api.py

@@ -10,7 +10,13 @@ from obs.api.db import connect_db
 
 
 def main():
-    app.run(host=app.config.HOST, port=app.config.PORT, debug=app.config.DEBUG)
+    debug = app.config.DEBUG
+    app.run(
+        host=app.config.HOST,
+        port=app.config.PORT,
+        debug=debug,
+        auto_reload=app.config.get("AUTO_RELOAD", debug),
+    )
 
 
 if __name__ == "__main__":

docker-compose.yaml

@@ -39,6 +39,7 @@ services:
       - ./api/config.dev.py:/opt/obs/api/config.py
       - ./frontend/build:/opt/obs/frontend/build
       - ./local/api-data:/data
+      - ./tile-generator/data/:/tiles
     links:
       - postgres
       - keycloak
@@ -140,17 +141,6 @@ services:
       PGHOST: postgres
       PGPORT: 5432
 
-  tileserver:
-    image: klokantech/tileserver-gl
-    ports:
-      - 3002:80
-    volumes:
-      - ./tile-generator/tileserver-gl-config.json:/config/tileserver.json
-      - ./tile-generator/data/:/data
-    command:
-      - --config
-      - /config/tileserver.json
-
   keycloak:
     image: jboss/keycloak
     ports:
@@ -166,19 +156,3 @@ services:
       DB_DATABASE: obs
       DB_USER: obs
       DB_PASSWORD: obs
-      # DB_SCHEMA: keycloak
-
-  prod-test:
-    image: openbikesensor-portal
-    build:
-      context: ./
-      dockerfile: Dockerfile
-    volumes:
-      - ./api/config.prod-test.py:/opt/obs/api/config.py
-      - ./local/api-data:/data
-    links:
-      - postgres
-      - keycloak
-    ports:
-      - '3000:3000'
-    restart: on-failure

frontend/config.dev.json

@@ -13,5 +13,10 @@
     "longitude": 7.8302,
     "latitude": 47.9755
   },
-  "obsMapSource": "http://localhost:3002/data/v3.json"
+  "obsMapSource": {
+    "type": "vector",
+    "tiles": ["http://localhost:3000/tiles/{z}/{x}/{y}.pbf"],
+    "minzoom": 12,
+    "maxzoom": 14
+  }
 }

frontend/config.example.json

@@ -13,5 +13,8 @@
     "longitude": 9.1797,
     "latitude": 48.7784
   },
-  "obsMapSource": "http://portal.example.com/tileserver/data/v3.json"
+  "obsMapSource": {
+    "type": "vector",
+    "url": "http://portal.example.com/tileserver/data/v3.json"
+  }
 }

frontend/public/config.json

@@ -12,5 +12,8 @@
     "longitude": 7.8302,
     "latitude": 47.9755
   },
-  "obsMapSource": "https://portal.example.com/tileset/data/v3.json"
+  "obsMapSource": {
+    "type": "vector",
+    "url": "https://portal.example.com/tileset/data/v3.json"
+  }
 }

frontend/src/config.ts

@@ -1,5 +1,15 @@
 import React from 'react'
 
+export type MapSoure = {
+  type: 'vector'
+  url: string,
+} | {
+  type: 'vector',
+  tiles: string[],
+  minzoom: number,
+  maxzoom: number,
+}
+
 export interface Config {
   apiUrl: string
   mapHome: {
@@ -7,7 +17,7 @@ export interface Config {
     longitude: number
     zoom: number
   }
-  obsMapSource?: string
+  obsMapSource?: MapSoure
   imprintUrl?: string
   privacyPolicyUrl?: string
   mapTileset?: {

frontend/src/mapstyles/index.js

@@ -3,8 +3,8 @@ import _ from 'lodash'
 import bright from './bright.json'
 import positron from './positron.json'
 
-function addRoadsStyle(style, sourceUrl = "http://localhost:3002/data/v3.json") {
+function addRoadsStyle(style, mapSource) {
-  style.sources.obs = {"type": "vector", "url": sourceUrl}
+  style.sources.obs = mapSource
 
   // insert before "road_oneway" layer
   let idx = style.layers.findIndex(l => l.id === 'road_oneway')

frontend/src/pages/HomePage.tsx

@@ -7,7 +7,7 @@ import {map, switchMap} from 'rxjs/operators'
 
 import api from 'api'
 import {Stats, Page} from 'components'
-import {useConfig} from 'config'
+import {useConfig, MapSource} from 'config'
 
 import {TrackListItem} from './TracksPage'
 import styles from './HomePage.module.scss'
@@ -16,7 +16,7 @@ import 'ol/ol.css'
 import {obsRoads} from '../mapstyles'
 import ReactMapGl from 'react-map-gl'
 
-function WelcomeMap({mapSource}: {mapSource: string}) {
+function WelcomeMap({mapSource}: {mapSource: MapSource}) {
   const mapStyle = React.useMemo(() => obsRoads(mapSource), [mapSource])
   const config = useConfig()
   const [viewport, setViewport] = React.useState({

frontend/src/pages/MapPage.tsx

@@ -12,7 +12,7 @@ import {obsRoads} from '../mapstyles'
 import ReactMapGl from 'react-map-gl'
 
 function BigMap({mapSource, config}: {mapSource: string ,config: Config}) {
-  const mapStyle = React.useMemo(() => obsRoads(mapSource), [mapSource])
+  const mapStyle = React.useMemo(() => mapSource && obsRoads(mapSource), [mapSource])
   const [viewport, setViewport] = React.useState({
     longitude: 0,
     latitude: 0,
@@ -25,6 +25,10 @@ function BigMap({mapSource, config}: {mapSource: string ,config: Config}) {
     }
   }, [config])
 
+  if (!mapStyle) {
+    return null
+  }
+
   return (
     <div className={styles.mapContainer}>
       <ReactMapGl mapStyle={mapStyle} width="100%" height="100%" onViewportChange={setViewport} {...viewport} />

tile-generator/README.md

@@ -74,11 +74,18 @@ make generate-tiles-pg
 
 ## Publish vector tiles
 
-The tool [tileserver-gl](http://tileserver.org/) is used to publish the vector
+The API is capable of serving the generated mbtiles file in XYZ scheme with PBF
-tiles separately through HTTP. The tileserver runs inside docker, so all you need to do for a development setup is start it:
+format. Set the config variable `TILES_FILE` to point to your generated file.
+
+The API might be inefficient at publishing the tiles. You might want to try a
+proper tileserver with caching and all if you run into trouble with its
+capabilities.
+
+The URL for the tiles is:
 
 ```
-docker compose up -d tileserver
+http://api.example.com/tiles/{z}/{x}/{y}.pbf
 ```
 
-It is now available at [http://localhost:3002/](http://localhost:3002/).
+The API generates this URL into the `/config.json` as `obsMapSource` if it is
+configured to serve tiles *and* the frontend.

tile-generator/tileserver-gl-config.json

@@ -1,39 +0,0 @@
-{
-  "options": {
-    "paths": {
-      "root": "/usr/src/app/node_modules/tileserver-gl-styles",
-      "fonts": "fonts",
-      "styles": "styles",
-      "mbtiles": "/data"
-    }
-  },
-  "styles": {
-    "klokantech-basic": {
-      "style": "klokantech-basic/style.json",
-      "tilejson": {
-        "bounds": [
-          7.487897,
-          47.80556,
-          8.212994,
-          48.224458
-        ]
-      }
-    },
-    "osm-bright": {
-      "style": "osm-bright/style.json",
-      "tilejson": {
-        "bounds": [
-          7.487897,
-          47.80556,
-          8.212994,
-          48.224458
-        ]
-      }
-    }
-  },
-  "data": {
-    "v3": {
-      "mbtiles": "tiles.mbtiles"
-    }
-  }
-}