diff --git a/api/Dockerfile b/api/Dockerfile index 94b55c2..a488df1 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -5,7 +5,8 @@ WORKDIR /opt/obs/api ADD scripts /opt/obs/scripts RUN pip install -e /opt/obs/scripts -ADD requirements.txt setup.py obs /opt/obs/api/ +ADD requirements.txt setup.py /opt/obs/api/ +ADD obs /opt/obs/api/obs/ RUN pip install -e . EXPOSE 8000 diff --git a/api/config.dev.py b/api/config.dev.py index 0c16c69..f48f737 100644 --- a/api/config.dev.py +++ b/api/config.dev.py @@ -1,44 +1,21 @@ -# Bind address of the server HOST = "0.0.0.0" PORT = 3000 - -# Extended log output, but slower -DEBUG = True - -# Required to encrypt or sign sessions, cookies, tokens, etc. -SECRET = "CHANGEME!!!!!!!!!!@##@!!$$$$$$$$$$$$$!!" - -# Connection to the database +DEBUG = False +AUTO_RESTART = True +SECRET = "!!!!!!!!!!!!CHANGE ME!!!!!!!!!!!!" POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs" - -# URL to the keycloak realm, as reachable by the API service. This is not -# necessarily its publicly reachable URL, keycloak advertises that iself. KEYCLOAK_URL = "http://keycloak:8080/auth/realms/OBS%20Dev/" - -# Auth client credentials KEYCLOAK_CLIENT_ID = "portal" KEYCLOAK_CLIENT_SECRET = "76b84224-dc24-4824-bb98-9e1ba15bd58f" - -# Whether the API should run the worker loop, or a dedicated worker is used DEDICATED_WORKER = True - -# The root of the frontend. Needed for redirecting after login, and for CORS. -# Set to None if frontend is served by the API. -FRONTEND_URL = "http://localhost:3000/" - -# Where to find the compiled frontend assets (must include index.html), or None -# to disable serving the frontend. +FRONTEND_URL = "http://localhost:3001/" FRONTEND_DIR = None - -# Can be an object or a JSON string FRONTEND_CONFIG = None - -# Path overrides: -# API_ROOT_DIR = "??" # default: api/ inside repository +TILES_FILE = "/tiles/tiles.mbtiles" DATA_DIR = "/data" -# PROCESSING_DIR = "??" # default: DATA_DIR/processing -# PROCESSING_OUTPUT_DIR = "??" # default: DATA_DIR/processing-output -# TRACKS_DIR = "??" # default: DATA_DIR/tracks -# OBS_FACE_CACHE_DIR = "??" # default: DATA_DIR/obs-face-cache +ADDITIONAL_CORS_ORIGINS = [ + "http://localhost:8880/", # for maputnik on 8880 + "http://localhost:8888/", # for maputnik on 8888 +] # vim: set ft=python : diff --git a/api/config.prod-test.py b/api/config.prod-test.py deleted file mode 100644 index 2469b6a..0000000 --- a/api/config.prod-test.py +++ /dev/null @@ -1,54 +0,0 @@ -# Bind address of the server -HOST = "0.0.0.0" -PORT = 3000 - -# Extended log output, but slower -DEBUG = True - -# Required to encrypt or sign sessions, cookies, tokens, etc. -SECRET = "CHANGEME!!!!!!!!!!@##@!!$$$$$$$$$$$$$!!" - -# Connection to the database -POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs" - -# URL to the keycloak realm, as reachable by the API service. This is not -# necessarily its publicly reachable URL, keycloak advertises that iself. -KEYCLOAK_URL = "http://keycloak:8080/auth/realms/OBS%20Dev/" - -# Auth client credentials -KEYCLOAK_CLIENT_ID = "portal" -KEYCLOAK_CLIENT_SECRET = "76b84224-dc24-4824-bb98-9e1ba15bd58f" - -# Whether the API should run the worker loop, or a dedicated worker is used -DEDICATED_WORKER = False - -# The root of the frontend. Needed for redirecting after login, and for CORS. -# Set to None if frontend is served by the API. -FRONTEND_URL = None - -# Where to find the compiled frontend assets (must include index.html), or None -# to disable serving the frontend. -FRONTEND_DIR = "../frontend/build/" - -# Can be an object or a JSON string -FRONTEND_CONFIG = { - "imprintUrl": "https://example.com/imprint", - "privacyPolicyUrl": "https://example.com/privacy", - "mapTileset": { - "url": "https://tiles.wmflabs.org/bw-mapnik/{z}/{x}/{y}.png", - "minZoom": 0, - "maxZoom": 18, - }, - "mapHome": {"zoom": 15, "longitude": 7.8302, "latitude": 47.9755}, - "obsMapSource": "http://localhost:3002/data/v3.json", -} - -# Path overrides: -# API_ROOT_DIR = "??" # default: api/ inside repository -DATA_DIR = "/data" -# PROCESSING_DIR = "??" # default: DATA_DIR/processing -# PROCESSING_OUTPUT_DIR = "??" # default: DATA_DIR/processing-output -# TRACKS_DIR = "??" # default: DATA_DIR/tracks -# OBS_FACE_CACHE_DIR = "??" # default: DATA_DIR/obs-face-cache - -# vim: set ft=python : diff --git a/api/config.py.example b/api/config.py.example index e1ad34c..55a39c1 100644 --- a/api/config.py.example +++ b/api/config.py.example @@ -4,6 +4,7 @@ PORT = 3000 # Extended log output, but slower DEBUG = False +AUTO_RESTART = DEBUG # Required to encrypt or sign sessions, cookies, tokens, etc. SECRET = "!!!<<>>!!!" @@ -40,9 +41,17 @@ FRONTEND_CONFIG = { "maxZoom": 18, }, "mapHome": {"zoom": 15, "longitude": 7.8302, "latitude": 47.9755}, - "obsMapSource": "http://localhost:3002/data/v3.json", + "obsMapSource": { + "type": "vector", + "url": "http://localhost:3002/data/v3.json", + }, } +# If the API should serve generated tiles, this is the path where the tiles are +# built. This is an experimental option and probably very inefficient, a proper +# tileserver should be prefered. Set to None to disable. +TILES_FILE = None + # Path overrides: # API_ROOT_DIR = "??" # default: api/ inside repository # DATA_DIR = "??" # default: $API_ROOT_DIR/.. @@ -51,4 +60,8 @@ FRONTEND_CONFIG = { # TRACKS_DIR = "??" # default: DATA_DIR/tracks # OBS_FACE_CACHE_DIR = "??" # default: DATA_DIR/obs-face-cache +# Additional allowed origins for CORS headers. The FRONTEND_URL is included by +# default. Python list, or whitespace separated string. +ADDITIONAL_CORS_ORIGINS = None + # vim: set ft=python : diff --git a/api/obs/api/app.py b/api/obs/api/app.py index 79db73c..098c55f 100644 --- a/api/obs/api/app.py +++ b/api/obs/api/app.py @@ -1,5 +1,6 @@ import logging import os +import re from json import JSONEncoder, dumps from functools import wraps, partial from urllib.parse import urlparse @@ -17,9 +18,6 @@ from sqlalchemy.orm import sessionmaker from obs.api.db import User, make_session, connect_db -from sanic_session.base import BaseSessionInterface -from sanic_session.utils import ExpiringDict - log = logging.getLogger(__name__) app = Sanic("OpenBikeSensor Portal API") @@ -30,26 +28,57 @@ api = Blueprint("api", url_prefix="/api") auth = Blueprint("auth", url_prefix="") # Configure paths -c.API_ROOT_DIR = c.get("API_ROOT_DIR") or abspath(join(dirname(__file__), "..", "..")) -c.DATA_DIR = c.get("DATA_DIR") or normpath(join(c.API_ROOT_DIR, "../data")) -c.PROCESSING_DIR = c.get("PROCESSING_DIR") or join(c.DATA_DIR, "processing") -c.PROCESSING_OUTPUT_DIR = c.get("PROCESSING_OUTPUT_DIR") or join( - c.DATA_DIR, "processing-output" -) -c.TRACKS_DIR = c.get("TRACKS_DIR") or join(c.DATA_DIR, "tracks") -c.OBS_FACE_CACHE_DIR = c.get("OBS_FACE_CACHE_DIR") or join(c.DATA_DIR, "obs-face-cache") -c.FRONTEND_DIR = c.get("FRONTEND_DIR") +def configure_paths(c): + c.API_ROOT_DIR = c.get("API_ROOT_DIR") or abspath( + join(dirname(__file__), "..", "..") + ) + c.DATA_DIR = c.get("DATA_DIR") or normpath(join(c.API_ROOT_DIR, "../data")) + c.PROCESSING_DIR = c.get("PROCESSING_DIR") or join(c.DATA_DIR, "processing") + c.PROCESSING_OUTPUT_DIR = c.get("PROCESSING_OUTPUT_DIR") or join( + c.DATA_DIR, "processing-output" + ) + c.TRACKS_DIR = c.get("TRACKS_DIR") or join(c.DATA_DIR, "tracks") + c.OBS_FACE_CACHE_DIR = c.get("OBS_FACE_CACHE_DIR") or join( + c.DATA_DIR, "obs-face-cache" + ) + c.FRONTEND_DIR = c.get("FRONTEND_DIR") + + +configure_paths(app.config) + + +def setup_cors(app): + frontend_url = app.config.get("FRONTEND_URL") + additional_origins = app.config.get("ADDITIONAL_CORS_ORIGINS") + if not frontend_url and not additional_origins: + # No CORS configured + return + + origins = [] + if frontend_url: + u = urlparse(frontend_url) + origins.append(f"{u.scheme}://{u.netloc}") + + if isinstance(additional_origins, str): + origins += re.split(r"\s+", additional_origins) + elif isinstance(additional_origins, list): + origins += additional_origins + elif additional_origins is not None: + raise ValueError( + "invalid option type for ADDITIONAL_CORS_ORIGINS, must be list or space separated str" + ) -if c.FRONTEND_URL: from sanic_cors import CORS - frontend_url = urlparse(c.FRONTEND_URL) CORS( app, - origins=[f"{frontend_url.scheme}://{frontend_url.netloc}"], + origins=origins, supports_credentials=True, ) + +setup_cors(app) + # TODO: use a different interface, maybe backed by the PostgreSQL, to allow # scaling the API Session(app, interface=InMemorySessionInterface()) @@ -57,7 +86,7 @@ Session(app, interface=InMemorySessionInterface()) @app.before_server_start async def app_connect_db(app, loop): - app.ctx._db_engine_ctx = connect_db(c.POSTGRES_URL) + app.ctx._db_engine_ctx = connect_db(app.config.POSTGRES_URL) app.ctx._db_engine = await app.ctx._db_engine_ctx.__aenter__() @@ -118,26 +147,37 @@ def json(*args, **kwargs): from . import routes -INDEX_HTML = join(c.FRONTEND_DIR, "index.html") -if exists(INDEX_HTML): +INDEX_HTML = ( + join(app.config.FRONTEND_DIR, "index.html") + if app.config.get("FRONTEND_DIR") + else None +) +if INDEX_HTML and exists(INDEX_HTML): @app.get("/config.json") def get_frontend_config(req): base_path = req.server_path.replace("config.json", "") - return json_response( - { - **req.app.config.FRONTEND_CONFIG, - "apiUrl": f"{req.scheme}://{req.host}{base_path}api", - "loginUrl": f"{req.scheme}://{req.host}{base_path}login", + result = { + **req.app.config.FRONTEND_CONFIG, + "apiUrl": f"{req.scheme}://{req.host}{base_path}api", + "loginUrl": f"{req.scheme}://{req.host}{base_path}login", + } + print(req.app.config) + if req.app.config.get("TILES_FILE"): + result["obsMapSource"] = { + "type": "vector", + "tiles": [req.app.url_for("tiles", zoom="{zoom}", x="{x}", y="{y}")], + "minzoom": 12, + "maxzoom": 14, } - ) + return json_response() @app.get("/") def get_frontend_static(req, path): if path.startswith("api/"): raise NotFound() - file = join(c.FRONTEND_DIR, path) + file = join(app.config.FRONTEND_DIR, path) if not exists(file) or not path or not isfile(file): file = INDEX_HTML return file_response(file) diff --git a/api/obs/api/routes/__init__.py b/api/obs/api/routes/__init__.py index 6c82c89..48a3ba2 100644 --- a/api/obs/api/routes/__init__.py +++ b/api/obs/api/routes/__init__.py @@ -1,7 +1,8 @@ from . import ( + info, login, stats, + tiles, tracks, - info, users, ) diff --git a/api/obs/api/routes/tiles.py b/api/obs/api/routes/tiles.py new file mode 100644 index 0000000..a00b118 --- /dev/null +++ b/api/obs/api/routes/tiles.py @@ -0,0 +1,57 @@ +import gzip +from sqlite3 import connect +from sanic.response import raw + +from obs.api.app import app + + +def get_tile(filename, zoom, x, y): + """ + Inspired by: + https://github.com/TileStache/TileStache/blob/master/TileStache/MBTiles.py + """ + + print(filename) + db = connect(filename) + db.text_factory = bytes + + fmt = db.execute("SELECT value FROM metadata WHERE name='format'").fetchone()[0] + if fmt != b"pbf": + print(repr(b"pbf"), " versus ", repr(fmt)) + raise ValueError("mbtiles file is in wrong format: %s" % fmt) + + content = db.execute( + "SELECT tile_data FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?", + (zoom, x, (2 ** zoom - 1) - y), + ).fetchone() + return content and content[0] or None + + +# regenerate approx. once each day +TILE_CACHE_MAX_AGE = 3600 * 24 + +if app.config.get("TILES_FILE"): + + @app.route(r"/tiles///") + async def tiles(req, zoom: int, x: int, y: str): + tile = get_tile(req.app.config.TILES_FILE, int(zoom), int(x), int(y)) + + gzip = "gzip" in req.headers["accept-encoding"] + + headers = {} + headers["Vary"] = "Accept-Encoding" + + if req.app.config.DEBUG: + headers["Cache-Control"] = "no-cache" + else: + headers["Cache-Control"] = f"public, max-age={TILE_CACHE_MAX_AGE}" + + # The tiles in the mbtiles file are gzip-compressed already, so we + # serve them actually as-is, and only decompress them if the browser + # doesn't accept gzip + if gzip: + headers["Content-Encoding"] = "gzip" + else: + tile = gzip.decompress(tile) + + return raw(tile, content_type="application/x-protobuf", headers=headers) diff --git a/api/obs/api/routes/users.py b/api/obs/api/routes/users.py index 14dc0ea..8fd3784 100644 --- a/api/obs/api/routes/users.py +++ b/api/obs/api/routes/users.py @@ -21,9 +21,8 @@ def user_to_json(user): @api.get("/user") -@require_auth async def get_user(req): - return json(user_to_json(req.ctx.user)) + return json(user_to_json(req.ctx.user) if req.ctx.user else None) @api.put("/user") diff --git a/api/obs/bin/openbikesensor_api.py b/api/obs/bin/openbikesensor_api.py index 7aa131b..4079901 100755 --- a/api/obs/bin/openbikesensor_api.py +++ b/api/obs/bin/openbikesensor_api.py @@ -10,7 +10,13 @@ from obs.api.db import connect_db def main(): - app.run(host=app.config.HOST, port=app.config.PORT, debug=app.config.DEBUG) + debug = app.config.DEBUG + app.run( + host=app.config.HOST, + port=app.config.PORT, + debug=debug, + auto_reload=app.config.get("AUTO_RELOAD", debug), + ) if __name__ == "__main__": diff --git a/docker-compose.yaml b/docker-compose.yaml index 798cafb..9472da5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -39,6 +39,7 @@ services: - ./api/config.dev.py:/opt/obs/api/config.py - ./frontend/build:/opt/obs/frontend/build - ./local/api-data:/data + - ./tile-generator/data/:/tiles links: - postgres - keycloak @@ -140,17 +141,6 @@ services: PGHOST: postgres PGPORT: 5432 - tileserver: - image: klokantech/tileserver-gl - ports: - - 3002:80 - volumes: - - ./tile-generator/tileserver-gl-config.json:/config/tileserver.json - - ./tile-generator/data/:/data - command: - - --config - - /config/tileserver.json - keycloak: image: jboss/keycloak ports: @@ -166,19 +156,3 @@ services: DB_DATABASE: obs DB_USER: obs DB_PASSWORD: obs - # DB_SCHEMA: keycloak - - prod-test: - image: openbikesensor-portal - build: - context: ./ - dockerfile: Dockerfile - volumes: - - ./api/config.prod-test.py:/opt/obs/api/config.py - - ./local/api-data:/data - links: - - postgres - - keycloak - ports: - - '3000:3000' - restart: on-failure diff --git a/frontend/config.dev.json b/frontend/config.dev.json index 9ae37aa..3a6cd61 100644 --- a/frontend/config.dev.json +++ b/frontend/config.dev.json @@ -13,5 +13,10 @@ "longitude": 7.8302, "latitude": 47.9755 }, - "obsMapSource": "http://localhost:3002/data/v3.json" + "obsMapSource": { + "type": "vector", + "tiles": ["http://localhost:3000/tiles/{z}/{x}/{y}.pbf"], + "minzoom": 12, + "maxzoom": 14 + } } diff --git a/frontend/config.example.json b/frontend/config.example.json index c034e7c..59eefd6 100644 --- a/frontend/config.example.json +++ b/frontend/config.example.json @@ -13,5 +13,8 @@ "longitude": 9.1797, "latitude": 48.7784 }, - "obsMapSource": "http://portal.example.com/tileserver/data/v3.json" + "obsMapSource": { + "type": "vector", + "url": "http://portal.example.com/tileserver/data/v3.json" + } } diff --git a/frontend/public/config.json b/frontend/public/config.json index 9c5082c..d2fabae 100644 --- a/frontend/public/config.json +++ b/frontend/public/config.json @@ -12,5 +12,8 @@ "longitude": 7.8302, "latitude": 47.9755 }, - "obsMapSource": "https://portal.example.com/tileset/data/v3.json" + "obsMapSource": { + "type": "vector", + "url": "https://portal.example.com/tileset/data/v3.json" + } } diff --git a/frontend/src/config.ts b/frontend/src/config.ts index 4e3e28a..04805a7 100644 --- a/frontend/src/config.ts +++ b/frontend/src/config.ts @@ -1,5 +1,15 @@ import React from 'react' +export type MapSoure = { + type: 'vector' + url: string, +} | { + type: 'vector', + tiles: string[], + minzoom: number, + maxzoom: number, +} + export interface Config { apiUrl: string mapHome: { @@ -7,7 +17,7 @@ export interface Config { longitude: number zoom: number } - obsMapSource?: string + obsMapSource?: MapSoure imprintUrl?: string privacyPolicyUrl?: string mapTileset?: { diff --git a/frontend/src/mapstyles/index.js b/frontend/src/mapstyles/index.js index 101eda8..0f17cd9 100644 --- a/frontend/src/mapstyles/index.js +++ b/frontend/src/mapstyles/index.js @@ -3,8 +3,8 @@ import _ from 'lodash' import bright from './bright.json' import positron from './positron.json' -function addRoadsStyle(style, sourceUrl = "http://localhost:3002/data/v3.json") { - style.sources.obs = {"type": "vector", "url": sourceUrl} +function addRoadsStyle(style, mapSource) { + style.sources.obs = mapSource // insert before "road_oneway" layer let idx = style.layers.findIndex(l => l.id === 'road_oneway') diff --git a/frontend/src/pages/HomePage.tsx b/frontend/src/pages/HomePage.tsx index ad300b6..8642af7 100644 --- a/frontend/src/pages/HomePage.tsx +++ b/frontend/src/pages/HomePage.tsx @@ -7,7 +7,7 @@ import {map, switchMap} from 'rxjs/operators' import api from 'api' import {Stats, Page} from 'components' -import {useConfig} from 'config' +import {useConfig, MapSource} from 'config' import {TrackListItem} from './TracksPage' import styles from './HomePage.module.scss' @@ -16,7 +16,7 @@ import 'ol/ol.css' import {obsRoads} from '../mapstyles' import ReactMapGl from 'react-map-gl' -function WelcomeMap({mapSource}: {mapSource: string}) { +function WelcomeMap({mapSource}: {mapSource: MapSource}) { const mapStyle = React.useMemo(() => obsRoads(mapSource), [mapSource]) const config = useConfig() const [viewport, setViewport] = React.useState({ diff --git a/frontend/src/pages/MapPage.tsx b/frontend/src/pages/MapPage.tsx index bb98682..b5bd8c9 100644 --- a/frontend/src/pages/MapPage.tsx +++ b/frontend/src/pages/MapPage.tsx @@ -12,7 +12,7 @@ import {obsRoads} from '../mapstyles' import ReactMapGl from 'react-map-gl' function BigMap({mapSource, config}: {mapSource: string ,config: Config}) { - const mapStyle = React.useMemo(() => obsRoads(mapSource), [mapSource]) + const mapStyle = React.useMemo(() => mapSource && obsRoads(mapSource), [mapSource]) const [viewport, setViewport] = React.useState({ longitude: 0, latitude: 0, @@ -25,6 +25,10 @@ function BigMap({mapSource, config}: {mapSource: string ,config: Config}) { } }, [config]) + if (!mapStyle) { + return null + } + return (
diff --git a/tile-generator/README.md b/tile-generator/README.md index 26b66df..4bbb037 100644 --- a/tile-generator/README.md +++ b/tile-generator/README.md @@ -74,11 +74,18 @@ make generate-tiles-pg ## Publish vector tiles -The tool [tileserver-gl](http://tileserver.org/) is used to publish the vector -tiles separately through HTTP. The tileserver runs inside docker, so all you need to do for a development setup is start it: +The API is capable of serving the generated mbtiles file in XYZ scheme with PBF +format. Set the config variable `TILES_FILE` to point to your generated file. + +The API might be inefficient at publishing the tiles. You might want to try a +proper tileserver with caching and all if you run into trouble with its +capabilities. + +The URL for the tiles is: ``` -docker compose up -d tileserver +http://api.example.com/tiles/{z}/{x}/{y}.pbf ``` -It is now available at [http://localhost:3002/](http://localhost:3002/). +The API generates this URL into the `/config.json` as `obsMapSource` if it is +configured to serve tiles *and* the frontend. diff --git a/tile-generator/tileserver-gl-config.json b/tile-generator/tileserver-gl-config.json deleted file mode 100644 index feae300..0000000 --- a/tile-generator/tileserver-gl-config.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "options": { - "paths": { - "root": "/usr/src/app/node_modules/tileserver-gl-styles", - "fonts": "fonts", - "styles": "styles", - "mbtiles": "/data" - } - }, - "styles": { - "klokantech-basic": { - "style": "klokantech-basic/style.json", - "tilejson": { - "bounds": [ - 7.487897, - 47.80556, - 8.212994, - 48.224458 - ] - } - }, - "osm-bright": { - "style": "osm-bright/style.json", - "tilejson": { - "bounds": [ - 7.487897, - 47.80556, - 8.212994, - 48.224458 - ] - } - } - }, - "data": { - "v3": { - "mbtiles": "tiles.mbtiles" - } - } -}