pub-solar/obs-portal
5
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat: publish tiles from API directly

Browse source
This commit is contained in:
Paul Bienkowski 2021-11-16 21:59:37 +01:00
parent c85f261292
commit 131afd5adc
19 changed files with 203 additions and 196 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
api/Dockerfile
View file

@ -5,7 +5,8 @@ WORKDIR /opt/obs/api
ADD scripts /opt/obs/scripts
RUN pip install -e /opt/obs/scripts
ADD requirements.txt setup.py obs /opt/obs/api/
ADD requirements.txt setup.py /opt/obs/api/
ADD obs /opt/obs/api/obs/
RUN pip install -e .
EXPOSE 8000

41
api/config.dev.py
View file

@ -1,44 +1,21 @@
# Bind address of the server
HOST = "0.0.0.0"
PORT = 3000
# Extended log output, but slower
DEBUG = True
# Required to encrypt or sign sessions, cookies, tokens, etc.
SECRET = "CHANGEME!!!!!!!!!!@##@!!$$$$$$$$$$$$$!!"
# Connection to the database
DEBUG = False
AUTO_RESTART = True
SECRET = "!!!!!!!!!!!!CHANGE ME!!!!!!!!!!!!"
POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs"
# URL to the keycloak realm, as reachable by the API service. This is not
# necessarily its publicly reachable URL, keycloak advertises that iself.
KEYCLOAK_URL = "http://keycloak:8080/auth/realms/OBS%20Dev/"
# Auth client credentials
KEYCLOAK_CLIENT_ID = "portal"
KEYCLOAK_CLIENT_SECRET = "76b84224-dc24-4824-bb98-9e1ba15bd58f"
# Whether the API should run the worker loop, or a dedicated worker is used
DEDICATED_WORKER = True
# The root of the frontend. Needed for redirecting after login, and for CORS.
# Set to None if frontend is served by the API.
FRONTEND_URL = "http://localhost:3000/"
# Where to find the compiled frontend assets (must include index.html), or None
# to disable serving the frontend.
FRONTEND_URL = "http://localhost:3001/"
FRONTEND_DIR = None
# Can be an object or a JSON string
FRONTEND_CONFIG = None
# Path overrides:
# API_ROOT_DIR = "??" # default: api/ inside repository
TILES_FILE = "/tiles/tiles.mbtiles"
DATA_DIR = "/data"
# PROCESSING_DIR = "??" # default: DATA_DIR/processing
# PROCESSING_OUTPUT_DIR = "??" # default: DATA_DIR/processing-output
# TRACKS_DIR = "??" # default: DATA_DIR/tracks
# OBS_FACE_CACHE_DIR = "??" # default: DATA_DIR/obs-face-cache
ADDITIONAL_CORS_ORIGINS = [
"http://localhost:8880/", # for maputnik on 8880
"http://localhost:8888/", # for maputnik on 8888
]
# vim: set ft=python :

54
api/config.prod-test.py
View file

@ -1,54 +0,0 @@
# Bind address of the server
HOST = "0.0.0.0"
PORT = 3000
# Extended log output, but slower
DEBUG = True
# Required to encrypt or sign sessions, cookies, tokens, etc.
SECRET = "CHANGEME!!!!!!!!!!@##@!!$$$$$$$$$$$$$!!"
# Connection to the database
POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs"
# URL to the keycloak realm, as reachable by the API service. This is not
# necessarily its publicly reachable URL, keycloak advertises that iself.
KEYCLOAK_URL = "http://keycloak:8080/auth/realms/OBS%20Dev/"
# Auth client credentials
KEYCLOAK_CLIENT_ID = "portal"
KEYCLOAK_CLIENT_SECRET = "76b84224-dc24-4824-bb98-9e1ba15bd58f"
# Whether the API should run the worker loop, or a dedicated worker is used
DEDICATED_WORKER = False
# The root of the frontend. Needed for redirecting after login, and for CORS.
# Set to None if frontend is served by the API.
FRONTEND_URL = None
# Where to find the compiled frontend assets (must include index.html), or None
# to disable serving the frontend.
FRONTEND_DIR = "../frontend/build/"
# Can be an object or a JSON string
FRONTEND_CONFIG = {
"imprintUrl": "https://example.com/imprint",
"privacyPolicyUrl": "https://example.com/privacy",
"mapTileset": {
"url": "https://tiles.wmflabs.org/bw-mapnik/{z}/{x}/{y}.png",
"minZoom": 0,
"maxZoom": 18,
},
"mapHome": {"zoom": 15, "longitude": 7.8302, "latitude": 47.9755},
"obsMapSource": "http://localhost:3002/data/v3.json",
}
# Path overrides:
# API_ROOT_DIR = "??" # default: api/ inside repository
DATA_DIR = "/data"
# PROCESSING_DIR = "??" # default: DATA_DIR/processing
# PROCESSING_OUTPUT_DIR = "??" # default: DATA_DIR/processing-output
# TRACKS_DIR = "??" # default: DATA_DIR/tracks
# OBS_FACE_CACHE_DIR = "??" # default: DATA_DIR/obs-face-cache
# vim: set ft=python :

15
api/config.py.example
View file

@ -4,6 +4,7 @@ PORT = 3000
# Extended log output, but slower
DEBUG = False
AUTO_RESTART = DEBUG
# Required to encrypt or sign sessions, cookies, tokens, etc.
SECRET = "!!!<<<CHANGEME>>>!!!"
@ -40,9 +41,17 @@ FRONTEND_CONFIG = {
"maxZoom": 18,
},
"mapHome": {"zoom": 15, "longitude": 7.8302, "latitude": 47.9755},
"obsMapSource": "http://localhost:3002/data/v3.json",
"obsMapSource": {
"type": "vector",
"url": "http://localhost:3002/data/v3.json",
},
}
# If the API should serve generated tiles, this is the path where the tiles are
# built. This is an experimental option and probably very inefficient, a proper
# tileserver should be prefered. Set to None to disable.
TILES_FILE = None
# Path overrides:
# API_ROOT_DIR = "??" # default: api/ inside repository
# DATA_DIR = "??" # default: $API_ROOT_DIR/..
@ -51,4 +60,8 @@ FRONTEND_CONFIG = {
# TRACKS_DIR = "??" # default: DATA_DIR/tracks
# OBS_FACE_CACHE_DIR = "??" # default: DATA_DIR/obs-face-cache
# Additional allowed origins for CORS headers. The FRONTEND_URL is included by
# default. Python list, or whitespace separated string.
ADDITIONAL_CORS_ORIGINS = None
# vim: set ft=python :

70
api/obs/api/app.py
View file

@ -1,5 +1,6 @@
import logging
import os
import re
from json import JSONEncoder, dumps
from functools import wraps, partial
from urllib.parse import urlparse
@ -17,9 +18,6 @@ from sqlalchemy.orm import sessionmaker
from obs.api.db import User, make_session, connect_db
from sanic_session.base import BaseSessionInterface
from sanic_session.utils import ExpiringDict
log = logging.getLogger(__name__)
app = Sanic("OpenBikeSensor Portal API")
@ -30,26 +28,57 @@ api = Blueprint("api", url_prefix="/api")
auth = Blueprint("auth", url_prefix="")
# Configure paths
c.API_ROOT_DIR = c.get("API_ROOT_DIR") or abspath(join(dirname(__file__), "..", ".."))
def configure_paths(c):
c.API_ROOT_DIR = c.get("API_ROOT_DIR") or abspath(
join(dirname(__file__), "..", "..")
)
c.DATA_DIR = c.get("DATA_DIR") or normpath(join(c.API_ROOT_DIR, "../data"))
c.PROCESSING_DIR = c.get("PROCESSING_DIR") or join(c.DATA_DIR, "processing")
c.PROCESSING_OUTPUT_DIR = c.get("PROCESSING_OUTPUT_DIR") or join(
c.DATA_DIR, "processing-output"
)
c.TRACKS_DIR = c.get("TRACKS_DIR") or join(c.DATA_DIR, "tracks")
c.OBS_FACE_CACHE_DIR = c.get("OBS_FACE_CACHE_DIR") or join(c.DATA_DIR, "obs-face-cache")
c.OBS_FACE_CACHE_DIR = c.get("OBS_FACE_CACHE_DIR") or join(
c.DATA_DIR, "obs-face-cache"
)
c.FRONTEND_DIR = c.get("FRONTEND_DIR")
if c.FRONTEND_URL:
configure_paths(app.config)
def setup_cors(app):
frontend_url = app.config.get("FRONTEND_URL")
additional_origins = app.config.get("ADDITIONAL_CORS_ORIGINS")
if not frontend_url and not additional_origins:
# No CORS configured
return
origins = []
if frontend_url:
u = urlparse(frontend_url)
origins.append(f"{u.scheme}://{u.netloc}")
if isinstance(additional_origins, str):
origins += re.split(r"\s+", additional_origins)
elif isinstance(additional_origins, list):
origins += additional_origins
elif additional_origins is not None:
raise ValueError(
"invalid option type for ADDITIONAL_CORS_ORIGINS, must be list or space separated str"
)
from sanic_cors import CORS
frontend_url = urlparse(c.FRONTEND_URL)
CORS(
app,
origins=[f"{frontend_url.scheme}://{frontend_url.netloc}"],
origins=origins,
supports_credentials=True,
)
setup_cors(app)
# TODO: use a different interface, maybe backed by the PostgreSQL, to allow
# scaling the API
Session(app, interface=InMemorySessionInterface())
@ -57,7 +86,7 @@ Session(app, interface=InMemorySessionInterface())
@app.before_server_start
async def app_connect_db(app, loop):
app.ctx._db_engine_ctx = connect_db(c.POSTGRES_URL)
app.ctx._db_engine_ctx = connect_db(app.config.POSTGRES_URL)
app.ctx._db_engine = await app.ctx._db_engine_ctx.__aenter__()
@ -118,26 +147,37 @@ def json(*args, **kwargs):
from . import routes
INDEX_HTML = join(c.FRONTEND_DIR, "index.html")
if exists(INDEX_HTML):
INDEX_HTML = (
join(app.config.FRONTEND_DIR, "index.html")
if app.config.get("FRONTEND_DIR")
else None
)
if INDEX_HTML and exists(INDEX_HTML):
@app.get("/config.json")
def get_frontend_config(req):
base_path = req.server_path.replace("config.json", "")
return json_response(
{
result = {
**req.app.config.FRONTEND_CONFIG,
"apiUrl": f"{req.scheme}://{req.host}{base_path}api",
"loginUrl": f"{req.scheme}://{req.host}{base_path}login",
}
)
print(req.app.config)
if req.app.config.get("TILES_FILE"):
result["obsMapSource"] = {
"type": "vector",
"tiles": [req.app.url_for("tiles", zoom="{zoom}", x="{x}", y="{y}")],
"minzoom": 12,
"maxzoom": 14,
}
return json_response()
@app.get("/<path:path>")
def get_frontend_static(req, path):
if path.startswith("api/"):
raise NotFound()
file = join(c.FRONTEND_DIR, path)
file = join(app.config.FRONTEND_DIR, path)
if not exists(file) or not path or not isfile(file):
file = INDEX_HTML
return file_response(file)

3
api/obs/api/routes/__init__.py
View file

@ -1,7 +1,8 @@
from . import (
info,
login,
stats,
tiles,
tracks,
info,
users,
)

57
api/obs/api/routes/tiles.py Normal file
View file

@ -0,0 +1,57 @@
import gzip
from sqlite3 import connect
from sanic.response import raw
from obs.api.app import app
def get_tile(filename, zoom, x, y):
"""
Inspired by:
https://github.com/TileStache/TileStache/blob/master/TileStache/MBTiles.py
"""
print(filename)
db = connect(filename)
db.text_factory = bytes
fmt = db.execute("SELECT value FROM metadata WHERE name='format'").fetchone()[0]
if fmt != b"pbf":
print(repr(b"pbf"), " versus ", repr(fmt))
raise ValueError("mbtiles file is in wrong format: %s" % fmt)
content = db.execute(
"SELECT tile_data FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?",
(zoom, x, (2 ** zoom - 1) - y),
).fetchone()
return content and content[0] or None
# regenerate approx. once each day
TILE_CACHE_MAX_AGE = 3600 * 24
if app.config.get("TILES_FILE"):
@app.route(r"/tiles/<zoom:int>/<x:int>/<y:(\d+)\.pbf>")
async def tiles(req, zoom: int, x: int, y: str):
tile = get_tile(req.app.config.TILES_FILE, int(zoom), int(x), int(y))
gzip = "gzip" in req.headers["accept-encoding"]
headers = {}
headers["Vary"] = "Accept-Encoding"
if req.app.config.DEBUG:
headers["Cache-Control"] = "no-cache"
else:
headers["Cache-Control"] = f"public, max-age={TILE_CACHE_MAX_AGE}"
# The tiles in the mbtiles file are gzip-compressed already, so we
# serve them actually as-is, and only decompress them if the browser
# doesn't accept gzip
if gzip:
headers["Content-Encoding"] = "gzip"
else:
tile = gzip.decompress(tile)
return raw(tile, content_type="application/x-protobuf", headers=headers)

3
api/obs/api/routes/users.py
View file

@ -21,9 +21,8 @@ def user_to_json(user):
@api.get("/user")
@require_auth
async def get_user(req):
return json(user_to_json(req.ctx.user))
return json(user_to_json(req.ctx.user) if req.ctx.user else None)
@api.put("/user")

8
api/obs/bin/openbikesensor_api.py
View file

@ -10,7 +10,13 @@ from obs.api.db import connect_db
def main():
app.run(host=app.config.HOST, port=app.config.PORT, debug=app.config.DEBUG)
debug = app.config.DEBUG
app.run(
host=app.config.HOST,
port=app.config.PORT,
debug=debug,
auto_reload=app.config.get("AUTO_RELOAD", debug),
)
if __name__ == "__main__":

28
docker-compose.yaml
View file

@ -39,6 +39,7 @@ services:
- ./api/config.dev.py:/opt/obs/api/config.py
- ./frontend/build:/opt/obs/frontend/build
- ./local/api-data:/data
- ./tile-generator/data/:/tiles
links:
- postgres
- keycloak
@ -140,17 +141,6 @@ services:
PGHOST: postgres
PGPORT: 5432
tileserver:
image: klokantech/tileserver-gl
ports:
- 3002:80
volumes:
- ./tile-generator/tileserver-gl-config.json:/config/tileserver.json
- ./tile-generator/data/:/data
command:
- --config
- /config/tileserver.json
keycloak:
image: jboss/keycloak
ports:
@ -166,19 +156,3 @@ services:
DB_DATABASE: obs
DB_USER: obs
DB_PASSWORD: obs
# DB_SCHEMA: keycloak
prod-test:
image: openbikesensor-portal
build:
context: ./
dockerfile: Dockerfile
volumes:
- ./api/config.prod-test.py:/opt/obs/api/config.py
- ./local/api-data:/data
links:
- postgres
- keycloak
ports:
- '3000:3000'
restart: on-failure

7
frontend/config.dev.json
View file

@ -13,5 +13,10 @@
"longitude": 7.8302,
"latitude": 47.9755
},
"obsMapSource": "http://localhost:3002/data/v3.json"
"obsMapSource": {
"type": "vector",
"tiles": ["http://localhost:3000/tiles/{z}/{x}/{y}.pbf"],
"minzoom": 12,
"maxzoom": 14
}
}

5
frontend/config.example.json
View file

@ -13,5 +13,8 @@
"longitude": 9.1797,
"latitude": 48.7784
},
"obsMapSource": "http://portal.example.com/tileserver/data/v3.json"
"obsMapSource": {
"type": "vector",
"url": "http://portal.example.com/tileserver/data/v3.json"
}
}

5
frontend/public/config.json
View file

@ -12,5 +12,8 @@
"longitude": 7.8302,
"latitude": 47.9755
},
"obsMapSource": "https://portal.example.com/tileset/data/v3.json"
"obsMapSource": {
"type": "vector",
"url": "https://portal.example.com/tileset/data/v3.json"
}
}

12
frontend/src/config.ts
View file

@ -1,5 +1,15 @@
import React from 'react'
export type MapSoure = {
type: 'vector'
url: string,
} | {
type: 'vector',
tiles: string[],
minzoom: number,
maxzoom: number,
}
export interface Config {
apiUrl: string
mapHome: {
@ -7,7 +17,7 @@ export interface Config {
longitude: number
zoom: number
}
obsMapSource?: string
obsMapSource?: MapSoure
imprintUrl?: string
privacyPolicyUrl?: string
mapTileset?: {

4
frontend/src/mapstyles/index.js
View file

@ -3,8 +3,8 @@ import _ from 'lodash'
import bright from './bright.json'
import positron from './positron.json'
function addRoadsStyle(style, sourceUrl = "http://localhost:3002/data/v3.json") {
style.sources.obs = {"type": "vector", "url": sourceUrl}
function addRoadsStyle(style, mapSource) {
style.sources.obs = mapSource
// insert before "road_oneway" layer
let idx = style.layers.findIndex(l => l.id === 'road_oneway')

4
frontend/src/pages/HomePage.tsx
View file

@ -7,7 +7,7 @@ import {map, switchMap} from 'rxjs/operators'
import api from 'api'
import {Stats, Page} from 'components'
import {useConfig} from 'config'
import {useConfig, MapSource} from 'config'
import {TrackListItem} from './TracksPage'
import styles from './HomePage.module.scss'
@ -16,7 +16,7 @@ import 'ol/ol.css'
import {obsRoads} from '../mapstyles'
import ReactMapGl from 'react-map-gl'
function WelcomeMap({mapSource}: {mapSource: string}) {
function WelcomeMap({mapSource}: {mapSource: MapSource}) {
const mapStyle = React.useMemo(() => obsRoads(mapSource), [mapSource])
const config = useConfig()
const [viewport, setViewport] = React.useState({

6
frontend/src/pages/MapPage.tsx
View file

@ -12,7 +12,7 @@ import {obsRoads} from '../mapstyles'
import ReactMapGl from 'react-map-gl'
function BigMap({mapSource, config}: {mapSource: string ,config: Config}) {
const mapStyle = React.useMemo(() => obsRoads(mapSource), [mapSource])
const mapStyle = React.useMemo(() => mapSource && obsRoads(mapSource), [mapSource])
const [viewport, setViewport] = React.useState({
longitude: 0,
latitude: 0,
@ -25,6 +25,10 @@ function BigMap({mapSource, config}: {mapSource: string ,config: Config}) {
}
}, [config])
if (!mapStyle) {
return null
}
return (
<div className={styles.mapContainer}>
<ReactMapGl mapStyle={mapStyle} width="100%" height="100%" onViewportChange={setViewport} {...viewport} />

15
tile-generator/README.md
View file

@ -74,11 +74,18 @@ make generate-tiles-pg
## Publish vector tiles
The tool [tileserver-gl](http://tileserver.org/) is used to publish the vector
tiles separately through HTTP. The tileserver runs inside docker, so all you need to do for a development setup is start it:
The API is capable of serving the generated mbtiles file in XYZ scheme with PBF
format. Set the config variable `TILES_FILE` to point to your generated file.
The API might be inefficient at publishing the tiles. You might want to try a
proper tileserver with caching and all if you run into trouble with its
capabilities.
The URL for the tiles is:
```
docker compose up -d tileserver
http://api.example.com/tiles/{z}/{x}/{y}.pbf
```
It is now available at [http://localhost:3002/](http://localhost:3002/).
The API generates this URL into the `/config.json` as `obsMapSource` if it is
configured to serve tiles *and* the frontend.

39
tile-generator/tileserver-gl-config.json
View file

@ -1,39 +0,0 @@
{
"options": {
"paths": {
"root": "/usr/src/app/node_modules/tileserver-gl-styles",
"fonts": "fonts",
"styles": "styles",
"mbtiles": "/data"
}
},
"styles": {
"klokantech-basic": {
"style": "klokantech-basic/style.json",
"tilejson": {
"bounds": [
7.487897,
47.80556,
8.212994,
48.224458
]
}
},
"osm-bright": {
"style": "osm-bright/style.json",
"tilejson": {
"bounds": [
7.487897,
47.80556,
8.212994,
48.224458
]
}
}
},
"data": {
"v3": {
"mbtiles": "tiles.mbtiles"
}
}
}