pub-solar/obs-portal
5
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Use logging in import_osm

Browse source
This commit is contained in:
Paul Bienkowski 2023-03-30 14:56:21 +02:00
parent 6fb5dfe6de
commit 0d9ddf4884
1 changed files with 14 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
api/tools/import_osm.py
View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from dataclasses import asdict, dataclass from dataclasses import dataclass
import asyncio import asyncio
from os.path import basename, splitext from os.path import basename, splitext
import sys import sys
@ -8,12 +8,8 @@ import logging
import msgpack import msgpack
import psycopg import psycopg
from psycopg.types import TypeInfo
from psycopg.types.shapely import register_shapely
from shapely.wkb import loads
from obs.api.app import app from obs.api.app import app
from obs.api.db import ZoneType, connect_db, make_session
from obs.api.utils import chunk from obs.api.utils import chunk
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -43,6 +39,7 @@ class Road:
oneway: int oneway: int
geometry: bytes geometry: bytes
data_types = {ROAD_TYPE: Road, REGION_TYPE: Region} data_types = {ROAD_TYPE: Road, REGION_TYPE: Region}
@ -79,9 +76,9 @@ async def import_osm(connection, filename, import_group=None):
region_ids.append(item.relation_id) region_ids.append(item.relation_id)
async with connection.cursor() as cursor: async with connection.cursor() as cursor:
print(f"Pass 1: Delete previous") log.info("Pass 1: Delete previously imported data")
print(f"Deleting import group {import_group}") log.debug("Delete import group %s", import_group)
await cursor.execute( await cursor.execute(
"DELETE FROM road WHERE import_group = %s", (import_group,) "DELETE FROM road WHERE import_group = %s", (import_group,)
) )
@ -89,15 +86,17 @@ async def import_osm(connection, filename, import_group=None):
"DELETE FROM region WHERE import_group = %s", (import_group,) "DELETE FROM region WHERE import_group = %s", (import_group,)
) )
print(f"Deleting roads by ID") log.debug("Delete roads by way_id")
for ids in chunk(road_ids, 10000): for ids in chunk(road_ids, 10000):
await cursor.execute("DELETE FROM road WHERE way_id = ANY(%s)", (ids,)) await cursor.execute("DELETE FROM road WHERE way_id = ANY(%s)", (ids,))
print(f"Deleting regions by ID") log.debug("Delete regions by region_id")
for ids in chunk(region_ids, 10000): for ids in chunk(region_ids, 10000):
await cursor.execute("DELETE FROM region WHERE relation_id = ANY(%s)", (ids,)) await cursor.execute(
"DELETE FROM region WHERE relation_id = ANY(%s)", (ids,)
)
# Pass 2: Import # Pass 2: Import
print(f"Pass 2: Import Roads") log.info("Pass 2: Import roads")
async with cursor.copy( async with cursor.copy(
"COPY road (way_id, name, zone, directionality, oneway, geometry, import_group) FROM STDIN" "COPY road (way_id, name, zone, directionality, oneway, geometry, import_group) FROM STDIN"
) as copy: ) as copy:
@ -114,7 +113,7 @@ async def import_osm(connection, filename, import_group=None):
) )
) )
print(f"Pass 2: Import Regions") log.info(f"Pass 2: Import regions")
async with cursor.copy( async with cursor.copy(
"COPY region (relation_id, name, geometry, admin_level, import_group) FROM STDIN" "COPY region (relation_id, name, geometry, admin_level, import_group) FROM STDIN"
) as copy: ) as copy:
@ -131,12 +130,14 @@ async def import_osm(connection, filename, import_group=None):
async def main(): async def main():
logging.basicConfig(level=logging.DEBUG, format="%(levelname)s: %(message)s")
url = app.config.POSTGRES_URL url = app.config.POSTGRES_URL
url = url.replace("+asyncpg", "") url = url.replace("+asyncpg", "")
async with await psycopg.AsyncConnection.connect(url) as connection: async with await psycopg.AsyncConnection.connect(url) as connection:
for filename in sys.argv[1:]: for filename in sys.argv[1:]:
print("Loading file", filename) log.debug("Loading file: %s", filename)
await import_osm(connection, filename) await import_osm(connection, filename)