pub-solar/obs-portal
5
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

import in chunks to avoid smaller systems chocking

Browse source
This commit is contained in:
gluap 2023-06-10 12:56:44 +02:00
parent 497e1b739a
commit 0233045959
No known key found for this signature in database
1 changed files with 18 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
api/tools/import_osm.py
View file

@ -71,21 +71,25 @@ async def import_osm(connection, filename, import_group=None):
# Pass 2: Import # Pass 2: Import
log.info("Pass 2: Import roads") log.info("Pass 2: Import roads")
async with cursor.copy( amount = 0
"COPY road (way_id, name, zone, directionality, oneway, geometry, import_group) FROM STDIN" for items in chunk(read_file(filename), 10000):
) as copy: amount += 10000
for item in read_file(filename): log.info(f"...{amount}/{len(ids)} ({100*amount/len(ids)}%)")
await copy.write_row( async with cursor.copy(
( "COPY road (way_id, name, zone, directionality, oneway, geometry, import_group) FROM STDIN"
item.way_id, ) as copy:
item.name, for item in items:
item.zone, await copy.write_row(
item.directionality, (
item.oneway, item.way_id,
bytes.hex(item.geometry), item.name,
import_group, item.zone,
item.directionality,
item.oneway,
bytes.hex(item.geometry),
import_group,
)
) )
)
async def main(): async def main():