pub-solar/obs-portal
5
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

import in chunks to avoid smaller systems chocking

Browse source
This commit is contained in:
gluap 2023-06-10 12:56:44 +02:00
parent 497e1b739a
commit 0233045959
No known key found for this signature in database
1 changed files with 18 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
api/tools/import_osm.py
View file

@ -71,10 +71,14 @@ async def import_osm(connection, filename, import_group=None):
# Pass 2: Import
log.info("Pass 2: Import roads")
amount = 0
for items in chunk(read_file(filename), 10000):
amount += 10000
log.info(f"...{amount}/{len(ids)} ({100*amount/len(ids)}%)")
async with cursor.copy(
"COPY road (way_id, name, zone, directionality, oneway, geometry, import_group) FROM STDIN"
) as copy:
for item in read_file(filename):
for item in items:
await copy.write_row(
(
item.way_id,