From 5ac2900e63b7580c8acf1c6dd13551d4781600b9 Mon Sep 17 00:00:00 2001
From: gluap <github@pgoergen.de>
Date: Tue, 8 Feb 2022 23:23:29 +0100
Subject: [PATCH] make pool_size and overflow configurable for worker and
 portal

---
 api/config.dev.py              | 2 ++
 api/config.py.example          | 2 ++
 api/obs/api/app.py             | 2 +-
 api/obs/api/db.py              | 4 ++--
 api/tools/prepare_sql_tiles.py | 2 +-
 api/tools/process_track.py     | 2 +-
 6 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/api/config.dev.py b/api/config.dev.py
index fe2f7cf..fae82f7 100644
--- a/api/config.dev.py
+++ b/api/config.dev.py
@@ -5,6 +5,8 @@ VERBOSE = False
 AUTO_RESTART = True
 SECRET = "!!!!!!!!!!!!CHANGE ME!!!!!!!!!!!!"
 POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs"
+POSTGRES_POOL_SIZE = 20
+POSTGRES_MAX_OVERFLOW = 2 * POSTGRES_POOL_SIZE
 KEYCLOAK_URL = "http://keycloak:8080/auth/realms/obs-dev/"
 KEYCLOAK_CLIENT_ID = "portal"
 KEYCLOAK_CLIENT_SECRET = "c385278e-bd2e-4f13-9937-34b0c0f44c2d"
diff --git a/api/config.py.example b/api/config.py.example
index 7e4553d..94f7280 100644
--- a/api/config.py.example
+++ b/api/config.py.example
@@ -12,6 +12,8 @@ SECRET = "!!!<<<CHANGEME>>>!!!"
 
 # Connection to the database
 POSTGRES_URL = "postgresql+asyncpg://user:pass@host/dbname"
+POSTGRES_POOL_SIZE = 20
+POSTGRES_MAX_OVERFLOW = 2 * POSTGRES_POOL_SIZE
 
 # URL to the keycloak realm, as reachable by the API service. This is not
 # necessarily its publicly reachable URL, keycloak advertises that iself.
diff --git a/api/obs/api/app.py b/api/obs/api/app.py
index dd52250..5f9e349 100644
--- a/api/obs/api/app.py
+++ b/api/obs/api/app.py
@@ -104,7 +104,7 @@ Session(app, interface=InMemorySessionInterface())
 
 @app.before_server_start
 async def app_connect_db(app, loop):
-    app.ctx._db_engine_ctx = connect_db(app.config.POSTGRES_URL)
+    app.ctx._db_engine_ctx = connect_db(app.config.POSTGRES_URL, app.config.POSTGRES_POOL_SIZE, app.config.POSTGRES_MAX_OVERFLOW)
     app.ctx._db_engine = await app.ctx._db_engine_ctx.__aenter__()
 
 
diff --git a/api/obs/api/db.py b/api/obs/api/db.py
index 22e4746..c9b28d8 100644
--- a/api/obs/api/db.py
+++ b/api/obs/api/db.py
@@ -65,10 +65,10 @@ def random_string(length):
 
 
 @asynccontextmanager
-async def connect_db(url):
+async def connect_db(url, pool_size=10, max_overflow=20):
     global engine, sessionmaker
 
-    engine = create_async_engine(url, echo=False)
+    engine = create_async_engine(url, echo=False, pool_size=pool_size, max_overflow=max_overflow)
     sessionmaker = SessionMaker(engine, class_=AsyncSession, expire_on_commit=False)
 
     yield engine
diff --git a/api/tools/prepare_sql_tiles.py b/api/tools/prepare_sql_tiles.py
index 833b32c..993205f 100755
--- a/api/tools/prepare_sql_tiles.py
+++ b/api/tools/prepare_sql_tiles.py
@@ -121,7 +121,7 @@ async def generate_sql(build_dir):
 
 async def import_sql(sql_snippets):
     statements = sum(map(sqlparse.split, sql_snippets), [])
-    async with connect_db(app.config.POSTGRES_URL):
+    async with connect_db(app.config.POSTGRES_URL, app.config.POSTGRES_POOL_SIZE, app.config.POSTGRES_MAX_OVERFLOW):
         for i, statement in enumerate(statements):
             clean_statement = sqlparse.format(
                 statement,
diff --git a/api/tools/process_track.py b/api/tools/process_track.py
index 9302612..eeb027e 100755
--- a/api/tools/process_track.py
+++ b/api/tools/process_track.py
@@ -35,7 +35,7 @@ async def main():
 
     args = parser.parse_args()
 
-    async with connect_db(app.config.POSTGRES_URL):
+    async with connect_db(app.config.POSTGRES_URL, app.config.POSTGRES_POOL_SIZE, app.config.POSTGRES_MAX_OVERFLOW):
         if args.tracks:
             await process_tracks(args.tracks)
         else: