make pool_size and overflow configurable for worker and portal

api/config.dev.py

@@ -5,6 +5,8 @@ VERBOSE = False
 AUTO_RESTART = True
 SECRET = "!!!!!!!!!!!!CHANGE ME!!!!!!!!!!!!"
 POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs"
+POSTGRES_POOL_SIZE = 20
+POSTGRES_MAX_OVERFLOW = 2 * POSTGRES_POOL_SIZE
 KEYCLOAK_URL = "http://keycloak:8080/auth/realms/obs-dev/"
 KEYCLOAK_CLIENT_ID = "portal"
 KEYCLOAK_CLIENT_SECRET = "c385278e-bd2e-4f13-9937-34b0c0f44c2d"

api/config.py.example

@@ -12,6 +12,8 @@ SECRET = "!!!<<<CHANGEME>>>!!!"
 
 # Connection to the database
 POSTGRES_URL = "postgresql+asyncpg://user:pass@host/dbname"
+POSTGRES_POOL_SIZE = 20
+POSTGRES_MAX_OVERFLOW = 2 * POSTGRES_POOL_SIZE
 
 # URL to the keycloak realm, as reachable by the API service. This is not
 # necessarily its publicly reachable URL, keycloak advertises that iself.

api/obs/api/app.py

@@ -104,7 +104,7 @@ Session(app, interface=InMemorySessionInterface())
 
 @app.before_server_start
 async def app_connect_db(app, loop):
-    app.ctx._db_engine_ctx = connect_db(app.config.POSTGRES_URL)
+    app.ctx._db_engine_ctx = connect_db(app.config.POSTGRES_URL, app.config.POSTGRES_POOL_SIZE, app.config.POSTGRES_MAX_OVERFLOW)
     app.ctx._db_engine = await app.ctx._db_engine_ctx.__aenter__()
 
 

api/obs/api/db.py

@@ -65,10 +65,10 @@ def random_string(length):
 
 
 @asynccontextmanager
-async def connect_db(url):
+async def connect_db(url, pool_size=10, max_overflow=20):
     global engine, sessionmaker
 
-    engine = create_async_engine(url, echo=False)
+    engine = create_async_engine(url, echo=False, pool_size=pool_size, max_overflow=max_overflow)
     sessionmaker = SessionMaker(engine, class_=AsyncSession, expire_on_commit=False)
 
     yield engine

api/tools/prepare_sql_tiles.py

@@ -121,7 +121,7 @@ async def generate_sql(build_dir):
 
 async def import_sql(sql_snippets):
     statements = sum(map(sqlparse.split, sql_snippets), [])
-    async with connect_db(app.config.POSTGRES_URL):
+    async with connect_db(app.config.POSTGRES_URL, app.config.POSTGRES_POOL_SIZE, app.config.POSTGRES_MAX_OVERFLOW):
         for i, statement in enumerate(statements):
             clean_statement = sqlparse.format(
                 statement,

api/tools/process_track.py

@@ -35,7 +35,7 @@ async def main():
 
     args = parser.parse_args()
 
-    async with connect_db(app.config.POSTGRES_URL):
+    async with connect_db(app.config.POSTGRES_URL, app.config.POSTGRES_POOL_SIZE, app.config.POSTGRES_MAX_OVERFLOW):
         if args.tracks:
             await process_tracks(args.tracks)
         else: