From 6aae2c7aae7ea770fe904c1bd3dc5cc5d1b385f3 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 9 Jan 2025 10:53:33 -0800 Subject: [PATCH 1/2] fix db connection pool --- app/__init__.py | 18 +++++++++++++++++- app/celery/scheduled_tasks.py | 2 ++ app/clients/__init__.py | 3 +-- app/config.py | 2 +- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 23c2399e1..0d617ee0c 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -18,6 +18,7 @@ from werkzeug.exceptions import HTTPException as WerkzeugHTTPException from werkzeug.local import LocalProxy +from app import config from app.clients import NotificationProviderClients from app.clients.cloudwatch.aws_cloudwatch import AwsCloudwatchClient from app.clients.document_download import DocumentDownloadClient @@ -25,6 +26,7 @@ from app.clients.email.aws_ses_stub import AwsSesStubClient from app.clients.pinpoint.aws_pinpoint import AwsPinpointClient from app.clients.sms.aws_sns import AwsSnsClient +from app.utils import hilite from notifications_utils import logging, request_helper from notifications_utils.clients.encryption.encryption_client import Encryption from notifications_utils.clients.redis.redis_client import RedisClient @@ -58,15 +60,29 @@ class SQLAlchemy(_SQLAlchemy): def apply_driver_hacks(self, app, info, options): sa_url, options = super().apply_driver_hacks(app, info, options) + print(hilite(f"OPTIONS {options}")) + if "connect_args" not in options: options["connect_args"] = {} options["connect_args"]["options"] = "-c statement_timeout={}".format( int(app.config["SQLALCHEMY_STATEMENT_TIMEOUT"]) * 1000 ) + return (sa_url, options) -db = SQLAlchemy() +# Set db engine settings here for now. +# They were not being set previous (despite environmental variables with appropriate +# sounding names) and were defaulting to low values +db = SQLAlchemy( + engine_options={ + "pool_size": config.Config.SQLALCHEMY_POOL_SIZE, + "max_overflow": 10, + "pool_timeout": config.Config.SQLALCHEMY_POOL_TIMEOUT, + "pool_recycle": config.Config.SQLALCHEMY_POOL_RECYCLE, + "pool_pre_ping": True, + } +) migrate = Migrate() ma = Marshmallow() notify_celery = NotifyCelery() diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index cb0e0886e..72806aa58 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -243,6 +243,8 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers(): bind=True, max_retries=7, default_retry_delay=3600, name="process-delivery-receipts" ) def process_delivery_receipts(self): + # If we need to check db settings do it here for convenience + # current_app.logger.info(f"POOL SIZE {app.db.engine.pool.size()}") """ Every eight minutes or so (see config.py) we run this task, which searches the last ten minutes of logs for delivery receipts and batch updates the db with the results. The overlap diff --git a/app/clients/__init__.py b/app/clients/__init__.py index 3392928e4..f185e45e2 100644 --- a/app/clients/__init__.py +++ b/app/clients/__init__.py @@ -13,8 +13,7 @@ "addressing_style": "virtual", }, use_fips_endpoint=True, - # This is the default but just for doc sake - max_pool_connections=10, + max_pool_connections=50, # This should be equal or greater than our celery concurrency ) diff --git a/app/config.py b/app/config.py index 580495731..9ae731290 100644 --- a/app/config.py +++ b/app/config.py @@ -81,7 +81,7 @@ class Config(object): SQLALCHEMY_DATABASE_URI = cloud_config.database_url SQLALCHEMY_RECORD_QUERIES = False SQLALCHEMY_TRACK_MODIFICATIONS = False - SQLALCHEMY_POOL_SIZE = int(getenv("SQLALCHEMY_POOL_SIZE", 5)) + SQLALCHEMY_POOL_SIZE = int(getenv("SQLALCHEMY_POOL_SIZE", 20)) SQLALCHEMY_POOL_TIMEOUT = 30 SQLALCHEMY_POOL_RECYCLE = 300 SQLALCHEMY_STATEMENT_TIMEOUT = 1200 From 2770f76431c2c5ebde3f461002f9a9d22d0e6adb Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 9 Jan 2025 11:14:51 -0800 Subject: [PATCH 2/2] cleanup --- app/__init__.py | 2 -- app/config.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 0d617ee0c..add218e5d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -26,7 +26,6 @@ from app.clients.email.aws_ses_stub import AwsSesStubClient from app.clients.pinpoint.aws_pinpoint import AwsPinpointClient from app.clients.sms.aws_sns import AwsSnsClient -from app.utils import hilite from notifications_utils import logging, request_helper from notifications_utils.clients.encryption.encryption_client import Encryption from notifications_utils.clients.redis.redis_client import RedisClient @@ -60,7 +59,6 @@ class SQLAlchemy(_SQLAlchemy): def apply_driver_hacks(self, app, info, options): sa_url, options = super().apply_driver_hacks(app, info, options) - print(hilite(f"OPTIONS {options}")) if "connect_args" not in options: options["connect_args"] = {} diff --git a/app/config.py b/app/config.py index 9ae731290..f7f08a36a 100644 --- a/app/config.py +++ b/app/config.py @@ -81,7 +81,7 @@ class Config(object): SQLALCHEMY_DATABASE_URI = cloud_config.database_url SQLALCHEMY_RECORD_QUERIES = False SQLALCHEMY_TRACK_MODIFICATIONS = False - SQLALCHEMY_POOL_SIZE = int(getenv("SQLALCHEMY_POOL_SIZE", 20)) + SQLALCHEMY_POOL_SIZE = int(getenv("SQLALCHEMY_POOL_SIZE", 40)) SQLALCHEMY_POOL_TIMEOUT = 30 SQLALCHEMY_POOL_RECYCLE = 300 SQLALCHEMY_STATEMENT_TIMEOUT = 1200