From 7c87b46213fb681b238218693bb90b41e1b4f1d4 Mon Sep 17 00:00:00 2001 From: Alex Kennedy Date: Wed, 4 Jan 2023 09:15:59 -0800 Subject: [PATCH] [AWSBaseActor] Respect boto3/AWS retry configurability (#522) * [AWSBaseActor] Respect boto3/AWS retry configurability * linter appeasement --- kingpin/actors/aws/base.py | 3 ++- kingpin/actors/aws/settings.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/kingpin/actors/aws/base.py b/kingpin/actors/aws/base.py index ac3a59d8..efb97625 100644 --- a/kingpin/actors/aws/base.py +++ b/kingpin/actors/aws/base.py @@ -119,7 +119,8 @@ def __init__(self, *args, **kwargs): boto_config = botocore_config.Config( region_name=self.region, retries={ - "mode": "adaptive", + "max_attempts": aws_settings.AWS_MAX_ATTEMPTS, + "mode": aws_settings.AWS_RETRY_MODE, }, ) diff --git a/kingpin/actors/aws/settings.py b/kingpin/actors/aws/settings.py index e7582ff2..245d15a1 100644 --- a/kingpin/actors/aws/settings.py +++ b/kingpin/actors/aws/settings.py @@ -32,3 +32,12 @@ AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", None) AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", None) AWS_SESSION_TOKEN = os.getenv("AWS_SESSION_TOKEN", None) + +# kingpin is pretty fast which can leads to API throttling. You can set you own +# boto3 configuration by using the standard AWS env vars, but in the absence of +# them, we try to set you some sane defaults based on the boto3 documentation +# and our experience with running kingpin as scale. +# +# Docs: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html +AWS_MAX_ATTEMPTS = os.getenv("AWS_MAX_ATTEMPTS", 10) +AWS_RETRY_MODE = os.getenv("AWS_RETRY_MODE", "standard")