diff --git a/CHANGELOG.md b/CHANGELOG.md index 32ae18bbf..f3ce81604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [7.1.1] +### Changed +- Reduced `start_execution_manager` batch size from 600 jobs to 500 jobs. Fixes [#2241](https://github.com/ASFHyP3/hyp3/issues/2241). + ## [7.1.0] ### Added - A `hyp3-its-live-test` deployment to [`deploy-enterprise-test.yml`](.github/workflows/deploy-enterprise-test.yml) for ITS_LIVE testing in preparation for some significant ITS_LIVE project development diff --git a/apps/start-execution-manager/src/start_execution_manager.py b/apps/start-execution-manager/src/start_execution_manager.py index e74911a37..0babf94ba 100644 --- a/apps/start-execution-manager/src/start_execution_manager.py +++ b/apps/start-execution-manager/src/start_execution_manager.py @@ -25,10 +25,10 @@ def lambda_handler(event, context) -> None: worker_function_arn = os.environ['START_EXECUTION_WORKER_ARN'] logger.info(f'Worker function ARN: {worker_function_arn}') - pending_jobs = dynamo.jobs.get_jobs_waiting_for_execution(limit=600) + pending_jobs = dynamo.jobs.get_jobs_waiting_for_execution(limit=500) logger.info(f'Got {len(pending_jobs)} pending jobs') - batch_size = 300 + batch_size = 250 for i in range(0, len(pending_jobs), batch_size): jobs = pending_jobs[i:i + batch_size] logger.info(f'Invoking worker for {len(jobs)} jobs') diff --git a/tests/test_start_execution_manager.py b/tests/test_start_execution_manager.py index f78603a8d..fb6768005 100644 --- a/tests/test_start_execution_manager.py +++ b/tests/test_start_execution_manager.py @@ -53,41 +53,41 @@ def test_invoke_worker(): ) -def test_lambda_handler_600_jobs(): +def test_lambda_handler_500_jobs(): with patch('dynamo.jobs.get_jobs_waiting_for_execution') as mock_get_jobs_waiting_for_execution, \ patch('start_execution_manager.invoke_worker') as mock_invoke_worker, \ patch.dict(os.environ, {'START_EXECUTION_WORKER_ARN': 'test-worker-function-arn'}, clear=True): - mock_jobs = list(range(600)) + mock_jobs = list(range(500)) mock_get_jobs_waiting_for_execution.return_value = mock_jobs mock_invoke_worker.return_value = {'StatusCode': None} start_execution_manager.lambda_handler(None, None) - mock_get_jobs_waiting_for_execution.assert_called_once_with(limit=600) + mock_get_jobs_waiting_for_execution.assert_called_once_with(limit=500) assert mock_invoke_worker.mock_calls == [ - call('test-worker-function-arn', mock_jobs[0:300]), - call('test-worker-function-arn', mock_jobs[300:600]), + call('test-worker-function-arn', mock_jobs[0:250]), + call('test-worker-function-arn', mock_jobs[250:500]), ] -def test_lambda_handler_500_jobs(): +def test_lambda_handler_400_jobs(): with patch('dynamo.jobs.get_jobs_waiting_for_execution') as mock_get_jobs_waiting_for_execution, \ patch('start_execution_manager.invoke_worker') as mock_invoke_worker, \ patch.dict(os.environ, {'START_EXECUTION_WORKER_ARN': 'test-worker-function-arn'}, clear=True): - mock_jobs = list(range(500)) + mock_jobs = list(range(400)) mock_get_jobs_waiting_for_execution.return_value = mock_jobs mock_invoke_worker.return_value = {'StatusCode': None} start_execution_manager.lambda_handler(None, None) - mock_get_jobs_waiting_for_execution.assert_called_once_with(limit=600) + mock_get_jobs_waiting_for_execution.assert_called_once_with(limit=500) assert mock_invoke_worker.mock_calls == [ - call('test-worker-function-arn', mock_jobs[0:300]), - call('test-worker-function-arn', mock_jobs[300:500]), + call('test-worker-function-arn', mock_jobs[0:250]), + call('test-worker-function-arn', mock_jobs[250:400]), ] @@ -102,7 +102,7 @@ def test_lambda_handler_50_jobs(): start_execution_manager.lambda_handler(None, None) - mock_get_jobs_waiting_for_execution.assert_called_once_with(limit=600) + mock_get_jobs_waiting_for_execution.assert_called_once_with(limit=500) assert mock_invoke_worker.mock_calls == [ call('test-worker-function-arn', mock_jobs), @@ -117,6 +117,6 @@ def test_lambda_handler_no_jobs(): start_execution_manager.lambda_handler(None, None) - mock_get_jobs_waiting_for_execution.assert_called_once_with(limit=600) + mock_get_jobs_waiting_for_execution.assert_called_once_with(limit=500) mock_invoke_worker.assert_not_called()