From 0bf82bfb6e8f6eeb54060260a3a9e1de4e653b27 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 7 Jan 2025 14:10:28 -0900 Subject: [PATCH 01/10] add mypy --- .github/workflows/static-analysis.yml | 3 +++ pyproject.toml | 12 ++++++++++++ requirements-all.txt | 1 + 3 files changed, 16 insertions(+) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 2ed4887b4..97dd510c6 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -7,6 +7,9 @@ jobs: # Docs: https://github.com/ASFHyP3/actions uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.12.0 + call-mypy-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-mypy.yml@v0.14.0 + cfn-lint: runs-on: ubuntu-latest strategy: diff --git a/pyproject.toml b/pyproject.toml index 5b7617bf7..e9e1f31da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,3 +31,15 @@ convention = "google" [tool.ruff.lint.isort] case-sensitive = true lines-after-imports = 2 + +[tool.mypy] +python_version = "3.9" +warn_redundant_casts = true +warn_unused_ignores = true +warn_unreachable = true +strict_equality = true +check_untyped_defs = true +exclude = [ + "/build/", + "/setup\\.py$", +] diff --git a/requirements-all.txt b/requirements-all.txt index 475929fe3..35b3d3ea6 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -12,6 +12,7 @@ pytest==8.3.4 PyYAML==6.0.2 responses==0.25.3 ruff +mypy setuptools==75.6.0 openapi-spec-validator==0.7.1 cfn-lint==1.22.3 From f7fe8471e8b35ce01500d2359feec3b0caa0e245 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 7 Jan 2025 15:00:04 -0900 Subject: [PATCH 02/10] fix mypy errors --- apps/api/src/hyp3_api/handlers.py | 2 +- apps/api/src/hyp3_api/openapi.py | 2 +- apps/api/src/hyp3_api/routes.py | 19 ++++++++++-------- apps/api/src/hyp3_api/validation.py | 6 +++--- .../src/check_processing_time.py | 2 +- .../src/disable_private_dns.py | 2 +- apps/get-files/src/get_files.py | 2 +- apps/render_cf.py | 10 ++++++---- apps/scale-cluster/src/scale_cluster.py | 4 ++-- .../src/set_batch_overrides.py | 4 +++- .../src/start_execution_manager.py | 2 +- .../src/start_execution_worker.py | 2 +- apps/upload-log/src/upload_log.py | 4 ++-- lib/dynamo/dynamo/user.py | 2 +- lib/dynamo/dynamo/util.py | 4 ++-- tests/conftest.py | 6 +++--- tests/test_api/test_list_jobs.py | 4 ++-- tests/test_api/test_submit_job.py | 2 +- tests/test_api/test_validation.py | 2 +- tests/test_dynamo/test_jobs.py | 6 +++--- tests/test_dynamo/test_user.py | 20 +++++++++---------- tests/test_render_cf.py | 10 +++++----- tests/test_start_execution_manager.py | 6 +++--- tests/test_upload_log.py | 2 +- 24 files changed, 66 insertions(+), 59 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index b26f13591..3d21cdf45 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -21,7 +21,7 @@ def post_jobs(body, user): try: validate_jobs(body['jobs']) - except requests.HTTPError as e: + except requests.HTTPError as e: # type: ignore[attr-defined] print(f'WARN: CMR search failed: {e}') except (BoundsValidationError, GranuleValidationError) as e: abort(problem_format(400, str(e))) diff --git a/apps/api/src/hyp3_api/openapi.py b/apps/api/src/hyp3_api/openapi.py index e81a0f4e8..fa5d19486 100644 --- a/apps/api/src/hyp3_api/openapi.py +++ b/apps/api/src/hyp3_api/openapi.py @@ -4,6 +4,6 @@ def get_spec_yaml(path_to_spec: Path): - parser = prance.ResolvingParser(str(path_to_spec.resolve())) + parser = prance.ResolvingParser(str(path_to_spec.resolve())) # type: ignore[attr-defined] parser.parse() return parser.specification diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index f7e53314a..8cca237b5 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import datetime import json from decimal import Decimal from os import environ from pathlib import Path +from typing import Any import yaml from flask import abort, g, jsonify, make_response, redirect, render_template, request from flask.json.provider import JSONProvider -from flask_cors import CORS +from flask_cors import CORS # type: ignore[attr-defined] from openapi_core import OpenAPI from openapi_core.contrib.flask.decorators import FlaskOpenAPIViewDecorator from openapi_core.contrib.flask.handlers import FlaskOpenAPIErrorsHandler @@ -57,7 +60,7 @@ def get_open_api_json(): @app.route('/openapi.yaml') def get_open_api_yaml(): - return yaml.dump(api_spec_dict) + return yaml.dump(api_spec_dict) # type: ignore[attr-defined] @app.route('/ui/') @@ -98,10 +101,10 @@ def default(self, o): class CustomJSONProvider(JSONProvider): - def dumps(self, o): - return json.dumps(o, cls=CustomEncoder) + def dumps(self, obj: Any, **kwargs) -> str: + return json.dumps(obj, cls=CustomEncoder) - def loads(self, s): + def loads(self, s: str | bytes, **kwargs) -> Any: return json.loads(s) @@ -111,7 +114,7 @@ def __init__(self): def __call__(self, errors): response = super().__call__(errors) - error = response.json['errors'][0] + error = response.json['errors'][0] # type: ignore[index] return handlers.problem_format(error['status'], error['title']) @@ -119,7 +122,7 @@ def __call__(self, errors): openapi = FlaskOpenAPIViewDecorator( api_spec, - response_cls=None, + response_cls=None, # type: ignore[arg-type] errors_handler_cls=ErrorHandler, ) @@ -138,7 +141,7 @@ def jobs_post(): @app.route('/jobs', methods=['GET']) @openapi def jobs_get(): - parameters = request.openapi.parameters.query + parameters = request.openapi.parameters.query # type: ignore[attr-defined] start = parameters.get('start') end = parameters.get('end') return jsonify( diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 1358b1feb..66b3aed64 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -6,7 +6,7 @@ import requests import yaml -from shapely.geometry import MultiPolygon, Polygon, shape +from shapely.geometry import MultiPolygon, Polygon, shape # type: ignore[attr-defined] from hyp3_api import CMR_URL from hyp3_api.util import get_granules @@ -24,7 +24,7 @@ class BoundsValidationError(Exception): with open(Path(__file__).parent / 'job_validation_map.yml') as f: - JOB_VALIDATION_MAP = yaml.safe_load(f.read()) + JOB_VALIDATION_MAP = yaml.safe_load(f.read()) # type: ignore[attr-defined] def has_sufficient_coverage(granule: Polygon): @@ -53,7 +53,7 @@ def get_cmr_metadata(granules): ], 'page_size': 2000, } - response = requests.post(CMR_URL, data=cmr_parameters) + response = requests.post(CMR_URL, data=cmr_parameters) # type: ignore[attr-defined] response.raise_for_status() granules = [ { diff --git a/apps/check-processing-time/src/check_processing_time.py b/apps/check-processing-time/src/check_processing_time.py index 2b0950499..cdf3dccd1 100644 --- a/apps/check-processing-time/src/check_processing_time.py +++ b/apps/check-processing-time/src/check_processing_time.py @@ -8,7 +8,7 @@ def get_time_from_result(result: Union[list, dict]) -> Union[list, float]: return (result['StoppedAt'] - result['StartedAt']) / 1000 -def lambda_handler(event, _) -> list[Union[list, float]]: +def lambda_handler(event, _) -> Union[list, float]: processing_results = event['processing_results'] result_list = [processing_results[key] for key in sorted(processing_results.keys())] return get_time_from_result(result_list) diff --git a/apps/disable-private-dns/src/disable_private_dns.py b/apps/disable-private-dns/src/disable_private_dns.py index 53c205744..fb3a11924 100644 --- a/apps/disable-private-dns/src/disable_private_dns.py +++ b/apps/disable-private-dns/src/disable_private_dns.py @@ -3,7 +3,7 @@ import boto3 -CLIENT = boto3.client('ec2') +CLIENT = boto3.client('ec2') # type: ignore[attr-defined] def get_endpoint(vpc_id, endpoint_name): diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index 585b3beb1..9d5b88fcc 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -9,7 +9,7 @@ import boto3 -S3_CLIENT = boto3.client('s3') +S3_CLIENT = boto3.client('s3') # type: ignore[attr-defined] def get_download_url(bucket, key): diff --git a/apps/render_cf.py b/apps/render_cf.py index eeb265479..aab0707f5 100644 --- a/apps/render_cf.py +++ b/apps/render_cf.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import json from pathlib import Path @@ -77,7 +79,7 @@ def get_map_state(job_spec: dict, step: dict) -> dict: def get_batch_submit_job_state(job_spec: dict, step: dict, filter_batch_params=False) -> dict: if filter_batch_params: - batch_job_parameters = get_batch_job_parameters(job_spec, step) + batch_job_parameters: dict | str = get_batch_job_parameters(job_spec, step) parameters_key = 'Parameters' else: batch_job_parameters = '$.batch_job_parameters' @@ -120,7 +122,7 @@ def parse_map_statement(map_statement: str) -> tuple[str, str]: return tokens[1], tokens[3] -def get_batch_job_parameters(job_spec: dict, step: dict, map_item: str = None) -> dict: +def get_batch_job_parameters(job_spec: dict, step: dict, map_item: str | None = None) -> dict: job_params = {'bucket_prefix', *job_spec['parameters'].keys()} step_params = get_batch_param_names_for_job_step(step) batch_params = {} @@ -171,7 +173,7 @@ def render_templates(job_types: dict, compute_envs: dict, security_environment: def get_compute_environments_for_deployment(job_types: dict, compute_env_file: Path) -> dict: - compute_envs = yaml.safe_load(compute_env_file.read_text())['compute_environments'] + compute_envs = yaml.safe_load(compute_env_file.read_text())['compute_environments'] # type: ignore[attr-defined] if 'Default' in compute_envs: raise ValueError("'Default' is a reserved compute environment name") @@ -252,7 +254,7 @@ def main(): job_types = {} for file in args.job_spec_files: - job_types.update(yaml.safe_load(file.read_text())) + job_types.update(yaml.safe_load(file.read_text())) # type: ignore[attr-defined] for job_type, job_spec in job_types.items(): validate_job_spec(job_type, job_spec) diff --git a/apps/scale-cluster/src/scale_cluster.py b/apps/scale-cluster/src/scale_cluster.py index 9f50aec01..a227c6de0 100644 --- a/apps/scale-cluster/src/scale_cluster.py +++ b/apps/scale-cluster/src/scale_cluster.py @@ -6,8 +6,8 @@ import dateutil.relativedelta -BATCH = boto3.client('batch') -COST_EXPLORER = boto3.client('ce') +BATCH = boto3.client('batch') # type: ignore[attr-defined] +COST_EXPLORER = boto3.client('ce') # type: ignore[attr-defined] def get_time_period(today: date): diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 9d419ec31..76d3c8d7e 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -1,3 +1,5 @@ +from typing import Optional + AUTORIFT_S2_MEMORY = '7875' AUTORIFT_LANDSAT_MEMORY = '15750' RTC_GAMMA_10M_MEMORY = '63200' @@ -19,7 +21,7 @@ } -def get_container_overrides(memory: str, omp_num_threads: str = None) -> dict: +def get_container_overrides(memory: str, omp_num_threads: Optional[str] = None) -> dict: container_overrides = { 'ResourceRequirements': [ { diff --git a/apps/start-execution-manager/src/start_execution_manager.py b/apps/start-execution-manager/src/start_execution_manager.py index 841c63577..3701f9d66 100644 --- a/apps/start-execution-manager/src/start_execution_manager.py +++ b/apps/start-execution-manager/src/start_execution_manager.py @@ -7,7 +7,7 @@ from lambda_logging import log_exceptions, logger -LAMBDA_CLIENT = boto3.client('lambda') +LAMBDA_CLIENT = boto3.client('lambda') # type: ignore[attr-defined] def invoke_worker(worker_function_arn: str, jobs: list[dict]) -> dict: diff --git a/apps/start-execution-worker/src/start_execution_worker.py b/apps/start-execution-worker/src/start_execution_worker.py index ac8b47b18..6587def87 100644 --- a/apps/start-execution-worker/src/start_execution_worker.py +++ b/apps/start-execution-worker/src/start_execution_worker.py @@ -8,7 +8,7 @@ from lambda_logging import log_exceptions, logger -STEP_FUNCTION = boto3.client('stepfunctions') +STEP_FUNCTION = boto3.client('stepfunctions') # type: ignore[attr-defined] batch_params_file = Path(__file__).parent / 'batch_params_by_job_type.json' if batch_params_file.exists(): diff --git a/apps/upload-log/src/upload_log.py b/apps/upload-log/src/upload_log.py index 38c04520b..2e1a51552 100644 --- a/apps/upload-log/src/upload_log.py +++ b/apps/upload-log/src/upload_log.py @@ -7,8 +7,8 @@ config = Config(retries={'max_attempts': 2, 'mode': 'standard'}) -CLOUDWATCH = boto3.client('logs', config=config) -S3 = boto3.client('s3') +CLOUDWATCH = boto3.client('logs', config=config) # type: ignore[attr-defined] +S3 = boto3.client('s3') # type: ignore[attr-defined] def get_log_stream(result: dict) -> Optional[str]: diff --git a/lib/dynamo/dynamo/user.py b/lib/dynamo/dynamo/user.py index 6198032c1..563c0d0ae 100644 --- a/lib/dynamo/dynamo/user.py +++ b/lib/dynamo/dynamo/user.py @@ -90,7 +90,7 @@ def _validate_access_code(access_code: str) -> None: def _get_edl_profile(user_id: str, edl_access_token: str) -> dict: url = f'https://urs.earthdata.nasa.gov/api/users/{user_id}' - response = requests.get(url, headers={'Authorization': f'Bearer {edl_access_token}'}) + response = requests.get(url, headers={'Authorization': f'Bearer {edl_access_token}'}) # type: ignore[attr-defined] response.raise_for_status() return response.json() diff --git a/lib/dynamo/dynamo/util.py b/lib/dynamo/dynamo/util.py index 7dab2f394..b52b9281c 100644 --- a/lib/dynamo/dynamo/util.py +++ b/lib/dynamo/dynamo/util.py @@ -3,10 +3,10 @@ import boto3 from boto3.dynamodb.conditions import Key -from dateutil.parser import parse +from dateutil.parser import parse # type: ignore[attr-defined] -DYNAMODB_RESOURCE = boto3.resource('dynamodb') +DYNAMODB_RESOURCE = boto3.resource('dynamodb') # type: ignore[attr-defined] def get_request_time_expression(start, end): diff --git a/tests/conftest.py b/tests/conftest.py index f0de52b5c..f51145cbe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,10 +19,10 @@ class TableProperties: def get_table_properties_from_template(resource_name): - yaml.SafeLoader.add_multi_constructor('!', lambda loader, suffix, node: None) + yaml.SafeLoader.add_multi_constructor('!', lambda loader, suffix, node: None) # type: ignore[attr-defined] template_file = path.join(path.dirname(__file__), '../apps/main-cf.yml') with open(template_file) as f: - template = yaml.safe_load(f) + template = yaml.safe_load(f) # type: ignore[attr-defined] table_properties = template['Resources'][resource_name]['Properties'] return table_properties @@ -53,7 +53,7 @@ class Tables: @pytest.fixture def approved_user(tables) -> str: - user = { + user: dict = { 'user_id': 'approved_user', 'remaining_credits': Decimal(0), 'application_status': APPLICATION_APPROVED, diff --git a/tests/test_api/test_list_jobs.py b/tests/test_api/test_list_jobs.py index 13c99862d..021d1bce9 100644 --- a/tests/test_api/test_list_jobs.py +++ b/tests/test_api/test_list_jobs.py @@ -22,7 +22,7 @@ def test_list_jobs(client, tables): }, ] browse_images = ['https://mybucket.s3.us-west-2.amazonaws.com/prefix/browse/foo.png'] - thumbnail_images = [] + thumbnail_images: list = [] items = [ make_db_record('0ddaeb98-7636-494d-9496-03ea4a7df266', user_id='user_with_jobs'), make_db_record( @@ -203,7 +203,7 @@ def test_bad_date_formats(client): def test_list_paging(client): login(client) - mock_response = ([], {'foo': 1, 'bar': 2}) + mock_response: tuple = ([], {'foo': 1, 'bar': 2}) with mock.patch('dynamo.jobs.query_jobs', return_value=mock_response): response = client.get(JOBS_URI) assert unquote(response.json['next']) == 'http://localhost/jobs?start_token=eyJmb28iOiAxLCAiYmFyIjogMn0=' diff --git a/tests/test_api/test_submit_job.py b/tests/test_api/test_submit_job.py index 239b94ed0..4a585d667 100644 --- a/tests/test_api/test_submit_job.py +++ b/tests/test_api/test_submit_job.py @@ -150,7 +150,7 @@ def test_submit_unapproved_user(client, tables): def test_submit_without_jobs(client): login(client) - batch = [] + batch: list = [] response = submit_batch(client, batch) assert response.status_code == HTTPStatus.BAD_REQUEST diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 556c4d707..eba982e1b 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -1,6 +1,6 @@ import responses from pytest import raises -from shapely.geometry import Polygon +from shapely.geometry import Polygon # type: ignore[attr-defined] from hyp3_api import CMR_URL, validation from test_api.conftest import setup_requests_mock_with_given_polygons diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index add1092e0..b41380b85 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -191,7 +191,7 @@ def test_query_jobs_by_type(tables): def test_get_credit_cost(): - costs = [ + costs: list[dict] = [ { 'job_type': 'RTC_GAMMA', 'cost_parameter': 'resolution', @@ -231,7 +231,7 @@ def test_get_credit_cost(): def test_get_credit_cost_validate_keys(): - costs = [ + costs: list[dict] = [ {'job_type': 'JOB_TYPE_A', 'cost_parameter': 'foo', 'cost_table': [{'parameter_value': 'bar', 'cost': 3.0}]}, {'job_type': 'JOB_TYPE_B', 'cost': 5.0}, {'job_type': 'JOB_TYPE_C', 'cost_parameter': ''}, @@ -353,7 +353,7 @@ def test_put_jobs_default_params(tables, approved_user): {'job_type': 'JOB_TYPE_B', 'cost': Decimal('1.0')}, {'job_type': 'JOB_TYPE_C', 'cost': Decimal('1.0')}, ] - payload = [ + payload: list[dict] = [ {}, {'job_type': 'JOB_TYPE_A'}, {'job_type': 'JOB_TYPE_A', 'job_parameters': {}}, diff --git a/tests/test_dynamo/test_user.py b/tests/test_dynamo/test_user.py index 9b4779c50..e617a583b 100644 --- a/tests/test_dynamo/test_user.py +++ b/tests/test_dynamo/test_user.py @@ -667,24 +667,24 @@ def test_reset_credits_failed_zero_credits(tables): def test_decrement_credits(tables): tables.users_table.put_item(Item={'user_id': 'foo', 'remaining_credits': Decimal(25)}) - dynamo.user.decrement_credits('foo', 1) + dynamo.user.decrement_credits('foo', Decimal(1)) assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(24)}] - dynamo.user.decrement_credits('foo', 4) + dynamo.user.decrement_credits('foo', Decimal(4)) assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(20)}] - dynamo.user.decrement_credits('foo', 20) + dynamo.user.decrement_credits('foo', Decimal(20)) assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(0)}] def test_decrement_credits_invalid_cost(tables): with pytest.raises(ValueError, match=r'^Cost 0 <= 0$'): - dynamo.user.decrement_credits('foo', 0) + dynamo.user.decrement_credits('foo', Decimal(0)) assert tables.users_table.scan()['Items'] == [] with pytest.raises(ValueError, match=r'^Cost -1 <= 0$'): - dynamo.user.decrement_credits('foo', -1) + dynamo.user.decrement_credits('foo', Decimal(-1)) assert tables.users_table.scan()['Items'] == [] @@ -693,16 +693,16 @@ def test_decrement_credits_cost_too_high(tables): tables.users_table.put_item(Item={'user_id': 'foo', 'remaining_credits': Decimal(1)}) with pytest.raises(DatabaseConditionException): - dynamo.user.decrement_credits('foo', 2) + dynamo.user.decrement_credits('foo', Decimal(2)) assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(1)}] - dynamo.user.decrement_credits('foo', 1) + dynamo.user.decrement_credits('foo', Decimal(1)) assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(0)}] with pytest.raises(DatabaseConditionException): - dynamo.user.decrement_credits('foo', 1) + dynamo.user.decrement_credits('foo', Decimal(1)) assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': Decimal(0)}] @@ -715,13 +715,13 @@ def test_decrement_credits_infinite_credits(tables): match=r'^An error occurred \(ValidationException\) when calling the UpdateItem operation:' r' An operand in the update expression has an incorrect data type$', ): - dynamo.user.decrement_credits('foo', 1) + dynamo.user.decrement_credits('foo', Decimal(1)) assert tables.users_table.scan()['Items'] == [{'user_id': 'foo', 'remaining_credits': None}] def test_decrement_credits_user_does_not_exist(tables): with pytest.raises(DatabaseConditionException): - dynamo.user.decrement_credits('foo', 1) + dynamo.user.decrement_credits('foo', Decimal(1)) assert tables.users_table.scan()['Items'] == [] diff --git a/tests/test_render_cf.py b/tests/test_render_cf.py index 50b3e49a5..daca1d11b 100644 --- a/tests/test_render_cf.py +++ b/tests/test_render_cf.py @@ -22,7 +22,7 @@ def test_parse_map_statement(): def test_get_batch_job_parameters(): - job_spec = {'parameters': {'param1': {}, 'param2': {}, 'param3': {}, 'param4': {}}} + job_spec: dict = {'parameters': {'param1': {}, 'param2': {}, 'param3': {}, 'param4': {}}} step = {'command': ['foo', 'Ref::param2', 'Ref::param3', 'bar', 'Ref::bucket_prefix']} assert render_cf.get_batch_job_parameters(job_spec, step) == { @@ -68,7 +68,7 @@ def test_get_compute_environments(tmp_path): ], }, } - compute_env_file_contents = { + compute_env_file_contents: dict = { 'compute_environments': { 'ComputeEnvironment1': {'key1': 'value1'}, 'ComputeEnvironment2': {'key2': 'value2'}, @@ -80,7 +80,7 @@ def test_get_compute_environments(tmp_path): 'ComputeEnvironment2': {'key2': 'value2'}, } compute_env_file = tmp_path / 'compute_environments.yml' - yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) + yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) # type: ignore[attr-defined] assert render_cf.get_compute_environments_for_deployment(job_types, compute_env_file) == expected_compute_envs compute_env_file_contents = { @@ -91,7 +91,7 @@ def test_get_compute_environments(tmp_path): 'Default': {'key', 'value'}, } } - yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) + yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) # type: ignore[attr-defined] with pytest.raises(ValueError, match="'Default' is a reserved compute environment name"): render_cf.get_compute_environments_for_deployment(job_types, compute_env_file) @@ -100,6 +100,6 @@ def test_get_compute_environments(tmp_path): 'ComputeEnvironment1': {'key1': 'value1'}, } } - yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) + yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) # type: ignore[attr-defined] with pytest.raises(KeyError, match='ComputeEnvironment2'): render_cf.get_compute_environments_for_deployment(job_types, compute_env_file) diff --git a/tests/test_start_execution_manager.py b/tests/test_start_execution_manager.py index 6d8731611..711ff3d10 100644 --- a/tests/test_start_execution_manager.py +++ b/tests/test_start_execution_manager.py @@ -7,7 +7,7 @@ def test_invoke_worker(): - jobs = [ + jobs: list[dict] = [ { 'job_id': 'job0', 'decimal_float_field': Decimal('10.1'), @@ -38,9 +38,9 @@ def test_invoke_worker(): } ) with patch('start_execution_manager.LAMBDA_CLIENT.invoke') as mock_invoke: - mock_invoke.return_value = 'test-response' + mock_invoke.return_value = {'foo': 'bar'} - assert start_execution_manager.invoke_worker('test-worker-arn', jobs) == 'test-response' + assert start_execution_manager.invoke_worker('test-worker-arn', jobs) == {'foo': 'bar'} mock_invoke.assert_called_once_with( FunctionName='test-worker-arn', diff --git a/tests/test_upload_log.py b/tests/test_upload_log.py index bd34bf5ec..c5a154bf0 100644 --- a/tests/test_upload_log.py +++ b/tests/test_upload_log.py @@ -22,7 +22,7 @@ def s3_stubber(): def test_get_log_stream(): - result = { + result: dict = { 'Container': { 'LogStreamName': 'mySucceededLogStream', }, From 2224efb62726102cff642d2d1f70b76f673f82ce Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 7 Jan 2025 15:25:36 -0900 Subject: [PATCH 03/10] upgrade to python 3.13 --- .github/workflows/deploy-daac.yml | 2 +- .github/workflows/deploy-enterprise-test.yml | 2 +- .github/workflows/deploy-enterprise.yml | 2 +- .github/workflows/deploy-multi-burst-sandbox.yml | 2 +- .github/workflows/static-analysis.yml | 8 ++++---- .github/workflows/tests.yml | 2 +- CHANGELOG.md | 3 +++ README.md | 2 +- apps/api/api-cf.yml.j2 | 2 +- .../check-processing-time/check-processing-time-cf.yml.j2 | 2 +- apps/disable-private-dns/disable-private-dns-cf.yml.j2 | 2 +- apps/get-files/get-files-cf.yml.j2 | 2 +- apps/handle-batch-event/handle-batch-event-cf.yml.j2 | 2 +- apps/scale-cluster/scale-cluster-cf.yml.j2 | 4 ++-- apps/set-batch-overrides/set-batch-overrides-cf.yml.j2 | 2 +- .../start-execution-manager-cf.yml.j2 | 2 +- .../start-execution-worker-cf.yml.j2 | 2 +- apps/update-db/update-db-cf.yml.j2 | 2 +- apps/upload-log/upload-log-cf.yml.j2 | 2 +- environment.yml | 2 +- lib/dynamo/setup.py | 2 +- lib/lambda_logging/setup.py | 2 +- pyproject.toml | 4 ++-- 23 files changed, 30 insertions(+), 27 deletions(-) diff --git a/.github/workflows/deploy-daac.yml b/.github/workflows/deploy-daac.yml index 8cad31f7c..af0a7dd05 100644 --- a/.github/workflows/deploy-daac.yml +++ b/.github/workflows/deploy-daac.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - uses: ./.github/actions/deploy-hyp3 if: github.ref == matrix.deploy_ref diff --git a/.github/workflows/deploy-enterprise-test.yml b/.github/workflows/deploy-enterprise-test.yml index 0872503ea..bfed131e4 100644 --- a/.github/workflows/deploy-enterprise-test.yml +++ b/.github/workflows/deploy-enterprise-test.yml @@ -113,7 +113,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - uses: ./.github/actions/deploy-hyp3 with: diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml index 7eb29e145..146caae65 100644 --- a/.github/workflows/deploy-enterprise.yml +++ b/.github/workflows/deploy-enterprise.yml @@ -270,7 +270,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - uses: ./.github/actions/deploy-hyp3 with: diff --git a/.github/workflows/deploy-multi-burst-sandbox.yml b/.github/workflows/deploy-multi-burst-sandbox.yml index b073dd11f..2aeaf3378 100644 --- a/.github/workflows/deploy-multi-burst-sandbox.yml +++ b/.github/workflows/deploy-multi-burst-sandbox.yml @@ -54,7 +54,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - uses: ./.github/actions/deploy-hyp3 with: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 97dd510c6..c2de9ae87 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4.2.2 - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - run: | python -m pip install --upgrade pip make install @@ -34,7 +34,7 @@ jobs: - uses: actions/checkout@v4.2.2 - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - run: | python -m pip install --upgrade pip make install @@ -51,7 +51,7 @@ jobs: - run: gem install statelint - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - run: | python -m pip install --upgrade pip make install @@ -68,7 +68,7 @@ jobs: - uses: snyk/actions/setup@0.4.0 - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - run: | python -m pip install --upgrade pip make install diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d7824a78c..cc9c06677 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.13 - run: | python -m pip install --upgrade pip diff --git a/CHANGELOG.md b/CHANGELOG.md index e74012291..833beb694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - OPERA_DISP_TMS job type is now available in EDC UAT deployment +### Changed +- Upgraded AWS Lambda functions and Github Actions workflows to Python 3.13 + ### Removed - hyp3-opera-disp-sandbox deployment diff --git a/README.md b/README.md index bd0c7fa32..35823595a 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ From the repository root, make install ``` -- Install Python dependencies for AWS Lambda functions (requires pip for python 3.9) +- Install Python dependencies for AWS Lambda functions (requires pip for python 3.13) ```sh make build ``` diff --git a/apps/api/api-cf.yml.j2 b/apps/api/api-cf.yml.j2 index 338ff01ec..b9b744dbc 100644 --- a/apps/api/api-cf.yml.j2 +++ b/apps/api/api-cf.yml.j2 @@ -196,7 +196,7 @@ Resources: Handler: hyp3_api.lambda_handler.handler MemorySize: 3008 Role: !GetAtt LambdaRole.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 30 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/apps/check-processing-time/check-processing-time-cf.yml.j2 b/apps/check-processing-time/check-processing-time-cf.yml.j2 index d07e2df2f..174400ace 100644 --- a/apps/check-processing-time/check-processing-time-cf.yml.j2 +++ b/apps/check-processing-time/check-processing-time-cf.yml.j2 @@ -66,7 +66,7 @@ Resources: Handler: check_processing_time.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 30 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/apps/disable-private-dns/disable-private-dns-cf.yml.j2 b/apps/disable-private-dns/disable-private-dns-cf.yml.j2 index d8e98d554..b2623891a 100644 --- a/apps/disable-private-dns/disable-private-dns-cf.yml.j2 +++ b/apps/disable-private-dns/disable-private-dns-cf.yml.j2 @@ -56,7 +56,7 @@ Resources: Handler: disable_private_dns.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 5 Environment: Variables: diff --git a/apps/get-files/get-files-cf.yml.j2 b/apps/get-files/get-files-cf.yml.j2 index 5e6268a99..ea2b4038a 100644 --- a/apps/get-files/get-files-cf.yml.j2 +++ b/apps/get-files/get-files-cf.yml.j2 @@ -86,7 +86,7 @@ Resources: Handler: get_files.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 30 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/apps/handle-batch-event/handle-batch-event-cf.yml.j2 b/apps/handle-batch-event/handle-batch-event-cf.yml.j2 index cf39fd7a2..3d252f1cd 100644 --- a/apps/handle-batch-event/handle-batch-event-cf.yml.j2 +++ b/apps/handle-batch-event/handle-batch-event-cf.yml.j2 @@ -80,7 +80,7 @@ Resources: Handler: handle_batch_event.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 30 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/apps/scale-cluster/scale-cluster-cf.yml.j2 b/apps/scale-cluster/scale-cluster-cf.yml.j2 index 0fa44cd90..eb70d19f3 100644 --- a/apps/scale-cluster/scale-cluster-cf.yml.j2 +++ b/apps/scale-cluster/scale-cluster-cf.yml.j2 @@ -104,7 +104,7 @@ Resources: Handler: scale_cluster.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 30 {% if security_environment == 'EDC' %} VpcConfig: @@ -161,7 +161,7 @@ Resources: Handler: scale_cluster.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 30 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/apps/set-batch-overrides/set-batch-overrides-cf.yml.j2 b/apps/set-batch-overrides/set-batch-overrides-cf.yml.j2 index 57362c9bf..3a63a0c0c 100644 --- a/apps/set-batch-overrides/set-batch-overrides-cf.yml.j2 +++ b/apps/set-batch-overrides/set-batch-overrides-cf.yml.j2 @@ -66,7 +66,7 @@ Resources: Handler: set_batch_overrides.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 30 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/apps/start-execution-manager/start-execution-manager-cf.yml.j2 b/apps/start-execution-manager/start-execution-manager-cf.yml.j2 index 57465a58b..ab760a447 100644 --- a/apps/start-execution-manager/start-execution-manager-cf.yml.j2 +++ b/apps/start-execution-manager/start-execution-manager-cf.yml.j2 @@ -78,7 +78,7 @@ Resources: Handler: start_execution_manager.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 10 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/apps/start-execution-worker/start-execution-worker-cf.yml.j2 b/apps/start-execution-worker/start-execution-worker-cf.yml.j2 index 172f363ae..808bd2abd 100644 --- a/apps/start-execution-worker/start-execution-worker-cf.yml.j2 +++ b/apps/start-execution-worker/start-execution-worker-cf.yml.j2 @@ -75,7 +75,7 @@ Resources: Handler: start_execution_worker.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 45 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/apps/update-db/update-db-cf.yml.j2 b/apps/update-db/update-db-cf.yml.j2 index 82ceb5ad0..0e4fac64b 100644 --- a/apps/update-db/update-db-cf.yml.j2 +++ b/apps/update-db/update-db-cf.yml.j2 @@ -75,7 +75,7 @@ Resources: Handler: main.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 30 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/apps/upload-log/upload-log-cf.yml.j2 b/apps/upload-log/upload-log-cf.yml.j2 index d1003832e..75992dcfa 100644 --- a/apps/upload-log/upload-log-cf.yml.j2 +++ b/apps/upload-log/upload-log-cf.yml.j2 @@ -80,7 +80,7 @@ Resources: Handler: upload_log.lambda_handler MemorySize: 128 Role: !GetAtt Role.Arn - Runtime: python3.9 + Runtime: python3.13 Timeout: 30 {% if security_environment == 'EDC' %} VpcConfig: diff --git a/environment.yml b/environment.yml index f0f595530..7bb9cf3ab 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - nodefaults dependencies: - - python=3.9 + - python=3.13 - pip - pip: - -r requirements-all.txt diff --git a/lib/dynamo/setup.py b/lib/dynamo/setup.py index 676e0abb6..c6a7400b2 100644 --- a/lib/dynamo/setup.py +++ b/lib/dynamo/setup.py @@ -10,7 +10,7 @@ 'python-dateutil', 'requests', ], - python_requires='~=3.9', + python_requires='~=3.13', packages=find_packages(), package_data={'dynamo': ['*.json']}, ) diff --git a/lib/lambda_logging/setup.py b/lib/lambda_logging/setup.py index 017984faf..76b3db846 100644 --- a/lib/lambda_logging/setup.py +++ b/lib/lambda_logging/setup.py @@ -5,6 +5,6 @@ name='lambda_logging', license='BSD', include_package_data=True, - python_requires='~=3.9', + python_requires='~=3.13', packages=find_packages(), ) diff --git a/pyproject.toml b/pyproject.toml index e9e1f31da..17429969b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -requires-python = "==3.9" +requires-python = "==3.13" [tool.ruff] line-length = 120 @@ -33,7 +33,7 @@ case-sensitive = true lines-after-imports = 2 [tool.mypy] -python_version = "3.9" +python_version = "3.13" warn_redundant_casts = true warn_unused_ignores = true warn_unreachable = true From 2914b9ddd8a990962607bac6bd263ef8eb836bb3 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 7 Jan 2025 15:33:09 -0900 Subject: [PATCH 04/10] remove some type ignores --- apps/api/src/hyp3_api/handlers.py | 2 +- apps/api/src/hyp3_api/openapi.py | 2 +- apps/api/src/hyp3_api/routes.py | 6 +++--- apps/api/src/hyp3_api/validation.py | 6 +++--- apps/disable-private-dns/src/disable_private_dns.py | 2 +- apps/get-files/src/get_files.py | 2 +- apps/render_cf.py | 4 ++-- apps/scale-cluster/src/scale_cluster.py | 4 ++-- apps/start-execution-manager/src/start_execution_manager.py | 2 +- apps/start-execution-worker/src/start_execution_worker.py | 2 +- apps/upload-log/src/upload_log.py | 4 ++-- lib/dynamo/dynamo/user.py | 2 +- lib/dynamo/dynamo/util.py | 4 ++-- tests/conftest.py | 4 ++-- tests/test_api/test_validation.py | 2 +- tests/test_render_cf.py | 6 +++--- 16 files changed, 27 insertions(+), 27 deletions(-) diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 3d21cdf45..b26f13591 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -21,7 +21,7 @@ def post_jobs(body, user): try: validate_jobs(body['jobs']) - except requests.HTTPError as e: # type: ignore[attr-defined] + except requests.HTTPError as e: print(f'WARN: CMR search failed: {e}') except (BoundsValidationError, GranuleValidationError) as e: abort(problem_format(400, str(e))) diff --git a/apps/api/src/hyp3_api/openapi.py b/apps/api/src/hyp3_api/openapi.py index fa5d19486..e81a0f4e8 100644 --- a/apps/api/src/hyp3_api/openapi.py +++ b/apps/api/src/hyp3_api/openapi.py @@ -4,6 +4,6 @@ def get_spec_yaml(path_to_spec: Path): - parser = prance.ResolvingParser(str(path_to_spec.resolve())) # type: ignore[attr-defined] + parser = prance.ResolvingParser(str(path_to_spec.resolve())) parser.parse() return parser.specification diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index 8cca237b5..80594edc0 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -10,7 +10,7 @@ import yaml from flask import abort, g, jsonify, make_response, redirect, render_template, request from flask.json.provider import JSONProvider -from flask_cors import CORS # type: ignore[attr-defined] +from flask_cors import CORS from openapi_core import OpenAPI from openapi_core.contrib.flask.decorators import FlaskOpenAPIViewDecorator from openapi_core.contrib.flask.handlers import FlaskOpenAPIErrorsHandler @@ -60,7 +60,7 @@ def get_open_api_json(): @app.route('/openapi.yaml') def get_open_api_yaml(): - return yaml.dump(api_spec_dict) # type: ignore[attr-defined] + return yaml.dump(api_spec_dict) @app.route('/ui/') @@ -141,7 +141,7 @@ def jobs_post(): @app.route('/jobs', methods=['GET']) @openapi def jobs_get(): - parameters = request.openapi.parameters.query # type: ignore[attr-defined] + parameters = request.openapi.parameters.query start = parameters.get('start') end = parameters.get('end') return jsonify( diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 66b3aed64..1358b1feb 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -6,7 +6,7 @@ import requests import yaml -from shapely.geometry import MultiPolygon, Polygon, shape # type: ignore[attr-defined] +from shapely.geometry import MultiPolygon, Polygon, shape from hyp3_api import CMR_URL from hyp3_api.util import get_granules @@ -24,7 +24,7 @@ class BoundsValidationError(Exception): with open(Path(__file__).parent / 'job_validation_map.yml') as f: - JOB_VALIDATION_MAP = yaml.safe_load(f.read()) # type: ignore[attr-defined] + JOB_VALIDATION_MAP = yaml.safe_load(f.read()) def has_sufficient_coverage(granule: Polygon): @@ -53,7 +53,7 @@ def get_cmr_metadata(granules): ], 'page_size': 2000, } - response = requests.post(CMR_URL, data=cmr_parameters) # type: ignore[attr-defined] + response = requests.post(CMR_URL, data=cmr_parameters) response.raise_for_status() granules = [ { diff --git a/apps/disable-private-dns/src/disable_private_dns.py b/apps/disable-private-dns/src/disable_private_dns.py index fb3a11924..53c205744 100644 --- a/apps/disable-private-dns/src/disable_private_dns.py +++ b/apps/disable-private-dns/src/disable_private_dns.py @@ -3,7 +3,7 @@ import boto3 -CLIENT = boto3.client('ec2') # type: ignore[attr-defined] +CLIENT = boto3.client('ec2') def get_endpoint(vpc_id, endpoint_name): diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index 9d5b88fcc..585b3beb1 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -9,7 +9,7 @@ import boto3 -S3_CLIENT = boto3.client('s3') # type: ignore[attr-defined] +S3_CLIENT = boto3.client('s3') def get_download_url(bucket, key): diff --git a/apps/render_cf.py b/apps/render_cf.py index aab0707f5..72c5ba26e 100644 --- a/apps/render_cf.py +++ b/apps/render_cf.py @@ -173,7 +173,7 @@ def render_templates(job_types: dict, compute_envs: dict, security_environment: def get_compute_environments_for_deployment(job_types: dict, compute_env_file: Path) -> dict: - compute_envs = yaml.safe_load(compute_env_file.read_text())['compute_environments'] # type: ignore[attr-defined] + compute_envs = yaml.safe_load(compute_env_file.read_text())['compute_environments'] if 'Default' in compute_envs: raise ValueError("'Default' is a reserved compute environment name") @@ -254,7 +254,7 @@ def main(): job_types = {} for file in args.job_spec_files: - job_types.update(yaml.safe_load(file.read_text())) # type: ignore[attr-defined] + job_types.update(yaml.safe_load(file.read_text())) for job_type, job_spec in job_types.items(): validate_job_spec(job_type, job_spec) diff --git a/apps/scale-cluster/src/scale_cluster.py b/apps/scale-cluster/src/scale_cluster.py index a227c6de0..9f50aec01 100644 --- a/apps/scale-cluster/src/scale_cluster.py +++ b/apps/scale-cluster/src/scale_cluster.py @@ -6,8 +6,8 @@ import dateutil.relativedelta -BATCH = boto3.client('batch') # type: ignore[attr-defined] -COST_EXPLORER = boto3.client('ce') # type: ignore[attr-defined] +BATCH = boto3.client('batch') +COST_EXPLORER = boto3.client('ce') def get_time_period(today: date): diff --git a/apps/start-execution-manager/src/start_execution_manager.py b/apps/start-execution-manager/src/start_execution_manager.py index 3701f9d66..841c63577 100644 --- a/apps/start-execution-manager/src/start_execution_manager.py +++ b/apps/start-execution-manager/src/start_execution_manager.py @@ -7,7 +7,7 @@ from lambda_logging import log_exceptions, logger -LAMBDA_CLIENT = boto3.client('lambda') # type: ignore[attr-defined] +LAMBDA_CLIENT = boto3.client('lambda') def invoke_worker(worker_function_arn: str, jobs: list[dict]) -> dict: diff --git a/apps/start-execution-worker/src/start_execution_worker.py b/apps/start-execution-worker/src/start_execution_worker.py index 6587def87..ac8b47b18 100644 --- a/apps/start-execution-worker/src/start_execution_worker.py +++ b/apps/start-execution-worker/src/start_execution_worker.py @@ -8,7 +8,7 @@ from lambda_logging import log_exceptions, logger -STEP_FUNCTION = boto3.client('stepfunctions') # type: ignore[attr-defined] +STEP_FUNCTION = boto3.client('stepfunctions') batch_params_file = Path(__file__).parent / 'batch_params_by_job_type.json' if batch_params_file.exists(): diff --git a/apps/upload-log/src/upload_log.py b/apps/upload-log/src/upload_log.py index 2e1a51552..38c04520b 100644 --- a/apps/upload-log/src/upload_log.py +++ b/apps/upload-log/src/upload_log.py @@ -7,8 +7,8 @@ config = Config(retries={'max_attempts': 2, 'mode': 'standard'}) -CLOUDWATCH = boto3.client('logs', config=config) # type: ignore[attr-defined] -S3 = boto3.client('s3') # type: ignore[attr-defined] +CLOUDWATCH = boto3.client('logs', config=config) +S3 = boto3.client('s3') def get_log_stream(result: dict) -> Optional[str]: diff --git a/lib/dynamo/dynamo/user.py b/lib/dynamo/dynamo/user.py index 563c0d0ae..6198032c1 100644 --- a/lib/dynamo/dynamo/user.py +++ b/lib/dynamo/dynamo/user.py @@ -90,7 +90,7 @@ def _validate_access_code(access_code: str) -> None: def _get_edl_profile(user_id: str, edl_access_token: str) -> dict: url = f'https://urs.earthdata.nasa.gov/api/users/{user_id}' - response = requests.get(url, headers={'Authorization': f'Bearer {edl_access_token}'}) # type: ignore[attr-defined] + response = requests.get(url, headers={'Authorization': f'Bearer {edl_access_token}'}) response.raise_for_status() return response.json() diff --git a/lib/dynamo/dynamo/util.py b/lib/dynamo/dynamo/util.py index b52b9281c..7dab2f394 100644 --- a/lib/dynamo/dynamo/util.py +++ b/lib/dynamo/dynamo/util.py @@ -3,10 +3,10 @@ import boto3 from boto3.dynamodb.conditions import Key -from dateutil.parser import parse # type: ignore[attr-defined] +from dateutil.parser import parse -DYNAMODB_RESOURCE = boto3.resource('dynamodb') # type: ignore[attr-defined] +DYNAMODB_RESOURCE = boto3.resource('dynamodb') def get_request_time_expression(start, end): diff --git a/tests/conftest.py b/tests/conftest.py index f51145cbe..28e16b1b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,10 +19,10 @@ class TableProperties: def get_table_properties_from_template(resource_name): - yaml.SafeLoader.add_multi_constructor('!', lambda loader, suffix, node: None) # type: ignore[attr-defined] + yaml.SafeLoader.add_multi_constructor('!', lambda loader, suffix, node: None) template_file = path.join(path.dirname(__file__), '../apps/main-cf.yml') with open(template_file) as f: - template = yaml.safe_load(f) # type: ignore[attr-defined] + template = yaml.safe_load(f) table_properties = template['Resources'][resource_name]['Properties'] return table_properties diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index eba982e1b..556c4d707 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -1,6 +1,6 @@ import responses from pytest import raises -from shapely.geometry import Polygon # type: ignore[attr-defined] +from shapely.geometry import Polygon from hyp3_api import CMR_URL, validation from test_api.conftest import setup_requests_mock_with_given_polygons diff --git a/tests/test_render_cf.py b/tests/test_render_cf.py index daca1d11b..79494f49a 100644 --- a/tests/test_render_cf.py +++ b/tests/test_render_cf.py @@ -80,7 +80,7 @@ def test_get_compute_environments(tmp_path): 'ComputeEnvironment2': {'key2': 'value2'}, } compute_env_file = tmp_path / 'compute_environments.yml' - yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) # type: ignore[attr-defined] + yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) assert render_cf.get_compute_environments_for_deployment(job_types, compute_env_file) == expected_compute_envs compute_env_file_contents = { @@ -91,7 +91,7 @@ def test_get_compute_environments(tmp_path): 'Default': {'key', 'value'}, } } - yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) # type: ignore[attr-defined] + yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) with pytest.raises(ValueError, match="'Default' is a reserved compute environment name"): render_cf.get_compute_environments_for_deployment(job_types, compute_env_file) @@ -100,6 +100,6 @@ def test_get_compute_environments(tmp_path): 'ComputeEnvironment1': {'key1': 'value1'}, } } - yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) # type: ignore[attr-defined] + yaml.dump(compute_env_file_contents, open(compute_env_file, 'w')) with pytest.raises(KeyError, match='ComputeEnvironment2'): render_cf.get_compute_environments_for_deployment(job_types, compute_env_file) From 19abecb855ba3c69108820863ff490db5962c368 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 7 Jan 2025 15:34:06 -0900 Subject: [PATCH 05/10] type ignore --- apps/api/src/hyp3_api/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index 80594edc0..a6a0c747c 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -141,7 +141,7 @@ def jobs_post(): @app.route('/jobs', methods=['GET']) @openapi def jobs_get(): - parameters = request.openapi.parameters.query + parameters = request.openapi.parameters.query # type: ignore[attr-defined] start = parameters.get('start') end = parameters.get('end') return jsonify( From 81be34fd29b808be82199cf5a68f746aae3d01b6 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 7 Jan 2025 15:38:00 -0900 Subject: [PATCH 06/10] fix ruff --- apps/api/src/hyp3_api/auth.py | 3 +-- apps/api/src/hyp3_api/routes.py | 2 -- apps/check-processing-time/src/check_processing_time.py | 5 ++--- apps/get-files/src/get_files.py | 3 +-- apps/render_cf.py | 2 -- apps/set-batch-overrides/src/set_batch_overrides.py | 4 ++-- apps/upload-log/src/upload_log.py | 3 +-- lib/dynamo/dynamo/jobs.py | 5 ++--- lib/dynamo/dynamo/user.py | 4 ++-- lib/dynamo/dynamo/util.py | 6 +++--- tests/test_dynamo/test_util.py | 2 +- 11 files changed, 15 insertions(+), 24 deletions(-) diff --git a/apps/api/src/hyp3_api/auth.py b/apps/api/src/hyp3_api/auth.py index ff75ae6ff..1cfd39450 100644 --- a/apps/api/src/hyp3_api/auth.py +++ b/apps/api/src/hyp3_api/auth.py @@ -1,11 +1,10 @@ import time from os import environ -from typing import Optional import jwt -def decode_token(token) -> Optional[dict]: +def decode_token(token) -> dict | None: try: return jwt.decode(token, environ['AUTH_PUBLIC_KEY'], algorithms=environ['AUTH_ALGORITHM']) except (jwt.ExpiredSignatureError, jwt.DecodeError): diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index a6a0c747c..9bb157d23 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import datetime import json from decimal import Decimal diff --git a/apps/check-processing-time/src/check_processing_time.py b/apps/check-processing-time/src/check_processing_time.py index cdf3dccd1..94818652f 100644 --- a/apps/check-processing-time/src/check_processing_time.py +++ b/apps/check-processing-time/src/check_processing_time.py @@ -1,14 +1,13 @@ -from typing import Union -def get_time_from_result(result: Union[list, dict]) -> Union[list, float]: +def get_time_from_result(result: list | dict) -> list | float: if isinstance(result, list): return [get_time_from_result(item) for item in result] return (result['StoppedAt'] - result['StartedAt']) / 1000 -def lambda_handler(event, _) -> Union[list, float]: +def lambda_handler(event, _) -> list | float: processing_results = event['processing_results'] result_list = [processing_results[key] for key in sorted(processing_results.keys())] return get_time_from_result(result_list) diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index 585b3beb1..c91d7a251 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -4,7 +4,6 @@ from os import environ from os.path import basename from pathlib import Path -from typing import Union import boto3 @@ -36,7 +35,7 @@ def get_object_file_type(bucket, key): return None -def visible_product(product_path: Union[str, Path]) -> bool: +def visible_product(product_path: str | Path) -> bool: return Path(product_path).suffix in ('.zip', '.nc', '.geojson') diff --git a/apps/render_cf.py b/apps/render_cf.py index 72c5ba26e..3e737461e 100644 --- a/apps/render_cf.py +++ b/apps/render_cf.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import argparse import json from pathlib import Path diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 76d3c8d7e..634c13215 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -1,4 +1,4 @@ -from typing import Optional + AUTORIFT_S2_MEMORY = '7875' AUTORIFT_LANDSAT_MEMORY = '15750' @@ -21,7 +21,7 @@ } -def get_container_overrides(memory: str, omp_num_threads: Optional[str] = None) -> dict: +def get_container_overrides(memory: str, omp_num_threads: str | None = None) -> dict: container_overrides = { 'ResourceRequirements': [ { diff --git a/apps/upload-log/src/upload_log.py b/apps/upload-log/src/upload_log.py index 38c04520b..cc29d08ee 100644 --- a/apps/upload-log/src/upload_log.py +++ b/apps/upload-log/src/upload_log.py @@ -1,6 +1,5 @@ import json from os import environ -from typing import Optional import boto3 from botocore.config import Config @@ -11,7 +10,7 @@ S3 = boto3.client('s3') -def get_log_stream(result: dict) -> Optional[str]: +def get_log_stream(result: dict) -> str | None: if 'Error' in result: result = json.loads(result['Cause']) return result['Container'].get('LogStreamName') diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index a9638db0b..d3564fafb 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -2,7 +2,6 @@ from decimal import Decimal from os import environ from pathlib import Path -from typing import Optional from uuid import uuid4 from boto3.dynamodb.conditions import Attr, Key @@ -86,8 +85,8 @@ def _prepare_job_for_database( job: dict, user_id: str, request_time: str, - remaining_credits: Optional[Decimal], - priority_override: Optional[int], + remaining_credits: Decimal | None, + priority_override: int | None, running_cost: Decimal, ) -> dict: if priority_override: diff --git a/lib/dynamo/dynamo/user.py b/lib/dynamo/dynamo/user.py index 6198032c1..ca6baf376 100644 --- a/lib/dynamo/dynamo/user.py +++ b/lib/dynamo/dynamo/user.py @@ -1,5 +1,5 @@ import os -from datetime import datetime, timezone +from datetime import UTC, datetime from decimal import Decimal from os import environ @@ -106,7 +106,7 @@ def get_or_create_user(user_id: str) -> dict: def _get_current_month() -> str: - return datetime.now(tz=timezone.utc).strftime('%Y-%m') + return datetime.now(tz=UTC).strftime('%Y-%m') def _create_user(user_id: str, users_table) -> dict: diff --git a/lib/dynamo/dynamo/util.py b/lib/dynamo/dynamo/util.py index 7dab2f394..3696f7cd4 100644 --- a/lib/dynamo/dynamo/util.py +++ b/lib/dynamo/dynamo/util.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import UTC, datetime from decimal import Decimal import boto3 @@ -25,12 +25,12 @@ def get_request_time_expression(start, end): def format_time(time: datetime) -> str: if time.tzinfo is None: raise ValueError(f'missing tzinfo for datetime {time}') - utc_time = time.astimezone(timezone.utc) + utc_time = time.astimezone(UTC) return utc_time.isoformat(timespec='seconds') def current_utc_time() -> str: - return format_time(datetime.now(timezone.utc)) + return format_time(datetime.now(UTC)) def convert_floats_to_decimals(element): diff --git a/tests/test_dynamo/test_util.py b/tests/test_dynamo/test_util.py index 875034f05..4e0b2c96b 100644 --- a/tests/test_dynamo/test_util.py +++ b/tests/test_dynamo/test_util.py @@ -22,7 +22,7 @@ def test_get_request_time_expression(): def test_format_time(): - date = datetime.datetime(2021, 2, 3, 4, 5, 6, 7, tzinfo=datetime.timezone.utc) + date = datetime.datetime(2021, 2, 3, 4, 5, 6, 7, tzinfo=datetime.UTC) assert dynamo.util.format_time(date) == '2021-02-03T04:05:06+00:00' offset = datetime.timedelta(hours=1) From e13c954099ba83bf823066949c67da98737814a8 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 7 Jan 2025 15:39:00 -0900 Subject: [PATCH 07/10] ruff format --- apps/check-processing-time/src/check_processing_time.py | 2 -- apps/set-batch-overrides/src/set_batch_overrides.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/apps/check-processing-time/src/check_processing_time.py b/apps/check-processing-time/src/check_processing_time.py index 94818652f..62eea5770 100644 --- a/apps/check-processing-time/src/check_processing_time.py +++ b/apps/check-processing-time/src/check_processing_time.py @@ -1,5 +1,3 @@ - - def get_time_from_result(result: list | dict) -> list | float: if isinstance(result, list): return [get_time_from_result(item) for item in result] diff --git a/apps/set-batch-overrides/src/set_batch_overrides.py b/apps/set-batch-overrides/src/set_batch_overrides.py index 634c13215..d1fedca4a 100644 --- a/apps/set-batch-overrides/src/set_batch_overrides.py +++ b/apps/set-batch-overrides/src/set_batch_overrides.py @@ -1,5 +1,3 @@ - - AUTORIFT_S2_MEMORY = '7875' AUTORIFT_LANDSAT_MEMORY = '15750' RTC_GAMMA_10M_MEMORY = '63200' From 29304b0de4c1b69a11cc4d775a757132ea80dba3 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 7 Jan 2025 15:43:12 -0900 Subject: [PATCH 08/10] changelog --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 833beb694..7b72b4ba5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [9.1.2] ### Added -- OPERA_DISP_TMS job type is now available in EDC UAT deployment +- Add `mypy` to [`static-analysis`](.github/workflows/static-analysis.yml) workflow +- `OPERA_DISP_TMS` job type is now available in EDC UAT deployment ### Changed -- Upgraded AWS Lambda functions and Github Actions workflows to Python 3.13 +- Upgrade to Python 3.13 ### Removed -- hyp3-opera-disp-sandbox deployment +- Remove `hyp3-opera-disp-sandbox` deployment ## [9.1.1] From 6ba9f746b5482ff191141b4843eeee13b59a6e96 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 8 Jan 2025 07:55:00 -0900 Subject: [PATCH 09/10] add perms to static analysis --- .github/workflows/static-analysis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index c2de9ae87..64461b3ac 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -1,5 +1,8 @@ name: Static code analysis +permissions: + contents: read + on: push jobs: From 95e812752d0cfb75935799462eae2f667998d3de Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 8 Jan 2025 10:01:51 -0900 Subject: [PATCH 10/10] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b72b4ba5..b19ae904d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [9.1.2] +## [9.2.0] ### Added - Add `mypy` to [`static-analysis`](.github/workflows/static-analysis.yml) workflow