Merge pull request #1186 from GSA/main

07/16/2024 Production Deploy
GSA · Jul 18, 2024 · 72dc0e3 · 72dc0e3
2 parents 2090022 + 8124d98
commit 72dc0e3
Show file tree

Hide file tree

Showing 8 changed files with 334 additions and 111 deletions.
diff --git a/.ds.baseline b/.ds.baseline
@@ -92,7 +92,7 @@
     },
     {
       "path": "detect_secrets.filters.common.is_baseline_file",
-      "filename": ".secrets.baseline"
+      "filename": ".ds.baseline"
     },
     {
       "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
@@ -239,7 +239,7 @@
         "filename": "tests/app/dao/test_services_dao.py",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 261,
+        "line_number": 265,
         "is_secret": false
       }
     ],
@@ -384,5 +384,5 @@
       }
     ]
   },
-  "generated_at": "2024-05-20T15:20:28Z"
+  "generated_at": "2024-07-10T20:12:22Z"
 }
diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py
@@ -692,7 +692,7 @@ def fetch_notification_stats_for_service_by_month_by_user(
     )
 
 
-def get_specific_days_stats(results, start_date, days=None, end_date=None):
+def get_specific_days_stats(data, start_date, days=None, end_date=None):
     if days is not None and end_date is not None:
         raise ValueError("Only set days OR set end_date, not both.")
     elif days is not None:
@@ -702,14 +702,14 @@ def get_specific_days_stats(results, start_date, days=None, end_date=None):
     else:
         raise ValueError("Either days or end_date must be set.")
 
-    grouped_results = {date: [] for date in gen_range} | {
-        day.date(): [notification_type, status, day, count]
-        for notification_type, status, day, count in results
+    grouped_data = {date: [] for date in gen_range} | {
+        day: [row for row in data if row.day.date() == day]
+        for day in {item.day.date() for item in data}
     }
 
     stats = {
         day.strftime("%Y-%m-%d"): statistics.format_statistics(rows)
-        for day, rows in grouped_results.items()
+        for day, rows in grouped_data.items()
     }
 
     return stats
diff --git a/app/user/rest.py b/app/user/rest.py
@@ -178,6 +178,17 @@ def activate_user(user_id):
     return jsonify(data=user.serialize()), 200
 
 
+@user_blueprint.route("/<uuid:user_id>/deactivate", methods=["POST"])
+def deactivate_user(user_id):
+    user = get_user_by_id(user_id=user_id)
+    if user.state == "pending":
+        raise InvalidRequest("User already inactive", status_code=400)
+
+    user.state = "pending"
+    save_model_user(user)
+    return jsonify(data=user.serialize()), 200
+
+
 @user_blueprint.route("/<uuid:user_id>/reset-failed-login-count", methods=["POST"])
 def user_reset_failed_login_count(user_id):
     user_to_update = get_user_by_id(user_id=user_id)

diff --git a/docs/all.md b/docs/all.md
@@ -432,6 +432,8 @@ Rules for use:
 
 ### Deploying to the sandbox
 
+If this is the first time you have used Terraform in this repository, you will first have to hook your copy of Terraform up to our remote state. Follow [Retrieving existing bucket credentials](https://github.com/GSA/notifications-api/tree/main/terraform#retrieving-existing-bucket-credentials).
+
 1. Set up services:
     ```
     $ cd terraform/sandbox
@@ -440,6 +442,7 @@ Rules for use:
     $ terraform plan
     $ terraform apply
     ```
+1. Change back to the project root directory: `cd ../..`
 1. start a poetry shell as a shortcut to load `.env` file variables: `$ poetry shell`
 1. Output requirements.txt file: `poetry export --without-hashes --format=requirements.txt > requirements.txt`
 1. Deploy the application:

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -11,8 +11,8 @@ python = "^3.12.2"
 alembic = "==1.13.2"
 amqp = "==5.2.0"
 beautifulsoup4 = "==4.12.3"
-boto3 = "^1.34.138"
-botocore = "^1.34.139"
+boto3 = "^1.34.143"
+botocore = "^1.34.144"
 cachetools = "==5.3.3"
 celery = {version = "==5.4.0", extras = ["redis"]}
 certifi = ">=2022.12.7"
@@ -34,7 +34,7 @@ flask-redis = "==0.4.0"
 flask-sqlalchemy = "==3.1.1"
 gunicorn = {version = "==22.0.0", extras = ["eventlet"]}
 iso8601 = "==2.1.0"
-jsonschema = {version = "==4.22.0", extras = ["format"]}
+jsonschema = {version = "==4.23.0", extras = ["format"]}
 lxml = "==5.2.2"
 marshmallow = "==3.21.3"
 marshmallow-sqlalchemy = "==1.0.0"
@@ -59,7 +59,7 @@ phonenumbers = "^8.13.40"
 python-json-logger = "^2.0.7"
 pytz = "^2024.1"
 regex = "^2024.5.15"
-shapely = "^2.0.4"
+shapely = "^2.0.5"
 smartypants = "^2.0.1"
 mistune = "0.8.4"
 blinker = "^1.8.2"
@@ -92,7 +92,7 @@ freezegun = "^1.5.1"
 honcho = "*"
 isort = "^5.13.2"
 jinja2-cli = {version = "==0.8.2", extras = ["yaml"]}
-moto = "==5.0.10"
+moto = "==5.0.11"
 pip-audit = "*"
 pre-commit = "^3.7.1"
 pytest = "^8.2.2"
@@ -102,7 +102,7 @@ pytest-cov = "^5.0.0"
 pytest-xdist = "^3.5.0"
 radon = "^6.0.1"
 requests-mock = "^1.11.0"
-setuptools = "^70.2.0"
+setuptools = "^70.3.0"
 sqlalchemy-utils = "^0.41.2"
 vulture = "^2.10"
 detect-secrets = "^1.5.0"

diff --git a/terraform/README.md b/terraform/README.md
@@ -1,8 +1,9 @@
 # Terraform
 
 This directory holds the Terraform modules for maintaining Notify.gov's API infrastructure. You might want to:
-* [read about the directory structure](#structure), or
-* [get set up to develop HCL code](#retrieving-existing-bucket-credentials).
+* [Set up](#retrieving-existing-bucket-credentials) the Sandbox and develop Terraform,
+* [Learn](#structure) about the directory structure, or
+* [Troubleshoot](#troubleshooting) error messages
 
 The Admin app repo [has its own terraform directory](https://github.com/GSA/notifications-admin/tree/main/terraform) but a lot of the below instructions apply to both apps.
 
@@ -228,3 +229,32 @@ The audit event logs may also provide insight. They are visible in web UI or [in
 Error: Error creating SES domain identity verification: Expected domain verification Success, but was in state Pending
 ```
 This error comes via the [Supplementary Service Broker](https://github.com/GSA/usnotify-ssb/) and originates from the [SMTP Brokerpak](https://github.com/GSA-TTS/datagov-brokerpak-smtp) it uses. You can run the [broker provisioning locally](https://github.com/GSA-TTS/datagov-brokerpak-smtp/tree/main/terraform/provision) to tinker with the error.
+
+### Validating provider credentials
+```
+Error: validating provider credentials: retrieving caller identity from STS: operation error STS: GetCallerIdentity, https response error StatusCode: 403
+```
+The steps in [Use bootstrap credentials](#use-bootstrap-credentials) may not be complete. Or the AWS CLI may have reverted to the default profile, in which case, re-run:
+```bash
+export AWS_PROFILE=notify-terraform-backend
+```
+
+### No valid credential sources
+```
+Error: No valid credential sources found
+Please see https://www.terraform.io/docs/language/settings/backends/s3.html for more information about providing credentials.
+
+Error: failed to refresh cached credentials, no EC2 IMDS role found, operation error ec2imds: GetMetadata, request canceled, context deadline exceeded
+```
+You are not hooked up to the remote backend that stores Terraform state
+Run steps in [Retrieving existing bucket credentials](#retrieving-existing-bucket-credentials).
+
+### Space Deployers will be updated in-place
+```
+# module.egress-space.cloudfoundry_space_users.deployers will be updated in-place
+  ~ resource "cloudfoundry_space_users" "deployers" {
+      ~ developers = [
+          - "xxx-GUID-xxx",
+          + "yyy-GUID-yyy",
+```
+The environment was last deployed by someone other than you, using a different Space Deployer account. If you are working in the Sandbox environment, this is fine; go ahead and apply the changes. After you do, the other person evidently also working in the Sandbox env will then see the same message. The two of you might play tug-of-war with different GUIDs, but this is inconsequential.
diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py
@@ -1,6 +1,7 @@
 import uuid
 from datetime import datetime, timedelta
 from unittest import mock
+from unittest.mock import Mock
 
 import pytest
 import sqlalchemy
@@ -38,6 +39,7 @@
     delete_service_and_all_associated_db_objects,
     get_live_services_with_organization,
     get_services_by_partial_name,
+    get_specific_days_stats,
 )
 from app.dao.users_dao import create_user_code, save_model_user
 from app.enums import (
@@ -48,6 +50,7 @@
     OrganizationType,
     PermissionType,
     ServicePermissionType,
+    StatisticsType,
     TemplateType,
 )
 from app.models import (
@@ -1580,3 +1583,180 @@ def test_get_live_services_with_organization(sample_organization):
         (live_service.name, sample_organization.name),
         (service_without_org.name, None),
     ]
+
+
+_this_date = utc_now() - timedelta(days=4)
+
+
+@pytest.mark.parametrize(
+    ["data", "start_date", "days", "end_date", "expected", "is_error"],
+    [
+        [None, _this_date, None, None, None, True],
+        [None, _this_date, 4, _this_date - timedelta(4), None, True],
+        [
+            [
+                {"day": _this_date, "something": "else"},
+                {"day": _this_date, "something": "new"},
+                {"day": _this_date + timedelta(days=1), "something": "borrowed"},
+                {"day": _this_date + timedelta(days=2), "something": "old"},
+                {"day": _this_date + timedelta(days=4), "something": "blue"},
+            ],
+            _this_date,
+            4,
+            None,
+            {
+                _this_date.date().strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 2,
+                    },
+                },
+                (_this_date.date() + timedelta(days=1)).strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 1,
+                    },
+                },
+                (_this_date.date() + timedelta(days=2)).strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 1,
+                    },
+                },
+                (_this_date.date() + timedelta(days=3)).strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                },
+                (_this_date.date() + timedelta(days=4)).strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 1,
+                    },
+                },
+            },
+            False,
+        ],
+        [
+            [
+                {"day": _this_date, "something": "else"},
+                {"day": _this_date, "something": "new"},
+                {"day": _this_date + timedelta(days=1), "something": "borrowed"},
+                {"day": _this_date + timedelta(days=2), "something": "old"},
+                {"day": _this_date + timedelta(days=4), "something": "blue"},
+            ],
+            _this_date,
+            None,
+            _this_date + timedelta(4),
+            {
+                _this_date.date().strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 2,
+                    },
+                },
+                (_this_date.date() + timedelta(days=1)).strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 1,
+                    },
+                },
+                (_this_date.date() + timedelta(days=2)).strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 1,
+                    },
+                },
+                (_this_date.date() + timedelta(days=3)).strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                },
+                (_this_date.date() + timedelta(days=4)).strftime("%Y-%m-%d"): {
+                    TemplateType.EMAIL: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 0,
+                    },
+                    TemplateType.SMS: {
+                        StatisticsType.DELIVERED: 0,
+                        StatisticsType.FAILURE: 0,
+                        StatisticsType.REQUESTED: 1,
+                    },
+                },
+            },
+            False,
+        ],
+    ],
+)
+def test_get_specific_days(data, start_date, days, end_date, expected, is_error):
+    if is_error:
+        with pytest.raises(ValueError):
+            get_specific_days_stats(data, start_date, days, end_date)
+    else:
+        new_data = []
+        for line in data:
+            new_line = Mock()
+            new_line.day = line["day"]
+            new_line.notification_type = NotificationType.SMS
+            new_line.count = 1
+            new_line.something = line["something"]
+            new_data.append(new_line)
+        results = get_specific_days_stats(new_data, start_date, days, end_date)
+        assert results == expected