Merge pull request #877 from GSA/main

API Production Deploy - 3/28/2024

.github/actions/setup-project/action.yml

@@ -9,10 +9,10 @@ runs:
         sudo apt-get update \
         && sudo apt-get install -y --no-install-recommends \
         libcurl4-openssl-dev
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v4
       with:
-        python-version: "3.9"
+        python-version: "3.12"
     - name: Install poetry
       shell: bash
       run: pip install --upgrade poetry

.github/workflows/deploy-demo.yml

@@ -57,6 +57,8 @@ jobs:
         NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
         NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
         NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
+        LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=01234567890123456789012345&prompt=select_account&redirect_uri=https://notify-demo.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=abcdefghijklmnopabcdefghijklmnop"
+
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
@@ -70,6 +72,7 @@ jobs:
           --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
           --var NOTIFY_E2E_TEST_EMAIL="$NOTIFY_E2E_TEST_EMAIL"
           --var NOTIFY_E2E_TEST_PASSWORD="$NOTIFY_E2E_TEST_PASSWORD"
+          --var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL"
 
     - name: Check for changes to egress config
       id: changed-egress-config

.github/workflows/deploy-prod.yml

@@ -61,6 +61,8 @@ jobs:
         NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
         NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
         NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
+        LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=01234567890123456789012345&prompt=select_account&redirect_uri=https://beta.notify.gov/set-up-your-profile&response_type=code&scope=openid+email&state=abcdefghijklmnopabcdefghijklmnop"
+
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
@@ -74,6 +76,7 @@ jobs:
           --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
           --var NOTIFY_E2E_TEST_EMAIL="$NOTIFY_E2E_TEST_EMAIL"
           --var NOTIFY_E2E_TEST_PASSWORD="$NOTIFY_E2E_TEST_PASSWORD"
+          --var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL"
 
     - name: Check for changes to egress config
       id: changed-egress-config

.github/workflows/deploy.yml

@@ -62,6 +62,8 @@ jobs:
         NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
         NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
         NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
+        LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=01234567890123456789012345&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=abcdefghijklmnopabcdefghijklmnop"
+
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
@@ -75,6 +77,7 @@ jobs:
           --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
           --var NOTIFY_E2E_TEST_EMAIL="$NOTIFY_E2E_TEST_EMAIL"
           --var NOTIFY_E2E_TEST_PASSWORD="$NOTIFY_E2E_TEST_PASSWORD"
+          --var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL"
 
     - name: Check for changes to egress config
       id: changed-egress-config

README.md

@@ -41,7 +41,7 @@ You will need the following items:
 This project currently works with these major versions of the following main
 components:
 
-- Python 3.9.x
+- Python 3.12.x
 - PostgreSQL 15.x (version 12.x is used in the hosted environments)
 
 These instructions will walk you through how to set your machine up with all of
@@ -175,12 +175,12 @@ session to make the changes take effect.
 Now we're ready to install the Python version we need with `pyenv`, like so:
 
 ```sh
-pyenv install 3.9
+pyenv install 3.12
 ```
 
-This will install the latest version of Python 3.9.
+This will install the latest version of Python 3.12.
 
-_NOTE: This project currently runs on Python 3.9.x._
+_NOTE: This project currently runs on Python 3.12.x._
 
 #### Python Dependency Installation
 
@@ -253,20 +253,20 @@ Once all of pre-requisites for the project are installed and you have a
 cloud.gov account, you can now set up the API project and get things running
 locally!
 
-First, clone the respository in the directory of your choosing on your machine:
+First, clone the repository in the directory of your choosing on your machine:
 
 ```sh
 git clone [email protected]:GSA/notifications-api.git
 ```
 
 Now go into the project directory (`notifications-api` by default), create a
 virtual environment, and set the local Python version to point to the virtual
-environment (assumes version Python `3.9.18` is what is installed on your
+environment (assumes version Python `3.12.2` is what is installed on your
 machine):
 
 ```sh
 cd notifications-api
-pyenv virtualenv 3.9.18 notify-api
+pyenv virtualenv 3.12.2 notify-api
 pyenv local notify-api
 ```
 
@@ -278,6 +278,9 @@ in the command line by using this command:
 ```sh
 cf login -a api.fr.cloud.gov --sso
 ```
+If you are offered a choice of orgs, select `gsa-tts-benefits-studio`.
+For the space, choose `notify-local-dev` to start with (assuming you are
+setting up local development).
 
 _REMINDER: Ensure you have access to the `notify-local-dev` and `notify-staging` spaces in cloud.gov_
 
@@ -289,9 +292,12 @@ cd terraform/development
 ./run.sh
 ```
 
-In addition to some infrastructure setup, this will also create a local `.env`
-file for you in the project's root directory, which will include a handful of
-project-specific environment variables.
+If this runs correctly, Terraform will ask you if you want to create some
+resources. Answer `yes`.
+
+The script will also create a local `.env` file for you in the project's
+root directory, which will include a handful of project-specific environment
+variables.
 
 Lastly, if you didn't already start PostgreSQL and Redis above, be sure to do
 so now:
@@ -301,6 +307,40 @@ brew services start postgresql@15
 brew services start redis
 ```
 
+#### Upgrading Python in existing projects
+
+If you're upgrading an existing project to a newer version of Python, you can
+follow these steps to get yourself up-to-date.
+
+First, use `pyenv` to install the newer version of Python you'd like to use;
+we'll use `3.12` in our example here since we recently upgraded to this version:
+
+```sh
+pyenv install 3.12
+```
+
+Next, delete the virtual environment you previously had set up.  If you followed
+the instructions above with the first-time set up, you can do this with `pyenv`:
+
+```sh
+pyenv virtualenv-delete notify-api
+```
+
+Now, make sure you are in your project directory and recreate the same virtual
+environment with the newer version of Python you just installed:
+
+```sh
+cd notifications-api
+pyenv virtualenv 3.12.2 notify-api
+pyenv local notify-api
+```
+
+At this point, proceed with the rest of the instructions here in the README and
+you'll be set with an upgraded version of Python.
+
+_If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._
+
+
 ### Final environment setup
 
 There's one final thing to adjust in the newly created `.env` file.  This
@@ -351,6 +391,9 @@ Now you can run the web server and background workers for asynchronous jobs:
 make run-procfile
 ```
 
+If it runs correctly, you will be able to visit http://127.0.0.1:6011/ and see
+JSON from the API in your web browser.
+
 This will run all of the services within the same shell session.  If you need to
 run them separately to help with debugging or tracing logs, you can do so by
 opening three sepearate shell sessions and running one of these commands in each

app/__init__.py

@@ -359,9 +359,9 @@ def checkout(dbapi_connection, connection_record, connection_proxy):  # noqa
                     connection_record.info["request_data"] = {
                         "method": request.method,
                         "host": request.host,
-                        "url_rule": request.url_rule.rule
-                        if request.url_rule
-                        else "No endpoint",
+                        "url_rule": (
+                            request.url_rule.rule if request.url_rule else "No endpoint"
+                        ),
                     }
                 # celery apps
                 elif current_task:

app/celery/process_ses_receipts_tasks.py

@@ -214,12 +214,12 @@ def handle_complaint(ses_message):
     complaint = Complaint(
         notification_id=notification.id,
         service_id=notification.service_id,
-        ses_feedback_id=ses_complaint.get("feedbackId", None)
-        if ses_complaint
-        else None,
-        complaint_type=ses_complaint.get("complaintFeedbackType", None)
-        if ses_complaint
-        else None,
+        ses_feedback_id=(
+            ses_complaint.get("feedbackId", None) if ses_complaint else None
+        ),
+        complaint_type=(
+            ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None
+        ),
         complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
     )
     save_complaint(complaint)

app/celery/provider_tasks.py

@@ -91,6 +91,7 @@ def check_sms_delivery_receipt(self, message_id, notification_id, sent_at):
     bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300
 )
 def deliver_sms(self, notification_id):
+    """Branch off to the final step in delivering the notification to sns and get delivery receipts."""
     try:
         current_app.logger.info(
             "Start sending SMS for notification id: {}".format(notification_id)
@@ -108,7 +109,7 @@ def deliver_sms(self, notification_id):
             current_app.logger.warning(
                 ansi_green + f"AUTHENTICATION CODE: {notification.content}" + ansi_reset
             )
-
+        # Code branches off to send_to_providers.py
         message_id = send_to_providers.send_sms_to_provider(notification)
         # We have to put it in UTC.  For other timezones, the delay
         # will be ignored and it will fire immediately (although this probably only affects developer testing)

app/celery/service_callback_tasks.py

@@ -120,12 +120,16 @@ def create_delivery_status_callback_data(notification, service_callback_api):
         "notification_status": notification.status,
         "notification_provider_response": notification.provider_response,  # TODO do we test for provider_response?
         "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
-        "notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT)
-        if notification.updated_at
-        else None,
-        "notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT)
-        if notification.sent_at
-        else None,
+        "notification_updated_at": (
+            notification.updated_at.strftime(DATETIME_FORMAT)
+            if notification.updated_at
+            else None
+        ),
+        "notification_sent_at": (
+            notification.sent_at.strftime(DATETIME_FORMAT)
+            if notification.sent_at
+            else None
+        ),
         "notification_type": notification.notification_type,
         "service_callback_api_url": service_callback_api.url,
         "service_callback_api_bearer_token": service_callback_api.bearer_token,

app/celery/tasks.py

@@ -31,6 +31,7 @@
 
 @notify_celery.task(name="process-job")
 def process_job(job_id, sender_id=None):
+    """Update job status, get csv data from s3, and begin processing csv rows."""
     start = datetime.utcnow()
     job = dao_get_job_by_id(job_id)
     current_app.logger.info(
@@ -74,6 +75,7 @@ def process_job(job_id, sender_id=None):
     for row in recipient_csv.get_rows():
         process_row(row, template, job, service, sender_id=sender_id)
 
+    # End point/Exit point for message send flow.
     job_complete(job, start=start)
 
 
@@ -109,6 +111,7 @@ def get_recipient_csv_and_template_and_sender_id(job):
 
 
 def process_row(row, template, job, service, sender_id=None):
+    """Branch off based on notification type, sms or email."""
     template_type = template.template_type
     encrypted = encryption.encrypt(
         {
@@ -121,6 +124,8 @@ def process_row(row, template, job, service, sender_id=None):
         }
     )
 
+    # Both save_sms and save_email have the same general
+    # persist logic.
     send_fns = {NotificationType.SMS: save_sms, NotificationType.EMAIL: save_email}
 
     send_fn = send_fns[template_type]
@@ -130,6 +135,7 @@ def process_row(row, template, job, service, sender_id=None):
         task_kwargs["sender_id"] = sender_id
 
     notification_id = create_uuid()
+    # Kick-off persisting notification in save_sms/save_email.
     send_fn.apply_async(
         (
             str(service.id),
@@ -163,7 +169,11 @@ def __total_sending_limits_for_job_exceeded(service, job, job_id):
 
 @notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300)
 def save_sms(self, service_id, notification_id, encrypted_notification, sender_id=None):
+    """Persist notification to db and place notification in queue to send to sns."""
     notification = encryption.decrypt(encrypted_notification)
+    # SerialisedService and SerialisedTemplate classes are
+    # used here to grab the same service and template from the cache
+    # to improve performance.
     service = SerialisedService.from_id(service_id)
     template = SerialisedTemplate.from_id_and_service_id(
         notification["template"],
@@ -177,7 +187,8 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i
         ).sms_sender
     else:
         reply_to_text = template.reply_to_text
-
+    # Return False when trial mode services try sending notifications
+    # to non-team and non-simulated recipients.
     if not service_allowed_to_send_to(notification["to"], service, KeyType.NORMAL):
         current_app.logger.debug(
             "SMS {} failed as restricted service".format(notification_id)
@@ -208,6 +219,7 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i
             reply_to_text=reply_to_text,
         )
 
+        # Kick off sns process in provider_tasks.py
         provider_tasks.deliver_sms.apply_async(
             [str(saved_notification.id)], queue=QueueNames.SEND_SMS
         )

app/commands.py

@@ -18,7 +18,7 @@
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
 
-from app import db
+from app import db, redis_store
 from app.aws import s3
 from app.celery.nightly_tasks import cleanup_unfinished_jobs
 from app.celery.tasks import process_row
@@ -799,6 +799,20 @@ def update_templates():
         data = json.load(f)
         for d in data:
             _update_template(d["id"], d["name"], d["type"], d["content"], d["subject"])
+    _clear_templates_from_cache()
+
+
+def _clear_templates_from_cache():
+    # When we update-templates in the db, we need to make sure to delete them
+    # from redis, otherwise the old versions will stick around forever.
+    CACHE_KEYS = [
+        "service-????????-????-????-????-????????????-templates",
+        "service-????????-????-????-????-????????????-template-????????-????-????-????-????????????-version-*",  # noqa
+        "service-????????-????-????-????-????????????-template-????????-????-????-????-????????????-versions",  # noqa
+    ]
+
+    num_deleted = sum(redis_store.delete_by_pattern(pattern) for pattern in CACHE_KEYS)
+    current_app.logger.info(f"Number of templates deleted from cache {num_deleted}")
 
 
 @notify_command(name="create-new-service")

app/config_files/templates.json

@@ -161,7 +161,7 @@
       "",
       "Open this link to create an account on Notify.gov:",
       "",
-      "((url))",
+      "[Join Organization](((url)))",
       "",
       "",
       "This invitation will stop working at midnight tomorrow. This is to keep ((organization_name)) secure."

app/dao/users_dao.py

@@ -36,7 +36,6 @@ def get_login_gov_user(login_uuid, email_address):
     should be removed.
     """
 
-    print(User.query.filter_by(login_uuid=login_uuid).first())
     user = User.query.filter_by(login_uuid=login_uuid).first()
     if user:
         if user.email_address != email_address: