Merge pull request #1249 from GSA/main

2/27/2024 Production Deployment

.github/workflows/checks.yml

@@ -7,12 +7,12 @@ permissions:
 
 env:
   NOTIFY_ENVIRONMENT: test
-  NEW_RELIC_CONFIG_FILE: newrelic.ini
-  NEW_RELIC_ENVIRONMENT: test
   FLASK_APP: application.py
   WERKZEUG_DEBUG_PIN: off
   REDIS_ENABLED: 0
   NODE_VERSION: 16.15.1
+  AWS_US_TOLL_FREE_NUMBER: "+18556438890"
+  ADMIN_BASE_URL: http://localhost:6012
 
 jobs:
   build:
@@ -44,29 +44,97 @@ jobs:
         run: npm test
       - name: Run py tests with coverage
         run: poetry run coverage run --omit=*/notifications_utils/* -m pytest --maxfail=10 --ignore=tests/end_to_end tests/
-      # - name: Run E2E tests
-      #   run: poetry run pytest -v --browser chromium --browser firefox --browser webkit tests/end_to_end
-      #   env:
-      #     NOTIFY_E2E_AUTH_STATE_PATH: ${{ secrets.NOTIFY_E2E_AUTH_STATE_PATH }}
-      #     NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
-      #     NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
-      #     NOTIFY_E2E_TEST_URI: ${{ secrets.NOTIFY_E2E_TEST_URI }}
       - name: Check coverage threshold
         run: poetry run coverage report --fail-under=90
-      # - name: Health check
-      #   run: |
-      #     response=$(curl -url ${{secrets.NOTIFY_E2E_TEST_URI}}_status)
-      #     if grep -q "ok" <<< "$response"; then
-      #       echo "Health check passed"
-      #     else
-      #       echo "Health check failed"
-      #       exit 1
-      #     fi
-      #   env:
-      #     NOTIFY_E2E_AUTH_STATE_PATH: ${{ secrets.NOTIFY_E2E_AUTH_STATE_PATH }}
-      #     NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
-      #     NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
-      #     NOTIFY_E2E_TEST_URI: ${{ secrets.NOTIFY_E2E_TEST_URI }}
+
+  end-to-end-tests:
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: write
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: user
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test_notification_api
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 6379 on service container to the host
+          - 6379:6379
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - uses: jwalton/gh-find-current-pr@v1
+        id: findPr
+      - name: Clone API
+        uses: actions/checkout@v3
+        with:
+         repository: GSA/notifications-api
+         path: 'notifications-api'
+      - name: Install API dependencies
+        working-directory: 'notifications-api'
+        run: make bootstrap
+        env:
+          DATABASE_URL: postgresql://user:password@localhost:5432/test_notification_api
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+          REDIS_URL: redis://localhost:6379
+          NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
+          NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
+          NOTIFY_ENVIRONMENT: development
+      - name: Run API server
+        working-directory: 'notifications-api'
+        run: make run-procfile &
+        env:
+          DATABASE_URL: postgresql://user:password@localhost:5432/test_notification_api
+          SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
+          REDIS_URL: redis://localhost:6379
+          NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
+          NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
+          NOTIFY_ENVIRONMENT: development
+      - name: Run Admin server
+        run: make run-flask &
+        env:
+          # API_HOST_NAME: https://notify-api-staging.app.cloud.gov
+          API_HOST_NAME: http://localhost:6011
+          DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
+          SECRET_KEY: ${{ secrets.SECRET_KEY }}
+          ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
+          ADMIN_CLIENT_USERNAME: notify-admin
+          NOTIFY_ENVIRONMENT: e2etest
+          NOTIFY_E2E_AUTH_STATE_PATH: ${{ secrets.NOTIFY_E2E_AUTH_STATE_PATH }}
+          NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
+          NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
+          NOTIFY_E2E_TEST_URI: http://localhost:6012
+      - name: Run E2E tests
+        # Run the E2E tests against the code found in this PR.
+        # run: poetry run pytest -v --browser chromium --browser firefox --browser webkit tests/end_to_end
+        # --browser webkit doesn't work at this time.
+        run: make e2e-test
+        # Debugging for now to troubleshoot a connectivity issue to the local servers
+        # run: curl --request GET --url "http://localhost:6012"
+        env:
+          NOTIFY_ENVIRONMENT: e2etest
+          NOTIFY_E2E_AUTH_STATE_PATH: ${{ secrets.NOTIFY_E2E_AUTH_STATE_PATH }}
+          NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}
+          NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
+          NOTIFY_E2E_TEST_URI: http://localhost:6012
 
   validate-new-relic-config:
     runs-on: ubuntu-latest
@@ -76,6 +144,7 @@ jobs:
       - uses: ./.github/actions/setup-project
       - name: Validate NewRelic config
         env:
+          NEW_RELIC_CONFIG_FILE: newrelic.ini
           NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
           # Need to set a NEW_RELIC_ENVIRONMENT with monitor_mode: true
           NEW_RELIC_ENVIRONMENT: staging

.github/workflows/deploy-demo.yml

@@ -56,12 +56,12 @@ jobs:
           NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }}
           LOGIN_PEM: ${{ secrets.LOGIN_PEM }}
           LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov"
-          LOGIN_DOT_GOV_USER_INFO_URL: "https://idp.int.identitysandbox.gov/api/openid_connect/userinfo"
-          LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://idp.int.identitysandbox.gov/api/openid_connect/token"
-          LOGIN_DOT_GOV_LOGOUT_URL: "https://idp.int.identitysandbox.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-demo.app.cloud.gov/sign-out"
-          LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://idp.int.identitysandbox.gov/openid_connect/logout?"
+          LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo"
+          LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token"
+          LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-demo.app.cloud.gov/sign-out"
+          LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
           LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-demo.app.cloud.gov/sign-out"
-          LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://idp.int.identitysandbox.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=01234567890123456789012345&prompt=select_account&redirect_uri=https://notify-demo.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=abcdefghijklmnopabcdefghijklmnop"
+          LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=01234567890123456789012345&prompt=select_account&redirect_uri=https://notify-demo.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=abcdefghijklmnopabcdefghijklmnop"
         with:
           cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
           cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}

.github/workflows/deploy.yml

@@ -62,12 +62,12 @@ jobs:
           NR_BROWSER_KEY: ${{ secrets.NR_BROWSER_KEY }}
           LOGIN_PEM: ${{ secrets.LOGIN_PEM }}
           LOGIN_DOT_GOV_CLIENT_ID: "urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov"
-          LOGIN_DOT_GOV_USER_INFO_URL: "https://idp.int.identitysandbox.gov/api/openid_connect/userinfo"
-          LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://idp.int.identitysandbox.gov/api/openid_connect/token"
-          LOGIN_DOT_GOV_LOGOUT_URL: "https://idp.int.identitysandbox.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out"
-          LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://idp.int.identitysandbox.gov/openid_connect/logout?"
+          LOGIN_DOT_GOV_USER_INFO_URL: "https://secure.login.gov/api/openid_connect/userinfo"
+          LOGIN_DOT_GOV_ACCESS_TOKEN_URL: "https://secure.login.gov/api/openid_connect/token"
+          LOGIN_DOT_GOV_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&post_logout_redirect_uri=https://notify-staging.app.cloud.gov/sign-out"
+          LOGIN_DOT_GOV_BASE_LOGOUT_URL: "https://secure.login.gov/openid_connect/logout?"
           LOGIN_DOT_GOV_SIGNOUT_REDIRECT: "https://notify-staging.app.cloud.gov/sign-out"
-          LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://idp.int.identitysandbox.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=01234567890123456789012345&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=abcdefghijklmnopabcdefghijklmnop"
+          LOGIN_DOT_GOV_INITIAL_SIGNIN_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=01234567890123456789012345&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/sign-in&response_type=code&scope=openid+email&state=abcdefghijklmnopabcdefghijklmnop"
         with:
           cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
           cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}

Makefile

@@ -80,8 +80,8 @@ dead-code:
 
 .PHONY: e2e-test
 e2e-test: export NEW_RELIC_ENVIRONMENT=test
-e2e-test: ## Run end-to-end integration tests
-	poetry run pytest -v --browser chromium --browser firefox --browser webkit tests/end_to_end
+e2e-test: ## Run end-to-end integration tests; note that --browser webkit isn't currently working
+	poetry run pytest -v --browser chromium --browser firefox tests/end_to_end
 
 .PHONY: js-lint
 js-lint: ## Run javascript linting scanners

app/assets/sass/uswds/_uswds-theme-custom-styles.scss

@@ -403,8 +403,11 @@ td.table-empty-message {
       width: 5%;
    }
    th {
-      padding: 0.5rem 0.5rem;
+      padding: 0.5rem 1rem
    }
+   td {
+      padding: 0.5rem 1rem
+  }
 }
 
 #template-list {

app/config.py

@@ -81,8 +81,7 @@ class Config(object):
         "IBAN": "GB33BUKB20201555555555",
         "swift": "ABCDEF12",
         "notify_billing_email_addresses": [
-            "[email protected]",
-            "[email protected]",
+            "[email protected]",
         ],
     }
 
@@ -152,6 +151,34 @@ class Staging(Production):
     HEADER_COLOUR = "#00ff00"  # $green
 
 
+class E2ETest(Staging):
+    """
+    An environment config that is intended to operate as if it were in the
+    staging environment but with the configuration of the development and test
+    environments so the E2E tests work.
+    """
+
+    # Borrowed from development environment
+    DEBUG = True
+    SESSION_COOKIE_SECURE = False
+    SESSION_PROTECTION = None
+    HTTP_PROTOCOL = "http"
+    ASSET_DOMAIN = ""
+    ASSET_PATH = "/static/"
+
+    # Borrowed from test environment
+    TESTING = True
+    WTF_CSRF_ENABLED = False
+
+    # buckets - mirror staging
+    CSV_UPLOAD_BUCKET = cloud_config.s3_credentials(
+        "notify-api-csv-upload-bucket-staging"
+    )
+    LOGO_UPLOAD_BUCKET = cloud_config.s3_credentials(
+        "notify-admin-logo-upload-bucket-staging"
+    )
+
+
 class Demo(Staging):
     HEADER_COLOUR = "#6F72AF"  # $mauve
 
@@ -173,6 +200,7 @@ class Scanning(Production):
 configs = {
     "development": Development,
     "test": Test,
+    "e2etest": E2ETest,
     "scanning": Scanning,
     "staging": Staging,
     "demo": Demo,

app/main/views/dashboard.py

@@ -49,7 +49,9 @@ def service_dashboard(service_id):
         return redirect(url_for("main.choose_template", service_id=service_id))
 
     job_response = job_api_client.get_jobs(service_id)["data"]
-    notifications_response = notification_api_client.get_notifications_for_service(service_id)["notifications"]
+    notifications_response = notification_api_client.get_notifications_for_service(
+        service_id
+    )["notifications"]
     service_data_retention_days = 7
 
     aggregate_notifications_by_job = defaultdict(list)
@@ -74,6 +76,7 @@ def service_dashboard(service_id):
             "notifications": aggregate_notifications_by_job.get(job["id"], []),
         }
         for job in job_response
+        if aggregate_notifications_by_job.get(job["id"], [])
     ]
     return render_template(
         "views/dashboard/dashboard.html",

app/main/views/sign_in.py

@@ -198,7 +198,7 @@ def sign_in():
         form=form,
         again=bool(redirect_url),
         other_device=other_device,
-        login_gov_enabled=bool(notify_env in ["development", "staging"]),
+        login_gov_enabled=bool(notify_env in ["development", "staging", "demo"]),
         password_reset_url=password_reset_url,
         initial_signin_url=initial_signin_url,
     )

app/templates/views/guidance/index.html

@@ -41,7 +41,7 @@ <h2 class="padding-top-1" id="format-content">Format your content</h2>
   <h3>To create and format your message</h3>
   <ol class="list">
     <li>All messages start from a template</li>
-    <li>Click “Send Messages”. You’ll see existing templates.</li>
+    <li>Click “<a href={{ url_for('.choose_template', service_id=current_service.id) }}>Send Messages</a>”. You’ll see existing templates.</li>
     <li>Add a new template or choose an existing template and select Edit.</li>
   </ol>
 
@@ -120,7 +120,7 @@ <h4>Examples</h4>
 
 
   {# Identify your program #}
-  <h2 class="padding-top-1" id="identify-program">Identify your program</h2>
+  <h2 class="padding-top-1" id="indentify-program">Identify your program</h2>
   <p>You can help your recipients identify your texts as legitimate by customizing your messages to clearly state who they
   are from. Consider using the program or benefit name that is most familiar to your recipients.</p>
 

app/templates/views/pricing/index.html

@@ -30,7 +30,7 @@ <h1 class="font-body-2xl margin-bottom-3">Pricing</h1>
 </ul>
 
 <h3 class="font-body-lg" id="long-text-messages">Long text messages</h3>
-<p>If a text message is longer than 160 characters (including spaces), it counts as more than one message.</p>
+<p>If a text message is longer than 160 characters (including spaces), it counts as more than one message part.</p>
 
 <div class="bottom-gutter-3-2">
   {% call mapping_table(

app/templates/views/roadmap.html

@@ -50,9 +50,9 @@ <h3 id="now">Now</h3>
     <li>Message send/failure analytics</li>
   </ul>
 
-  <h3id="next">Next</h3>
+  <h3 id="next">Next</h3>
 
-  <p>If the pilot is successful, we hope to recruit additional high-impact partners to improve outcomes for low-income individuals and families.</p>
+  <p>If the pilot is successful, we hope to recruit additional partners to improve outcomes for low-income individuals and families.</p>
 
   <p>Goals during this stage:</p>
 

app/templates/views/security.html

@@ -65,9 +65,9 @@ <h2 id="technical-security">Technical security</h2>
   <h3 class="font-body-lg">Protect sensitive information</h3>
   <p>Some messages include sensitive information like security codes or password reset links.</p>
   <p>If you’re sending a message with sensitive information, you can choose to hide those details on the Notify dashboard once the message has been sent. This means that only the message recipient will be able to see that information.</p>
+  <img src="{{ asset_url('images/product/security-review-message.png') }}"
+  alt="Screenshot of a test message in review with the link to 'hide personalization after sending' circled.">
 
-  <h2 class="font-body-lg" id="user-permissions-signing-in">User permissions and signing in</h2>
-  <p>You can set different user permissions in Notify. This lets you control who in your team has access to certain parts of the service.</p>
   <h3 class="font-body-lg">Two-factor authentication</h3>
   <p>To sign in to Notify, you’ll need to enter:</p>
   <ul class="list list-bullet">
@@ -76,11 +76,6 @@ <h3 class="font-body-lg">Two-factor authentication</h3>
   </ul>
   <p>If signing in with a text message is a problem for your team, <a class="usa-link" href="{{ url_for('main.support') }}">contact us</a> to find out about using an email link instead.</p>
 
-  <img src="{{ asset_url('images/product/security-review-message.png') }}"
-    alt="Screenshot of a teat message in review with the link to 'hide personalization after sending' circled.">
-
-  <h4>How to hide PII after sending a message</h4>
-
   <h3>User permissions and signing in</h3>
   <p>You can set different user permissions in Notify. This lets you control who in your team has access to certain parts of
   the service.</p>
@@ -93,32 +88,4 @@ <h3>Multi-factor authentication (MFA)</h3>
   </ul>
   <p>If signing in with a text message is a problem for your team, <a href="https://beta.notify.gov/support">contact us</a> to find out about using an email link instead.</p>
 
-  <!-- <h2 class="font-body-lg" id="information-risk-management">Information risk management</h2>
-  <p>Our approach to information risk management follows NCSC guidance. It assesses:</p>
-  <ul class="list list-bullet">
-    <li>how Notify is built</li>
-    <li>the infrastructure Notify is built upon</li>
-    <li>support for the Notify service</li>
-  </ul>
-  <p>This approach also applies to the service providers Notify uses to send messages.</p> -->
-
-  <!-- <h2 class="font-body-lg" id="how-we-manage-risk">How we manage risks on Notify</h2>
-  <p>Things we do to manage risks on Notify include:</p>
-  <ul class="list list-bullet">
-    <li>formal risk assessments based on <a class="usa-link" href="http://www.iso.org/iso/catalogue_detail?csnumber=56742">ISO 27005:2011</a> and National Cyber Security Centre guidance</li>
-    <li><a class="usa-link" href="https://www.ncsc.gov.uk/information/check-penetration-testing">CHECK</a>-based testing, both annually and when any major changes are made to Notify</li>
-    <li>residual risk statement preparation and active management of the risk treatment plan</li>
-    <li>regular updates to the Privacy Impact Assessment</li>
-    <li>security impact assessments</li>
-  </ul> -->
-
-  <!-- <h2 class="font-body-lg" id="cabinet-office-approval">Cabinet Office approval</h2>
-  <p>Notify has been assessed and approved by the Cabinet Office Senior Information Risk Officer (SIRO). The SIRO checks this approval once a year.</p>
-  <p>Notify also has approval from the Office of the Government’s SIRO to host data within the EEA.</p>
-
-  <h2 class="font-body-lg" id="classifications-and-security-vetting">Classifications and security vetting</h2>
-  <p>You can use Notify to send messages classified as ‘OFFICIAL’ or ‘OFFICIAL-SENSITIVE’ under the <a class="usa-link" href="https://www.gov.uk/government/publications/government-security-classifications">Government Security Classifications</a> policy.</p>
-  <p>Notify does not process data classified as ‘SECRET’ or ‘TOP SECRET’.</p>
-  <p>The Notify team has Security Check (SC) level clearance from <a class="usa-link" href="https://www.gov.uk/government/organizations/united-kingdom-security-vetting">United Kingdom Security Vetting</a> (UKSV).</p> -->
-
 {% endblock %}