Nightly Testing #170
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2024 Defense Unicorns | |
| # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial | |
| name: Nightly Testing | |
| on: | |
| schedule: | |
| - cron: '0 6 * * *' # Runs at midnight Mountain every day | |
| workflow_dispatch: | |
| # Abort prior jobs in the same workflow / PR | |
| concurrency: | |
| group: test-fleeting-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| nightly-testing: | |
| name: Test Fleeting GitLab Runners | |
| runs-on: uds-swf-ubuntu-big-boy-8-core | |
| strategy: | |
| matrix: | |
| flavor: [upstream, registry1] | |
| fail-fast: false | |
| env: | |
| SHA: ${{ github.sha }} | |
| FLAVOR: ${{ matrix.flavor }} | |
| UDS_REGION: us-gov-west-1 | |
| UDS_STATE_BUCKET_NAME: uds-ci-govcloud-us-gov-west-1-tfstate | |
| UDS_STATE_DYNAMODB_TABLE_NAME: uds-ci-govcloud-tfstate-lock | |
| TF_VAR_ami_id: ${{ secrets.AMI_ID }} | |
| TF_VAR_permissions_boundary_name: ${{ secrets.PERMISSIONS_BOUNDARY_NAME }} | |
| steps: | |
| - name: Set ENV | |
| run: | | |
| printf "%s\n" \ | |
| "UDS_GITLAB_RUNNER_AUTOSCALING_GROUP=uds-package-gitlab-runner-ci-${FLAVOR}-${SHA:0:7}" \ | |
| "UDS_STATE_KEY=tfstate/ci/install/${SHA:0:7}-gitlab-runner-${FLAVOR}-aws.tfstate" \ | |
| "TF_VAR_runner_ip=$(curl -s https://api.ipify.org)" \ | |
| "TF_VAR_region=${UDS_REGION}" \ | |
| "TF_VAR_name=uds-package-gitlab-runner-ci-${FLAVOR}-${SHA:0:7}" \ | |
| >> "$GITHUB_ENV" | |
| shell: bash | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_GOVCLOUD_ROLE_TO_ASSUME }} | |
| role-session-name: ${{ github.job || github.event.client_payload.pull_request.head.sha || github.sha }} | |
| aws-region: ${{ env.UDS_REGION }} | |
| role-duration-seconds: 21600 | |
| - name: Install UDS CLI | |
| uses: defenseunicorns/setup-uds@b987a32bac3baeb67bfb08f5e1544e2f9076ee8a # v1.0.0 | |
| with: | |
| # renovate: datasource=github-tags depName=defenseunicorns/uds-cli versioning=semver | |
| version: v0.19.2 | |
| - name: Environment setup | |
| run: | | |
| uds run actions:setup-environment \ | |
| --set REGISTRY1_USERNAME="${{ secrets.IRON_BANK_ROBOT_USERNAME }}" \ | |
| --set REGISTRY1_PASSWORD="${{ secrets.IRON_BANK_ROBOT_PASSWORD }}" \ | |
| --set GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" \ | |
| --set CHAINGUARD_IDENTITY="${{ secrets.CHAINGUARD_IDENTITY }}" | |
| shell: bash | |
| - name: Setup Tofu | |
| uses: opentofu/setup-opentofu@592200bd4b9bbf4772ace78f887668b1aee8f716 # v1.0.5 | |
| with: | |
| tofu_version: 1.8.1 | |
| tofu_wrapper: false | |
| - name: Apply IAC | |
| run: uds run -f tasks/iac.yaml apply-iac --no-progress | |
| timeout-minutes: 20 | |
| - name: IRSA Setup | |
| run: | | |
| echo "Getting OIDC keypair" | |
| mkdir irsa | |
| aws secretsmanager get-secret-value --secret-id govcloud-ci-oidc-private-key | jq -r '.SecretString' > irsa/signer.key | |
| aws secretsmanager get-secret-value --secret-id govcloud-ci-oidc-public-key | jq -r '.SecretString' > irsa/signer.key.pub | |
| timeout-minutes: 20 | |
| - name: Test Fleeting Bundle | |
| run: | | |
| pushd .github/test-infra/asg-iac || exit | |
| tofu output -raw jumpbox_private_key > ~/.ssh/id_rsa_jump | |
| chmod 600 ~/.ssh/id_rsa_jump | |
| ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa_jump \ | |
| -o ServerAliveInterval=60 -o ServerAliveCountMax=3 \ | |
| -R 8080:localhost:443 "ubuntu@$(tofu output -raw jumpbox_public_ip)" \ | |
| 'sudo apt install socat -y && sudo socat TCP-LISTEN:443,reuseaddr,keepalive,nodelay,fork TCP:localhost:8080' > /tmp/debug-tunnel.log 2>&1 & | |
| popd || exit | |
| UDS_GITLAB_RUNNER_AUTOSCALING_ROLE_ARN=$(cd .github/test-infra/asg-iac && tofu output -raw asg_role_arn) uds run test-fleeting --set FLAVOR=${{ matrix.flavor }} --no-progress | |
| shell: bash | |
| timeout-minutes: 25 | |
| env: | |
| UDS_K3D_EXTRA_ARGS: --k3s-arg=--kube-apiserver-arg=service-account-key-file=/irsa/signer.key.pub@server:* --k3s-arg=--kube-apiserver-arg=service-account-signing-key-file=/irsa/signer.key@server:* --k3s-arg=--kube-apiserver-arg=api-audiences=kubernetes.svc.default@server:* --k3s-arg=--kube-apiserver-arg=service-account-issuer=https://govcloud-ci-oidc.s3.us-gov-west-1.amazonaws.com@server:* --k3s-arg=--kube-apiserver-arg=audit-log-path=/var/log/kubernetes/audit/audit.log@server:* --volume ${{ github.workspace }}/irsa:/irsa@server:* | |
| - name: Debug Output | |
| if: ${{ always() }} | |
| run: uds run actions:debug-output | |
| shell: bash | |
| - name: Save logs | |
| if: ${{ always() }} | |
| run: uds run actions:save-logs | |
| shell: bash | |
| - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| if: ${{ always() }} | |
| with: | |
| name: debug-log-nightly-${{ matrix.flavor }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ runner.arch }} | |
| path: | | |
| /tmp/zarf-*.log | |
| /tmp/uds-*.log | |
| /tmp/maru-*.log | |
| /tmp/debug-*.log | |
| /tmp/uds-containerd-logs | |
| /tmp/k3d-uds-*.log | |
| oscal-assessment-results.yaml | |
| - name: Destroy IAC | |
| if: always() | |
| run: uds run -f tasks/iac.yaml destroy-iac --no-progress | |
| timeout-minutes: 10 | |
| continue-on-error: true |