From 70307f4c0099955e99dabdb6fcec3328e0d1c7c7 Mon Sep 17 00:00:00 2001 From: sweavers-ukhsa <135218414+sweavers-ukhsa@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:58:09 +0000 Subject: [PATCH 1/5] Add missing needs dependency (#42) #major - Added logic for filtering matrix objects based on whether there are planned changes or not. - Added sequential apply workflow for use of initial setup of environments. - Fixed the post deployment QA workflow. --- .github/workflows/terraform-code-check.yml | 17 +- .github/workflows/terraform-core.yml | 145 ++++++++++---- .github/workflows/terraform-destroy.yml | 15 +- .github/workflows/terraform-initial-apply.yml | 187 ++++++++++++++++++ .github/workflows/terraform-plan-apply.yml | 105 +++++++--- .../terraform-post-deployment-qa.yml | 20 +- 6 files changed, 399 insertions(+), 90 deletions(-) create mode 100644 .github/workflows/terraform-initial-apply.yml diff --git a/.github/workflows/terraform-code-check.yml b/.github/workflows/terraform-code-check.yml index b88fa56..3c0fc73 100644 --- a/.github/workflows/terraform-code-check.yml +++ b/.github/workflows/terraform-code-check.yml @@ -25,11 +25,6 @@ on: type: string default: ${{ github.ref }} description: "Specify the branch of the Terraform code. Normally left blank to use calling ref." - runner_label: - required: false - type: string - default: "ubuntu-latest" - description: "Specifies a targetted runner for the job. Use \"self-hosted\" if you've configured a self hosted runner, otherwise leave as the default." secrets: TF_MODULES_SSH_DEPLOY_KEY: @@ -42,7 +37,7 @@ on: jobs: recursive-checks: name: Lint and Scan Terraform code - runs-on: ${{ inputs.runner_label }} + runs-on: "ubuntu-latest" container: image: ghcr.io/ukhsa-internal/devops-terraform-ci:latest # Volume is workaround for https://github.com/actions/runner/issues/716 @@ -56,7 +51,7 @@ jobs: ssh-key: ${{ secrets.REPO_SSH_DEPLOY_KEY }} - name: Create pending check - uses: UKHSA-Internal/devops-github-actions/.github/actions/github-status-check@main + uses: UKHSA-Internal/devops-github-actions/.github/actions/github-status-check@v0.7.0 id: create_check with: name: "terraform-checks" @@ -71,7 +66,7 @@ jobs: | Checkov Scan | :hourglass_flowing_sand: Pending | - name: Find Terraform version - uses: UKHSA-Internal/devops-github-actions/.github/actions/parse-terraform-version@main + uses: UKHSA-Internal/devops-github-actions/.github/actions/parse-terraform-version@v0.7.0 id: terraform_version with: tf_file: "${{ inputs.stack_root_directory }}/terraform.tf" @@ -92,14 +87,14 @@ jobs: - name: Terraform Linting with tflint id: tf_lint continue-on-error: true - uses: UKHSA-Internal/devops-github-actions/.github/actions/terraform-tflint@main + uses: UKHSA-Internal/devops-github-actions/.github/actions/terraform-tflint@v0.7.0 with: tf_directory: ${{ inputs.stack_root_directory }} scan_modules: ${{ inputs.tflint_module_scan }} - name: SAST Scan Terraform code id: tf_scan - uses: UKHSA-Internal/devops-github-actions/.github/actions/terraform-checkov-scan@main + uses: UKHSA-Internal/devops-github-actions/.github/actions/terraform-checkov-scan@v0.7.0 with: scan_directory: ${{ inputs.stack_root_directory }} scan_type: "${{ inputs.checkov_deep_scan && 'deep' || 'light' }}" @@ -126,7 +121,7 @@ jobs: echo "tf_scan=$tf_scan" >> $GITHUB_ENV - name: Update check with result - uses: UKHSA-Internal/devops-github-actions/.github/actions/github-status-check@main + uses: UKHSA-Internal/devops-github-actions/.github/actions/github-status-check@v0.7.0 with: name: "terraform-checks" status: "completed" diff --git a/.github/workflows/terraform-core.yml b/.github/workflows/terraform-core.yml index 978b68a..8f7d898 100644 --- a/.github/workflows/terraform-core.yml +++ b/.github/workflows/terraform-core.yml @@ -24,8 +24,8 @@ on: type: string default: "apply" description: "Which Terraform action to run. 'apply' or 'destroy'" - directories: - description: "The list of Terraform stacks to run Terraform in" + stack_config: + description: "A detailed matrix containing the Terraform stack configuration and dependencies" required: true type: string max_parallel: @@ -53,11 +53,6 @@ on: type: boolean default: false description: "Download an artifact containing an existing Terraform plan created by a previous run. Incompatible with upload_plan" - runner_label: - required: false - type: string - default: "ubuntu-latest" - description: "Specifies a targetted runner for the job. Use \"self-hosted\" if you've configured a self hosted runner, otherwise leave as the default." secrets: AWS_ROLE_NAME: @@ -95,15 +90,15 @@ on: jobs: initialise: - name: "Initialise and run Terraform for ${{ matrix.directory }}" - runs-on: ${{ inputs.runner_label }} + name: "Initialise and run Terraform for ${{ matrix.stack.directory }}" + runs-on: "${{ matrix.stack.runner_label }}" environment: ${{ inputs.environment_name }} defaults: run: shell: bash strategy: matrix: - directory: "${{ fromJSON(inputs.directories) }}" + stack: "${{ fromJSON(inputs.stack_config) }}" max-parallel: ${{ inputs.max_parallel }} steps: - name: Checkout code @@ -114,7 +109,7 @@ jobs: ssh-key: ${{ secrets.REPO_SSH_DEPLOY_KEY }} - name: Export variables - run: echo "state_name=$(basename ${{ matrix.directory }})" >> $GITHUB_ENV + run: echo "state_name=$(basename ${{ matrix.stack.directory }})" >> $GITHUB_ENV - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -146,7 +141,7 @@ jobs: - name: Copy required files from root directory env: - DIRECTORY: "${{ matrix.directory }}" + DIRECTORY: "${{ matrix.stack.directory }}" run: | files_to_copy=("providers.tf" "terraform.tf") @@ -160,27 +155,27 @@ jobs: done - name: Find Terraform version - uses: UKHSA-Internal/devops-github-actions/.github/actions/parse-terraform-version@main + uses: UKHSA-Internal/devops-github-actions/.github/actions/parse-terraform-version@v0.7.0 id: terraform_version with: - tf_file: "${{ matrix.directory }}/terraform.tf" + tf_file: "${{ matrix.stack.directory }}/terraform.tf" - name: Setup Terraform for self-hosted runner - if: ${{ inputs.runner_label == 'self-hosted' }} + if: ${{ contains(matrix.stack.runner_label, 'self-hosted') }} run: | echo "Terraform Version: ${{ steps.terraform_version.outputs.tf_version }}" tfenv use ${{ steps.terraform_version.outputs.tf_version }} terraform --version - working-directory: ${{ matrix.directory }} + working-directory: ${{ matrix.stack.directory }} - name: Setup Terraform for GHE runner - if: ${{ inputs.runner_label != 'self-hosted' }} + if: ${{ !contains(matrix.stack.runner_label, 'self-hosted') }} uses: hashicorp/setup-terraform@v3 with: terraform_version: "${{ steps.terraform_version.outputs.tf_version }}" - name: Determine Backend Type - working-directory: "${{ matrix.directory }}" + working-directory: "${{ matrix.stack.directory }}" id: backend run: | backend_type=$(grep -oP 'backend\s+"?\K[^"\s]+' ./backend.tf) @@ -203,13 +198,13 @@ jobs: echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV - name: Terraform Init with AWS S3 Backend - working-directory: "${{ matrix.directory }}" + working-directory: "${{ matrix.stack.directory }}" if: ${{ steps.backend.outputs.backend_type == 's3' }} env: AWS_ACCOUNT_ID: "${{ secrets.AWS_ACCOUNT_ID }}" AWS_REGION: "${{ inputs.aws_region }}" ENVIRONMENT_NAME: "${{ inputs.environment_name }}" - DIRECTORY: "${{ matrix.directory }}" + DIRECTORY: "${{ matrix.stack.directory }}" run: | s3_key="${ENVIRONMENT_NAME}"/"$state_name"/terraform.tfstate dynamodb_table=${AWS_REGION}-state-locks @@ -221,10 +216,10 @@ jobs: - name: Terraform Init with Azure Backend if: ${{ steps.backend.outputs.backend_type == 'azurerm' }} - working-directory: "${{ matrix.directory }}" + working-directory: "${{ matrix.stack.directory }}" env: ENVIRONMENT_NAME: "${{ inputs.environment_name }}" - DIRECTORY: "${{ matrix.directory }}" + DIRECTORY: "${{ matrix.stack.directory }}" ARM_USE_OIDC: true ARM_CLIENT_ID: "${{ secrets.AZURE_CLIENT_ID }}" ARM_SUBSCRIPTION_ID: "${{ secrets.AZURE_SUBSCRIPTION_ID }}" @@ -236,11 +231,11 @@ jobs: terraform init \ -backend-config=storage_account_name="${storage_account_name}" \ -backend-config=container_name="$container_name" \ - -backend-config=key="${ENVIRONMENT_NAME}"/"$state_name"/terraform.tfstate \ + -backend-config=key=$ENVIRONMENT_NAME/"$state_name"/terraform.tfstate \ -backend-config=resource_group_name="${{ secrets.AZURE_RESOURCE_GROUP_NAME }}" - name: Check if the state file is empty - working-directory: "${{ matrix.directory }}" + working-directory: "${{ matrix.stack.directory }}" id: state_empty shell: bash env: @@ -267,7 +262,7 @@ jobs: id: variables if: steps.state_empty.outputs.skip_workflow == 'false' env: - DIRECTORY: "${{ matrix.directory }}" + DIRECTORY: "${{ matrix.stack.directory }}" ENVIRONMENT_NAME: "${{ inputs.environment_name }}" run: | find_app_var_files() { @@ -313,11 +308,13 @@ jobs: id: download_plan if: ( inputs.download_existing_plan == true && steps.state_empty.outputs.state_empty == 'false' ) with: - name: "tfplan-${{ env.state_name }}" - path: ${{ matrix.directory }} + name: "${{ env.state_name }}-artefacts" + path: ${{ matrix.stack.directory }} - name: Terraform Plan - working-directory: "${{ matrix.directory }}" + id: tf_plan + working-directory: "${{ matrix.stack.directory }}" + continue-on-error: true if: steps.download_plan.conclusion == 'skipped' && steps.state_empty.outputs.skip_workflow == 'false' env: ENVIRONMENT_NAME: "${{ inputs.environment_name }}" @@ -329,25 +326,93 @@ jobs: GH_TOKEN: ${{ secrets.GH_TOKEN }} TERRAFORM_ACTION: ${{ inputs.terraform_action }} run: | + # Required, otherwise GitHub will exit immediately when the tf plan exit code is non-zero (2 = changes) + set +e + if [[ "$TERRAFORM_ACTION" == "destroy" ]]; then - terraform plan -lock-timeout=5m -destroy -no-color -input=false -out=tfplan -compact-warnings ${TERRAFORM_VARIABLES} + terraform plan -lock-timeout=5m -destroy -no-color -input=false -out=tfplan -detailed-exitcode -compact-warnings ${TERRAFORM_VARIABLES} else - terraform plan -lock-timeout=5m -no-color -input=false -out=tfplan -compact-warnings ${TERRAFORM_VARIABLES} + terraform plan -lock-timeout=5m -no-color -input=false -out=tfplan -detailed-exitcode -compact-warnings ${TERRAFORM_VARIABLES} fi + + terraform_exit_code=$? + + echo "Terraform exit code: $terraform_exit_code" + echo "terraform_exit_code=$terraform_exit_code" >> $GITHUB_OUTPUT + terraform show -json tfplan | jq > tfplan.json - - name: Upload Terraform Plan + # If no changes are required, we can skip subsequent steps for this stack. + if [ $terraform_exit_code -eq 0 ]; then + # If there are resources to be destroyed, the -detailed-exitcode returned is 0, despite changes being present. + # Additional logic required to inspect the tfplan.json file for if any destroy actions are present. + destroy_count=$(jq '[.resource_changes[] | select(.change.actions | index("delete"))] | length' tfplan.json) + echo "Destroy count: $destroy_count" + if [ "$destroy_count" -gt 0 ]; then + planned_changes=true + else + planned_changes=false + fi + elif [ $terraform_exit_code -eq 2 ]; then + planned_changes=true + else + exit $terraform_exit_code + fi + + echo "planned_changes=$planned_changes" >> $GITHUB_OUTPUT + + - name: Check Terraform plan exit code + if: steps.tf_plan.conclusion == 'success' || steps.tf_plan.conclusion == 'failure' + env: + terraform_exit_code: "${{ steps.tf_plan.outputs.terraform_exit_code }}" + run: | + # Separate step required otherwise GitHub will exit on non-zero exit code. + if [ $terraform_exit_code -eq 1 ]; then + echo "Terraform encountered an error. Exiting workflow." + exit 1 + fi + + - name: Update stack matrix + working-directory: "${{ matrix.stack.directory }}" + run: | + # If the tf_plan step was skipped because of no state, persist the stack configuration with planned_changes. + if [ "${{ steps.tf_plan.conclusion }}" == "skipped" ]; then + planned_changes="${{ matrix.stack.planned_changes }}" + else + planned_changes="${{ steps.tf_plan.outputs.planned_changes }}" + fi + + echo "runner_label is: ${{ matrix.stack.runner_label }}" + + jq -n \ + --arg dir "${{ matrix.stack.directory }}" \ + --argjson runner_label '${{ toJSON(matrix.stack.runner_label) }}' \ + --argjson planned_changes "$planned_changes" \ + --argjson order "${{ matrix.stack.order }}" \ + '{ + directory: $dir, + runner_label: ($runner_label | if type=="array" then . else [.] end), + planned_changes: $planned_changes, + order: $order + }' \ + > updated_matrix.json + + cat updated_matrix.json + + - name: Upload Terraform Plan and matrix uses: actions/upload-artifact@v4 - if: inputs.upload_plan && steps.state_empty.outputs.skip_workflow == 'false' + if: ${{ inputs.upload_plan }} with: - name: "tfplan-${{ env.state_name }}" - path: "${{ matrix.directory }}/tfplan*" - if-no-files-found: error + name: "${{ env.state_name }}-artefacts" + path: | + ${{ matrix.stack.directory }}/tfplan* + ${{ matrix.stack.directory }}/updated_matrix.json + if-no-files-found: warn compression-level: 1 retention-days: 1 - name: Terraform Destructive Actions Check - working-directory: "${{ matrix.directory }}" + working-directory: "${{ matrix.stack.directory }}" if: >- ${{ ( inputs.destructive_action_check || inputs.environment_name == 'pre' || @@ -362,8 +427,10 @@ jobs: fi - name: Terraform Apply - if: inputs.execute_terraform_plan && steps.state_empty.outputs.skip_workflow == 'false' - working-directory: "${{ matrix.directory }}" + if: >- + ${{ inputs.execute_terraform_plan && + steps.state_empty.outputs.skip_workflow == 'false' }} + working-directory: "${{ matrix.stack.directory }}" env: ENVIRONMENT_NAME: "${{ inputs.environment_name }}" TERRAFORM_VARIABLES: "${{ steps.variables.outputs.tf_vars }}" @@ -372,4 +439,4 @@ jobs: ARM_SUBSCRIPTION_ID: "${{ secrets.AZURE_SUBSCRIPTION_ID }}" ARM_TENANT_ID: "${{ secrets.AZURE_TENANT_ID }}" GH_TOKEN: "${{ secrets.GH_TOKEN }}" - run: terraform apply -lock-timeout=5m -no-color -input=false tfplan + run: terraform apply -lock-timeout=5m -no-color -input=false tfplan \ No newline at end of file diff --git a/.github/workflows/terraform-destroy.yml b/.github/workflows/terraform-destroy.yml index 6ab7258..221ed52 100644 --- a/.github/workflows/terraform-destroy.yml +++ b/.github/workflows/terraform-destroy.yml @@ -24,11 +24,6 @@ on: type: string default: ${{ github.ref }} description: "Specify the branch of the Terraform code. Normally left blank to use calling ref." - runner_label: - required: false - type: string - default: "ubuntu-latest" - description: "Specifies a targetted runner for the job. Use \"self-hosted\" if you've configured a self hosted runner, otherwise leave as the default." secrets: AWS_ROLE_NAME: @@ -60,9 +55,9 @@ on: jobs: define_matrix: name: Define directory matrix for destroy - runs-on: ${{ inputs.runner_label }} + runs-on: ubuntu-latest outputs: - directories: "${{ steps.directories.outputs.json_directory_list }}" + stack_config: "${{ steps.stack_config.outputs.json_directory_list }}" steps: - uses: actions/checkout@v4 with: @@ -71,8 +66,8 @@ jobs: ssh-key: ${{ secrets.REPO_SSH_DEPLOY_KEY }} - name: Determine order to run Terraform stacks uses: >- - UKHSA-Internal/devops-github-actions/.github/actions/terraform-dependency-sort@main - id: directories + UKHSA-Internal/devops-github-actions/.github/actions/terraform-dependency-sort@v0.7.0 + id: stack_config with: reverse: true @@ -86,7 +81,7 @@ jobs: aws_region: ${{ inputs.aws_region }} repo: ${{ inputs.repo }} ref: ${{ inputs.ref }} - directories: "${{ needs.define_matrix.outputs.directories }}" + stack_config: "${{ needs.define_matrix.outputs.stack_config }}" terraform_action: "destroy" execute_terraform_plan: ${{ inputs.execute_terraform_plan }} secrets: inherit \ No newline at end of file diff --git a/.github/workflows/terraform-initial-apply.yml b/.github/workflows/terraform-initial-apply.yml new file mode 100644 index 0000000..768f289 --- /dev/null +++ b/.github/workflows/terraform-initial-apply.yml @@ -0,0 +1,187 @@ +name: "[CI] Sequential Terraform apply" +on: + workflow_call: + inputs: + environment_name: + required: false + default: dev + type: string + aws_region: + required: false + default: eu-west-2 + type: string + destructive_action_check: + required: false + type: boolean + default: false + tflint_module_scan: + required: false + type: boolean + default: false + execute_terraform_plan: + required: false + type: boolean + default: false + repo: + required: false + type: string + default: ${{ github.repository }} + description: "Specify the org/repo of the repo containing Terraform code. Normally left blank to clone calling repo." + ref: + required: false + type: string + default: ${{ github.ref }} + description: "Specify the branch of the Terraform code. Normally left blank to use calling ref." + runner_label: + required: false + type: string + default: "ubuntu-latest" + description: "Specifies a targetted runner for the job. Use \"self-hosted\" if you've configured a self hosted runner, otherwise leave as the default." + + secrets: + AWS_ROLE_NAME: + required: false + AWS_ACCOUNT_ID: + required: false + AWS_ACCESS_KEY_ID: + required: false + AWS_SECRET_ACCESS_KEY: + required: false + AZURE_SUBSCRIPTION_ID: + required: false + AZURE_TENANT_ID: + required: false + AZURE_CLIENT_ID: + required: false + AZURE_RESOURCE_GROUP_NAME: + required: false + TF_MODULES_SSH_DEPLOY_KEY: + required: false + description: "The SSH key used to clone Terraform modules downloaded as part of the Terraform init" + REPO_SSH_DEPLOY_KEY: + required: false + description: "The SSH key used to checkout private remote repos" + SSH_DEPLOY_KEY: + required: false + description: "Deprecated: Use either TF_MODULES_SSH_DEPLOY_KEY or REPO_SSH_DEPLOY_KEY instead." + +jobs: + define_matrix: + name: Define directory matrix for build + runs-on: ubuntu-latest + outputs: + stack_config: "${{ steps.stack_config.outputs.json_directory_list }}" + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ inputs.repo }} + ref: ${{ inputs.ref }} + ssh-key: ${{ secrets.REPO_SSH_DEPLOY_KEY }} + - name: Determine order to run Terraform stacks + uses: >- + UKHSA-Internal/devops-github-actions/.github/actions/terraform-dependency-sort@v0.7.0 + id: stack_config + + filter_matrix: + name: Filter matrix for only planned changes + needs: + - define_matrix + runs-on: ubuntu-latest + outputs: + filtered_matrix: ${{ steps.filter_matrix.outputs.filtered_matrix }} + skip_remaining: ${{ steps.check_empty.outputs.skip_remaining }} + steps: + - name: Filter Stacks + env: + ENVIRONMENT_NAME: ${{ inputs.environment_name }} + id: filter_matrix + run: | + echo '${{ needs.define_matrix.outputs.stack_config }}' > initial_matrix.json + + filtered_matrix=$(jq -c --arg env_name "${ENVIRONMENT_NAME}" ' + [ .[] + | select(.planned_changes == true) + | if .runner_label == "self-hosted" + then .runner_label = ["self-hosted", $env_name] + else . + end + ] + ' initial_matrix.json) + + echo "filtered_matrix=$filtered_matrix" >> $GITHUB_OUTPUT + - name: Check if filtered matrix is empty + id: check_matrix + run: | + if [ "$filtered_matrix" = "[]" ]; then + echo "skip_remaining=true" >> $GITHUB_OUTPUT + else + echo "skip_remaining=false" >> $GITHUB_OUTPUT + fi + + check: + name: Lint and SAST Scan Terraform code + uses: ./.github/workflows/terraform-code-check.yml + needs: + - filter_matrix + strategy: + matrix: + stack: ${{ fromJSON(needs.filter_matrix.outputs.filtered_matrix) }} + with: + stack_root_directory: "${{ matrix.stack.directory }}" + repo: ${{ inputs.repo }} + ref: ${{ inputs.ref }} + tflint_module_scan: ${{ inputs.tflint_module_scan }} + secrets: + REPO_SSH_DEPLOY_KEY: ${{ secrets.REPO_SSH_DEPLOY_KEY }} + + apply: + name: Run Terraform + if: ${{ needs.filter_matrix.outputs.skip_remaining == 'false' }} + uses: ./.github/workflows/terraform-core.yml + needs: + - check + - filter_matrix + with: + environment_name: ${{ inputs.environment_name }} + aws_region: ${{ inputs.aws_region }} + repo: ${{ inputs.repo }} + ref: ${{ inputs.ref }} + stack_config: "${{ needs.filter_matrix.outputs.filtered_matrix }}" + destructive_action_check: ${{ inputs.destructive_action_check }} + terraform_action: "apply" + execute_terraform_plan: ${{ inputs.execute_terraform_plan }} + upload_plan: true + download_existing_plan: false + secrets: inherit + + # update-deployment: + # name: Update Github deployment + # if: always() + # runs-on: ubuntu-latest + # steps: + # - run: | + # deployid=$(gh api -H "Accept: application/vnd.github+json" \ + # --method GET \ + # -H "X-GitHub-Api-Version: 2022-11-28" \ + # -f 'q=sha:${{ github.sha }};environment:${{ inputs.environment_name }}' \ + # /repos/UKHSA-Internal/${{ github.event.repository.name }}/deployments) + + # gh api \ + # --method POST \ + # -H "Accept: application/vnd.github+json" \ + # -H "X-GitHub-Api-Version: 2022-11-28" \ + # /repos/OWNER/${{ github.event.repository.name }}/deployments/"$deployid"/statuses \ + # -f "state=success" \ + # -f "description=Deployment finished successfully." + + post-deployment-qa-checks: + name: Run post deployment QA checks. + uses: ./.github/workflows/terraform-post-deployment-qa.yml + needs: + - apply + - filter_matrix_2 + with: + environment_name: ${{ inputs.environment_name }} + aws_region: ${{ inputs.aws_region }} + stack_config: "${{ needs.filter_matrix_2.outputs.filtered_matrix }}" + secrets: inherit diff --git a/.github/workflows/terraform-plan-apply.yml b/.github/workflows/terraform-plan-apply.yml index efc6400..6c4fae4 100644 --- a/.github/workflows/terraform-plan-apply.yml +++ b/.github/workflows/terraform-plan-apply.yml @@ -32,11 +32,6 @@ on: type: string default: ${{ github.ref }} description: "Specify the branch of the Terraform code. Normally left blank to use calling ref." - runner_label: - required: false - type: string - default: "ubuntu-latest" - description: "Specifies a targetted runner for the job. Use \"self-hosted\" if you've configured a self hosted runner, otherwise leave as the default." secrets: AWS_ROLE_NAME: @@ -70,7 +65,7 @@ jobs: name: Define directory matrix for build runs-on: ubuntu-latest outputs: - directories: "${{ steps.directories.outputs.json_directory_list }}" + stack_config: "${{ steps.stack_config.outputs.json_directory_list }}" steps: - uses: actions/checkout@v4 with: @@ -79,19 +74,47 @@ jobs: ssh-key: ${{ secrets.REPO_SSH_DEPLOY_KEY }} - name: Determine order to run Terraform stacks uses: >- - UKHSA-Internal/devops-github-actions/.github/actions/terraform-dependency-sort@main - id: directories + UKHSA-Internal/devops-github-actions/.github/actions/terraform-dependency-sort@v0.7.0 + id: stack_config + + filter_matrix_1: + name: Filter matrix for only planned changes + needs: + - define_matrix + runs-on: ubuntu-latest + outputs: + filtered_matrix: ${{ steps.filter_matrix.outputs.filtered_matrix }} + steps: + - name: Filter Stacks + env: + ENVIRONMENT_NAME: ${{ inputs.environment_name }} + id: filter_matrix + run: | + echo '${{ needs.define_matrix.outputs.stack_config }}' > initial_matrix.json + + filtered_matrix=$(jq -c --arg env_name "${ENVIRONMENT_NAME}" ' + [ .[] + | select(.planned_changes == true) + | if .runner_label == "self-hosted" + then .runner_label = ["self-hosted", $env_name] + else . + end + ] + ' initial_matrix.json) + echo "Filtered Matrix: $filtered_matrix" + echo "filtered_matrix=$filtered_matrix" >> $GITHUB_OUTPUT + check: name: Lint and SAST Scan Terraform code uses: ./.github/workflows/terraform-code-check.yml needs: - - define_matrix + - filter_matrix_1 strategy: matrix: - directory: ${{ fromJSON(needs.define_matrix.outputs.directories) }} + stack: ${{ fromJSON(needs.filter_matrix_1.outputs.filtered_matrix) }} with: - stack_root_directory: "${{ matrix.directory }}" + stack_root_directory: "${{ matrix.stack.directory }}" repo: ${{ inputs.repo }} ref: ${{ inputs.ref }} tflint_module_scan: ${{ inputs.tflint_module_scan }} @@ -102,20 +125,19 @@ jobs: name: Plan Terraform uses: ./.github/workflows/terraform-core.yml needs: - - define_matrix - check + - filter_matrix_1 with: environment_name: ${{ inputs.environment_name }} aws_region: ${{ inputs.aws_region }} repo: ${{ inputs.repo }} ref: ${{ inputs.ref }} - directories: "${{ needs.define_matrix.outputs.directories }}" + stack_config: "${{ needs.filter_matrix_1.outputs.filtered_matrix }}" destructive_action_check: ${{ inputs.destructive_action_check }} terraform_action: "apply" execute_terraform_plan: false upload_plan: true download_existing_plan: false - runner_label: ${{ inputs.runner_label }} secrets: inherit approve: @@ -126,9 +148,9 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} needs: - - define_matrix - check - plan + - filter_matrix_1 steps: - name: "Check approval requirement" run: | @@ -141,25 +163,63 @@ jobs: echo "Environment ${{ inputs.environment_name }} doesn't require manual approval. Automatically proceeding..." fi + filter_matrix_2: + name: Filter matrix for only planned changes + needs: + - plan + - approve + runs-on: ubuntu-latest + outputs: + filtered_matrix: ${{ steps.filter_matrix.outputs.filtered_matrix }} + skip_remaining: ${{ steps.check_matrix.outputs.skip_remaining }} + steps: + - uses: actions/download-artifact@v4 + with: + path: "./artifacts" + - name: Filter Stacks + id: filter_matrix + run: | + jq -s '.' ./artifacts/**/updated_matrix.json > combined_matrix.json + cat combined_matrix.json + + filtered_matrix=$(jq -c ' + [ .[] + | select(.planned_changes == true) + ] | sort_by(.order) + ' combined_matrix.json) + + echo "Filtered Matrix: $filtered_matrix" + echo "filtered_matrix=$filtered_matrix" >> $GITHUB_OUTPUT + - name: Check if filtered matrix is empty + id: check_matrix + run: | + filtered_matrix='${{ steps.filter_matrix.outputs.filtered_matrix }}' + if [ "$filtered_matrix" = "[]" ]; then + echo "Matrix is empty" + echo "skip_remaining=true" >> $GITHUB_OUTPUT + else + echo "Matrix is not empty" + echo "skip_remaining=false" >> $GITHUB_OUTPUT + fi + apply: name: Run Terraform + if: ${{ needs.filter_matrix_2.outputs.skip_remaining == 'false' }} uses: ./.github/workflows/terraform-core.yml needs: - - define_matrix - - check - approve + - filter_matrix_2 with: environment_name: ${{ inputs.environment_name }} aws_region: ${{ inputs.aws_region }} repo: ${{ inputs.repo }} ref: ${{ inputs.ref }} - directories: "${{ needs.define_matrix.outputs.directories }}" + stack_config: "${{ needs.filter_matrix_2.outputs.filtered_matrix }}" destructive_action_check: ${{ inputs.destructive_action_check }} terraform_action: "apply" execute_terraform_plan: ${{ inputs.execute_terraform_plan }} upload_plan: false download_existing_plan: true - runner_label: ${{ inputs.runner_label }} secrets: inherit # update-deployment: @@ -185,10 +245,11 @@ jobs: post-deployment-qa-checks: name: Run post deployment QA checks. uses: ./.github/workflows/terraform-post-deployment-qa.yml - needs: apply + needs: + - apply + - filter_matrix_2 with: environment_name: ${{ inputs.environment_name }} aws_region: ${{ inputs.aws_region }} - directories: "${{ needs.define_matrix.outputs.directories }}" - runner_label: ${{ inputs.runner_label }} + stack_config: "${{ needs.filter_matrix_2.outputs.filtered_matrix }}" secrets: inherit diff --git a/.github/workflows/terraform-post-deployment-qa.yml b/.github/workflows/terraform-post-deployment-qa.yml index d7fb006..7d98a19 100644 --- a/.github/workflows/terraform-post-deployment-qa.yml +++ b/.github/workflows/terraform-post-deployment-qa.yml @@ -15,8 +15,8 @@ on: type: string default: "ubuntu-latest" description: "Specifies a targetted runner for the job. Use \"self-hosted\" if you've configured a self hosted runner, otherwise leave as the default." - directories: - description: "The list of Terraform stacks to run Terraform in" + stack_config: + description: "A detailed matrix containing the Terraform stack configuration and dependencies" required: true type: string max_parallel: @@ -63,27 +63,31 @@ jobs: deep-sast-scan: name: Run a deep SAST scan against Terraform code. runs-on: "ubuntu-latest" + environment: ${{ inputs.environment_name }} + defaults: + run: + shell: bash strategy: matrix: - directory: "${{ fromJSON(inputs.directories) }}" + stack: "${{ fromJSON(inputs.stack_config) }}" max-parallel: ${{ inputs.max_parallel }} steps: - uses: actions/checkout@v4 - name: Export variables - run: echo "state_name=$(basename ${{ matrix.directory }})" >> $GITHUB_ENV + run: echo "state_name=$(basename ${{ matrix.stack.directory }})" >> $GITHUB_ENV - uses: actions/download-artifact@v4 id: download_plan with: - name: "tfplan-${{ env.state_name }}" - path: ${{ matrix.directory }} + name: "${{ env.state_name }}-artefacts" + path: ${{ matrix.stack.directory }} - name: Deep SAST Scan Terraform code id: tf_scan - uses: UKHSA-Internal/devops-github-actions/.github/actions/terraform-checkov-scan@main + uses: UKHSA-Internal/devops-github-actions/.github/actions/terraform-checkov-scan@v0.7.0 with: - scan_directory: ${{ matrix.directory }} + scan_directory: ${{ matrix.stack.directory }} scan_type: "deep" tfplan_file: "tfplan.json" From 7d8dbd889f89e64558b8c1eb2122623e72b17ed9 Mon Sep 17 00:00:00 2001 From: sweavers-ukhsa <135218414+sweavers-ukhsa@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:30:02 +0000 Subject: [PATCH 2/5] Updated jq query to count all actions to better handle ambiguity of tf plan exit code (#46) --- .github/workflows/terraform-core.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/terraform-core.yml b/.github/workflows/terraform-core.yml index 8f7d898..3d30c17 100644 --- a/.github/workflows/terraform-core.yml +++ b/.github/workflows/terraform-core.yml @@ -344,11 +344,11 @@ jobs: # If no changes are required, we can skip subsequent steps for this stack. if [ $terraform_exit_code -eq 0 ]; then - # If there are resources to be destroyed, the -detailed-exitcode returned is 0, despite changes being present. - # Additional logic required to inspect the tfplan.json file for if any destroy actions are present. - destroy_count=$(jq '[.resource_changes[] | select(.change.actions | index("delete"))] | length' tfplan.json) - echo "Destroy count: $destroy_count" - if [ "$destroy_count" -gt 0 ]; then + # If there are only resources to be destroyed or updated, the -detailed-exitcode returned is 0. + # Additional logic required to inspect the tfplan.json file for if any actions are present and ensure planned changes are applied. + action_count=$(jq '[.resource_changes[] | select(.change.actions | any(. == "create" or . == "update" or . == "delete"))] | length' tfplan.json) + echo "Action count: $action_count" + if [ "$action_count" -gt 0 ]; then planned_changes=true else planned_changes=false From 558aa6d953e65e050269a7db59c6d179ce78d2df Mon Sep 17 00:00:00 2001 From: Sam Ainsworth Date: Tue, 5 Nov 2024 12:27:22 +0000 Subject: [PATCH 3/5] DOE-282 Update README --- README.md | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 162 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4229aae..b69f8ef 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,166 @@ # devops-github-reusable-workflows -A collection of reusable Workflows to help standardise the CI pipelines of teams across the organisation. +The devops-github-reusable-workflows repository provides a collection of reusable GitHub Actions workflows designed to simplify and standardise DevOps tasks across projects. -## `[CI] Lint and Deploy Terraform stacks` +## Check Terraform Code +##### terraform-code-check.yml -Deploys Terraform code that uses our standard folder structure. The Terraform code will be linted, scanned, validated, planned and deployed. This currently only supports using an AWS S3 backend. Support for using an Azure backend is planned. +This workflow performs a series of checks on Terraform code, ensuring it meets formatting and security standards. It supports various configuration options, allowing users to tailor the workflow to their specific Terraform setup. The workflow includes linting, formatting, and security scanning steps, providing detailed status feedback. + +#### Workflow Inputs + +`stack_root_directory` (required): Specifies the root directory of the Terraform stack for code checks. + +`tflint_module_scan` (optional): Boolean value indicating whether TFLint should download Terraform modules during the linting process. + +`checkov_deep_scan` (optional, default: false): Boolean value determining if Checkov should perform a deep scan, which includes external modules. + +`repo` (optional, default: ${{ github.repository }}): The organisation/repository name of the repository containing Terraform code. Typically left blank to clone the calling repository. + +`ref` (optional, default: ${{ github.ref }}): Specifies the branch or tag of the Terraform code to check. By default, it uses the branch of the calling workflow. + +`runner_label` (optional, default: ubuntu-latest): Defines the runner environment for executing the workflow. Use "self-hosted" if a custom runner is configured. + +#### Secrets + +`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key for cloning Terraform modules during terraform init. + +`REPO_SSH_DEPLOY_KEY` (optional): SSH key for cloning the repository with Terraform stacks. + +## Destroy Terraform stacks +##### terrraform-destroy.yml + +This workflow facilitates the destruction of Terraform-managed resources, designed to run as part of a continuous integration pipeline. It provides configuration options for the environment and AWS region, with additional controls for executing the Terraform plan. This workflow executes a reverse-ordered destruction process for Terraform stacks, utilising defined directory dependencies for resource teardown. + +#### Workflow Inputs + +`environment_name` (optional, default: dev): Specifies the environment in which to destroy resources (e.g., dev, prd). + +`aws_region` (optional, default: eu-west-2): AWS region for executing the destruction process. + +`execute_terraform_plan` (optional, default: false): Boolean to determine whether to perform a Terraform plan execution before destroy. + +`repo` (optional, default: ${{ github.repository }}): Organisation/repository name containing the Terraform code. + +`ref` (optional, default: ${{ github.ref }}): Branch or tag of the Terraform code. Defaults to the calling workflow’s branch. + +`runner_label` (optional, default: ubuntu-latest): Target runner environment. Use "self-hosted" for custom runners, or leave as default. + +#### Secrets + +`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): Credentials for AWS account access. Not compatiable with Azure credentials. + +`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for authentication and resource management. Not compatiable with AWS credentials. + +`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key to clone Terraform modules as part of initialisation. + +`REPO_SSH_DEPLOY_KEY` (optional): SSH key to checkout private repositories with remote Terraform configurations. + +## Deploy Terraform stacks +##### terraform-plan-apply.yml + +This workflow handles the deployment of Terraform stacks, with various checks and approvals integrated into the process. It includes options for region, environment, and other deployment settings, designed to ensure safe and organised stack management within a CI/CD pipeline. This workflow includes dependency sorting for stack deployment, code linting, a Terraform plan step with optional manual approval, and post-deployment quality checks. + +#### Workflow Inputs + +`environment_name` (optional, default: dev): Specifies the target environment for the deployment (e.g., dev, prd). + +`aws_region` (optional, default: eu-west-2): AWS region for deployment. + +`destructive_action_check` (optional, default: false): Boolean to prevent unintentional destructive actions in Terraform plans. + +`tflint_module_scan` (optional, default: false): Boolean to enable or disable TFLint’s module scan during code checks. + +`execute_terraform_plan` (optional, default: false): Determines if the workflow should execute a Terraform apply after planning. + +`repo` (optional, default: ${{ github.repository }}): The organisation/repository containing the Terraform code. + +`ref` (optional, default: ${{ github.ref }}): Specifies the branch or tag of the Terraform code. + +`runner_label` (optional, default: ubuntu-latest): Sets the runner for executing jobs. Use "self-hosted" if you have a self-hosted runner configured. + +#### Secrets + +`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): Credentials required for access to AWS. + +`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for resource management and authentication. + +`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key for cloning Terraform modules. + +`REPO_SSH_DEPLOY_KEY` (optional): SSH key for checking out private repositories. + +## Terraform Core +##### terraform-core.yml + +This workflow is generally not intended to be called directly by users. Instead, it is integrated into broader workflows within a CI/CD pipeline to streamline Terraform setup, configuration, and deployment across multiple directories, with conditional execution, artifact handling, and destruction checks built-in. + + +#### Workflow Inputs + +`environment_name` (optional, default: dev): Specifies the deployment environment (e.g., dev, prd). + +`aws_region` (optional, default: eu-west-2): The AWS region for the deployment. + +`destructive_action_check` (optional, default: false): Boolean to enable a check for destructive actions during planning. + +`execute_terraform_plan` (optional, default: false): Indicates whether to apply the Terraform plan or just create it. + +`terraform_action` (optional, default: apply): Specifies the Terraform action to execute (apply or destroy). + +`directories` (required): List of directories for Terraform stacks. + +`max_parallel` (optional, default: 1): Sets the maximum number of parallel jobs to avoid conflicts. + +`repo` (optional, default: ${{ github.repository }}): Organisation/repository containing the Terraform code. + +`ref` (optional, default: ${{ github.ref }}): Branch or tag for the Terraform code. + +`upload_plan` (optional, default: false): If true, uploads the generated Terraform plan as an artifact. + +`download_existing_plan` (optional, default: false): If true, downloads an existing plan from a previous workflow. + +`runner_label` (optional, default: ubuntu-latest): Specifies the runner for job execution. Use "self-hosted" for custom runners. + +#### Secrets + +`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): AWS credentials for interaction with AWS resources. + +`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for managing resources. + +`GH_TOKEN` (optional): GitHub token for managing resources in GitHub. + +`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key for cloning Terraform modules. + +`REPO_SSH_DEPLOY_KEY` (optional): SSH key for checking out private repositories. + + +## Run post deployment QA checks +##### terraform-post-deployment-qa.yml + +This workflow performs post-deployment quality assurance checks on Terraform-managed resources. It includes options for security, compliance, and threat model scans to ensure that deployed resources meet security and compliance standards. + +#### Workflow Inputs + +`environment_name` (optional, default: dev): Specifies the environment for the QA checks. + +`aws_region` (optional, default: eu-west-2): The AWS region for running checks. + +`runner_label` (optional, default: ubuntu-latest): Specifies the runner for executing checks. Use "self-hosted" for custom runners. + +`directories` (required): List of directories for Terraform stacks to analyse. + +`max_parallel` (optional, default: 5): Sets the maximum number of parallel jobs. + +`zap_api_scan` (optional, default: false): Boolean to enable OWASP ZAP API scans. + +`zap_endpoint_scan` (optional, default: false): Boolean to enable OWASP ZAP endpoint scans. + +`prowler_scan` (optional, default: true): Enables a Prowler scan for AWS environment security checks. + +`create_threat_model` (optional, default: true): Option to generate a threat model for the deployment. + +#### Secrets + +`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): AWS credentials for accessing and scanning resources. + +`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for resource management and authentication. \ No newline at end of file From 26e38e594d3e003141cb9b563867d748eb733f1a Mon Sep 17 00:00:00 2001 From: Sam Ainsworth Date: Tue, 5 Nov 2024 14:03:16 +0000 Subject: [PATCH 4/5] DOE-282 Update READMEs --- .github/workflows/README.md | 7 +-- .../workflows/docs/terraform-code-check.md | 24 +++++++++++ .github/workflows/docs/terraform-core.md | 43 +++++++++++++++++++ .github/workflows/docs/terraform-destroy.md | 28 ++++++++++++ .../workflows/docs/terraform-plan-apply.md | 32 ++++++++++++++ 5 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/docs/terraform-code-check.md create mode 100644 .github/workflows/docs/terraform-core.md create mode 100644 .github/workflows/docs/terraform-destroy.md create mode 100644 .github/workflows/docs/terraform-plan-apply.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md index dbd01f7..8146e74 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -4,6 +4,7 @@ This repository contains several reusable GitHub Actions workflows. Below is an ## Workflows -1. [Local Development Workflow](docs/terraform-local-workflow.md) -2. [Remote Light Check Workflow](docs/terraform-code-check.md) -3. [Remote Full Check Workflow](docs/terraform-plan-apply.md) \ No newline at end of file +1. [Terraform Code Check](docs/terraform-code-check.md) +2. [Terraform Plan Apply](docs/terraform-plan-apply.md) +3. [Terraform Destroy](docs/terraform-destroy.md) +4. [Terraform Core](docs/terraform-core.md) \ No newline at end of file diff --git a/.github/workflows/docs/terraform-code-check.md b/.github/workflows/docs/terraform-code-check.md new file mode 100644 index 0000000..0e944df --- /dev/null +++ b/.github/workflows/docs/terraform-code-check.md @@ -0,0 +1,24 @@ +## Terraform Code Check +##### terraform-code-check.yml + +This workflow performs a series of checks on Terraform code, ensuring it meets formatting and security standards. It supports various configuration options, allowing users to tailor the workflow to their specific Terraform setup. The workflow includes linting, formatting, and security scanning steps, providing detailed status feedback. + +#### Workflow Inputs + +`stack_root_directory` (required): Specifies the root directory of the Terraform stack for code checks. + +`tflint_module_scan` (optional): Boolean value indicating whether TFLint should download Terraform modules during the linting process. + +`checkov_deep_scan` (optional, default: false): Boolean value determining if Checkov should perform a deep scan, which includes external modules. + +`repo` (optional, default: ${{ github.repository }}): The organisation/repository name of the repository containing Terraform code. Typically left blank to clone the calling repository. + +`ref` (optional, default: ${{ github.ref }}): Specifies the branch or tag of the Terraform code to check. By default, it uses the branch of the calling workflow. + +`runner_label` (optional, default: ubuntu-latest): Defines the runner environment for executing the workflow. Use "self-hosted" if a custom runner is configured. + +#### Secrets + +`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key for cloning Terraform modules during terraform init. + +`REPO_SSH_DEPLOY_KEY` (optional): SSH key for cloning the repository with Terraform stacks. \ No newline at end of file diff --git a/.github/workflows/docs/terraform-core.md b/.github/workflows/docs/terraform-core.md new file mode 100644 index 0000000..407c5b5 --- /dev/null +++ b/.github/workflows/docs/terraform-core.md @@ -0,0 +1,43 @@ +## Terraform Core +##### terraform-core.yml + +This workflow is generally not intended to be called directly by users. Instead, it is integrated into broader workflows within a CI/CD pipeline to streamline Terraform setup, configuration, and deployment across multiple directories, with conditional execution, artifact handling, and destruction checks built-in. + + +#### Workflow Inputs + +`environment_name` (optional, default: dev): Specifies the deployment environment (e.g., dev, prd). + +`aws_region` (optional, default: eu-west-2): The AWS region for the deployment. + +`destructive_action_check` (optional, default: false): Boolean to enable a check for destructive actions during planning. + +`execute_terraform_plan` (optional, default: false): Indicates whether to apply the Terraform plan or just create it. + +`terraform_action` (optional, default: apply): Specifies the Terraform action to execute (apply or destroy). + +`directories` (required): List of directories for Terraform stacks. + +`max_parallel` (optional, default: 1): Sets the maximum number of parallel jobs to avoid conflicts. + +`repo` (optional, default: ${{ github.repository }}): Organisation/repository containing the Terraform code. + +`ref` (optional, default: ${{ github.ref }}): Branch or tag for the Terraform code. + +`upload_plan` (optional, default: false): If true, uploads the generated Terraform plan as an artifact. + +`download_existing_plan` (optional, default: false): If true, downloads an existing plan from a previous workflow. + +`runner_label` (optional, default: ubuntu-latest): Specifies the runner for job execution. Use "self-hosted" for custom runners. + +#### Secrets + +`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): AWS credentials for interaction with AWS resources. + +`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for managing resources. + +`GH_TOKEN` (optional): GitHub token for managing resources in GitHub. + +`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key for cloning Terraform modules. + +`REPO_SSH_DEPLOY_KEY` (optional): SSH key for checking out private repositories. \ No newline at end of file diff --git a/.github/workflows/docs/terraform-destroy.md b/.github/workflows/docs/terraform-destroy.md new file mode 100644 index 0000000..57a4397 --- /dev/null +++ b/.github/workflows/docs/terraform-destroy.md @@ -0,0 +1,28 @@ +## Terraform Destroy +##### terrraform-destroy.yml + +This workflow facilitates the destruction of Terraform-managed resources, designed to run as part of a continuous integration pipeline. It provides configuration options for the environment and AWS region, with additional controls for executing the Terraform plan. This workflow executes a reverse-ordered destruction process for Terraform stacks, utilising defined directory dependencies for resource teardown. + +#### Workflow Inputs + +`environment_name` (optional, default: dev): Specifies the environment in which to destroy resources (e.g., dev, prd). + +`aws_region` (optional, default: eu-west-2): AWS region for executing the destruction process. + +`execute_terraform_plan` (optional, default: false): Boolean to determine whether to perform a Terraform plan execution before destroy. + +`repo` (optional, default: ${{ github.repository }}): Organisation/repository name containing the Terraform code. + +`ref` (optional, default: ${{ github.ref }}): Branch or tag of the Terraform code. Defaults to the calling workflow’s branch. + +`runner_label` (optional, default: ubuntu-latest): Target runner environment. Use "self-hosted" for custom runners, or leave as default. + +#### Secrets + +`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): Credentials for AWS account access. Not compatiable with Azure credentials. + +`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for authentication and resource management. Not compatiable with AWS credentials. + +`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key to clone Terraform modules as part of initialisation. + +`REPO_SSH_DEPLOY_KEY` (optional): SSH key to checkout private repositories with remote Terraform configurations. \ No newline at end of file diff --git a/.github/workflows/docs/terraform-plan-apply.md b/.github/workflows/docs/terraform-plan-apply.md new file mode 100644 index 0000000..9f9075f --- /dev/null +++ b/.github/workflows/docs/terraform-plan-apply.md @@ -0,0 +1,32 @@ +## Terraform Plan Apply +##### terraform-plan-apply.yml + +This workflow handles the deployment of Terraform stacks, with various checks and approvals integrated into the process. It includes options for region, environment, and other deployment settings, designed to ensure safe and organised stack management within a CI/CD pipeline. This workflow includes dependency sorting for stack deployment, code linting, a Terraform plan step with optional manual approval, and post-deployment quality checks. + +#### Workflow Inputs + +`environment_name` (optional, default: dev): Specifies the target environment for the deployment (e.g., dev, prd). + +`aws_region` (optional, default: eu-west-2): AWS region for deployment. + +`destructive_action_check` (optional, default: false): Boolean to prevent unintentional destructive actions in Terraform plans. + +`tflint_module_scan` (optional, default: false): Boolean to enable or disable TFLint’s module scan during code checks. + +`execute_terraform_plan` (optional, default: false): Determines if the workflow should execute a Terraform apply after planning. + +`repo` (optional, default: ${{ github.repository }}): The organisation/repository containing the Terraform code. + +`ref` (optional, default: ${{ github.ref }}): Specifies the branch or tag of the Terraform code. + +`runner_label` (optional, default: ubuntu-latest): Sets the runner for executing jobs. Use "self-hosted" if you have a self-hosted runner configured. + +#### Secrets + +`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): Credentials required for access to AWS. + +`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for resource management and authentication. + +`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key for cloning Terraform modules. + +`REPO_SSH_DEPLOY_KEY` (optional): SSH key for checking out private repositories. From 6d6ec6077fc13d7cc7b0d2d8e5a51439ac98456e Mon Sep 17 00:00:00 2001 From: Sam Ainsworth Date: Tue, 5 Nov 2024 14:04:19 +0000 Subject: [PATCH 5/5] Move files around --- .github/workflows/README.md | 10 --- README.md | 170 ++---------------------------------- 2 files changed, 7 insertions(+), 173 deletions(-) delete mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md deleted file mode 100644 index 8146e74..0000000 --- a/.github/workflows/README.md +++ /dev/null @@ -1,10 +0,0 @@ - # GitHub Actions Workflows Documentation - -This repository contains several reusable GitHub Actions workflows. Below is an overview and links to detailed documentation for each workflow. - -## Workflows - -1. [Terraform Code Check](docs/terraform-code-check.md) -2. [Terraform Plan Apply](docs/terraform-plan-apply.md) -3. [Terraform Destroy](docs/terraform-destroy.md) -4. [Terraform Core](docs/terraform-core.md) \ No newline at end of file diff --git a/README.md b/README.md index b69f8ef..8146e74 100644 --- a/README.md +++ b/README.md @@ -1,166 +1,10 @@ -# devops-github-reusable-workflows + # GitHub Actions Workflows Documentation -The devops-github-reusable-workflows repository provides a collection of reusable GitHub Actions workflows designed to simplify and standardise DevOps tasks across projects. +This repository contains several reusable GitHub Actions workflows. Below is an overview and links to detailed documentation for each workflow. -## Check Terraform Code -##### terraform-code-check.yml +## Workflows -This workflow performs a series of checks on Terraform code, ensuring it meets formatting and security standards. It supports various configuration options, allowing users to tailor the workflow to their specific Terraform setup. The workflow includes linting, formatting, and security scanning steps, providing detailed status feedback. - -#### Workflow Inputs - -`stack_root_directory` (required): Specifies the root directory of the Terraform stack for code checks. - -`tflint_module_scan` (optional): Boolean value indicating whether TFLint should download Terraform modules during the linting process. - -`checkov_deep_scan` (optional, default: false): Boolean value determining if Checkov should perform a deep scan, which includes external modules. - -`repo` (optional, default: ${{ github.repository }}): The organisation/repository name of the repository containing Terraform code. Typically left blank to clone the calling repository. - -`ref` (optional, default: ${{ github.ref }}): Specifies the branch or tag of the Terraform code to check. By default, it uses the branch of the calling workflow. - -`runner_label` (optional, default: ubuntu-latest): Defines the runner environment for executing the workflow. Use "self-hosted" if a custom runner is configured. - -#### Secrets - -`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key for cloning Terraform modules during terraform init. - -`REPO_SSH_DEPLOY_KEY` (optional): SSH key for cloning the repository with Terraform stacks. - -## Destroy Terraform stacks -##### terrraform-destroy.yml - -This workflow facilitates the destruction of Terraform-managed resources, designed to run as part of a continuous integration pipeline. It provides configuration options for the environment and AWS region, with additional controls for executing the Terraform plan. This workflow executes a reverse-ordered destruction process for Terraform stacks, utilising defined directory dependencies for resource teardown. - -#### Workflow Inputs - -`environment_name` (optional, default: dev): Specifies the environment in which to destroy resources (e.g., dev, prd). - -`aws_region` (optional, default: eu-west-2): AWS region for executing the destruction process. - -`execute_terraform_plan` (optional, default: false): Boolean to determine whether to perform a Terraform plan execution before destroy. - -`repo` (optional, default: ${{ github.repository }}): Organisation/repository name containing the Terraform code. - -`ref` (optional, default: ${{ github.ref }}): Branch or tag of the Terraform code. Defaults to the calling workflow’s branch. - -`runner_label` (optional, default: ubuntu-latest): Target runner environment. Use "self-hosted" for custom runners, or leave as default. - -#### Secrets - -`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): Credentials for AWS account access. Not compatiable with Azure credentials. - -`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for authentication and resource management. Not compatiable with AWS credentials. - -`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key to clone Terraform modules as part of initialisation. - -`REPO_SSH_DEPLOY_KEY` (optional): SSH key to checkout private repositories with remote Terraform configurations. - -## Deploy Terraform stacks -##### terraform-plan-apply.yml - -This workflow handles the deployment of Terraform stacks, with various checks and approvals integrated into the process. It includes options for region, environment, and other deployment settings, designed to ensure safe and organised stack management within a CI/CD pipeline. This workflow includes dependency sorting for stack deployment, code linting, a Terraform plan step with optional manual approval, and post-deployment quality checks. - -#### Workflow Inputs - -`environment_name` (optional, default: dev): Specifies the target environment for the deployment (e.g., dev, prd). - -`aws_region` (optional, default: eu-west-2): AWS region for deployment. - -`destructive_action_check` (optional, default: false): Boolean to prevent unintentional destructive actions in Terraform plans. - -`tflint_module_scan` (optional, default: false): Boolean to enable or disable TFLint’s module scan during code checks. - -`execute_terraform_plan` (optional, default: false): Determines if the workflow should execute a Terraform apply after planning. - -`repo` (optional, default: ${{ github.repository }}): The organisation/repository containing the Terraform code. - -`ref` (optional, default: ${{ github.ref }}): Specifies the branch or tag of the Terraform code. - -`runner_label` (optional, default: ubuntu-latest): Sets the runner for executing jobs. Use "self-hosted" if you have a self-hosted runner configured. - -#### Secrets - -`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): Credentials required for access to AWS. - -`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for resource management and authentication. - -`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key for cloning Terraform modules. - -`REPO_SSH_DEPLOY_KEY` (optional): SSH key for checking out private repositories. - -## Terraform Core -##### terraform-core.yml - -This workflow is generally not intended to be called directly by users. Instead, it is integrated into broader workflows within a CI/CD pipeline to streamline Terraform setup, configuration, and deployment across multiple directories, with conditional execution, artifact handling, and destruction checks built-in. - - -#### Workflow Inputs - -`environment_name` (optional, default: dev): Specifies the deployment environment (e.g., dev, prd). - -`aws_region` (optional, default: eu-west-2): The AWS region for the deployment. - -`destructive_action_check` (optional, default: false): Boolean to enable a check for destructive actions during planning. - -`execute_terraform_plan` (optional, default: false): Indicates whether to apply the Terraform plan or just create it. - -`terraform_action` (optional, default: apply): Specifies the Terraform action to execute (apply or destroy). - -`directories` (required): List of directories for Terraform stacks. - -`max_parallel` (optional, default: 1): Sets the maximum number of parallel jobs to avoid conflicts. - -`repo` (optional, default: ${{ github.repository }}): Organisation/repository containing the Terraform code. - -`ref` (optional, default: ${{ github.ref }}): Branch or tag for the Terraform code. - -`upload_plan` (optional, default: false): If true, uploads the generated Terraform plan as an artifact. - -`download_existing_plan` (optional, default: false): If true, downloads an existing plan from a previous workflow. - -`runner_label` (optional, default: ubuntu-latest): Specifies the runner for job execution. Use "self-hosted" for custom runners. - -#### Secrets - -`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): AWS credentials for interaction with AWS resources. - -`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for managing resources. - -`GH_TOKEN` (optional): GitHub token for managing resources in GitHub. - -`TF_MODULES_SSH_DEPLOY_KEY` (optional): SSH key for cloning Terraform modules. - -`REPO_SSH_DEPLOY_KEY` (optional): SSH key for checking out private repositories. - - -## Run post deployment QA checks -##### terraform-post-deployment-qa.yml - -This workflow performs post-deployment quality assurance checks on Terraform-managed resources. It includes options for security, compliance, and threat model scans to ensure that deployed resources meet security and compliance standards. - -#### Workflow Inputs - -`environment_name` (optional, default: dev): Specifies the environment for the QA checks. - -`aws_region` (optional, default: eu-west-2): The AWS region for running checks. - -`runner_label` (optional, default: ubuntu-latest): Specifies the runner for executing checks. Use "self-hosted" for custom runners. - -`directories` (required): List of directories for Terraform stacks to analyse. - -`max_parallel` (optional, default: 5): Sets the maximum number of parallel jobs. - -`zap_api_scan` (optional, default: false): Boolean to enable OWASP ZAP API scans. - -`zap_endpoint_scan` (optional, default: false): Boolean to enable OWASP ZAP endpoint scans. - -`prowler_scan` (optional, default: true): Enables a Prowler scan for AWS environment security checks. - -`create_threat_model` (optional, default: true): Option to generate a threat model for the deployment. - -#### Secrets - -`AWS_ROLE_NAME, AWS_ACCOUNT_ID` (optional): AWS credentials for accessing and scanning resources. - -`AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_RESOURCE_GROUP_NAME` (optional): Azure credentials for resource management and authentication. \ No newline at end of file +1. [Terraform Code Check](docs/terraform-code-check.md) +2. [Terraform Plan Apply](docs/terraform-plan-apply.md) +3. [Terraform Destroy](docs/terraform-destroy.md) +4. [Terraform Core](docs/terraform-core.md) \ No newline at end of file