diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 000000000..2907f7f1e --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,61 @@ +# **what?** +# Verifies python build on all code commited to the repository. This workflow +# should not require any secrets since it runs for PRs from forked repos. By +# default, secrets are not passed to workflows running from a forked repos. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to main, and when manually triggered. +name: "Build release" + +on: + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + archive-name: + description: "The name to use for the upload archive, leave blank for no upload" + type: string + default: "" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + archive-name: + description: "The name to use for the upload archive, leave blank for no upload" + type: string + default: "" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}" + cancel-in-progress: true + +jobs: + build: + name: "Build a release" + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Build ${{ github.event.repository.name }}" + uses: dbt-labs/dbt-adapters/.github/actions/build-artifacts@update-workflows + with: + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/changelog-check.yml b/.github/workflows/changelog-check.yml new file mode 100644 index 000000000..dd4b11d76 --- /dev/null +++ b/.github/workflows/changelog-check.yml @@ -0,0 +1,46 @@ +# **what?** +# Checks that a file has been committed under the /.changes directory +# as a new CHANGELOG entry. Cannot check for a specific filename as +# it is dynamically generated by change type and timestamp. +# This workflow should not require any secrets since it runs for PRs +# from forked repos. +# By default, secrets are not passed to workflows running from +# a forked repo. +# +# **why?** +# Ensure code changes are reflected in the CHANGELOG. +# +# **when?** +# This will run for all PRs going into main and *.latest. It will run when: +# - the PR is opened or reopened +# - labels are updated on the PR +# - new code is pushed to the branch +# The action will get skipped if the 'Skip Changelog' label is present. +name: Check Changelog Entry + +on: + pull_request: + types: + - opened + - reopened + - labeled + - unlabeled + - synchronize + workflow_dispatch: + +defaults: + run: + shell: bash + +permissions: + contents: read + pull-requests: write + +jobs: + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: "Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see the [dbt-redshift contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md)." + skip_label: "Skip Changelog" + # this is only acceptable because we own the action we're calling + secrets: inherit diff --git a/.github/workflows/clean-repo.yml b/.github/workflows/clean-repo.yml new file mode 100644 index 000000000..2ce2837b3 --- /dev/null +++ b/.github/workflows/clean-repo.yml @@ -0,0 +1,27 @@ +# **what?** +# Cleanup branches left over from automation and testing. +# Also cleanup draft releases from release testing. +# +# **why?** +# The automations are leaving behind branches and releases that clutter the repository. +# Sometimes we need them to debug processes so we don't want them immediately deleted. +# Running on Saturday to avoid running at the same time as an actual release +# to prevent breaking a release mid-release. +# +# **when?** +# - every Saturday at noon UTC +# - manually +name: "Clean repo" + +on: + schedule: + - cron: '0 12 * * SAT' # noon UTC on Saturday - details in `why` above + workflow_dispatch: + +permissions: + contents: write + +jobs: + cleanup-repo: + uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main + secrets: inherit diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 000000000..800de1a6f --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,86 @@ +# **what?** +# Run code quality checks, e.g. `black`, `flake8`, `mypy`, via `pre-commit` +# +# **why?** +# Ensure code quality meets dbt Labs standards +# +# **when?** +# - runs on all PRs into protected branches +# - runs indirectly during releases and release tests +# - can be manually run +name: "Code quality" + +on: + pull_request: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run code quality on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run code quality on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + code-quality: + name: "Code quality" + runs-on: ubuntu-latest + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + + - name: "Run code quality" + shell: bash + run: hatch run code-quality diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..2873de21b --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,145 @@ +name: "Integration tests" + +on: + pull_request_target: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + require-approval-on-forks: + # if it's a pull request from a fork that has not been approved yet, don't run tests + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository && + !contains(github.event.pull_request.labels.*.name, 'ok to test') + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: "Author requires permission to run tests" + uses: unsplash/comment-on-pr@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + msg: | + "Thanks for your contribution! "\ + "Since this is a fork, integration tests need to be approved prior to running. "\ + "@dbt-labs/core will review this PR and approve running integration tests." + check_for_duplicate_msg: true + + integration-tests: + name: "Integration tests" + needs: require-approval-on-forks + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} + include: + - os: macos-latest + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + - os: windows-latest + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + env: + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} + DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} + DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} + DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: ${{ vars.DD_SITE }} + DD_ENV: ${{ vars.DD_ENV }} + DD_SERVICE: ${{ github.event.repository.name }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ matrix.python-version }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Run integration tests" + shell: bash + run: hatch run integration-tests:all + + aggregate-results: + name: "Successful integration tests" + needs: integration-tests + if: ${{ !failure() && !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: "Integration tests completed successfully" + shell: bash + run: | + title="Integration tests" + message="Integration tests completed successfully!" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml new file mode 100644 index 000000000..cd5099492 --- /dev/null +++ b/.github/workflows/scheduled-tests.yml @@ -0,0 +1,72 @@ +# **what?** +# The purpose of this workflow is to trigger CI to run for each release +# branch on a regular cadence. If the CI workflow fails for a branch, it +# will post to dev-core-alerts to raise awareness. +# +# **why?** +# Ensures release branches are always shippable and not broken. +# Also, can catch any dependencies shifting beneath us that might +# introduce breaking changes (could also impact Cloud). + +# **when?** +# - scheduled with the repository variable DBT_SCHEDULED_TESTS_SCHEDULE +# - manually +name: "Scheduled tests" + +on: + schedule: + - cron: ${{ vars.DBT_SCHEDULED_TESTS_SCHEDULE }} + workflow_dispatch: + +permissions: read-all + +jobs: + code-quality: + name: "Code quality" + uses: dbt-labs/dbt-redshift/.github/workflows/code-quality.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + integration-tests: + name: "Integration tests" + uses: dbt-labs/dbt-redshift/.github/workflows/integration-tests.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + unit-tests: + name: "Unit tests" + uses: dbt-labs/dbt-redshift/.github/workflows/unit-tests.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + verify-build-artifacts: + name: "Unit tests" + uses: dbt-labs/dbt-redshift/.github/workflows/build-artifacts.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + post-failure: + runs-on: ubuntu-latest + needs: test + if: ${{ failure() }} + steps: + - name: Posting scheduled run failures + uses: ravsamhq/notify-slack-action@v2 + if: ${{ github.event_name == 'schedule' }} + with: + notification_title: 'Redshift nightly integration test failed' + status: ${{ job.status }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml new file mode 100644 index 000000000..e2ecd89dd --- /dev/null +++ b/.github/workflows/stale-issues.yml @@ -0,0 +1,13 @@ +name: "Close stale issues and PRs" + +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 000000000..6da8914b6 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,102 @@ +name: "Unit tests" + +on: + pull_request_target: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + unit-tests: + name: "Unit tests" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Run unit tests" + shell: bash + run: hatch run unit-tests:all + + aggregate-results: + name: "Successful unit tests" + needs: unit-tests + if: ${{ !failure() && !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: "Unit tests completed successfully" + shell: bash + run: | + title="Unit tests" + message="Unit tests completed successfully!" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"