From c2da70c77cc8b89b9b5cb21abd766f43e65bc8a8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 20 Apr 2024 11:07:35 -0400 Subject: [PATCH] draft commit --- .bumpversion.cfg | 35 -- .../actions/update-dev-branches/action.yml | 38 ++ .../update-dev-branches/update_dev_branch.sh | 15 + .github/scripts/integration-test-matrix.js | 87 ---- .github/scripts/update_dbt_core_branch.sh | 20 - .github/workflows/backport.yml | 3 +- .github/workflows/bot-changelog.yml | 1 + .github/workflows/build-artifacts.yml | 61 +++ .github/workflows/changelog-check.yml | 46 ++ .github/workflows/clean-repo.yml | 27 ++ .github/workflows/code-quality.yml | 86 ++++ .github/workflows/docs-issues.yml | 0 .github/workflows/integration-tests.yml | 145 ++++++ .github/workflows/integration.yml | 244 ---------- .github/workflows/main.yml | 213 --------- .github/workflows/nightly-release.yml | 116 ----- .github/workflows/publish-pypi-internal.yml | 36 ++ .github/workflows/release-branch-tests.yml | 31 -- .github/workflows/release-internal.yml | 49 -- .github/workflows/release.yml | 418 +++++++++--------- .github/workflows/scheduled-test-releases.yml | 72 +++ .github/workflows/scheduled-tests.yml | 72 +++ .github/workflows/stale-issues.yml | 13 + .github/workflows/triage-labels.yml | 0 .github/workflows/unit-tests.yml | 102 +++++ .github/workflows/version-bump.yml | 28 -- MANIFEST.in | 1 - Makefile | 67 --- dev-requirements.txt | 26 -- pyproject.toml | 137 ++++++ pytest.ini | 9 - setup.py | 87 ---- test.env.example | 11 +- tox.ini | 34 -- 34 files changed, 1065 insertions(+), 1265 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .github/actions/update-dev-branches/action.yml create mode 100755 .github/actions/update-dev-branches/update_dev_branch.sh delete mode 100644 .github/scripts/integration-test-matrix.js delete mode 100755 .github/scripts/update_dbt_core_branch.sh create mode 100644 .github/workflows/build-artifacts.yml create mode 100644 .github/workflows/changelog-check.yml create mode 100644 .github/workflows/clean-repo.yml create mode 100644 .github/workflows/code-quality.yml create mode 100644 .github/workflows/docs-issues.yml create mode 100644 .github/workflows/integration-tests.yml delete mode 100644 .github/workflows/integration.yml delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/nightly-release.yml create mode 100644 .github/workflows/publish-pypi-internal.yml delete mode 100644 .github/workflows/release-branch-tests.yml delete mode 100644 .github/workflows/release-internal.yml create mode 100644 .github/workflows/scheduled-test-releases.yml create mode 100644 .github/workflows/scheduled-tests.yml create mode 100644 .github/workflows/stale-issues.yml create mode 100644 .github/workflows/triage-labels.yml create mode 100644 .github/workflows/unit-tests.yml delete mode 100644 .github/workflows/version-bump.yml delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 dev-requirements.txt create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 setup.py delete mode 100644 tox.ini diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 52cc0acca..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[bumpversion] -current_version = 1.8.0b3 -parse = (?P[\d]+) # major version number - \.(?P[\d]+) # minor version number - \.(?P[\d]+) # patch version number - (?P # optional pre-release - ex: a1, b2, rc25 - (?Pa|b|rc) # pre-release type - (?P[\d]+) # pre-release version number - )? - ( # optional nightly release indicator - \.(?Pdev[0-9]+) # ex: .dev02142023 - )? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0` -serialize = - {major}.{minor}.{patch}{prekind}{num}.{nightly} - {major}.{minor}.{patch}.{nightly} - {major}.{minor}.{patch}{prekind}{num} - {major}.{minor}.{patch} -commit = False -tag = False - -[bumpversion:part:prekind] -first_value = a -optional_value = final -values = - a - b - rc - final - -[bumpversion:part:num] -first_value = 1 - -[bumpversion:part:nightly] - -[bumpversion:file:dbt/adapters/redshift/__version__.py] diff --git a/.github/actions/update-dev-branches/action.yml b/.github/actions/update-dev-branches/action.yml new file mode 100644 index 000000000..a177e69d2 --- /dev/null +++ b/.github/actions/update-dev-branches/action.yml @@ -0,0 +1,38 @@ +name: "Update dev branches" +description: "Update dev branches for testing cross-repo changes" + +inputs: + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + default: "" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + default: "" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + default: "" + +runs: + using: composite + steps: + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo dbt-adapters : ${{ inputs.dbt-adapters-branch }} + echo dbt-common : ${{ inputs.dbt-common-branch }} + echo dbt-core : ${{ inputs.dbt-core-branch }} + + - name: "Update `dbt-adapters` branch" + if: ${{ inputs.dbt-adapters-branch }} + shell: bash + run: ./.github/actions/update-dev-branches/update_dev_branch.sh "dbt-adapters" ${{ inputs.dbt-adapters-branch }} + + - name: "Update `dbt-common` branch" + if: ${{ inputs.dbt-common-branch }} + shell: bash + run: ./.github/actions/update-dev-branches/update_dev_branch.sh "dbt-common" ${{ inputs.dbt-common-branch }} + + - name: "Update `dbt-core` branch" + if: ${{ inputs.dbt-core-branch }} + shell: bash + run: ./.github/actions/update-dev-branches/update_dev_branch.sh "dbt-core" ${{ inputs.dbt-core-branch }} diff --git a/.github/actions/update-dev-branches/update_dev_branch.sh b/.github/actions/update-dev-branches/update_dev_branch.sh new file mode 100755 index 000000000..4d51f9211 --- /dev/null +++ b/.github/actions/update-dev-branches/update_dev_branch.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e +set -e + +package=$1 +branch=$2 + +file="pyproject.toml" +sed_pattern="s|${package}.git@main*|${package}.git@${branch}|g" + +# mac ships with a different version of sed that requires a delimiter arg +if [[ "$OSTYPE" == darwin* ]]; then + sed -i "" "$sed_pattern" $file +else + sed -i "$sed_pattern" $file +fi diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js deleted file mode 100644 index 7db445d9e..000000000 --- a/.github/scripts/integration-test-matrix.js +++ /dev/null @@ -1,87 +0,0 @@ -module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11"]; - const supportedAdapters = ["redshift"]; - - // if PR, generate matrix based on files changed and PR labels - if (context.eventName.includes("pull_request")) { - // `changes` is a list of adapter names that have related - // file changes in the PR - // ex: ['postgres', 'snowflake'] - const labels = context.payload.pull_request.labels.map(({ name }) => name); - console.log("labels", labels); - const testAllLabel = labels.includes("test all"); - const include = []; - - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - if ( - pythonVersion === defaultPythonVersion || - labels.includes(`test python${pythonVersion}`) || - testAllLabel - ) { - // always run tests on ubuntu by default - include.push({ - os: "ubuntu-latest", - adapter, - "python-version": pythonVersion, - }); - - if (labels.includes("test windows") || testAllLabel) { - include.push({ - os: "windows-latest", - adapter, - "python-version": pythonVersion, - }); - } - - if (labels.includes("test macos") || testAllLabel) { - include.push({ - os: "macos-latest", - adapter, - "python-version": pythonVersion, - }); - } - } - } - } - - console.log("matrix", { include }); - - return { - include, - }; - } - // if not PR, generate matrix of python version, adapter, and operating - // system to run integration tests on - - const include = []; - // run for all adapters and python versions on ubuntu - for (const adapter of supportedAdapters) { - for (const pythonVersion of supportedPythonVersions) { - include.push({ - os: 'ubuntu-latest', - adapter: adapter, - "python-version": pythonVersion, - }); - } - } - - // additionally include runs for all adapters, on macos and windows, - // but only for the default python version - for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-latest"]) { - include.push({ - os: operatingSystem, - adapter: adapter, - "python-version": defaultPythonVersion, - }); - } - } - - console.log("matrix", { include }); - - return { - include, - }; -}; diff --git a/.github/scripts/update_dbt_core_branch.sh b/.github/scripts/update_dbt_core_branch.sh deleted file mode 100755 index d28a40c35..000000000 --- a/.github/scripts/update_dbt_core_branch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -set -e - -git_branch=$1 -target_req_file="dev-requirements.txt" -core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g" -postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g" -tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g" -if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$core_req_sed_pattern" $target_req_file - sed -i "" "$postgres_req_sed_pattern" $target_req_file - sed -i "" "$tests_req_sed_pattern" $target_req_file -else - sed -i "$core_req_sed_pattern" $target_req_file - sed -i "$postgres_req_sed_pattern" $target_req_file - sed -i "$tests_req_sed_pattern" $target_req_file -fi -core_version=$(curl "https://raw.githubusercontent.com/dbt-labs/dbt-core/${git_branch}/core/dbt/version.py" | grep "__version__ = *"|cut -d'=' -f2) -bumpversion --allow-dirty --new-version "$core_version" major diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index d91fd5e6f..454ab12c9 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -19,7 +19,8 @@ on: - labeled permissions: - pull-requests: write # to create the PR + + pull-requests: write# to create the PR contents: write # to create the comment if the process fails jobs: diff --git a/.github/workflows/bot-changelog.yml b/.github/workflows/bot-changelog.yml index d7d80b2e8..92d857636 100644 --- a/.github/workflows/bot-changelog.yml +++ b/.github/workflows/bot-changelog.yml @@ -21,6 +21,7 @@ name: "Bot changelog" on: pull_request: + types: - opened - labeled diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 000000000..2907f7f1e --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,61 @@ +# **what?** +# Verifies python build on all code commited to the repository. This workflow +# should not require any secrets since it runs for PRs from forked repos. By +# default, secrets are not passed to workflows running from a forked repos. + +# **why?** +# Ensure code for dbt meets a certain quality standard. + +# **when?** +# This will run for all PRs, when code is pushed to main, and when manually triggered. +name: "Build release" + +on: + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + archive-name: + description: "The name to use for the upload archive, leave blank for no upload" + type: string + default: "" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + archive-name: + description: "The name to use for the upload archive, leave blank for no upload" + type: string + default: "" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ inputs.archive-name }}" + cancel-in-progress: true + +jobs: + build: + name: "Build a release" + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Build ${{ github.event.repository.name }}" + uses: dbt-labs/dbt-adapters/.github/actions/build-artifacts@update-workflows + with: + archive-name: ${{ inputs.archive-name }} diff --git a/.github/workflows/changelog-check.yml b/.github/workflows/changelog-check.yml new file mode 100644 index 000000000..dd4b11d76 --- /dev/null +++ b/.github/workflows/changelog-check.yml @@ -0,0 +1,46 @@ +# **what?** +# Checks that a file has been committed under the /.changes directory +# as a new CHANGELOG entry. Cannot check for a specific filename as +# it is dynamically generated by change type and timestamp. +# This workflow should not require any secrets since it runs for PRs +# from forked repos. +# By default, secrets are not passed to workflows running from +# a forked repo. +# +# **why?** +# Ensure code changes are reflected in the CHANGELOG. +# +# **when?** +# This will run for all PRs going into main and *.latest. It will run when: +# - the PR is opened or reopened +# - labels are updated on the PR +# - new code is pushed to the branch +# The action will get skipped if the 'Skip Changelog' label is present. +name: Check Changelog Entry + +on: + pull_request: + types: + - opened + - reopened + - labeled + - unlabeled + - synchronize + workflow_dispatch: + +defaults: + run: + shell: bash + +permissions: + contents: read + pull-requests: write + +jobs: + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: "Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see the [dbt-redshift contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md)." + skip_label: "Skip Changelog" + # this is only acceptable because we own the action we're calling + secrets: inherit diff --git a/.github/workflows/clean-repo.yml b/.github/workflows/clean-repo.yml new file mode 100644 index 000000000..2ce2837b3 --- /dev/null +++ b/.github/workflows/clean-repo.yml @@ -0,0 +1,27 @@ +# **what?** +# Cleanup branches left over from automation and testing. +# Also cleanup draft releases from release testing. +# +# **why?** +# The automations are leaving behind branches and releases that clutter the repository. +# Sometimes we need them to debug processes so we don't want them immediately deleted. +# Running on Saturday to avoid running at the same time as an actual release +# to prevent breaking a release mid-release. +# +# **when?** +# - every Saturday at noon UTC +# - manually +name: "Clean repo" + +on: + schedule: + - cron: '0 12 * * SAT' # noon UTC on Saturday - details in `why` above + workflow_dispatch: + +permissions: + contents: write + +jobs: + cleanup-repo: + uses: dbt-labs/actions/.github/workflows/repository-cleanup.yml@main + secrets: inherit diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 000000000..800de1a6f --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,86 @@ +# **what?** +# Run code quality checks, e.g. `black`, `flake8`, `mypy`, via `pre-commit` +# +# **why?** +# Ensure code quality meets dbt Labs standards +# +# **when?** +# - runs on all PRs into protected branches +# - runs indirectly during releases and release tests +# - can be manually run +name: "Code quality" + +on: + pull_request: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run code quality on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run code quality on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + code-quality: + name: "Code quality" + runs-on: ubuntu-latest + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + + - name: "Run code quality" + shell: bash + run: hatch run code-quality diff --git a/.github/workflows/docs-issues.yml b/.github/workflows/docs-issues.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..2873de21b --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,145 @@ +name: "Integration tests" + +on: + pull_request_target: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + require-approval-on-forks: + # if it's a pull request from a fork that has not been approved yet, don't run tests + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository && + !contains(github.event.pull_request.labels.*.name, 'ok to test') + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: "Author requires permission to run tests" + uses: unsplash/comment-on-pr@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + msg: | + "Thanks for your contribution! "\ + "Since this is a fork, integration tests need to be approved prior to running. "\ + "@dbt-labs/core will review this PR and approve running integration tests." + check_for_duplicate_msg: true + + integration-tests: + name: "Integration tests" + needs: require-approval-on-forks + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} + include: + - os: macos-latest + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + - os: windows-latest + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + env: + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_PORT: ${{ vars.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_DBNAME: ${{ vars.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_USER: ${{ vars.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + DBT_TEST_USER_1: ${{ vars.REDSHIFT_TEST_USER_1 }} + DBT_TEST_USER_2: ${{ vars.REDSHIFT_TEST_USER_2 }} + DBT_TEST_USER_3: ${{ vars.REDSHIFT_TEST_USER_3 }} + DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: ${{ vars.DD_SITE }} + DD_ENV: ${{ vars.DD_ENV }} + DD_SERVICE: ${{ github.event.repository.name }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ matrix.python-version }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Run integration tests" + shell: bash + run: hatch run integration-tests:all + + aggregate-results: + name: "Successful integration tests" + needs: integration-tests + if: ${{ !failure() && !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: "Integration tests completed successfully" + shell: bash + run: | + title="Integration tests" + message="Integration tests completed successfully!" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 9d1fe0807..000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,244 +0,0 @@ -# **what?** -# This workflow runs all integration tests for supported OS -# and python versions and core adapters. If triggered by PR, -# the workflow will only run tests for adapters related -# to code changes. Use the `test all` and `test ${adapter}` -# label to run all or additional tests. Use `ok to test` -# label to mark PRs from forked repositories that are safe -# to run integration tests for. Requires secrets to run -# against different warehouses. - -# **why?** -# This checks the functionality of dbt from a user's perspective -# and attempts to catch functional regressions. - -# **when?** -# This workflow will run on every push to a protected branch -# and when manually triggered. It will also run for all PRs, including -# PRs from forks. The workflow will be skipped until there is a label -# to mark the PR as safe to run. - -name: Adapter Integration Tests - -on: - # pushes to release branches - push: - branches: - - "main" - - "develop" - - "*.latest" - - "releases/*" - # all PRs, important to note that `pull_request_target` workflows - # will run in the context of the target branch of a PR - pull_request_target: - # manual trigger - workflow_dispatch: - inputs: - dbt-core-branch: - description: "branch of dbt-core to use in dev-requirements.txt" - required: false - type: string - -# explicitly turn off permissions for `GITHUB_TOKEN` -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -# sets default shell to bash, for all operating systems -defaults: - run: - shell: bash - -jobs: - # generate test metadata about what files changed and the testing matrix to use - test-metadata: - # run if not a PR from a forked repository or has a label to mark as safe to test - if: >- - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - runs-on: ubuntu-latest - - outputs: - matrix: ${{ steps.generate-matrix.outputs.result }} - - steps: - - name: Check out the repository (non-PR) - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check if relevant files changed - if: github.event_name == 'pull_request_target' - # https://github.com/marketplace/actions/paths-changes-filter - # For each filter, it sets output variable named by the filter to the text: - # 'true' - if any of changed files matches any of filter rules - # 'false' - if none of changed files matches any of filter rules - # also, returns: - # `changes` - JSON array with names of all filters matching any of the changed files - uses: dorny/paths-filter@v2 - id: get-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - redshift: - - 'dbt/**' - - 'tests/**' - - 'dev-requirements.txt' - - - name: Generate integration test matrix - id: generate-matrix - uses: actions/github-script@v6 - env: - CHANGES: ${{ steps.get-changes.outputs.changes }} - with: - script: | - const script = require('./.github/scripts/integration-test-matrix.js') - const matrix = script({ context }) - console.log(matrix) - return matrix - test: - name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }} - - # run if not a PR from a forked repository or has a label to mark as safe to test - # also checks that the matrix generated is not empty - if: >- - needs.test-metadata.outputs.matrix && - fromJSON( needs.test-metadata.outputs.matrix ).include[0] && - ( - github.event_name != 'pull_request_target' || - github.event.pull_request.head.repo.full_name == github.repository || - contains(github.event.pull_request.labels.*.name, 'ok to test') - ) - runs-on: ${{ matrix.os }} - - needs: test-metadata - - strategy: - fail-fast: false - max-parallel: 3 - matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }} - - env: - TOXENV: integration-${{ matrix.adapter }} - PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" - DBT_INVOCATION_ENV: github-actions - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: datadoghq.com - DD_ENV: ci - DD_SERVICE: ${{ github.event.repository.name }} - - steps: - - name: Check out the repository - if: github.event_name != 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - - # explicity checkout the branch for the PR, - # this is necessary for the `pull_request_target` event - - name: Check out the repository (PR) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v3 - with: - persist-credentials: false - ref: ${{ github.event.pull_request.head.sha }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Update dev_requirements.txt - if: inputs.dbt-core-branch != '' - run: | - pip install bumpversion - ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} - - - name: Run tox (redshift) - if: matrix.adapter == 'redshift' - env: - REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} - REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} - REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} - REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} - REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} - DBT_TEST_USER_1: dbt_test_user_1 - DBT_TEST_USER_2: dbt_test_user_2 - DBT_TEST_USER_3: dbt_test_user_3 - run: tox -- --ddtrace - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: logs - path: ./logs - - - name: Get current date - if: always() - id: date - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv - path: integration_results.csv - - require-label-comment: - runs-on: ubuntu-latest - - needs: test - - permissions: - pull-requests: write - - steps: - - name: Needs permission PR comment - if: >- - needs.test.result == 'skipped' && - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - "You do not have permissions to run integration tests, @dbt-labs/core "\ - "needs to label this PR with `ok to test` in order to run integration tests!" - check_for_duplicate_msg: true - - post-failure: - runs-on: ubuntu-latest - needs: test - if: ${{ failure() }} - - steps: - - name: Posting scheduled run failures - uses: ravsamhq/notify-slack-action@v2 - if: ${{ github.event_name == 'schedule' }} - with: - notification_title: 'Redshift nightly integration test failed' - status: ${{ job.status }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 389f55a88..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,213 +0,0 @@ -# **what?** -# Runs code quality checks, unit tests, and verifies python build on -# all code commited to the repository. This workflow should not -# require any secrets since it runs for PRs from forked repos. -# By default, secrets are not passed to workflows running from -# a forked repo. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to a release -# branch, and when manually triggered. - -name: Tests and Code Checks - -on: - push: - branches: - - "main" - - "develop" - - "*.latest" - - "releases/*" - pull_request: - workflow_dispatch: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - code-quality: - name: code-quality - - runs-on: ubuntu-latest - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install -r dev-requirements.txt - python -m pip --version - pre-commit --version - - - name: pre-commit hooks - run: pre-commit run --all-files --show-diff-on-failure - - unit: - name: unit test / python ${{ matrix.python-version }} - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - - env: - TOXENV: "unit" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install tox - python -m pip --version - tox --version - - - name: Run tox - run: tox - - - name: Get current date - if: always() - id: date - #no colons allowed for artifacts - run: | - echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv - path: unit_results.csv - - build: - name: build packages - - runs-on: ubuntu-latest - - outputs: - is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }} - - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade setuptools wheel twine check-wheel-contents - python -m pip --version - - - name: Build distributions - run: ./scripts/build-dist.sh - - - name: Show distributions - run: ls -lh dist/ - - - name: Check distribution descriptions - run: | - twine check dist/* - - - name: Check wheel contents - run: | - check-wheel-contents dist/*.whl --ignore W007,W008 - - - name: Check if this is an alpha version - id: check-is-alpha - run: | - export is_alpha=0 - if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi - echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist/ - - test-build: - name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} - - if: needs.build.outputs.is_alpha == 0 - - needs: build - - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11'] - - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel - python -m pip --version - - - uses: actions/download-artifact@v3 - with: - name: dist - path: dist/ - - - name: Show distributions - run: ls -lh dist/ - - - name: Install wheel distributions - run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check wheel distributions - run: | - python -c "import dbt.adapters.redshift" - - - name: Install source distributions - run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check source distributions - run: | - python -c "import dbt.adapters.redshift" diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml deleted file mode 100644 index 46db5b749..000000000 --- a/.github/workflows/nightly-release.yml +++ /dev/null @@ -1,116 +0,0 @@ -# **what?** -# Nightly releases to GitHub and PyPI. This workflow produces the following outcome: -# - generate and validate data for night release (commit SHA, version number, release branch); -# - pass data to release workflow; -# - night release will be pushed to GitHub as a draft release; -# - night build will be pushed to test PyPI; -# -# **why?** -# Ensure an automated and tested release process for nightly builds -# -# **when?** -# This workflow runs on schedule or can be run manually on demand. - -name: Nightly Test Release to GitHub and PyPI - -on: - workflow_dispatch: # for manual triggering - schedule: - - cron: 0 9 * * * - -permissions: - contents: write # this is the permission that allows creating a new release - -defaults: - run: - shell: bash - -env: - RELEASE_BRANCH: "1.5.latest" # must test against most recent .latest branch to have parity for dependency with core - -jobs: - aggregate-release-data: - runs-on: ubuntu-latest - - outputs: - commit_sha: ${{ steps.resolve-commit-sha.outputs.release_commit }} - version_number: ${{ steps.nightly-release-version.outputs.number }} - release_branch: ${{ steps.release-branch.outputs.name }} - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}" - uses: actions/checkout@v3 - with: - ref: ${{ env.RELEASE_BRANCH }} - - - name: "Resolve Commit To Release" - id: resolve-commit-sha - run: | - commit_sha=$(git rev-parse HEAD) - echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT - - - name: "Get Current Version Number" - id: version-number-sources - run: | - current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '` - echo "current_version=$current_version" >> $GITHUB_OUTPUT - - - name: "Audit Version And Parse Into Parts" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ steps.version-number-sources.outputs.current_version }} - - - name: "Get Current Date" - id: current-date - run: echo "date=$(date +'%m%d%Y')" >> $GITHUB_OUTPUT - - # Bump to the next patch because when this is a previously released patch, the changelog - # markdown will already exist and cause a failure in another step - - name: "Bump Patch Number" - id: bump_patch - run: | - echo "patch=$((${{ steps.semver.outputs.patch }}+1))" >> $GITHUB_OUTPUT - - - name: "Generate Nightly Release Version Number" - id: nightly-release-version - run: | - number="${{ steps.semver.outputs.major }}.${{ steps.semver.outputs.minor }}.${{ steps.bump_patch.outputs.patch }}.dev${{ steps.current-date.outputs.date }}" - echo "number=$number" >> $GITHUB_OUTPUT - - - name: "Audit Nightly Release Version And Parse Into Parts" - uses: dbt-labs/actions/parse-semver@v1.1.0 - with: - version: ${{ steps.nightly-release-version.outputs.number }} - - - name: "Set Release Branch" - id: release-branch - run: | - echo "name=${{ env.RELEASE_BRANCH }}" >> $GITHUB_OUTPUT - - log-outputs-aggregate-release-data: - runs-on: ubuntu-latest - needs: [aggregate-release-data] - - steps: - - name: "[DEBUG] Log Outputs" - run: | - echo commit_sha : ${{ needs.aggregate-release-data.outputs.commit_sha }} - echo version_number: ${{ needs.aggregate-release-data.outputs.version_number }} - echo release_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} - - release-github-pypi: - needs: [aggregate-release-data] - - uses: ./.github/workflows/release.yml - with: - sha: ${{ needs.aggregate-release-data.outputs.commit_sha }} - target_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} - version_number: ${{ needs.aggregate-release-data.outputs.version_number }} - build_script_path: "scripts/build-dist.sh" - env_setup_script_path: "scripts/env-setup.sh" - s3_bucket_name: "core-team-artifacts" - package_test_command: "dbt -h" - test_run: true - nightly_release: true - secrets: inherit diff --git a/.github/workflows/publish-pypi-internal.yml b/.github/workflows/publish-pypi-internal.yml new file mode 100644 index 000000000..8bcad76eb --- /dev/null +++ b/.github/workflows/publish-pypi-internal.yml @@ -0,0 +1,36 @@ +# What? +# Tag and release an arbitrary ref. Uploads to an internal archive. +# +# How? +# After checking out and testing the provided ref, the image is built and uploaded. +# +# When? +# - manually +name: "Publish to internal PyPI" + +on: + workflow_dispatch: + inputs: + version: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + ref: + description: "The ref (sha or branch name) to use" + type: string + required: true + test-command: + description: "Package test command" + type: string + default: "python -c \"import dbt.adapters.redshift\"" + +jobs: + build-and-publish: + name: "Build and release to internal PyPI" + uses: dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main + with: + version_number: "${{ inputs.version }}" + package_test_command: "${{ inputs.test-command }}" + dbms_name: "redshift" + ref: ${{ inputs.ref }} + secrets: inherit diff --git a/.github/workflows/release-branch-tests.yml b/.github/workflows/release-branch-tests.yml deleted file mode 100644 index 004c6fb29..000000000 --- a/.github/workflows/release-branch-tests.yml +++ /dev/null @@ -1,31 +0,0 @@ -# **what?** -# The purpose of this workflow is to trigger CI to run for each release -# branch on a regular cadence. If the CI workflow fails for a branch, it -# will post to dev-core-alerts to raise awareness. The - -# **why?** -# Ensures release branches are always shippable and not broken. -# Also, can catch any dependencies shifting beneath us that might -# introduce breaking changes (could also impact Cloud). - -# **when?** -# Once each morning. Manual trigger can also test on demand - -name: Release branch scheduled testing - -on: - # run this once per night to ensure no regressions from latest dbt-core changes - schedule: - - cron: '0 5 * * *' # 5 UTC - - workflow_dispatch: # for manual triggering - -# no special access is needed -permissions: read-all - -jobs: - run_tests: - uses: dbt-labs/actions/.github/workflows/release-branch-tests.yml@main - with: - workflows_to_run: '["main.yml", "integration.yml"]' - secrets: inherit diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml deleted file mode 100644 index 0ce0f9adf..000000000 --- a/.github/workflows/release-internal.yml +++ /dev/null @@ -1,49 +0,0 @@ -# What? -# -# Tag and release an arbitrary ref. Uploads to an internal archive for further processing. -# -# How? -# -# After checking out and testing the provided ref, the image is built and uploaded. -# -# When? -# -# Manual trigger. - -name: "Release internal patch" - -on: - workflow_dispatch: - inputs: - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - ref: - description: "The ref (sha or branch name) to use" - type: string - default: "main" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - required: true - -defaults: - run: - shell: "bash" - -jobs: - invoke-reusable-workflow: - name: "Build and Release Internally" - - uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" - - with: - version_number: "${{ inputs.version_number }}" - package_test_command: "${{ inputs.package_test_command }}" - dbms_name: "redshift" - ref: "${{ inputs.ref }}" - - secrets: "inherit" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f68eb3320..bdee63147 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,216 +13,220 @@ # # **when?** # This workflow can be run manually on demand or can be called by other workflows -name: Release to GitHub and PyPI +name: "Release pipeline" +run-name: "Release `${{ inputs.version }}` to `${{ inputs.deploy-environment }}`" on: - workflow_dispatch: - inputs: - sha: - description: "The last commit sha in the release" - type: string - required: true - target_branch: - description: "The branch to release from" - type: string - required: true - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - build_script_path: - description: "Build script path" - type: string - default: "scripts/build-dist.sh" - required: true - env_setup_script_path: - description: "Environment setup script path" - type: string - default: "scripts/env-setup.sh" - required: false - s3_bucket_name: - description: "AWS S3 bucket name" - type: string - default: "core-team-artifacts" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - required: true - test_run: - description: "Test run (Publish release as draft)" - type: boolean - default: true - required: false - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - workflow_call: - inputs: - sha: - description: "The last commit sha in the release" - type: string - required: true - target_branch: - description: "The branch to release from" - type: string - required: true - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - build_script_path: - description: "Build script path" - type: string - default: "scripts/build-dist.sh" - required: true - env_setup_script_path: - description: "Environment setup script path" - type: string - default: "scripts/env-setup.sh" - required: false - s3_bucket_name: - description: "AWS S3 bucket name" - type: string - default: "core-team-artifacts" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.redshift\"" - required: true - test_run: - description: "Test run (Publish release as draft)" - type: boolean - default: true - required: false - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - + workflow_call: + inputs: + sha: + description: "The commit sha to release" + type: string + required: true + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The release version (e.g. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "Where to publish" + type: string + default: "prod" + publish-slack-override: + description: "Use to publish a Slack notification for non-prod deploy environments" + type: boolean + default: false + workflow_dispatch: + inputs: + sha: + description: "The commit sha to release" + type: string + required: true + branch: + description: "The branch to release from" + type: string + default: "main" + version: + description: "The release version (e.g. 1.0.0b1)" + type: string + required: true + deploy-environment: + description: "Where to publish" + type: environment + publish-slack-override: + description: "Use to publish a Slack notification for non-prod deploy environments" + type: boolean + default: false + +# this is the permission that allows creating a new release to both GitHub and PyPI permissions: - contents: write # this is the permission that allows creating a new release + contents: write + id-token: write -defaults: - run: - shell: bash +# deploying the same version of a package to the same environment should override any previous deployment with those attributes +concurrency: + group: "${{ github.workflow }}-${{ inputs.version }}-${{ inputs.deploy-environment }}" + cancel-in-progress: true jobs: - log-inputs: - name: Log Inputs - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The branch to release from: ${{ inputs.target_branch }} - echo The release version number: ${{ inputs.version_number }} - echo Build script path: ${{ inputs.build_script_path }} - echo Environment setup script path: ${{ inputs.env_setup_script_path }} - echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} - echo Package test command: ${{ inputs.package_test_command }} - echo Test run: ${{ inputs.test_run }} - echo Nightly release: ${{ inputs.nightly_release }} - - bump-version-generate-changelog: - name: Bump package version, Generate changelog - - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main - - with: - sha: ${{ inputs.sha }} - version_number: ${{ inputs.version_number }} - target_branch: ${{ inputs.target_branch }} - env_setup_script_path: ${{ inputs.env_setup_script_path }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: inherit - - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog] - - runs-on: ubuntu-latest - - steps: - - name: Print variables - run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-test-package: - name: Build, Test, Package - if: ${{ !failure() && !cancelled() }} - needs: [bump-version-generate-changelog] - - uses: dbt-labs/dbt-release/.github/workflows/build.yml@main - - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - build_script_path: ${{ inputs.build_script_path }} - s3_bucket_name: ${{ inputs.s3_bucket_name }} - package_test_command: ${{ inputs.package_test_command }} - test_run: ${{ inputs.test_run }} - nightly_release: ${{ inputs.nightly_release }} - - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - github-release: - name: GitHub Release - if: ${{ !failure() && !cancelled() }} - - needs: [bump-version-generate-changelog, build-test-package] - - uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main - - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.test_run }} - - pypi-release: - name: PyPI Release - - needs: [github-release] - - uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main - - with: - version_number: ${{ inputs.version_number }} - test_run: ${{ inputs.test_run }} - - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - - slack-notification: - name: Slack Notification - if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} - - needs: - [ - bump-version-generate-changelog, - build-test-package, - github-release, - pypi-release, - ] - - uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main - with: - status: "failure" - - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} + release-inputs: + name: "Release inputs" + runs-on: ubuntu-latest + outputs: + archive-name: ${{ steps.computed-inputs.outputs.archive-name }} + steps: + - name: "Set: archive name" + id: archive + shell: bash + run: | + archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-environment }} + echo "name=$archive_name" >> $GITHUB_OUTPUT + + - name: "[DEBUG] Inputs" + shell: bash + run: | + echo branch : ${{ inputs.branch }} + echo package : ${{ inputs.package }} + echo version : ${{ inputs.version }} + echo deploy-environment : ${{ inputs.deploy-environment }} + echo archive-name : ${{ steps.archive.outputs.name }} + echo dbt-adapters-branch : ${{ inputs.dbt-adapters-branch }} + echo dbt-common-branch : ${{ inputs.dbt-common-branch }} + echo dbt-core-branch : ${{ inputs.dbt-core-branch }} + + release-branch: + name: "Create a release branch" + needs: release-inputs + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-create.yml@update-workflows + with: + branch: ${{ inputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + version-bump: + name: "Bump the version" + runs-on: ubuntu-latest + needs: release-branch + steps: + - name: "Check out ${{ github.event.repository.name }}@${{ inputs.branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.release-branch.outputs.branch }} + + - name: "Bump version to `${{ inputs.version }}`" + shell: bash + run: hatch version ${{ inputs.version }} + + - name: "Commit and push changes" + shell: bash + run: commit + + changelog: + name: "Generate a new changelog" + needs: release-branch + uses: dbt-labs/dbt-adapters/.github/workflows/build-changelog.yml@update-workflows + with: + branch: ${{ needs.release-branch.outputs.branch }} + version: ${{ inputs.version }} + secrets: inherit + + code-quality: + name: "Run code quality" + needs: + - release-branch + - version-bump + - changelog + uses: ./.github/workflows/code-quality.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + + unit-tests: + name: "Run unit tests" + needs: + - release-branch + - version-bump + - changelog + uses: ./.github/workflows/unit-tests.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + + integration-tests: + name: "Run unit tests" + needs: + - release-branch + - version-bump + - changelog + uses: ./.github/workflows/integration-tests.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + + build-artifacts: + name: "Build artifacts" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - release-branch + - code-quality + - unit-tests + - integration-tests + uses: ./.github/workflows/build-artifacts.yml + with: + branch: ${{ needs.release-branch.outputs.branch }} + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + + merge-changes: + name: "Merge the version bump and changelog updates" + needs: + - build-artifacts + - release-branch + if: >- + !failure() && !cancelled() && + inputs.deploy-environment == 'prod' && + needs.release-branch.outputs.name + uses: dbt-labs/dbt-adapters/.github/workflows/release-branch-merge.yml@update-workflows + with: + branch: ${{ inputs.branch }} + release-branch: ${{ needs.release-branch.outputs.branch }} + secrets: inherit + + publish-github: + name: "Publish to GitHub" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - changelog + - merge-changes + uses: dbt-labs/dbt-adapters/.github/workflows/publish-github.yml@update-workflows + with: + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-environment: ${{ inputs.deploy-environment }} + sha: ${{ needs.merge-changes.outputs.sha }} + changelog-path: ${{ needs.changelog.outputs.changelog-path }} + + publish-pypi: + name: "Publish to PyPI" + if: ${{ !failure() && !cancelled() }} + needs: + - release-inputs + - merge-changes + uses: dbt-labs/dbt-adapters/.github/workflows/publish-pypi.yml@update-workflows + with: + archive-name: ${{ needs.release-inputs.outputs.archive-name }} + version: ${{ inputs.version }} + deploy-environment: ${{ inputs.deploy-environment }} + + slack-notification: + name: "Slack notification" + if: ${{ failure() && (inputs.deploy-environment == 'prod' || inputs.publish-slack-override) }} + needs: + - github-release + - pypi-release + uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main + with: + status: "failure" + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/scheduled-test-releases.yml b/.github/workflows/scheduled-test-releases.yml new file mode 100644 index 000000000..1537d7b42 --- /dev/null +++ b/.github/workflows/scheduled-test-releases.yml @@ -0,0 +1,72 @@ +# **what?** +# Nightly releases to GitHub and PyPI. This workflow produces the following outcome: +# - generate and validate data for night release (commit SHA, version number, release branch); +# - pass data to release workflow; +# - night release will be pushed to GitHub as a draft release; +# - night build will be pushed to test PyPI; +# +# **why?** +# Ensure an automated and tested release process for nightly builds +# +# **when?** +# - scheduled with the repository variable DBT_SCHEDULED_TEST_RELEASES_SCHEDULE +# - manually +name: "Scheduled test releases" + +on: + workflow_dispatch: # for manual triggering + schedule: + - cron: ${{ vars.DBT_SCHEDULED_TEST_RELEASES_SCHEDULE }} + +permissions: + contents: write # this is the permission that allows creating a new release + +defaults: + run: + shell: bash + +jobs: + release-inputs: + name: "Get test release inputs" + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.commit.outputs.sha }} + version: ${{ steps.release-version.outputs.version }} + + steps: + - name: "Checkout ${{ github.repository }}" + uses: actions/checkout@v3 + + - name: "Set: commit" + id: commit + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: "Set: version" + id: current-version + run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT + + - name: "Audit Version And Parse Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ steps.current-version.outputs.version }} + + - name: "Set: release version" + id: release-version + run: | + version="${{ steps.semver.outputs.major }}" + version+=".${{ steps.semver.outputs.minor }}" + version+=".$(${{ steps.semver.outputs.patch }}+1)" + version+="dev$(date +'%m%d%Y')" + echo "version=$version" >> $GITHUB_OUTPUT + + release: + name: "Release test release" + needs: release-inputs + uses: ./.github/workflows/release.yml + with: + sha: ${{ needs.release-inputs.outputs.sha }} + version: ${{ needs.release-inputs.outputs.version }} + branch: "main" + deploy-environment: "test" + publish-slack-override: true diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml new file mode 100644 index 000000000..cd5099492 --- /dev/null +++ b/.github/workflows/scheduled-tests.yml @@ -0,0 +1,72 @@ +# **what?** +# The purpose of this workflow is to trigger CI to run for each release +# branch on a regular cadence. If the CI workflow fails for a branch, it +# will post to dev-core-alerts to raise awareness. +# +# **why?** +# Ensures release branches are always shippable and not broken. +# Also, can catch any dependencies shifting beneath us that might +# introduce breaking changes (could also impact Cloud). + +# **when?** +# - scheduled with the repository variable DBT_SCHEDULED_TESTS_SCHEDULE +# - manually +name: "Scheduled tests" + +on: + schedule: + - cron: ${{ vars.DBT_SCHEDULED_TESTS_SCHEDULE }} + workflow_dispatch: + +permissions: read-all + +jobs: + code-quality: + name: "Code quality" + uses: dbt-labs/dbt-redshift/.github/workflows/code-quality.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + integration-tests: + name: "Integration tests" + uses: dbt-labs/dbt-redshift/.github/workflows/integration-tests.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + unit-tests: + name: "Unit tests" + uses: dbt-labs/dbt-redshift/.github/workflows/unit-tests.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + verify-build-artifacts: + name: "Unit tests" + uses: dbt-labs/dbt-redshift/.github/workflows/build-artifacts.yml + strategy: + matrix: + branch: ${{ fromJSON(vars.DBT_SCHEDULED_TESTS_BRANCHES) }} + with: + branch: ${{ matrix.branch }} + + post-failure: + runs-on: ubuntu-latest + needs: test + if: ${{ failure() }} + steps: + - name: Posting scheduled run failures + uses: ravsamhq/notify-slack-action@v2 + if: ${{ github.event_name == 'schedule' }} + with: + notification_title: 'Redshift nightly integration test failed' + status: ${{ job.status }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml new file mode 100644 index 000000000..e2ecd89dd --- /dev/null +++ b/.github/workflows/stale-issues.yml @@ -0,0 +1,13 @@ +name: "Close stale issues and PRs" + +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/triage-labels.yml b/.github/workflows/triage-labels.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 000000000..6da8914b6 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,102 @@ +name: "Unit tests" + +on: + pull_request_target: + push: + branches: + - "main" + - "*.latest" + workflow_call: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + workflow_dispatch: + inputs: + branch: + description: "The branch/tag to run integration tests on" + type: string + default: "main" + dbt-adapters-branch: + description: "The branch/tag of `dbt-adapters` to use" + type: string + default: "main" + dbt-common-branch: + description: "The branch/tag of `dbt-common` to use" + type: string + default: "main" + dbt-core-branch: + description: "The branch/tag of `dbt-core` to use" + type: string + default: "main" + +permissions: read-all + +concurrency: + group: "${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}" + cancel-in-progress: true + +jobs: + unit-tests: + name: "Unit tests" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(vars.DBT_TEST_PYTHON_ALL_VERSIONS) }} + steps: + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (non-PR)" + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + persist-credentials: false + + - name: "Check out ${{ github.repository }}@${{ inputs.branch }} (PR)" + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Setup environment" + uses: dbt-labs/dbt-adapters/.github/actions/setup-environment@update-workflows + with: + python-version: ${{ vars.DBT_TEST_PYTHON_VERSION }} + + - name: "Update development branches" + if: ${{ contains(github.event_name, 'workflow_') }} + uses: ./.github/actions/update-dev-branches + with: + dbt-adapters-branch: ${{ inputs.dbt-adapters-branch }} + dbt-common-branch: ${{ inputs.dbt-common-branch }} + dbt-core-branch: ${{ inputs.dbt-core-branch }} + + - name: "Run unit tests" + shell: bash + run: hatch run unit-tests:all + + aggregate-results: + name: "Successful unit tests" + needs: unit-tests + if: ${{ !failure() && !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: "Unit tests completed successfully" + shell: bash + run: | + title="Unit tests" + message="Unit tests completed successfully!" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml deleted file mode 100644 index bde34d683..000000000 --- a/.github/workflows/version-bump.yml +++ /dev/null @@ -1,28 +0,0 @@ -# **what?** -# This workflow will take the new version number to bump to. With that -# it will run versionbump to update the version number everywhere in the -# code base and then run changie to create the corresponding changelog. -# A PR will be created with the changes that can be reviewed before committing. - -# **why?** -# This is to aid in releasing dbt and making sure we have updated -# the version in all places and generated the changelog. - -# **when?** -# This is triggered manually - -name: Version Bump - -on: - workflow_dispatch: - inputs: - version_number: - description: 'The version number to bump to (ex. 1.2.0, 1.3.0b1)' - required: true - -jobs: - version_bump_and_changie: - uses: dbt-labs/actions/.github/workflows/version-bump.yml@main - with: - version_number: ${{ inputs.version_number }} - secrets: inherit # ok since what we are calling is internally maintained diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index cfbc714ed..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -recursive-include dbt/include *.sql *.yml *.md diff --git a/Makefile b/Makefile deleted file mode 100644 index efd23b806..000000000 --- a/Makefile +++ /dev/null @@ -1,67 +0,0 @@ -.DEFAULT_GOAL:=help - -.PHONY: dev -dev: ## Installs adapter in develop mode along with development dependencies - @\ - pip install -e . -r dev-requirements.txt && pre-commit install - -.PHONY: dev-uninstall -dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment - ## Useful when updating versions, or if you accidentally installed into the system interpreter - pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y - pip uninstall -y dbt-redshift - -.PHONY: mypy -mypy: ## Runs mypy against staged changes for static type checking. - @\ - pre-commit run --hook-stage manual mypy-check | grep -v "INFO" - -.PHONY: flake8 -flake8: ## Runs flake8 against staged changes to enforce style guide. - @\ - pre-commit run --hook-stage manual flake8-check | grep -v "INFO" - -.PHONY: black -black: ## Runs black against staged changes to enforce style guide. - @\ - pre-commit run --hook-stage manual black-check -v | grep -v "INFO" - -.PHONY: lint -lint: ## Runs flake8 and mypy code checks against staged changes. - @\ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run mypy-check --hook-stage manual | grep -v "INFO" - -.PHONY: linecheck -linecheck: ## Checks for all Python lines 100 characters or more - @\ - find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \; - -.PHONY: unit -unit: ## Runs unit tests with py38. - @\ - tox -e py38 - -.PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. - @\ - tox -p -e py38; \ - pre-commit run black-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ - pre-commit run mypy-check --hook-stage manual | grep -v "INFO" - -.PHONY: integration -integration: ## Runs redshift integration tests with py38. - @\ - tox -e py38-redshift -- - -.PHONY: clean - @echo "cleaning repo" - @git clean -f -X - -.PHONY: help -help: ## Show this help message. - @echo 'usage: make [target]' - @echo - @echo 'targets:' - @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index d02863ae0..000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -# install latest changes in dbt-core + dbt-postgres -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core -git+https://github.com/dbt-labs/dbt-postgres.git - -# dev -ipdb~=0.13.13 -pre-commit==3.7.0;python_version >="3.9" -pre-commit==3.5.0;python_version <"3.9" - -# test -ddtrace==2.3.0 -freezegun~=1.3 -pytest~=7.4 -pytest-csv~=3.0 -pytest-dotenv~=0.5.2 -pytest-logbook~=1.2 -pytest-xdist~=3.5 -tox~=4.11 - -# build -bumpversion~=0.6.0 -twine~=4.0 -wheel~=0.42 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..7a6fcca54 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,137 @@ +[project] +dynamic = ["version"] +name = "dbt-redshift" +description = "The Redshift adapter plugin for dbt" +readme = "README.md" +keywords = ["dbt", "redshift", "dbt-redshift", "adapter", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs"] +requires-python = ">=3.8.0" +authors = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +maintainers = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +dependencies = [ + "dbt-adapters>=0.1.0a1,<2.0", + "dbt-common>=0.1.0a1,<2.0", + # `dbt-redshift` depends deeply on `redshift-connector` + # This package has experienced breaking changes in previous patch releases + # Pin to the patch version and bump version cautiously + "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", + "dbt-postgres>=1.8.0a1,<1.9", + "sqlparse>=0.5.0,<0.6.0", + "agate>=1.0,<2.0", + # Add `dbt-core` to ensure backwards compatibility of installation + # This is not a functional dependency + "dbt-core>=1.8.0b3", +] + +[project.urls] +Homepage = "https://github.com/dbt-labs/dbt-redshift" +Documentation = "https://docs.getdbt.com" +Repository = "https://github.com/dbt-labs/dbt-redshift.git" +Issues = "https://github.com/dbt-labs/dbt-redshift/issues" +Changelog = "https://github.com/dbt-labs/dbt-redshift/blob/main/CHANGELOG.md" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.sdist] +include = ["dbt"] + +[tool.hatch.build.targets.wheel] +packages = ["dbt"] + +[tool.hatch.version] +path = "dbt/adapters/redshift/__version__.py" + +[tool.hatch.envs.default] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-postgres @ git+https://github.com/dbt-labs/dbt-postgres.git", + "ipdb==0.13.13", + 'pre-commit==3.7.0;python_version>="3.9"', + 'pre-commit==3.5.0;python_version=="3.8"', +] +[tool.hatch.envs.default.scripts] +dev = "pre-commit install" +code-quality = "pre-commit run {args:--all-files}" +clean = "git clean -f -X" + +[tool.hatch.envs.unit-tests] +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "dbt-postgres @ git+https://github.com/dbt-labs/dbt-postgres.git", + "ddtrace==2.3.0", + "freezegun", + "pytest", + "pytest-csv", + "pytest-dotenv", + "pytest-logbook", + "pytest-mock", + "pytest-xdist", +] +[tool.hatch.envs.unit-tests.scripts] +all = "python -m pytest {args:tests/unit}" + +[tool.hatch.envs.integration-tests] +template = "unit-tests" +extra-dependencies = [ + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", +] +[tool.hatch.envs.integration-tests.scripts] +all = [ + 'python -m pytest --dist=loadscope tests/functional -k "not tests/functional/adapter/utils and not tests/functional/adapter/incremental"', + 'python -m pytest --dist=loadscope tests/functional/adapter/incremental', + 'python -m pytest --dist=loadscope tests/functional/adapter/utils', +] +[tool.hatch.envs.build] +detached = true +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] +[tool.hatch.envs.build.scripts] +check-all = [ + "- check-wheel", + "- check-sdist", +] +check-wheel = [ + "twine check dist/*", + "find ./dist/dbt*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-redshift", +] +check-sdist = [ + "check-wheel-contents dist/*.whl --ignore W007,W008", + "find ./dist/dbt*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", + "pip freeze | grep dbt-redshift", +] + +[tool.pytest] +color = true +verbose = true +env_files = ["test.env"] +testpaths = [ + "tests/functional", + "tests/unit", +] +filterwarnings = [ + "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", + "ignore:unclosed file .*:ResourceWarning", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b3d74bc14..000000000 --- a/pytest.ini +++ /dev/null @@ -1,9 +0,0 @@ -[pytest] -filterwarnings = - ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning - ignore:unclosed file .*:ResourceWarning -env_files = - test.env -testpaths = - tests/unit - tests/functional diff --git a/setup.py b/setup.py deleted file mode 100644 index dbb3913b9..000000000 --- a/setup.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -import re -import sys - -if sys.version_info < (3, 8): - print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") - sys.exit(1) - - -try: - from setuptools import find_namespace_packages -except ImportError: - print("Error: dbt requires setuptools v40.1.0 or higher.") - print('Please upgrade setuptools with "pip install --upgrade setuptools" and try again') - sys.exit(1) - - -from pathlib import Path -from setuptools import setup - - -# pull the long description from the README -README = Path(__file__).parent / "README.md" - - -# used for this adapter's version and in determining the compatible dbt-core version -VERSION = Path(__file__).parent / "dbt/adapters/redshift/__version__.py" - - -def _plugin_version() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - return attributes["version"] - - -def _plugin_version_trim() -> str: - """ - Pull the package version from the main package version file - """ - attributes = {} - exec(VERSION.read_text(), attributes) - pattern = r"\+build\d+$" - return re.sub(pattern, "", attributes["version"]) - - -setup( - name="dbt-redshift", - version=_plugin_version(), - description="The Redshift adapter plugin for dbt", - long_description=README.read_text(), - long_description_content_type="text/markdown", - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt-redshift", - packages=find_namespace_packages(include=["dbt", "dbt.*"]), - include_package_data=True, - install_requires=[ - "dbt-common>=0.1.0a1,<2.0", - "dbt-adapters>=0.1.0a1,<2.0", - f"dbt-postgres~={_plugin_version_trim()}", - # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases - # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. - "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", - # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0b3", - # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core - "sqlparse>=0.5.0,<0.6.0", - "agate", - ], - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - ], - python_requires=">=3.8", -) diff --git a/test.env.example b/test.env.example index 4de05edab..44b34f5b1 100644 --- a/test.env.example +++ b/test.env.example @@ -1,18 +1,9 @@ -# Note: Make sure you have a Redshift account that is set up so these fields are easy to complete. - -### Test Environment field definitions -# These will all be gathered from account information or created by you. -# Endpoint for Redshift connection REDSHIFT_TEST_HOST= -# Username on your account REDSHIFT_TEST_USER= -# Password for Redshift account REDSHIFT_TEST_PASS= -# Local port to connect on REDSHIFT_TEST_PORT= -# Name of Redshift database in your account to test against REDSHIFT_TEST_DBNAME= -# Users for testing + DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index c490fed9a..000000000 --- a/tox.ini +++ /dev/null @@ -1,34 +0,0 @@ -[tox] -skipsdist = True -envlist = py38,py39,py310,py311 - -[testenv:{unit,py38,py39,py310,py311,py}] -description = unit testing -skip_install = true -passenv = - DBT_* - PYTEST_ADDOPTS -commands = {envpython} -m pytest {posargs} tests/unit -deps = - -rdev-requirements.txt - -e. - -[testenv:{integration,py38,py39,py310,py311,py}-{redshift}] -description = adapter plugin integration testing -skip_install = true -passenv = - DBT_* - REDSHIFT_TEST_* - PYTEST_ADDOPTS - DD_CIVISIBILITY_AGENTLESS_ENABLED - DD_API_KEY - DD_SITE - DD_ENV - DD_SERVICE -commands = - {envpython} -m pytest --dist=loadscope {posargs} tests/functional -k "not tests/functional/adapter/utils and not tests/functional/adapter/incremental" - {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/utils - {envpython} -m pytest --dist=loadscope {posargs} tests/functional/adapter/incremental -deps = - -rdev-requirements.txt - -e.