Merge pull request #17 from fivetran/bugfix/keyword-id-addition

bugfix/keyword-id-addition

.buildkite/hooks/pre-command

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Export secrets for Docker containers.
+# Restrict exposing secrets only to the steps that need them
+export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -0,0 +1,73 @@
+steps:
+  - label: ":postgres: Run Tests - Postgres"
+    key: "run-dbt-postgres"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_POSTGRES_DBT_DBNAME"
+            - "CI_POSTGRES_DBT_HOST"
+            - "CI_POSTGRES_DBT_PASS"
+            - "CI_POSTGRES_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh postgres 
+
+  - label: ":snowflake-db: Run Tests - Snowflake"
+    key: "run_dbt_snowflake"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_SNOWFLAKE_DBT_ACCOUNT"
+            - "CI_SNOWFLAKE_DBT_DATABASE"
+            - "CI_SNOWFLAKE_DBT_PASS"
+            - "CI_SNOWFLAKE_DBT_ROLE"
+            - "CI_SNOWFLAKE_DBT_USER"
+            - "CI_SNOWFLAKE_DBT_WAREHOUSE"
+    commands: |
+      bash .buildkite/scripts/run_models.sh snowflake
+
+  - label: ":gcloud: Run Tests - BigQuery"
+    key: "run_dbt_bigquery"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "GCLOUD_SERVICE_KEY"
+    commands: |
+      bash .buildkite/scripts/run_models.sh bigquery
+
+  - label: ":amazon-redshift: Run Tests - Redshift"
+    key: "run_dbt_redshift"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_REDSHIFT_DBT_DBNAME"
+            - "CI_REDSHIFT_DBT_HOST"
+            - "CI_REDSHIFT_DBT_PASS"
+            - "CI_REDSHIFT_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":bricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks

.buildkite/scripts/run_models.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+
+apt-get update
+apt-get install libsasl2-dev
+
+python3 -m venv venv
+. venv/bin/activate
+pip install --upgrade pip setuptools
+pip install -r integration_tests/requirements.txt
+mkdir -p ~/.dbt
+cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
+
+db=$1
+echo `pwd`
+cd integration_tests
+dbt deps
+dbt seed --target "$db" --full-refresh
+dbt run --target "$db" --full-refresh
+dbt test --target "$db"
+dbt run --vars '{twitter_ads__using_keywords: False}' --target "$db" --full-refresh
+dbt test --target "$db"

.github/pull_request_template.md

@@ -28,9 +28,9 @@
 
 **How did you test the PR changes?** 
 <!--- Proof of testing is required in order for the PR to be approved. -->
-<!--- To check a box, remove the space and insert an x in the box (eg. [x] CircleCi). --> 
+<!--- To check a box, remove the space and insert an x in the box (eg. [x] BuildKite). --> 
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] CircleCi <!--- CircleCi testing is only applicable to Fivetran employees. --> 
+- [ ] BuildKite <!--- BuildKite testing is only applicable to Fivetran employees. --> 
 - [ ] Local (please provide additional testing details below)
 
 **Select which warehouse(s) were used to test the PR**

CHANGELOG.md

@@ -1,3 +1,18 @@
+# dbt_twitter_source v0.5.1
+
+## Fixes
+- Fix the package name in the README ([#18](https://github.com/fivetran/dbt_twitter_source/pull/18))
+
+## Feature Enhancement
+- Addition of the `keyword_id` surrogate key within the `stg_twitter_ads__line_item_keywords_report`. This field is a combination of the account_id, line_item_id, segment, and placement fields. ([#17](https://github.com/fivetran/dbt_twitter_source/pull/17))
+
+## Under the Hood
+- Buildkite testing integration. ([#17](https://github.com/fivetran/dbt_twitter_source/pull/17))
+
+## Contributors
+- [@johnf](https://github.com/johnf) ([#18](https://github.com/fivetran/dbt_twitter_source/pull/18))
+
+
 # dbt_twitter_source v0.5.0
 
 ## 🎉 Feature Enhancements 🎉

README.md

@@ -41,7 +41,7 @@ Include the following twitter_source package version in your `packages.yml` file
 ```yml
 # packages.yml
 packages:
-  - package: fivetran/twitter_source
+  - package: fivetran/twitter_ads_source
     version: [">=0.5.0", "<0.6.0"]
 ```
 
@@ -137,4 +137,4 @@ A small team of analytics engineers at Fivetran develops these dbt packages. How
 # 🏪 Are there any resources available?
 - If you have questions or want to reach out for help, please refer to the [GitHub Issue](https://github.com/fivetran/dbt_twitter_source/issues/new/choose) section to find the right avenue of support for you.
 - If you would like to provide feedback to the dbt package team at Fivetran or would like to request a new dbt package, fill out our [Feedback Form](https://www.surveymonkey.com/r/DQ7K7WW).
-- Have questions or want to just say hi? Book a time during our office hours [on Calendly](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) or email us at [email protected].
+- Have questions or want to just say hi? Book a time during our office hours [on Calendly](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) or email us at [email protected].

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'twitter_ads_source'
-version: '0.5.0'
+version: '0.5.1'
 
 config-version: 2
 require-dbt-version: [">=1.0.0", "<2.0.0"]
@@ -27,4 +27,4 @@ vars:
     twitter_ads__campaign_report_passthrough_metrics: []
     twitter_ads__line_item_report_passthrough_metrics: []
     twitter_ads__line_item_keywords_report_passthrough_metrics: []
-    twitter_ads__promoted_tweet_report_passthrough_metrics: []
+    twitter_ads__promoted_tweet_report_passthrough_metrics: []

integration_tests/ci/sample.profiles.yml

@@ -1,13 +1,13 @@
 
-# HEY! This file is used in the Twitter integrations tests with CircleCI.
+# HEY! This file is used in the dbt package integrations tests with Buildkite.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
 config:
     send_anonymous_usage_stats: False
     use_colors: True
 
 integration_tests:
-  target: snowflake
+  target: redshift
   outputs:
     redshift:
       type: redshift
@@ -16,15 +16,15 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: twitter_source_integration_tests
+      schema: twitter_ads_source_integration_tests
       threads: 8
     bigquery:
       type: bigquery
-      method: service-account
-      keyfile: "{{ env_var('GCLOUD_SERVICE_KEY_PATH') }}"
+      method: service-account-json
       project: 'dbt-package-testing'
-      schema: twitter_source_integration_tests
+      schema: twitter_ads_source_integration_tests
       threads: 8
+      keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
       type: snowflake
       account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}"
@@ -33,28 +33,22 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: twitter_source_integration_tests
+      schema: twitter_ads_source_integration_tests
       threads: 8
-    spark:
-      type: spark
-      method: http
-      schema: twitter_source_integration_tests
-      host: "{{ env_var('CI_SPARK_DBT_HOST') }}"
-      organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}"
-      token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}"
-      cluster: "{{ env_var('CI_SPARK_DBT_CLUSTER') }}"
-      port: 443
-      connect_timeout: 60
-      connect_retries: 5
-      threads: 4
     postgres:
       type: postgres
       host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}"
       user: "{{ env_var('CI_POSTGRES_DBT_USER') }}"
-      password: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
+      pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
+      dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      dbname: "{{ env_var('CI_POSTGRES_DBT_DATABASE') }}"
-      schema: twitter_source_integration_tests
+      schema: twitter_ads_source_integration_tests
       threads: 8
-      keepalives_idle: 0
-      sslmode: prefer
+    databricks:
+      catalog: null
+      host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
+      http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
+      schema: twitter_ads_source_integration_tests
+      threads: 2
+      token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
+      type: databricks

integration_tests/dbt_project.yml

@@ -1,13 +1,13 @@
 name: 'twitter_source_integration_tests'
-version: '0.5.0'
+version: '0.5.1'
 
 profile: 'integration_tests'
 config-version: 2
 snapshot-paths: ["snapshots"]
 
 vars:
   # twitter_source:
-  twitter_ads_schema: twitter_source_integration_tests
+  twitter_ads_schema: twitter_ads_source_integration_tests
 
   twitter_ads_account_history_identifier: "twitter_account_history_data"
   twitter_ads_campaign_history_identifier: "twitter_campaign_history_data"
@@ -20,7 +20,7 @@ vars:
   twitter_ads_line_item_report_identifier: "twitter_line_item_report_data"
   twitter_ads_account_report_identifier: "twitter_account_report_data"
   twitter_ads_line_item_keywords_report_identifier: "twitter_line_item_keywords_report_data"
-    
+
 seeds:
   twitter_source_integration_tests:
     +column_types:
@@ -32,7 +32,7 @@ seeds:
       +column_types:
         tweet_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}"
 
-        
+
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']

integration_tests/requirements.txt

@@ -1,6 +1,7 @@
-dbt-snowflake>=1.0.0
-dbt-bigquery>=1.0.0
-dbt-redshift>=1.0.0
-dbt-postgres>=1.0.0
-dbt-spark>=1.0.0
-dbt-spark[PyHive]>=1.0.0
+dbt-snowflake>=1.0.0,<1.3.0
+dbt-bigquery>=1.0.0,<1.3.0
+dbt-redshift>=1.0.0,<1.3.0
+dbt-postgres>=1.0.0,<1.3.0
+dbt-spark>=1.0.0,<1.3.0
+dbt-spark[PyHive]>=1.0.0,<1.3.0
+dbt-databricks>=1.0.0,<1.3.0