add var for CALL + update way we disable source freshness

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:
@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks

.buildkite/scripts/run_models.sh

@@ -19,7 +19,7 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{using_twilio_messaging_service: false}' --target "$db" --full-refresh
+dbt run --vars '{using_twilio_call: false, using_twilio_messaging_service: false}' --target "$db" --full-refresh
 dbt test --target "$db"
 
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md

@@ -4,48 +4,26 @@
 **This PR will result in the following new package version:**
 <!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
 
-**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:**
+**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:**
+<!--- Copy/paste the CHANGELOG for this version below. -->
 
 ## PR Checklist
 ### Basic Validation
 Please acknowledge that you have successfully performed the following commands locally:
-- [ ] dbt compile
-- [ ] dbt run –full-refresh
-- [ ] dbt run
-- [ ] dbt test
-- [ ] dbt run –vars (if applicable)
+- [ ] dbt run –full-refresh && dbt test
+- [ ] dbt run (if incremental models are present) && dbt test
 
 Before marking this PR as "ready for review" the following have been applied:
-- [ ] The appropriate issue has been linked and tagged
-- [ ] You are assigned to the corresponding issue and this PR
+- [ ] The appropriate issue has been linked, tagged, and properly assigned
+- [ ] All necessary documentation and version upgrades have been applied
+- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
 - [ ] BuildKite integration tests are passing
+- [ ] Detailed validation steps have been provided below
 
 ### Detailed Validation
-Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review":
-- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature.
-- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages.
-- [ ] You have provided details below around the validation steps performed to gain confidence in these changes.
+Please share any and all of your validation steps:
 <!--- Provide the steps you took to validate your changes below. -->
 
-### Standard Updates
-Please acknowledge that your PR contains the following standard updates:
-- Package versioning has been appropriately indexed in the following locations:
-    - [ ] indexed within dbt_project.yml
-    - [ ] indexed within integration_tests/dbt_project.yml
-- [ ] CHANGELOG has individual entries for each respective change in this PR
-    <!--- If there is a parallel upstream change, remember to reference the corresponding CHANGELOG as an individual entry.  -->
-- [ ] README updates have been applied (if applicable)
-    <!--- Remember to check the following README locations for common updates. →
-        <!--- Suggested install range (needed for breaking changes) →
-        <!--- Dependency matrix is appropriately updated (if applicable) →
-        <!--- New variable documentation (if applicable) -->
-- [ ] DECISIONLOG updates have been updated (if applicable)
-- [ ] Appropriate yml documentation has been added (if applicable)
-
-### dbt Docs
-Please acknowledge that after the above were all completed the below were applied to your branch:
-- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
-
 ### If you had to summarize this PR in an emoji, which would it be?
 <!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
-:dancer:
+:dancer:

.github/pull_request_template.md

@@ -30,4 +30,4 @@ We are so excited you decided to contribute to the Fivetran community dbt packag
 **PR Template** 
 - [Community Pull Request Template](?expand=1&template=pull_request_template.md) (default)
 
-- [Maintainer Pull Request Template](?expand=1&template=maintainer_pull_request_template.md) (to be used by maintainers)
+- [Maintainer Pull Request Template](?expand=1&template=maintainer_pull_request_template.md) (to be used by maintainers)

.github/workflows/auto-release.yml

@@ -0,0 +1,13 @@
+name: 'auto release'
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  call-workflow-passing-data:
+    if: github.event.pull_request.merged
+    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
+    secrets: inherit

CHANGELOG.md

@@ -1,3 +1,22 @@
+# dbt_twilio_source v0.2.0
+
+## 🚨 Breaking Changes 🚨
+- Under the hood, we've updated the `_tmp` models to use the `dbt_utils.star` macro instead of a basic `select *` ([PR #6](https://github.com/fivetran/dbt_twilio_source/pull/6)).
+  - This means that you can no longer use `var(<table_name>)` to override the source tables we create staging models from. Instead, see the [README](https://github.com/fivetran/dbt_twilio_source?tab=readme-ov-file#change-the-source-table-references) for how to use our `_identifier` variables.
+- Removed the deprecated `twilio_using_message` variable. This is a breaking change because you could previously use this variable to disable freshness tests on the `MESSAGE` source table. To continue to do so, leverage dbt [overrides](https://docs.getdbt.com/reference/resource-properties/overrides#configure-your-own-source-freshness-for-a-source-table-in-a-package) to set `message`'s freshness to `null` ([PR #6](https://github.com/fivetran/dbt_twilio_source/pull/6)).
+
+## Features
+- Added the ability to disable models related to the `CALL` source table. Refer to the [README](https://github.com/fivetran/dbt_twilio_source?tab=readme-ov-file#step-4-enablingdisabling-models) for more details ([PR #5](https://github.com/fivetran/dbt_twilio_source/pull/5)).
+
+## Under the Hood
+- Adjusted the case of loader name from `fivetran` to `Fivetran` in the `src_twilio.yml` file ([PR #5](https://github.com/fivetran/dbt_twilio_source/pull/5)).
+- Adjusted the way we dynamically disable source freshness tests for tables that may be missing to use dbt's native `config.enabled` [flag](https://docs.getdbt.com/reference/resource-configs/enabled) ([PR #6](https://github.com/fivetran/dbt_twilio_source/pull/6)).
+- Updated the pull request [templates](/.github) ([PR #6](https://github.com/fivetran/dbt_twilio_source/pull/6)).
+- Included auto-releaser GitHub Actions workflow to automate future releases ([PR #6](https://github.com/fivetran/dbt_twilio_source/pull/6)).
+
+## Contributors
+- [@raphaelvarieras](https://github.com/raphaelvarieras) ([PR #5](https://github.com/fivetran/dbt_twilio_source/pull/5))
+
 # dbt_twilio_source v0.1.0
 
 ## Initial Release

README.md

@@ -44,7 +44,7 @@ Include the following Twilio package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/twilio_source
-    version: [">=0.1.0", "<0.2.0"]
+    version: [">=0.2.0", "<0.3.0"]
 ```
 
 ## Step 3: Define database and schema variables
@@ -55,6 +55,7 @@ vars:
   twilio_database: your_database_name
   twilio_schema: your_schema_name 
 ```
+
 ## Step 4: Enabling/Disabling Models
 
 Your Twilio connector might not sync every table that this package expects, for example if you are not using the Twilio messaging service feature. If your syncs exclude certain tables, it is either because you do not use that functionality in Twilio or have actively excluded some tables from your syncs. In order to enable or disable the relevant tables in the package, you will need to add the following variable(s) to your `dbt_project.yml` file.
@@ -63,12 +64,13 @@ By default, all variables are assumed to be `true`.
 
 ```yml
 vars:
+  using_twilio_call: False # Disable this if not using call
   using_twilio_messaging_service: False # Disable this if not using messaging_service
 ```
 
 ## (Optional) Step 5: Additional configurations
 
-<details><summary>Expand for details</summary>
+<details open><summary>Expand/Collapse details</summary>
 <br>
 
 ### Changing the Build Schema
@@ -80,8 +82,9 @@ models:
     twilio_source:
         +schema: my_new_schema_name # leave blank for just the target_schema
 ```
+
 ### Change the source table references
-If an individual source table has a different name than what the package expects, add the table name as it appears in your destination to the respective variable:
+If an individual source table has a different name than what the package expects (but is in the same schema and database as the other tables), add the table name as it appears in your destination to the respective variable:
 > IMPORTANT: See this project's [`dbt_project.yml`](https://github.com/fivetran/dbt_twilio_source/blob/main/dbt_project.yml) variable declarations to see the expected names.
 
 ```yml
@@ -119,11 +122,10 @@ packages:
 The Fivetran team maintaining this package _only_ maintains the latest version of the package. We highly recommend you stay consistent with the [latest version](https://hub.getdbt.com/fivetran/twilio_source/latest/) of the package and refer to the [CHANGELOG](https://github.com/fivetran/dbt_twilio_source/blob/main/CHANGELOG.md) and release notes for more information on changes across versions.
 
 ## Contributions
-A small team of analytics engineers at Fivetran develops these dbt packages. However, the packages are made better by community contributions! 
+A small team of analytics engineers at Fivetran develops these dbt packages. However, the packages are made better by community contributions!
 
 We highly encourage and welcome contributions to this package. Check out [this dbt Discourse article](https://discourse.getdbt.com/t/contributing-to-a-dbt-package/657) on the best workflow for contributing to a package!
 
 # 🏪 Are there any resources available?
 - If you have questions or want to reach out for help, please refer to the [GitHub Issue](https://github.com/fivetran/dbt_twilio_source/issues/new/choose) section to find the right avenue of support for you.
-- If you would like to provide feedback to the dbt package team at Fivetran or would like to request a new dbt package, fill out our [Feedback Form](https://www.surveymonkey.com/r/DQ7K7WW).
-- Have questions or want to be part of the community discourse? Create a post in the [Fivetran community](https://community.fivetran.com/t5/user-group-for-dbt/gh-p/dbt-user-group) and our team along with the community can join in on the discussion!
+- If you would like to provide feedback to the dbt package team at Fivetran or would like to request a new dbt package, fill out our [Feedback Form](https://www.surveymonkey.com/r/DQ7K7WW).

dbt_project.yml

@@ -1,6 +1,6 @@
 
 name: 'twilio_source'
-version: '0.1.0'
+version: '0.2.0'
 config-version: 2
 
 models:

integration_tests/.gitignore

@@ -2,4 +2,5 @@
 target/
 dbt_modules/
 logs/
-.DS_Store
+.DS_Store
+package-lock.yml

integration_tests/ci/sample.profiles.yml

@@ -45,7 +45,7 @@ integration_tests:
       schema: twilio_source_integration_tests_1
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: twilio_source_integration_tests_1

integration_tests/dbt_project.yml

@@ -1,7 +1,6 @@
 config-version: 2
 name: 'twilio_source_integration_tests'
-
-version: '0.1.0'
+version: '0.2.0'
 profile: 'integration_tests'
 
 vars:
@@ -13,9 +12,8 @@ vars:
     twilio_messaging_service_identifier:  "twilio_messaging_service_data"
     twilio_outgoing_caller_id_identifier:   "twilio_outgoing_caller_id_data"
     twilio_usage_record_identifier:   "twilio_usage_record_data"
-    call: "{{ fivetran_utils.snowflake_seed_data('twilio_call_data') }}" 
-    message: "{{ fivetran_utils.snowflake_seed_data('twilio_message_data') }}" 
-
+    twilio_call_identifier:   "twilio_call_data"
+    twilio_message_identifier:   "twilio_message_data"
 
 dispatch:
   - macro_namespace: dbt_utils
@@ -35,5 +33,7 @@ seeds:
       +enabled: "{{ true if target.type != 'snowflake' else false }}"
     twilio_message_data_snowflake:
       +enabled: "{{ true if target.type == 'snowflake' else false }}"
+      +alias: twilio_message_data
     twilio_call_data_snowflake:
-      +enabled: "{{ true if target.type == 'snowflake' else false }}"
+      +enabled: "{{ true if target.type == 'snowflake' else false }}"
+      +alias: twilio_call_data

integration_tests/requirements.txt

@@ -3,5 +3,5 @@ dbt-bigquery>=1.3.0,<2.0.0
 dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
-dbt-spark[PyHive]>=1.3.0,<2.0.0
+dbt-spark[PyHive]>=1.3.0,<1.8.0
 dbt-databricks>=1.3.0,<2.0.0

models/src_twilio.yml

@@ -5,7 +5,7 @@ sources:
     database: "{{ var('twilio_database', target.database)}}"
     schema: "{{ var('twilio_schema', 'twilio')}}"
 
-    loader: fivetran
+    loader: Fivetran
     loaded_at_field: _fivetran_synced
 
     tables:
@@ -96,8 +96,8 @@ sources:
       - name: call
         identifier: "{{ var('twilio_call_identifier', 'call' )}}"
         description: Each record represents a phone call that occurred or is scheduled to occur.
-        meta:
-          is_enabled: "{{ var('using_twilio_call', true) }}"
+        config:
+          enabled: "{{ var('using_twilio_call', true) }}"
         freshness:
           warn_after: {count: 72, period: hour}
           error_after: {count: 168, period: hour}
@@ -283,8 +283,6 @@ sources:
       - name: message
         identifier: "{{ var('twilio_message_identifier', 'message' )}}"
         description:  Each record represents an individual outbound or inbound message.
-        meta:
-          is_enabled: "{{ var('using_twilio_message', true) }}"
         freshness:
           warn_after: {count: 72, period: hour}
           error_after: {count: 168, period: hour}
@@ -346,8 +344,8 @@ sources:
       - name: messaging_service
         identifier: "{{ var('twilio_messaging_service_identifier', 'messaging_service' )}}"
         description: Each record represents information about each messaging service. You can think of messaging services as a higher-level “bundling” of messaging functionality around a common set of senders, features, and configuration
-        meta:
-          is_enabled: "{{ var('using_twilio_messaging_service', true) }}"
+        config:
+          enabled: "{{ var('using_twilio_messaging_service', true) }}"
         columns:
           - name: _fivetran_synced
             description: Timestamp of when fivetran synced a record. 

models/stg_twilio__call.sql

@@ -1,3 +1,6 @@
+--To disable this model, set the using_twilio_call variable within your dbt_project.yml file to False.
+{{ config(enabled=var('using_twilio_call', True)) }}
+
 with base as (
 
     select * 

models/stg_twilio__messaging_service.sql

@@ -1,4 +1,4 @@
---To disable this model, set the using_domain_names variable within your dbt_project.yml file to False.
+--To disable this model, set the using_twilio_messaging_service variable within your dbt_project.yml file to False.
 {{ config(enabled=var('using_twilio_messaging_service', True)) }}
 
 with base as (

models/tmp/stg_twilio__account_history_tmp.sql

@@ -1,2 +1,2 @@
-select * 
+select {{ dbt_utils.star(source('twilio', 'account_history')) }}
 from {{ var('account_history') }}

models/tmp/stg_twilio__address_tmp.sql

@@ -1,2 +1,2 @@
-select * 
+select {{ dbt_utils.star(source('twilio', 'address')) }}
 from {{ var('address') }}

models/tmp/stg_twilio__call_tmp.sql

@@ -1,2 +1,5 @@
-select * 
+--To disable this model, set the using_twilio_call variable within your dbt_project.yml file to False.
+{{ config(enabled=var('using_twilio_call', True)) }}
+
+select {{ dbt_utils.star(source('twilio', 'call')) }}
 from {{ var('call') }}

models/tmp/stg_twilio__incoming_phone_number_tmp.sql

@@ -1,2 +1,2 @@
-select * 
+select {{ dbt_utils.star(source('twilio', 'incoming_phone_number')) }}
 from {{ var('incoming_phone_number') }}

models/tmp/stg_twilio__message_tmp.sql

@@ -1,2 +1,2 @@
-select * 
+select {{ dbt_utils.star(source('twilio', 'message')) }}
 from {{ var('message') }}

models/tmp/stg_twilio__messaging_service_tmp.sql

@@ -1,5 +1,5 @@
---To disable this model, set the using_domain_names variable within your dbt_project.yml file to False.
+--To disable this model, set the using_twilio_messaging_service variable within your dbt_project.yml file to False.
 {{ config(enabled=var('using_twilio_messaging_service', True)) }}
 
-select * 
+select {{ dbt_utils.star(source('twilio', 'messaging_service')) }}
 from {{ var('messaging_service') }}

models/tmp/stg_twilio__outgoing_caller_id_tmp.sql

@@ -1,2 +1,2 @@
-select * 
+select {{ dbt_utils.star(source('twilio', 'outgoing_caller_id')) }}
 from {{ var('outgoing_caller_id') }}

models/tmp/stg_twilio__usage_record_tmp.sql

@@ -1,2 +1,2 @@
-select * 
+select {{ dbt_utils.star(source('twilio', 'usage_record')) }}
 from {{ var('usage_record') }}