bugfix/too-many-partitions (#165)

* bugfix/too-many-partitions

* docs regen

* Update CHANGELOG.md

Co-authored-by: fivetran-catfritz <[email protected]>

---------

Co-authored-by: fivetran-catfritz <[email protected]>

CHANGELOG.md

@@ -1,3 +1,15 @@
+# dbt_zendesk v0.17.0
+
+## Breaking Changes (Full refresh required after upgrading)
+- Incremental models running on BigQuery have had the `partition_by` logic adjusted to include a granularity of a month. This change only impacts BigQuery warehouses and was applied to avoid the common `too many partitions` error some users have experienced when partitioning by day. Therefore, adjusting the partition to a month granularity will decrease the number of partitions created and allow for more performant querying and incremental loads. This change was applied to the following models:
+  - `int_zendesk__field_calendar_spine`
+  - `int_zendesk__field_history_pivot`
+  - `zendesk__ticket_field_history`
+
+## Under the Hood
+- Updated seed files to reflect a real world ticket field history update scenario.
+- Modified the `consistency_sla_policy_count` validation test to group by `ticket_id` for more accurate testing.
+
 # dbt_zendesk v0.16.0
 ## 🚨 Minor Upgrade 🚨
 Although this update is not a breaking change, it will likely impact the output of the `zendesk__sla_policies` and `zendesk__sla_metrics` models. [PR #154](https://github.com/fivetran/dbt_zendesk/pull/154) includes the following changes:

README.md

@@ -64,7 +64,7 @@ Include the following zendesk package version in your `packages.yml` file:
 ```yml
 packages:
   - package: fivetran/zendesk
-    version: [">=0.16.0", "<0.17.0"]
+    version: [">=0.17.0", "<0.18.0"]
 
 ```
 > **Note**: Do not include the Zendesk Support source package. The Zendesk Support transform package already has a dependency on the source in its own `packages.yml` file.

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'zendesk'
-version: '0.16.0'
+version: '0.17.0'
 
 
 config-version: 2

integration_tests/dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'zendesk_integration_tests'
-version: '0.16.0'
+version: '0.17.0'
 
 profile: 'integration_tests'
 
@@ -33,8 +33,7 @@ vars:
   # using_organization_tags: false
   # fivetran_integrity_sla_first_reply_time_exclusion_tickets: (1,56,80)
   # fivetran_consistency_ticket_metrics_exclusion_tickets: (11092,11093,11094)
-  # fivetran_integrity_sla_count_match_tickets: (76)
-
+  # fivetran_integrity_sla_count_match_tickets: (76,11106) ## Upcoming PR will address this.
 
 models:
   +schema: "zendesk_{{ var('directed_schema','dev') }}"

integration_tests/seeds/ticket_data.csv

@@ -2,7 +2,7 @@ id,_fivetran_synced,allow_channelback,assignee_id,brand_id,created_at,descriptio
 1595,2020-03-20 02:32:49.426,FALSE,,360003529474,2020-02-19 01:54:52,description1,,,,360006965034,FALSE,TRUE,370295712714,,,[email protected],396331237134,deleted,subject1,396331237134,,360002048693,incident,2020-02-19 01:55:11,https://zendesk.com/api/v2/tickets/1595.json,web,,,,[email protected],,[],,,
 16988,2021-01-13 20:09:16.325,FALSE,418284131934,360003529474,2020-12-22 00:19:23,description1,,,,360013366274,FALSE,TRUE,370469077513,,,[email protected],1500656884401,solved,subject1,1500656884401,,360002048693,,2021-01-13 18:42:39,https://zendesk.com/api/v2/tickets/16988.json,email,,,,[email protected],Support,[],,[],
 14173,2020-11-11 20:08:45.130,FALSE,396371699653,360003529474,2020-10-28 12:03:02,description1,,,,360006965034,FALSE,TRUE,370321120273,,,[email protected],424883466453,closed,subject1,424883466453,,360002048693,,2020-11-11 17:01:32,https://zendesk.com/api/v2/tickets/14173.json,email,,,,[email protected],Support,[],,,
-11071,2020-10-02 14:08:33.216,FALSE,,360003529474,2020-08-28 18:06:36,description1,,,,,FALSE,TRUE,,,,[email protected],419755385214,deleted,subject1,419755385214,,360002048693,,2020-09-02 11:01:27,https://zendesk.com/api/v2/tickets/11071.json,email,,,,X,Support,[],,,
+11071,2020-10-02 14:08:33.216,FALSE,1111,360003529474,2020-08-28 18:06:36,Ticket to test field history changes,,,,360006965034,FALSE,TRUE,,urgent,,[email protected],2222,closed,subject1,2222,,360002048693,,2020-11-15 11:01:27,https://zendesk.com/api/v2/tickets/11071.json,email,,,,X,Support,[],,,
 1966,2020-03-25 20:32:23.617,FALSE,396315360434,360003529474,2020-02-27 06:05:08,description1,,,,360006965034,FALSE,TRUE,370295721514,,,[email protected],402813302773,closed,subject1,402813302773,,360002048693,,2020-03-25 16:03:26,https://zendesk.com/api/v2/tickets/1966.json,email,,,,[email protected],Support,[1967],,,
 11013,2020-10-02 20:08:20.449,FALSE,402851697393,360003529474,2020-08-27 23:09:52,description1,,,,360008376313,FALSE,TRUE,370297881854,,,[email protected],419688934974,deleted,subject1,419688934974,,360002048693,,2020-09-02 15:53:16,https://zendesk.com/api/v2/tickets/11013.json,email,,,,X,Support,[],,,
 1404,2020-03-05 04:53:46.466,FALSE,396371699653,360003529474,2020-02-13 21:43:58,description1,,,,360006965034,FALSE,TRUE,370295709874,,,[email protected],403125197514,closed,subject1,403125197514,,360002048693,,2020-02-28 01:01:57,https://zendesk.com/api/v2/tickets/1404.json,email,,,,[email protected],Support,,,,

integration_tests/seeds/ticket_field_history_data.csv

@@ -1,8 +1,25 @@
 field_name,ticket_id,updated,_fivetran_synced,user_id,value
+status,11071,2020-08-28 11:01:27,2020-10-02 14:08:33.216,,open
+assignee_id,11071,2020-08-28 11:01:27,2020-03-11 14:32:23.872,,1111
+priority,11071,2020-08-28 11:01:27,2020-03-11 14:32:23.872,,normal
+status,11071,2020-08-29 11:01:27,2020-10-02 14:08:33.216,,pending
+status,11071,2020-09-02 11:01:27,2020-10-02 14:08:33.216,,on-hold
+status,11071,2020-09-05 11:01:27,2020-10-02 14:08:33.216,,pending
+status,11071,2020-09-15 11:01:27,2020-10-02 14:08:33.216,,open
+status,11071,2020-09-30 11:01:27,2020-10-02 14:08:33.216,,pending
+priority,11071,2020-09-30 11:01:27,2020-03-11 14:32:23.872,,medium
+status,11071,2020-10-01 11:01:27,2020-10-02 14:08:33.216,,on-hold
+status,11071,2020-10-15 11:01:27,2020-10-02 14:08:33.216,,open
+priority,11071,2020-10-15 11:01:27,2020-03-11 14:32:23.872,,high
+status,11071,2020-10-17 11:01:27,2020-10-02 14:08:33.216,,on-hold
+status,11071,2020-10-20 11:01:27,2020-10-02 14:08:33.216,,pending
+status,11071,2020-11-02 11:01:27,2020-10-02 14:08:33.216,,open
+priority,11071,2020-11-02 11:01:27,2020-03-11 14:32:23.872,,urgent
+status,11071,2020-11-14 11:01:27,2020-10-02 14:08:33.216,,solved
+status,11071,2020-11-15 11:01:27,2020-10-02 14:08:33.216,,closed
 status,6964,2020-06-01 21:11:59,2020-07-02 02:09:05.984,,solved
 status,974,2020-02-10 21:47:41,2020-03-12 02:32:23.808,,solved
 priority,980,2020-02-10 22:06:57,2020-03-12 02:32:23.808,,solved
-status,11071,2020-09-02 11:01:27,2020-10-02 14:08:33.216,,solved
 status,8205,2020-07-07 23:01:47,2020-08-07 02:09:08.192,,solved
 status,103,2020-02-10 08:36:38,2020-03-11 14:32:23.872,,solved
 assignee_id,108,2020-02-10 08:36:38,2020-03-11 14:32:23.872,,1111

integration_tests/seeds/user_data.csv

@@ -1,6 +1,6 @@
 id,_fivetran_synced,active,alias,authenticity_token,chat_only,created_at,details,email,external_id,last_login_at,locale,locale_id,moderator,name,notes,only_private_comments,organization_id,phone,remote_photo_url,restricted_agent,role,shared,shared_agent,signature,suspended,ticket_restriction,time_zone,two_factor_auth_enabled,updated_at,url,verified
-403958466973,2020-03-05 05:03:34.208,TRUE,,,FALSE,2020-02-26 22:55:12,,[email protected],,2023-02-26 22:55:12,en-US,1,FALSE,John Doe,,FALSE,370297696174,,,TRUE,end-user,FALSE,FALSE,,FALSE,requested,Pacific Time (US & Canada),FALSE,2020-02-26 22:55:12,https://fivetran1813.zendesk.com/api/v2/users/403958466973.json,TRUE
-403969371634,2020-03-05 05:03:34.208,TRUE,,,FALSE,2020-02-26 22:41:37,,[email protected],,2023-02-26 22:55:12,en-US,1,FALSE,John Doe,,FALSE,,,,TRUE,end-user,FALSE,FALSE,,FALSE,requested,Pacific Time (US & Canada),FALSE,2020-02-26 22:41:37,https://fivetran1813.zendesk.com/api/v2/users/403969371634.json,TRUE
+1111,2020-03-05 05:03:34.208,TRUE,,,FALSE,2020-02-26 22:55:12,,[email protected],,2023-02-26 22:55:12,en-US,1,FALSE,Arthur Agent,,FALSE,370326203233,,,TRUE,agent,FALSE,FALSE,,FALSE,requested,Pacific Time (US & Canada),FALSE,2020-02-26 22:55:12,https://fivetran1813.zendesk.com/api/v2/users/403958466973.json,TRUE
+2222,2020-03-05 05:03:34.208,TRUE,,,FALSE,2020-02-26 22:41:37,,[email protected],,2023-02-26 22:55:12,en-US,1,FALSE,Earnest End User,,FALSE,370297696174,,,TRUE,end-user,FALSE,FALSE,,FALSE,requested,Pacific Time (US & Canada),FALSE,2020-02-26 22:41:37,https://fivetran1813.zendesk.com/api/v2/users/403969371634.json,TRUE
 403957746773,2020-03-05 05:03:34.208,TRUE,,,FALSE,2020-02-26 22:35:14,,[email protected],,2023-02-26 22:55:12,en-US,1,FALSE,John Doe,,FALSE,370319191913,,,TRUE,end-user,FALSE,FALSE,,FALSE,requested,Pacific Time (US & Canada),FALSE,2020-02-26 22:35:15,https://fivetran1813.zendesk.com/api/v2/users/403957746773.json,TRUE
 403970285734,2020-03-05 05:03:34.208,TRUE,,,FALSE,2020-02-26 23:07:41,,[email protected],,2023-02-26 22:55:12,en-US,1,FALSE,John Doe,,FALSE,370326203233,,,TRUE,end-user,FALSE,FALSE,,FALSE,requested,Pacific Time (US & Canada),FALSE,2020-02-26 23:07:41,https://fivetran1813.zendesk.com/api/v2/users/403970285734.json,TRUE
 403969943274,2020-03-05 05:03:34.208,TRUE,,,FALSE,2020-02-26 22:57:51,,[email protected],,2023-02-26 22:55:12,en-US,1,FALSE,John Doe,,FALSE,370326203233,,,TRUE,end-user,FALSE,FALSE,,FALSE,requested,Pacific Time (US & Canada),FALSE,2020-02-26 22:57:52,https://fivetran1813.zendesk.com/api/v2/users/403969943274.json,TRUE

integration_tests/tests/consistency/consistency_sla_policy_count.sql

@@ -6,28 +6,29 @@
 
 with prod as (
     select
-        1 as join_key,
+        ticket_id,
         count(*) as total_slas
     from {{ target.schema }}_zendesk_prod.zendesk__sla_policies
     group by 1
 ),
 
 dev as (
     select
-        1 as join_key,
+        ticket_id,
         count(*) as total_slas
     from {{ target.schema }}_zendesk_dev.zendesk__sla_policies
     group by 1
 ),
 
 final as (
     select 
-        prod.join_key,
+        prod.ticket_id,
+        dev.ticket_id,
         prod.total_slas as prod_sla_total,
         dev.total_slas as dev_sla_total
     from prod
     full outer join dev 
-        on dev.join_key = prod.join_key
+        on dev.ticket_id = prod.ticket_id
 )
 
 select *

models/ticket_history/int_zendesk__field_calendar_spine.sql

@@ -1,7 +1,7 @@
 {{
     config(
         materialized='incremental',
-        partition_by = {'field': 'date_day', 'data_type': 'date'} if target.type not in ['spark', 'databricks'] else ['date_day'],
+        partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'month'} if target.type not in ['spark', 'databricks'] else ['date_day'],
         unique_key='ticket_day_id',
         incremental_strategy = 'merge' if target.type not in ('snowflake', 'postgres', 'redshift') else 'delete+insert',
         file_format='delta'

models/ticket_history/int_zendesk__field_history_pivot.sql

@@ -3,7 +3,7 @@
 {{ 
     config(
         materialized='incremental',
-        partition_by = {'field': 'date_day', 'data_type': 'date'} if target.type not in ['spark', 'databricks'] else ['date_day'],
+        partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'month'} if target.type not in ['spark', 'databricks'] else ['date_day'],
         unique_key='ticket_day_id',
         incremental_strategy = 'merge' if target.type not in ('snowflake', 'postgres', 'redshift') else 'delete+insert',
         file_format='delta'

models/zendesk__ticket_field_history.sql

@@ -1,7 +1,7 @@
 {{ 
     config(
         materialized='incremental',
-        partition_by = {'field': 'date_day', 'data_type': 'date'} if target.type not in ['spark', 'databricks'] else ['date_day'],
+        partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'month' } if target.type not in ['spark', 'databricks'] else ['date_day'],
         unique_key='ticket_day_id',
         incremental_strategy = 'merge' if target.type not in ('snowflake', 'postgres', 'redshift') else 'delete+insert',
         file_format='delta'