From 546882dbeb5d81e5229ce3b7d7b1ab5ceb90b7c5 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 25 May 2021 14:48:43 -0400 Subject: [PATCH 1/7] union example draft --- models/stg_shopify__customer.sql | 4 +++ models/tmp/stg_shopify__customer_tmp.sql | 44 ++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index cdb9180..908a12a 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -22,6 +22,10 @@ renamed as ( {% endif %} + {% if var('union_schemas', none) or var('union_databases', none) %} + , _dbt_source_relation as source_relation + {% endif %} + from source ) diff --git a/models/tmp/stg_shopify__customer_tmp.sql b/models/tmp/stg_shopify__customer_tmp.sql index 0698d7e..9a17e8c 100644 --- a/models/tmp/stg_shopify__customer_tmp.sql +++ b/models/tmp/stg_shopify__customer_tmp.sql @@ -1,2 +1,42 @@ -select * -from {{ var('customer_source') }} \ No newline at end of file +{% if var('union_schemas', none) %} + + {% set relations = [] %} + + {% for schema in var('union_schemas') %} + + {% set relation=adapter.get_relation( + database=var('shopify_database', target.database), + schema=schema, + identifier="customer" + ) -%} + + {% do relations.append(relation) %} + + {% endfor %} + + {{ dbt_utils.union_relations(relations) }} + +{% elif var('union_databases', none) %} + + {% set relations = [] %} + + {% for database in var('union_databases') %} + + {% set relation=adapter.get_relation( + database=database, + schema=var('shopify_schema', 'shopify'), + identifier="customer" + ) -%} + + {% do relations.append(relation) %} + + {% endfor %} + + {{ dbt_utils.union_relations(relations) }} + +{% else %} + + select * + from {{ var('customer_source') }} + +{% endif %} \ No newline at end of file From 6d639c9282716eac91f3275c896f621546fb5487 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 28 May 2021 11:32:26 -0400 Subject: [PATCH 2/7] union all models --- macros/source_relation.sql | 19 +++++++ macros/union_data.sql | 46 ++++++++++++++++ models/stg_shopify__customer.sql | 4 +- models/stg_shopify__order.sql | 2 + models/stg_shopify__order_line.sql | 2 + models/stg_shopify__order_line_refund.sql | 2 + models/stg_shopify__product.sql | 2 + models/stg_shopify__product_variant.sql | 2 + models/tmp/stg_shopify__customer_tmp.sql | 52 ++++--------------- .../stg_shopify__order_line_refund_tmp.sql | 12 ++++- models/tmp/stg_shopify__order_line_tmp.sql | 12 ++++- models/tmp/stg_shopify__order_tmp.sql | 12 ++++- models/tmp/stg_shopify__product_tmp.sql | 12 ++++- .../tmp/stg_shopify__product_variant_tmp.sql | 12 ++++- 14 files changed, 136 insertions(+), 55 deletions(-) create mode 100644 macros/source_relation.sql create mode 100644 macros/union_data.sql diff --git a/macros/source_relation.sql b/macros/source_relation.sql new file mode 100644 index 0000000..084a6d7 --- /dev/null +++ b/macros/source_relation.sql @@ -0,0 +1,19 @@ +{% macro source_relation() %} + +{% if var('union_schemas', none) %} +, case + {% for schema in var('union_schemas') %} + when lower(replace(_dbt_source_relation,'"','')) like '%.{{ schema|lower }}.%' then '{{ schema|lower }}' + {% endfor %} + end as source_relation +{% elif var('union_databases', none) %} +, case + {% for database in var('union_databases') %} + when lower(replace(_dbt_source_relation,'"','')) like '%{{ database|lower }}.%' then '{{ database|lower }}' + {% endfor %} + end as source_relation +{% else %} +, '' as source_relation +{% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/macros/union_data.sql b/macros/union_data.sql new file mode 100644 index 0000000..9b6db35 --- /dev/null +++ b/macros/union_data.sql @@ -0,0 +1,46 @@ +{% macro union_data(table_identifier, database_variable, schema_variable, default_database, default_schema, default_variable) %} + +{% if var('union_schemas', none) %} + + {% set relations = [] %} + + {% for schema in var('union_schemas') %} + + {% set relation=adapter.get_relation( + database=var(database_variable, default_database), + schema=schema, + identifier=table_identifier + ) -%} + + {% do relations.append(relation) %} + + {% endfor %} + + {{ dbt_utils.union_relations(relations) }} + +{% elif var('union_databases', none) %} + + {% set relations = [] %} + + {% for database in var('union_databases') %} + + {% set relation=adapter.get_relation( + database=database, + schema=var(schema_variable, default_schema), + identifier=table_identifier + ) -%} + + {% do relations.append(relation) %} + + {% endfor %} + + {{ dbt_utils.union_relations(relations) }} + +{% else %} + + select * + from {{ var(default_variable) }} + +{% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index 908a12a..3accd0a 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -22,9 +22,7 @@ renamed as ( {% endif %} - {% if var('union_schemas', none) or var('union_databases', none) %} - , _dbt_source_relation as source_relation - {% endif %} + {{ source_relation() }} from source diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql index 2fde46d..6e4fdab 100644 --- a/models/stg_shopify__order.sql +++ b/models/stg_shopify__order.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ source_relation() }} + from source ) diff --git a/models/stg_shopify__order_line.sql b/models/stg_shopify__order_line.sql index ec00c13..26d971f 100644 --- a/models/stg_shopify__order_line.sql +++ b/models/stg_shopify__order_line.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ source_relation() }} + from source ) diff --git a/models/stg_shopify__order_line_refund.sql b/models/stg_shopify__order_line_refund.sql index 043b53a..e7c6f0c 100644 --- a/models/stg_shopify__order_line_refund.sql +++ b/models/stg_shopify__order_line_refund.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ source_relation() }} + from source ) diff --git a/models/stg_shopify__product.sql b/models/stg_shopify__product.sql index a541453..10269a2 100644 --- a/models/stg_shopify__product.sql +++ b/models/stg_shopify__product.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ source_relation() }} + from source ) diff --git a/models/stg_shopify__product_variant.sql b/models/stg_shopify__product_variant.sql index 1d12cd4..4bf5ade 100644 --- a/models/stg_shopify__product_variant.sql +++ b/models/stg_shopify__product_variant.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ source_relation() }} + from source ) diff --git a/models/tmp/stg_shopify__customer_tmp.sql b/models/tmp/stg_shopify__customer_tmp.sql index 9a17e8c..b32518c 100644 --- a/models/tmp/stg_shopify__customer_tmp.sql +++ b/models/tmp/stg_shopify__customer_tmp.sql @@ -1,42 +1,10 @@ -{% if var('union_schemas', none) %} - - {% set relations = [] %} - - {% for schema in var('union_schemas') %} - - {% set relation=adapter.get_relation( - database=var('shopify_database', target.database), - schema=schema, - identifier="customer" - ) -%} - - {% do relations.append(relation) %} - - {% endfor %} - - {{ dbt_utils.union_relations(relations) }} - -{% elif var('union_databases', none) %} - - {% set relations = [] %} - - {% for database in var('union_databases') %} - - {% set relation=adapter.get_relation( - database=database, - schema=var('shopify_schema', 'shopify'), - identifier="customer" - ) -%} - - {% do relations.append(relation) %} - - {% endfor %} - - {{ dbt_utils.union_relations(relations) }} - -{% else %} - - select * - from {{ var('customer_source') }} - -{% endif %} \ No newline at end of file +{{ + union_data( + table_identifier='customer', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='customer_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__order_line_refund_tmp.sql b/models/tmp/stg_shopify__order_line_refund_tmp.sql index 57fbb3c..8fdf278 100644 --- a/models/tmp/stg_shopify__order_line_refund_tmp.sql +++ b/models/tmp/stg_shopify__order_line_refund_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('order_line_refund_source') }} \ No newline at end of file +{{ + union_data( + table_identifier='order_line_refund', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='order_line_refund_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__order_line_tmp.sql b/models/tmp/stg_shopify__order_line_tmp.sql index 7c6b189..1de5dc2 100644 --- a/models/tmp/stg_shopify__order_line_tmp.sql +++ b/models/tmp/stg_shopify__order_line_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('order_line_source') }} \ No newline at end of file +{{ + union_data( + table_identifier='order_line', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='order_line_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__order_tmp.sql b/models/tmp/stg_shopify__order_tmp.sql index 49cf535..5cf6a11 100644 --- a/models/tmp/stg_shopify__order_tmp.sql +++ b/models/tmp/stg_shopify__order_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('order_source') }} \ No newline at end of file +{{ + union_data( + table_identifier='order', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='order_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__product_tmp.sql b/models/tmp/stg_shopify__product_tmp.sql index 2f45797..1135201 100644 --- a/models/tmp/stg_shopify__product_tmp.sql +++ b/models/tmp/stg_shopify__product_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('product_source') }} \ No newline at end of file +{{ + union_data( + table_identifier='product', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='product_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__product_variant_tmp.sql b/models/tmp/stg_shopify__product_variant_tmp.sql index 684fa8f..a5e9a16 100644 --- a/models/tmp/stg_shopify__product_variant_tmp.sql +++ b/models/tmp/stg_shopify__product_variant_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('product_variant_source') }} \ No newline at end of file +{{ + union_data( + table_identifier='product_variant', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='product_variant_source' + ) +}} \ No newline at end of file From 2a9e7feb1bea559658c6273c95dfc8d37f558573 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 28 May 2021 11:50:24 -0400 Subject: [PATCH 3/7] merge and fix tests --- models/stg_shopify.yml | 80 ++++++++++++------- models/stg_shopify__order_adjustment.sql | 2 + models/stg_shopify__refund.sql | 2 + models/stg_shopify__transaction.sql | 2 + .../tmp/stg_shopify__order_adjustment_tmp.sql | 11 ++- models/tmp/stg_shopify__refund_tmp.sql | 11 ++- models/tmp/stg_shopify__transaction_tmp.sql | 12 ++- 7 files changed, 89 insertions(+), 31 deletions(-) diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index b913382..e8c066e 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -3,6 +3,11 @@ version: 2 models: - name: stg_shopify__customer description: Each record represents a customer in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - customer_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -17,9 +22,6 @@ models: - name: first_name description: The customer's first name. - name: customer_id - tests: - - unique - - not_null description: A unique identifier for the customer. - name: last_name description: The customer's last name. @@ -37,15 +39,18 @@ models: description: The date and time when the customer information was last updated. - name: is_verified_email description: Whether the customer has verified their email address. + - name: stg_shopify__order_line_refund description: Each record represents a line item from an order in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_line_refund_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" - name: order_line_refund_id - tests: - - unique - - not_null description: The unique identifier of the line item in the refund. - name: location_id description: TThe unique identifier of the location where the items will be restockedBD @@ -61,8 +66,14 @@ models: description: Subtotal amount of the order line refund - name: total_tax description: The total tax applied to the refund. + - name: stg_shopify__order_line description: Each record represents a line item from an order in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_line_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -77,9 +88,6 @@ models: - name: grams description: The weight of the item in grams. - name: order_line_id - tests: - - unique - - not_null description: The ID of the line item. - name: name description: The name of the product variant. @@ -105,8 +113,14 @@ models: description: The ID of the product variant. - name: vendor description: The name of the item's supplier. + - name: stg_shopify__order description: Each record represents an order in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_id + - source_relation columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -167,9 +181,6 @@ models: - name: fulfillment_status description: The order's status in terms of fulfilled line items. - name: order_id - tests: - - unique - - not_null description: The ID of the order, used for API purposes. This is different from the order_number property, which is the ID used by the shop owner and customer. - name: landing_site_base_url description: The URL for the page where the buyer landed when they entered the shop. @@ -249,8 +260,14 @@ models: description: The index associated with the order. - name: pre_tax_price description: The total pre tax price of the order. + - name: stg_shopify__product description: Each record represents a product in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - product_id + - source_relation columns: - name: _fivetran_deleted description: Whether the record has been deleted in the source system. @@ -261,9 +278,6 @@ models: - name: handle description: A unique human-friendly string for the product. Automatically generated from the product's title. - name: product_id - tests: - - unique - - not_null description: An unsigned 64-bit integer that's used as a unique identifier for the product. Each id is unique across the Shopify system. No two products will have the same id, even if they're from different shops. - name: product_type description: A categorization for the product used for filtering and searching products. @@ -277,8 +291,14 @@ models: description: The date and time when the product was last modified. - name: vendor description: The name of the product's vendor. + - name: stg_shopify__product_variant description: Each record represents a product variant in Shopify + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - variant_id + - source_relation columns: - name: barcode description: The barcode, UPC, or ISBN number for the product. @@ -292,9 +312,6 @@ models: description: The weight of the product variant in grams. - name: variant_id description: The unique numeric identifier for the product variant. - tests: - - not_null - - unique - name: image_id description: The unique numeric identifier for a product's image. The image must be associated to the same product as the variant. - name: inventory_item_id @@ -337,13 +354,16 @@ models: description: "The unit of measurement that applies to the product variant's weight. If you don't specify a value for weight_unit, then the shop's default unit of measurement is applied. Valid values: g, kg, oz, and lb." - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" + - name: stg_shopify__transaction description: Each record represents a transaction in Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - transaction_id + - source_relation columns: - name: transaction_id - tests: - - unique - - not_null description: The ID for the transaction. - name: order_id description: The ID for the order that the transaction is associated with. @@ -405,14 +425,17 @@ models: description: The ID for the user who was logged into the Shopify POS device when the order was processed, if applicable. - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" + - name: stg_shopify__refund description: Each record represents a refund within Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - refund_id + - source_relation columns: - name: refund_id description: The unique numeric identifier for the refund. - tests: - - unique - - not_null - name: created_at description: Timestamp of the date when the refund was created. - name: processed_at @@ -429,14 +452,17 @@ models: description: Record representing total duties set for the refund. - name: order_id description: Reference to the order which the refund is associated. + - name: stg_shopify__order_adjustment description: Each record represents and adjustment to and order within Shopify. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_adjustment_id + - source_relation columns: - name: order_adjustment_id description: The unique numeric identifier for the order adjustment. - tests: - - unique - - not_null - name: order_id description: Reference to the order which the adjustment is associated. - name: refund_id diff --git a/models/stg_shopify__order_adjustment.sql b/models/stg_shopify__order_adjustment.sql index 1f363c0..7d0d8e1 100644 --- a/models/stg_shopify__order_adjustment.sql +++ b/models/stg_shopify__order_adjustment.sql @@ -15,6 +15,8 @@ renamed as ( staging_columns=get_order_adjustment_columns() ) }} + + {{ source_relation() }} from source ) diff --git a/models/stg_shopify__refund.sql b/models/stg_shopify__refund.sql index 87e476d..3d48f84 100644 --- a/models/stg_shopify__refund.sql +++ b/models/stg_shopify__refund.sql @@ -15,6 +15,8 @@ renamed as ( staging_columns=get_refund_columns() ) }} + + {{ source_relation() }} from source ) diff --git a/models/stg_shopify__transaction.sql b/models/stg_shopify__transaction.sql index 25e57f3..ba39ce5 100644 --- a/models/stg_shopify__transaction.sql +++ b/models/stg_shopify__transaction.sql @@ -22,6 +22,8 @@ renamed as ( {% endif %} + {{ source_relation() }} + from source where not test diff --git a/models/tmp/stg_shopify__order_adjustment_tmp.sql b/models/tmp/stg_shopify__order_adjustment_tmp.sql index 0661b40..e936ba3 100644 --- a/models/tmp/stg_shopify__order_adjustment_tmp.sql +++ b/models/tmp/stg_shopify__order_adjustment_tmp.sql @@ -1 +1,10 @@ -select * from {{ var('order_adjustment_source') }} +{{ + union_data( + table_identifier='order_adjustment', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='order_adjustment_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__refund_tmp.sql b/models/tmp/stg_shopify__refund_tmp.sql index ac2559d..f99fc58 100644 --- a/models/tmp/stg_shopify__refund_tmp.sql +++ b/models/tmp/stg_shopify__refund_tmp.sql @@ -1 +1,10 @@ -select * from {{ var('refund_source') }} +{{ + union_data( + table_identifier='refund', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='refund_source' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__transaction_tmp.sql b/models/tmp/stg_shopify__transaction_tmp.sql index 3e1abb8..f727b58 100644 --- a/models/tmp/stg_shopify__transaction_tmp.sql +++ b/models/tmp/stg_shopify__transaction_tmp.sql @@ -1,2 +1,10 @@ -select * -from {{ var('transaction_source') }} +{{ + union_data( + table_identifier='transaction', + database_variable='shopify_database', + schema_variable='shopify_schema', + default_database=target.database, + default_schema='shopify', + default_variable='transaction_source' + ) +}} \ No newline at end of file From cc89db7773c02bf18172af6bd1503d4a2f38a3b6 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Wed, 16 Jun 2021 14:58:03 -0400 Subject: [PATCH 4/7] move macro to fivetran_utils --- macros/union_data.sql | 46 ------------------- models/tmp/stg_shopify__customer_tmp.sql | 2 +- .../tmp/stg_shopify__order_adjustment_tmp.sql | 2 +- .../stg_shopify__order_line_refund_tmp.sql | 2 +- models/tmp/stg_shopify__order_line_tmp.sql | 2 +- models/tmp/stg_shopify__order_tmp.sql | 2 +- models/tmp/stg_shopify__product_tmp.sql | 2 +- .../tmp/stg_shopify__product_variant_tmp.sql | 2 +- models/tmp/stg_shopify__refund_tmp.sql | 2 +- models/tmp/stg_shopify__transaction_tmp.sql | 2 +- packages.yml | 1 + 11 files changed, 10 insertions(+), 55 deletions(-) delete mode 100644 macros/union_data.sql diff --git a/macros/union_data.sql b/macros/union_data.sql deleted file mode 100644 index 9b6db35..0000000 --- a/macros/union_data.sql +++ /dev/null @@ -1,46 +0,0 @@ -{% macro union_data(table_identifier, database_variable, schema_variable, default_database, default_schema, default_variable) %} - -{% if var('union_schemas', none) %} - - {% set relations = [] %} - - {% for schema in var('union_schemas') %} - - {% set relation=adapter.get_relation( - database=var(database_variable, default_database), - schema=schema, - identifier=table_identifier - ) -%} - - {% do relations.append(relation) %} - - {% endfor %} - - {{ dbt_utils.union_relations(relations) }} - -{% elif var('union_databases', none) %} - - {% set relations = [] %} - - {% for database in var('union_databases') %} - - {% set relation=adapter.get_relation( - database=database, - schema=var(schema_variable, default_schema), - identifier=table_identifier - ) -%} - - {% do relations.append(relation) %} - - {% endfor %} - - {{ dbt_utils.union_relations(relations) }} - -{% else %} - - select * - from {{ var(default_variable) }} - -{% endif %} - -{% endmacro %} \ No newline at end of file diff --git a/models/tmp/stg_shopify__customer_tmp.sql b/models/tmp/stg_shopify__customer_tmp.sql index b32518c..53cb4c5 100644 --- a/models/tmp/stg_shopify__customer_tmp.sql +++ b/models/tmp/stg_shopify__customer_tmp.sql @@ -1,5 +1,5 @@ {{ - union_data( + fivetran_utils.union_data( table_identifier='customer', database_variable='shopify_database', schema_variable='shopify_schema', diff --git a/models/tmp/stg_shopify__order_adjustment_tmp.sql b/models/tmp/stg_shopify__order_adjustment_tmp.sql index e936ba3..0bb9d95 100644 --- a/models/tmp/stg_shopify__order_adjustment_tmp.sql +++ b/models/tmp/stg_shopify__order_adjustment_tmp.sql @@ -1,5 +1,5 @@ {{ - union_data( + fivetran_utils.union_data( table_identifier='order_adjustment', database_variable='shopify_database', schema_variable='shopify_schema', diff --git a/models/tmp/stg_shopify__order_line_refund_tmp.sql b/models/tmp/stg_shopify__order_line_refund_tmp.sql index 8fdf278..0adc837 100644 --- a/models/tmp/stg_shopify__order_line_refund_tmp.sql +++ b/models/tmp/stg_shopify__order_line_refund_tmp.sql @@ -1,5 +1,5 @@ {{ - union_data( + fivetran_utils.union_data( table_identifier='order_line_refund', database_variable='shopify_database', schema_variable='shopify_schema', diff --git a/models/tmp/stg_shopify__order_line_tmp.sql b/models/tmp/stg_shopify__order_line_tmp.sql index 1de5dc2..1d91d74 100644 --- a/models/tmp/stg_shopify__order_line_tmp.sql +++ b/models/tmp/stg_shopify__order_line_tmp.sql @@ -1,5 +1,5 @@ {{ - union_data( + fivetran_utils.union_data( table_identifier='order_line', database_variable='shopify_database', schema_variable='shopify_schema', diff --git a/models/tmp/stg_shopify__order_tmp.sql b/models/tmp/stg_shopify__order_tmp.sql index 5cf6a11..71723fd 100644 --- a/models/tmp/stg_shopify__order_tmp.sql +++ b/models/tmp/stg_shopify__order_tmp.sql @@ -1,5 +1,5 @@ {{ - union_data( + fivetran_utils.union_data( table_identifier='order', database_variable='shopify_database', schema_variable='shopify_schema', diff --git a/models/tmp/stg_shopify__product_tmp.sql b/models/tmp/stg_shopify__product_tmp.sql index 1135201..aaec57d 100644 --- a/models/tmp/stg_shopify__product_tmp.sql +++ b/models/tmp/stg_shopify__product_tmp.sql @@ -1,5 +1,5 @@ {{ - union_data( + fivetran_utils.union_data( table_identifier='product', database_variable='shopify_database', schema_variable='shopify_schema', diff --git a/models/tmp/stg_shopify__product_variant_tmp.sql b/models/tmp/stg_shopify__product_variant_tmp.sql index a5e9a16..3b1e479 100644 --- a/models/tmp/stg_shopify__product_variant_tmp.sql +++ b/models/tmp/stg_shopify__product_variant_tmp.sql @@ -1,5 +1,5 @@ {{ - union_data( + fivetran_utils.union_data( table_identifier='product_variant', database_variable='shopify_database', schema_variable='shopify_schema', diff --git a/models/tmp/stg_shopify__refund_tmp.sql b/models/tmp/stg_shopify__refund_tmp.sql index f99fc58..8e7e631 100644 --- a/models/tmp/stg_shopify__refund_tmp.sql +++ b/models/tmp/stg_shopify__refund_tmp.sql @@ -1,5 +1,5 @@ {{ - union_data( + fivetran_utils.union_data( table_identifier='refund', database_variable='shopify_database', schema_variable='shopify_schema', diff --git a/models/tmp/stg_shopify__transaction_tmp.sql b/models/tmp/stg_shopify__transaction_tmp.sql index f727b58..659c81a 100644 --- a/models/tmp/stg_shopify__transaction_tmp.sql +++ b/models/tmp/stg_shopify__transaction_tmp.sql @@ -1,5 +1,5 @@ {{ - union_data( + fivetran_utils.union_data( table_identifier='transaction', database_variable='shopify_database', schema_variable='shopify_schema', diff --git a/packages.yml b/packages.yml index 7feb803..82bc926 100644 --- a/packages.yml +++ b/packages.yml @@ -8,3 +8,4 @@ packages: - git: https://github.com/fivetran/dbt_fivetran_utils.git warn-unpinned: false + revision: "union-data-macro" From c303c3773275dcf6f89ac41c4074a7bccfffe4bf Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Wed, 16 Jun 2021 15:15:55 -0400 Subject: [PATCH 5/7] move source_relation to fivetran_utils --- macros/source_relation.sql | 19 ------------------- models/stg_shopify__customer.sql | 2 +- models/stg_shopify__order.sql | 2 +- models/stg_shopify__order_adjustment.sql | 2 +- models/stg_shopify__order_line.sql | 2 +- models/stg_shopify__order_line_refund.sql | 2 +- models/stg_shopify__product.sql | 2 +- models/stg_shopify__product_variant.sql | 2 +- models/stg_shopify__refund.sql | 2 +- models/stg_shopify__transaction.sql | 2 +- 10 files changed, 9 insertions(+), 28 deletions(-) delete mode 100644 macros/source_relation.sql diff --git a/macros/source_relation.sql b/macros/source_relation.sql deleted file mode 100644 index 084a6d7..0000000 --- a/macros/source_relation.sql +++ /dev/null @@ -1,19 +0,0 @@ -{% macro source_relation() %} - -{% if var('union_schemas', none) %} -, case - {% for schema in var('union_schemas') %} - when lower(replace(_dbt_source_relation,'"','')) like '%.{{ schema|lower }}.%' then '{{ schema|lower }}' - {% endfor %} - end as source_relation -{% elif var('union_databases', none) %} -, case - {% for database in var('union_databases') %} - when lower(replace(_dbt_source_relation,'"','')) like '%{{ database|lower }}.%' then '{{ database|lower }}' - {% endfor %} - end as source_relation -{% else %} -, '' as source_relation -{% endif %} - -{% endmacro %} \ No newline at end of file diff --git a/models/stg_shopify__customer.sql b/models/stg_shopify__customer.sql index 3accd0a..d4cf5d3 100644 --- a/models/stg_shopify__customer.sql +++ b/models/stg_shopify__customer.sql @@ -22,7 +22,7 @@ renamed as ( {% endif %} - {{ source_relation() }} + {{ fivetran_utils.source_relation() }} from source diff --git a/models/stg_shopify__order.sql b/models/stg_shopify__order.sql index 6e4fdab..7215216 100644 --- a/models/stg_shopify__order.sql +++ b/models/stg_shopify__order.sql @@ -22,7 +22,7 @@ renamed as ( {% endif %} - {{ source_relation() }} + {{ fivetran_utils.source_relation() }} from source diff --git a/models/stg_shopify__order_adjustment.sql b/models/stg_shopify__order_adjustment.sql index 7d0d8e1..63b90ab 100644 --- a/models/stg_shopify__order_adjustment.sql +++ b/models/stg_shopify__order_adjustment.sql @@ -16,7 +16,7 @@ renamed as ( ) }} - {{ source_relation() }} + {{ fivetran_utils.source_relation() }} from source ) diff --git a/models/stg_shopify__order_line.sql b/models/stg_shopify__order_line.sql index 26d971f..1861790 100644 --- a/models/stg_shopify__order_line.sql +++ b/models/stg_shopify__order_line.sql @@ -22,7 +22,7 @@ renamed as ( {% endif %} - {{ source_relation() }} + {{ fivetran_utils.source_relation() }} from source diff --git a/models/stg_shopify__order_line_refund.sql b/models/stg_shopify__order_line_refund.sql index e7c6f0c..d7d7d1a 100644 --- a/models/stg_shopify__order_line_refund.sql +++ b/models/stg_shopify__order_line_refund.sql @@ -22,7 +22,7 @@ renamed as ( {% endif %} - {{ source_relation() }} + {{ fivetran_utils.source_relation() }} from source diff --git a/models/stg_shopify__product.sql b/models/stg_shopify__product.sql index 10269a2..190efab 100644 --- a/models/stg_shopify__product.sql +++ b/models/stg_shopify__product.sql @@ -22,7 +22,7 @@ renamed as ( {% endif %} - {{ source_relation() }} + {{ fivetran_utils.source_relation() }} from source diff --git a/models/stg_shopify__product_variant.sql b/models/stg_shopify__product_variant.sql index 0156f8d..a39fcd6 100644 --- a/models/stg_shopify__product_variant.sql +++ b/models/stg_shopify__product_variant.sql @@ -22,7 +22,7 @@ renamed as ( {% endif %} - {{ source_relation() }} + {{ fivetran_utils.source_relation() }} from source diff --git a/models/stg_shopify__refund.sql b/models/stg_shopify__refund.sql index 3d48f84..4783213 100644 --- a/models/stg_shopify__refund.sql +++ b/models/stg_shopify__refund.sql @@ -16,7 +16,7 @@ renamed as ( ) }} - {{ source_relation() }} + {{ fivetran_utils.source_relation() }} from source ) diff --git a/models/stg_shopify__transaction.sql b/models/stg_shopify__transaction.sql index ba39ce5..28c78fe 100644 --- a/models/stg_shopify__transaction.sql +++ b/models/stg_shopify__transaction.sql @@ -22,7 +22,7 @@ renamed as ( {% endif %} - {{ source_relation() }} + {{ fivetran_utils.source_relation() }} from source where not test From b121af63b55bdc41685876e57b8f7f3aaf2711c2 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Wed, 16 Jun 2021 15:25:47 -0400 Subject: [PATCH 6/7] update README --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index b9bb865..1ffcca1 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,19 @@ vars: shopify_schema: your_schema_name ``` +If you have multiple Shopify connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `union_schemas` or `union_databases` variables: + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + union_schema: ['shopify_usa','shopify_canada'] # use this if the data is in different schemas/datasets of the same database/project + union_databases: ['shopify_usa','shopify_canada'] # use this if the data is in different databases/projects but uses the same schema name +``` + This package includes all source columns defined in the staging_columns.sql macro. To add additional columns to this package, do so using our pass-through column variables. This is extremely useful if you'd like to include custom fields to the package. ```yml From 68bfa0d749746ed57f02c2176a4e84dabf7af4db Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Tue, 22 Jun 2021 17:36:50 -0500 Subject: [PATCH 7/7] fivetran_utils dep switch and version upgrade --- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- packages.yml | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 7ef21e4..7d80a5f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ name: 'shopify_source' -version: '0.3.0' +version: '0.3.1' config-version: 2 require-dbt-version: [">=0.18.0", "<0.20.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index f7107c4..303b2b1 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'shopify_source_integration_tests' -version: '0.3.0' +version: '0.3.1' profile: 'integration_tests' config-version: 2 diff --git a/packages.yml b/packages.yml index 82bc926..7feb803 100644 --- a/packages.yml +++ b/packages.yml @@ -8,4 +8,3 @@ packages: - git: https://github.com/fivetran/dbt_fivetran_utils.git warn-unpinned: false - revision: "union-data-macro"