add line item standardized model

dbt_project.yml

@@ -1,10 +1,10 @@
 name: 'shopify'
-version: '0.12.2'
+version: '0.13.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   shopify:
-    +schema: shopify
+    +schema: "{{ 'shopify_sql_' ~ var('directed_schema','dev') if target.name == 'databricks-sql' else 'shopify_' ~ var('directed_schema','dev') }}"
     +materialized: table
     intermediate:
       +materialized: ephemeral

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_integration_tests_8
+      schema: shopify_integration_tests_9
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_integration_tests_8
+      schema: shopify_integration_tests_9
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_integration_tests_8
+      schema: shopify_integration_tests_9
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_integration_tests_8
+      schema: shopify_integration_tests_9
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_integration_tests_8
+      schema: shopify_integration_tests_9
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify_integration_tests'
-version: '0.12.2'
+version: '0.13.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -11,7 +11,7 @@ clean-targets:         # directories to be removed by `dbt clean`
 vars:
   # shopify_using_fulfillment_event: true # set to true when regenerating docs
   # shopify_using_all_metafields: true # set to true when regenerating docs
-  shopify_schema: shopify_integration_tests_8
+  shopify_schema: shopify_integration_tests_9
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"
     shopify_order_line_refund_identifier: "shopify_order_line_refund_data"

integration_tests/tests/consistency/consistency_line_item_enhanced.sql

@@ -0,0 +1,31 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_shopify_prod.shopify__line_item_enhanced
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_shopify_dev.shopify__line_item_enhanced
+), 
+
+final as (
+    -- test will fail if any rows from prod are not found in dev
+    (select * from prod
+    except distinct
+    select * from dev)
+
+    union all -- union since we only care if rows are produced
+
+    -- test will fail if any rows from dev are not found in prod
+    (select * from dev
+    except distinct
+    select * from prod)
+    )
+
+select *
+from final

integration_tests/tests/consistency/consistency_line_item_enhanced_count.sql

@@ -0,0 +1,21 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test is to make sure the rows counts are the same between versions
+with prod as (
+    select count(*) as prod_rows
+    from {{ target.schema }}_shopify_prod.shopify__line_item_enhanced
+),
+
+dev as (
+    select count(*) as dev_rows
+    from {{ target.schema }}_shopify_dev.shopify__line_item_enhanced
+)
+
+-- test will return values and fail if the row counts don't match
+select *
+from prod
+join dev
+    on prod.prod_rows != dev.dev_rows

integration_tests/tests/integrity/integrity_line_item_enhanced.sql

@@ -0,0 +1,36 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test is to make sure there is no fanout between the staging order_line_table and the line_item_enhanced model.
+with stg_order_line as (
+    select
+        1 as join_key,
+        count(*) as order_line_count,
+        count(distinct order_id) as order_count
+    from {{ target.schema }}_shopify_dev.stg_shopify__order_line
+),
+
+line_item_enhanced as (
+    select
+        1 as join_key,
+        count(*) as line_item_enhanced_count
+    from {{ target.schema }}_shopify_dev.shopify__line_item_enhanced
+),
+
+-- test will return values and fail if the row counts don't match
+
+final as (
+    select 
+        stg_order_line.join_key,
+        stg_order_line.order_line_count + stg_order_line.order_count as total_line_and_order_count,
+        line_item_enhanced.line_item_enhanced_count
+    from stg_order_line
+    join line_item_enhanced
+        on stg_order_line.join_key = line_item_enhanced.join_key
+) 
+
+select *
+from final
+where total_line_and_order_count != line_item_enhanced_count

models/common_data_models/shopify__common_data_models.yml

@@ -0,0 +1,82 @@
+version: 2
+
+models:
+  - name: shopify__line_item_enhanced
+    description: Add description
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - header_id
+            - line_item_id
+            - source_relation
+    columns:
+      - name: header_id
+        description: Id of corresponding order.
+      - name: line_item_id
+        description: Order line item id.
+      - name: line_item_index
+        description: Unique index of each line item id for each order.
+      - name: record_type
+        description: header or line_item
+      - name: created_at
+        description: When the order was created
+      - name: header_status
+        description: Status of the order.
+      - name: billing_type
+        description: NA for Shopify.
+      - name: currency
+        description: Currency used in the order.
+      - name: product_id
+        description: Product ID
+      - name: product_name
+        description: Product name
+      - name: product_type
+        description: Product type
+      - name: transaction_type
+        description: Transaction type (Charge, Refund, etc)
+      - name: quantity
+        description: The number of units of this line item.
+      - name: unit_amount
+        description: Amount per unit, or the price.
+      - name: discount_amount
+        description: The discount amount associated with this order.
+      - name: tax_amount
+        description: The tax amount associated with this order.
+      - name: total_amount
+        description: Order total
+      - name: payment_id
+        description: Transaction ID
+      - name: payment_method
+        description: Transaction gateway for Shopify ('gift card', 'shopify payments', etc)
+      - name: payment_method_id
+        description: NA for Shopify
+      - name: payment_at
+        description: Transaction processed timestamp
+      - name: fee_amount
+        description: NA for Shopify
+      - name: refund_amount
+        description: Refund amount associated with this order.
+      - name: subscription_id
+        description: NA for Shopify
+      - name: subscription_period_started_at
+        description: NA for Shopify
+      - name: subscription_period_ended_at
+        description: NA for Shopify
+      - name: subscription_status
+        description: NA for Shopify
+      - name: customer_id
+        description: Customer associated with this order.
+      - name: customer_level
+        description: Whether 'account' or 'customer'. For Shopify, the level is 'customer'.
+      - name: customer_name
+        description: Customer name
+      - name: customer_company
+        description: Associated customer's company noted on the shipping address if applicable.
+      - name: customer_email
+        description: Customer email
+      - name: customer_city
+        description: Customer's shipping address city
+      - name: customer_country
+        description: Customer's shipping address country
+      - name: source_relation
+        description: The source of the record if the unioning functionality is being used. If not this field will be empty.