From d487f8d83e823fbf86c70cead58b9be01dadb2a6 Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Tue, 27 Feb 2024 19:52:31 +0100 Subject: [PATCH 01/10] created tox testenv:integration_sqlserver added target 'sqlserver' to integration_test profiles.yml --- integration_test_project/profiles.yml | 11 +++++++++++ tox.ini | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index b24ad80d..b5ded769 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -52,3 +52,14 @@ dbt_artifacts: dbname: postgres schema: public threads: 8 + sqlserver: + type: sqlserver + driver: 'ODBC Driver 18 for SQL Server' + server: localhost + port: 1433 + database: dbt_artifact_integrationtests + schema: dbo + windows_login: False + trust_cert: True + user: dbt + password: "123" \ No newline at end of file diff --git a/tox.ini b/tox.ini index 542d6e21..858cd755 100644 --- a/tox.ini +++ b/tox.ini @@ -322,4 +322,12 @@ commands = dbt deps dbt build --target postgres +[testenv:integration_sqlserver] +changedir = integration_test_project +deps = dbt-sqlserver~=1.4.3 +commands = + dbt clean + dbt deps + dbt build --target sqlserver + From a567f8ad14ced192f6532c0628c7a5cac1f5fe94 Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Tue, 27 Feb 2024 19:59:15 +0100 Subject: [PATCH 02/10] remove persist_docs flag for sqlserver don't create Column-Store-Index for source-models because it contains column-types that aren't supported in a CCI --- dbt_project.yml | 3 ++- integration_test_project/dbt_project.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 60bd2117..c6ac1c0e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -19,4 +19,5 @@ models: +full_refresh: false +persist_docs: # Databricks doesn't offer column-level support for persisting docs - columns: '{{ target.name != "databricks" }}' + columns: '{{ target.name != "databricks" and target.name != "sqlserver" }}' + +as_columnstore: False \ No newline at end of file diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index dba89a50..aae6f8df 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -25,8 +25,8 @@ vars: models: +persist_docs: - relation: true - columns: true + relation: '{{ target.name != "sqlserver" }}' # sqlserver-adapter doesn't support this + columns: '{{ target.name != "sqlserver" }}' # sqlserver-adapter doesn't support this seeds: +quote_columns: false From a1301159d4ff55df6c467913ccd60b6234bd64e7 Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Tue, 27 Feb 2024 20:10:42 +0100 Subject: [PATCH 03/10] use doublequotes for columns named "schema" and "database" because these are reserved words in T-SQL --- models/dim_dbt__models.sql | 4 ++-- models/dim_dbt__seeds.sql | 4 ++-- models/dim_dbt__snapshots.sql | 4 ++-- models/dim_dbt__sources.sql | 4 ++-- models/fct_dbt__model_executions.sql | 4 ++-- models/fct_dbt__seed_executions.sql | 2 +- models/fct_dbt__snapshot_executions.sql | 2 +- models/sources/model_executions.sql | 2 +- models/sources/models.sql | 4 ++-- models/sources/seed_executions.sql | 2 +- models/sources/seeds.sql | 4 ++-- models/sources/snapshot_executions.sql | 2 +- models/sources/snapshots.sql | 4 ++-- models/sources/sources.sql | 4 ++-- models/staging/stg_dbt__model_executions.sql | 2 +- models/staging/stg_dbt__models.sql | 4 ++-- models/staging/stg_dbt__seed_executions.sql | 2 +- models/staging/stg_dbt__seeds.sql | 4 ++-- models/staging/stg_dbt__snapshot_executions.sql | 2 +- models/staging/stg_dbt__snapshots.sql | 4 ++-- models/staging/stg_dbt__sources.sql | 4 ++-- 21 files changed, 34 insertions(+), 34 deletions(-) diff --git a/models/dim_dbt__models.sql b/models/dim_dbt__models.sql index 2722ce83..0b254e72 100644 --- a/models/dim_dbt__models.sql +++ b/models/dim_dbt__models.sql @@ -12,8 +12,8 @@ models as ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, depends_on_nodes, package_name, diff --git a/models/dim_dbt__seeds.sql b/models/dim_dbt__seeds.sql index d7a461ab..efdd4ea7 100644 --- a/models/dim_dbt__seeds.sql +++ b/models/dim_dbt__seeds.sql @@ -12,8 +12,8 @@ seeds as ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, package_name, path, diff --git a/models/dim_dbt__snapshots.sql b/models/dim_dbt__snapshots.sql index 13f05d0e..7e524652 100644 --- a/models/dim_dbt__snapshots.sql +++ b/models/dim_dbt__snapshots.sql @@ -12,8 +12,8 @@ snapshots as ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, depends_on_nodes, package_name, diff --git a/models/dim_dbt__sources.sql b/models/dim_dbt__sources.sql index 84b76d4d..4351ac2c 100644 --- a/models/dim_dbt__sources.sql +++ b/models/dim_dbt__sources.sql @@ -12,8 +12,8 @@ sources as ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", source_name, loader, name, diff --git a/models/fct_dbt__model_executions.sql b/models/fct_dbt__model_executions.sql index f78a7f93..c7dd7b31 100644 --- a/models/fct_dbt__model_executions.sql +++ b/models/fct_dbt__model_executions.sql @@ -23,8 +23,8 @@ model_executions as ( bytes_processed, {% endif %} materialization, - schema, -- noqa - name, + "schema", -- noqa + "name", alias, message from base diff --git a/models/fct_dbt__seed_executions.sql b/models/fct_dbt__seed_executions.sql index e8e38365..086198d8 100644 --- a/models/fct_dbt__seed_executions.sql +++ b/models/fct_dbt__seed_executions.sql @@ -20,7 +20,7 @@ seed_executions as ( total_node_runtime, rows_affected, materialization, - schema, + "schema", name, alias, message diff --git a/models/fct_dbt__snapshot_executions.sql b/models/fct_dbt__snapshot_executions.sql index 83c656f1..b93ddd73 100644 --- a/models/fct_dbt__snapshot_executions.sql +++ b/models/fct_dbt__snapshot_executions.sql @@ -20,7 +20,7 @@ snapshot_executions as ( total_node_runtime, rows_affected, materialization, - schema, + "schema", name, alias, message diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql index 09bec105..d5d259ba 100644 --- a/models/sources/model_executions.sql +++ b/models/sources/model_executions.sql @@ -18,7 +18,7 @@ select cast(null as {{ type_int() }}) as bytes_processed, {% endif %} cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as "schema", cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as alias, cast(null as {{ type_string() }}) as message, diff --git a/models/sources/models.sql b/models/sources/models.sql index f4ec8f01..94a960ec 100644 --- a/models/sources/models.sql +++ b/models/sources/models.sql @@ -7,8 +7,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as "database", + cast(null as {{ type_string() }}) as "schema", cast(null as {{ type_string() }}) as name, cast(null as {{ type_array() }}) as depends_on_nodes, cast(null as {{ type_string() }}) as package_name, diff --git a/models/sources/seed_executions.sql b/models/sources/seed_executions.sql index dbb93977..3eae3f77 100644 --- a/models/sources/seed_executions.sql +++ b/models/sources/seed_executions.sql @@ -15,7 +15,7 @@ select cast(null as {{ type_float() }}) as total_node_runtime, cast(null as {{ type_int() }}) as rows_affected, cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as "schema", cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as alias, cast(null as {{ type_string() }}) as message, diff --git a/models/sources/seeds.sql b/models/sources/seeds.sql index 7c52ecf7..5727971c 100644 --- a/models/sources/seeds.sql +++ b/models/sources/seeds.sql @@ -7,8 +7,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as "database", + cast(null as {{ type_string() }}) as "schema", cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as package_name, cast(null as {{ type_string() }}) as path, diff --git a/models/sources/snapshot_executions.sql b/models/sources/snapshot_executions.sql index dbb93977..3eae3f77 100644 --- a/models/sources/snapshot_executions.sql +++ b/models/sources/snapshot_executions.sql @@ -15,7 +15,7 @@ select cast(null as {{ type_float() }}) as total_node_runtime, cast(null as {{ type_int() }}) as rows_affected, cast(null as {{ type_string() }}) as materialization, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as "schema", cast(null as {{ type_string() }}) as name, cast(null as {{ type_string() }}) as alias, cast(null as {{ type_string() }}) as message, diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql index e7832de6..2190b728 100644 --- a/models/sources/snapshots.sql +++ b/models/sources/snapshots.sql @@ -7,8 +7,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as "database", + cast(null as {{ type_string() }}) as "schema", cast(null as {{ type_string() }}) as name, cast(null as {{ type_array() }}) as depends_on_nodes, cast(null as {{ type_string() }}) as package_name, diff --git a/models/sources/sources.sql b/models/sources/sources.sql index ad08f0d7..9e671681 100644 --- a/models/sources/sources.sql +++ b/models/sources/sources.sql @@ -7,8 +7,8 @@ select cast(null as {{ type_string() }}) as command_invocation_id, cast(null as {{ type_string() }}) as node_id, cast(null as {{ type_timestamp() }}) as run_started_at, - cast(null as {{ type_string() }}) as database, - cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as "database", + cast(null as {{ type_string() }}) as "schema", cast(null as {{ type_string() }}) as source_name, cast(null as {{ type_string() }}) as loader, cast(null as {{ type_string() }}) as name, diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index 1ab0b5a5..53d3b731 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -23,7 +23,7 @@ enhanced as ( bytes_processed, {% endif %} materialization, - schema, -- noqa + "schema", -- noqa name, alias, message, diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index db896a4e..0751cb00 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -12,8 +12,8 @@ enhanced as ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, depends_on_nodes, package_name, diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index a8bba7a7..d5afb5f6 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -20,7 +20,7 @@ enhanced as ( total_node_runtime, rows_affected, materialization, - schema, -- noqa + "schema", -- noqa name, alias, message, diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 3a2a66fa..95db7a2a 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -12,8 +12,8 @@ enhanced as ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, package_name, path, diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index 3ee0c050..a4986841 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -20,7 +20,7 @@ enhanced as ( total_node_runtime, rows_affected, materialization, - schema, -- noqa + "schema", -- noqa name, alias, message, diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql index f5236e87..48c9cfb0 100644 --- a/models/staging/stg_dbt__snapshots.sql +++ b/models/staging/stg_dbt__snapshots.sql @@ -12,8 +12,8 @@ enhanced as ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, depends_on_nodes, package_name, diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index f18a80e9..6b89c3e6 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -12,8 +12,8 @@ enhanced as ( command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", source_name, loader, name, From 6833cf24ee913ae26da66c527c6458bfbb5f0496 Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Tue, 27 Feb 2024 20:11:21 +0100 Subject: [PATCH 04/10] alias column in sub-select (T-SQL) --- integration_test_project/tests/singular_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test_project/tests/singular_test.sql b/integration_test_project/tests/singular_test.sql index a1e49e8c..acc5eb12 100644 --- a/integration_test_project/tests/singular_test.sql +++ b/integration_test_project/tests/singular_test.sql @@ -1 +1 @@ -select 1 as failures from (select 2) as foo where 1 = 2 +select 1 as failures from (select 2 as two) as foo where 1 = 2 From f193215b2d9e98ef0c562bde702ed1b4181f056b Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Tue, 27 Feb 2024 20:21:33 +0100 Subject: [PATCH 05/10] make dim_dbt__current_models T-SQL compatible --- models/dim_dbt__current_models.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index 4d7a3c46..6873a4c5 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -45,9 +45,9 @@ latest_models_runs as ( latest_model_stats as ( select node_id - , max(case when was_full_refresh then query_completed_at end) as last_full_refresh_run_completed_at - , max(case when was_full_refresh then total_node_runtime end) as last_full_refresh_run_total_runtime - , max(case when was_full_refresh then rows_affected end) as last_full_refresh_run_rows_affected + , max(case when was_full_refresh = 1 then query_completed_at end) as last_full_refresh_run_completed_at + , max(case when was_full_refresh = 1 then total_node_runtime end) as last_full_refresh_run_total_runtime + , max(case when was_full_refresh = 1 then rows_affected end) as last_full_refresh_run_rows_affected {% if target.type == 'bigquery' %} , max(case when was_full_refresh then bytes_processed end) as last_full_refresh_run_bytes_processed {% endif %} @@ -57,15 +57,15 @@ latest_model_stats as ( {% if target.type == 'bigquery' %} , max(case when run_idx_id_only = 1 then bytes_processed end) as last_run_bytes_processed {% endif %} - , max(case when not was_full_refresh then query_completed_at end) as last_incremental_run_completed_at - , max(case when not was_full_refresh then total_node_runtime end) as last_incremental_run_total_runtime - , max(case when not was_full_refresh then rows_affected end) as last_incremental_run_rows_affected + , max(case when not was_full_refresh = 1 then query_completed_at end) as last_incremental_run_completed_at + , max(case when not was_full_refresh = 1 then total_node_runtime end) as last_incremental_run_total_runtime + , max(case when not was_full_refresh = 1 then rows_affected end) as last_incremental_run_rows_affected {% if target.type == 'bigquery' %} , max(case when not was_full_refresh then bytes_processed end) as last_incremental_run_bytes_processed {% endif %} from latest_models_runs where run_idx = 1 - group by 1 + group by node_id ), final as ( From 9eb599be2fb90ddf31a0f655a5c9e9fcfaabc515 Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Wed, 28 Feb 2024 11:17:20 +0100 Subject: [PATCH 06/10] insert_into_metadata_table: add T-SQL Adapter Version --- macros/upload_results/insert_into_metadata_table.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 24f1eb77..2c9d8a24 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -59,5 +59,16 @@ {%- endmacro %} +{% macro sqlserver__insert_into_metadata_table(relation, fields, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + {% macro default__insert_into_metadata_table(relation, fields, content) -%} {%- endmacro %} From 82c6f5afe959941881f600e8c174640c2e7d3a95 Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Wed, 28 Feb 2024 11:20:50 +0100 Subject: [PATCH 07/10] quote columns named "schema" and "database" for T-SQL compatibility --- .../upload_results/get_column_name_lists.sql | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/macros/upload_results/get_column_name_lists.sql b/macros/upload_results/get_column_name_lists.sql index 7911e866..a994cfcc 100644 --- a/macros/upload_results/get_column_name_lists.sql +++ b/macros/upload_results/get_column_name_lists.sql @@ -66,7 +66,7 @@ bytes_processed, {% endif %} materialization, - schema, + "schema", name, alias, message, @@ -79,8 +79,8 @@ command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, depends_on_nodes, package_name, @@ -108,7 +108,7 @@ total_node_runtime, rows_affected, materialization, - schema, + "schema", name, alias, message, @@ -121,8 +121,8 @@ command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, package_name, path, @@ -146,7 +146,7 @@ total_node_runtime, rows_affected, materialization, - schema, + "schema", name, alias, message, @@ -159,8 +159,8 @@ command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", name, depends_on_nodes, package_name, @@ -178,8 +178,8 @@ command_invocation_id, node_id, run_started_at, - database, - schema, + "database", + "schema", source_name, loader, name, From 01eb793d512a357a7dceb0354402ead60be2b6ec Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Wed, 28 Feb 2024 13:51:12 +0100 Subject: [PATCH 08/10] Added SQL-Server specifiv upload_*-Macros --- .../upload_exposures.sql | 39 +++++++++ .../upload_invocations.sql | 80 +++++++++++++++++++ .../upload_model_executions.sql | 46 +++++++++++ .../upload_models.sql | 43 ++++++++++ .../upload_seed_executions.sql | 47 +++++++++++ .../upload_seeds.sql | 37 +++++++++ .../upload_snapshot_executions.sql | 47 +++++++++++ .../upload_snapshots.sql | 40 ++++++++++ .../upload_sources.sql | 38 +++++++++ .../upload_test_executions.sql | 44 ++++++++++ .../upload_tests.sql | 35 ++++++++ 11 files changed, 496 insertions(+) diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql index 9f0ec5d3..2376edd0 100644 --- a/macros/upload_individual_datasets/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -118,3 +118,42 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_exposures_dml_sql(exposures) -%} + + {% if exposures != [] %} + {% set exposure_values %} + select "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14" + from ( values + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ exposure.unique_id | replace("'","''") }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ exposure.name | replace("'","''") }}', {# name #} + '{{ exposure.type }}', {# type #} + '{{ tojson(exposure.owner) }}', {# owner #} + '{{ exposure.maturity }}', {# maturity #} + '{{ exposure.original_file_path }}', {# path #} + '{{ exposure.description | replace("'","''") }}', {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + '{{ tojson(exposure.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ tojson(exposure.tags) }}', {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(exposure) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14") + + {% endset %} + {{ exposure_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql index 21c5574c..e0de6cac 100644 --- a/macros/upload_individual_datasets/upload_invocations.sql +++ b/macros/upload_individual_datasets/upload_invocations.sql @@ -221,3 +221,83 @@ {{ invocation_values }} {% endmacro -%} + + +{% macro sqlserver__get_invocations_dml_sql() -%} + {% set invocation_values %} + select + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + nullif("11", ''), + nullif("12", ''), + nullif("13", ''), + nullif("14", ''), + nullif("15", ''), + "16", + "17", + "18", + "19" + from (values + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + '{{ run_started_at }}', {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + '{{ flags.FULL_REFRESH }}', {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + '{{ env_var('DBT_CLOUD_PROJECT_ID', '') }}', {# dbt_cloud_project_id #} + '{{ env_var('DBT_CLOUD_JOB_ID', '') }}', {# dbt_cloud_job_id #} + '{{ env_var('DBT_CLOUD_RUN_ID', '') }}', {# dbt_cloud_run_id #} + '{{ env_var('DBT_CLOUD_RUN_REASON_CATEGORY', '') }}', {# dbt_cloud_run_reason_category #} + '{{ env_var('DBT_CLOUD_RUN_REASON', '') | replace("'","''") }}', {# dbt_cloud_run_reason #} + + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, '') | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(env_vars_dict) }}', {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} + + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, '') | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(dbt_vars_dict) }}', {# dbt_vars #} + {% else %} + null, {# dbt_vars #} + {% endif %} + + '{{ tojson(invocation_args_dict) | replace("'", "''") }}', {# invocation_args #} + + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: (value | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(metadata_env) }}' {# dbt_custom_envs #} + + ) + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19") + + + {% endset %} + {{ invocation_values }} + +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index bca26fea..11a996b9 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -203,3 +203,49 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_model_executions_dml_sql(models) -%} + {% if models != [] %} + {% set model_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" + from ( values + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake & BigQuery #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + ) v ( "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" ) + + {% endset %} + {{ model_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index 7570b06a..674b9a87 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -127,3 +127,46 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_models_dml_sql(models) -%} + + {% if models != [] %} + {% set model_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15" + from ( values + {% for model in models -%} + {% set model_copy = model.copy() -%} + {% do model_copy.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model_copy.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ model_copy.database }}', {# database #} + '{{ model_copy.schema }}', {# schema #} + '{{ model_copy.name }}', {# name #} + '{{ tojson(model_copy.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ model_copy.package_name }}', {# package_name #} + '{{ model_copy.original_file_path }}', {# path #} + '{{ model_copy.checksum.checksum }}', {# checksum #} + '{{ model_copy.config.materialized }}', {# materialization #} + '{{ tojson(model_copy.tags) }}', {# tags #} + '{{ tojson(model_copy.config.meta) | replace("'","''") }}', {# meta #} + '{{ model_copy.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(model_copy) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15") + + {% endset %} + {{ model_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index 1ccbfe2a..d638bdd9 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -217,3 +217,50 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_seed_executions_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" + from ( values + {% for model in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16") + + {% endset %} + {{ seed_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql index 32e67383..8eb946fd 100644 --- a/macros/upload_individual_datasets/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -109,3 +109,40 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_seeds_dml_sql(seeds) -%} + + {% if seeds != [] %} + {% set seed_values %} + select "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12" + from ( values + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ seed.original_file_path }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + '{{ tojson(seed.config.meta) | replace("'","''") }}', {# meta #} + '{{ seed.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(seed) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12") + + {% endset %} + {{ seed_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index 2006b168..70d4401d 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -217,3 +217,50 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_snapshot_executions_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16" + from ( values + {% for model in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Only available in Snowflake #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + '{{ model.message | replace("'", "''") }}', {# message #} + '{{ tojson(model.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16") + + {% endset %} + {{ snapshot_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql index 0be6759b..467e870a 100644 --- a/macros/upload_individual_datasets/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -119,3 +119,43 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_snapshots_dml_sql(snapshots) -%} + + {% if snapshots != [] %} + {% set snapshot_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14" + from ( values + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + '{{ tojson(snapshot.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ snapshot.original_file_path }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + '{{ tojson(snapshot.config.meta) | replace("'","''") }}', {# meta #} + '{{ snapshot.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(snapshot) | replace("'","''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14") + + {% endset %} + {{ snapshot_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_individual_datasets/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql index dbcb49e4..c6b8f984 100644 --- a/macros/upload_individual_datasets/upload_sources.sql +++ b/macros/upload_individual_datasets/upload_sources.sql @@ -109,3 +109,41 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_sources_dml_sql(sources) -%} + + {% if sources != [] %} + {% set source_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12" + from ( values + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + '{{ source.loaded_at_field | replace("'","''") }}', {# loaded_at_field #} + '{{ tojson(source.freshness) | replace("'","''") }}', {# freshness #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(source) | replace("'", "''") }}' {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12") + + {% endset %} + {{ source_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index ea3553ae..d2796bdf 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -148,3 +148,47 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro sqlserver__get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_execution_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13" + from ( values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ test.execution_time }}, {# total_node_runtime #} + null, {# rows_affected not available in Databricks #} + {{ 'null' if test.failures is none else test.failures }}, {# failures #} + '{{ test.message | replace("'", "''") }}', {# message #} + '{{ tojson(test.adapter_response) | replace("'", "''") }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) AS v ("1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13") + + {% endset %} + {{ test_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file diff --git a/macros/upload_individual_datasets/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql index cf75e9ba..1d8afff1 100644 --- a/macros/upload_individual_datasets/upload_tests.sql +++ b/macros/upload_individual_datasets/upload_tests.sql @@ -97,3 +97,38 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro sqlserver__get_tests_dml_sql(tests) -%} + + {% if tests != [] %} + {% set test_values %} + select + "1", "2", "3", "4", "5", "6", "7", "8", "9" + from ( values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ test.name }}', {# name #} + '{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ test.original_file_path }}', {# test_path #} + '{{ tojson(test.tags) }}', {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + '{{ tojson(test) | replace("'","''") }}' {# all_fields #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + + ) v ("1", "2", "3", "4", "5", "6", "7", "8", "9") + + {% endset %} + {{ test_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} From b2205629aa79d32db69241724db755cb82efebbb Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Fri, 22 Mar 2024 10:46:41 +0100 Subject: [PATCH 09/10] Add integration_sqlserver_1_4_3 and integration_sqlserver_1_7_4 --- tox.ini | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 858cd755..ce3e4186 100644 --- a/tox.ini +++ b/tox.ini @@ -324,10 +324,24 @@ commands = [testenv:integration_sqlserver] changedir = integration_test_project -deps = dbt-sqlserver~=1.4.3 +deps = dbt-sqlserver~=1.7.4 commands = dbt clean dbt deps dbt build --target sqlserver +[testenv:integration_sqlserver_1_4_3] +changedir = integration_test_project +deps = dbt-sqlserver~=1.4.3 +commands = + dbt clean + dbt deps + dbt build --target sqlserver +[testenv:integration_sqlserver_1_7_4] +changedir = integration_test_project +deps = dbt-sqlserver~=1.7.4 +commands = + dbt clean + dbt deps + dbt build --target sqlserver From 6c00e67ca1481bf5f8434c42846482bf5d847254 Mon Sep 17 00:00:00 2001 From: "philipp.kurz@avanade.com" Date: Fri, 22 Mar 2024 11:05:47 +0100 Subject: [PATCH 10/10] we now support SQL Server ;) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6350d743..fc1710a7 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ The package currently supports - Snowflake :white_check_mark: - Google BigQuery :white_check_mark: - Postgres :white_check_mark: +- SQL Server :white_check_mark: Models included: