diff --git a/README.md b/README.md index 2d1b310..e64ae6e 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # Snowflake Utils -This [dbt](https://github.com/fishtown-analytics/dbt) package contains Snowflake-specific macros that can be (re)used across dbt projects. +This [dbt](https://github.com/dbt-labs/dbt-core) package contains Snowflake-specific macros that can be (re)used across dbt projects. ## Installation Instructions Check [dbt Hub](https://hub.getdbt.com/montreal-analytics/snowflake_utils/latest/) for the latest installation instructions, or [read the docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. ## Prerequisites -Snowflake Utils is compatible with dbt 0.15.0 and later. +Snowflake Utils is compatible with dbt 0.20.0 and later. ---- @@ -15,8 +15,9 @@ Snowflake Utils is compatible with dbt 0.15.0 and later. ### snowflake_utils.warehouse_size() ([source](macros/warehouse_size.sql)) This macro returns an alternative warehouse if conditions are met. It will, in order, check the following conditions for incremental models: -- The relation doesn't exist (initial run) _and_ a warehouse has been configured - Full refresh run _and_ a warehouse has been configured +- Incremental run _and_ a warehouse has been configured +- The relation doesn't exist (initial run) _and_ a warehouse has been configured Otherwise, it returns the target warehouse configured in the profile. @@ -47,14 +48,10 @@ An example `dbt_project.yml` configuration: # dbt_project.yml ... - -models: - my_project: - vars: - 'snowflake_utils:initial_run_warehouse': "transforming_xl_wh" - 'snowflake_utils:full_refresh_run_warehouse': "transforming_xl_wh" - - +vars: + 'snowflake_utils:initial_run_warehouse': "transforming_xl_wh" + 'snowflake_utils:full_refresh_run_warehouse': "transforming_xl_wh" + 'snowflake_utils:incremental_run_warehouse': "transforming_m_wh" ``` #### Console Output @@ -67,6 +64,8 @@ When a variable is configured for a conditon _and_ that condition is matched whe 12:00:00 | 1 of 1 START incremental model DBT_MGUINDON.fct_orders... [RUN] 12:00:00 + Initial Run - Using warehouse TRANSFORMING_XL_WH ``` +#### Known Issues +When compiling or generating docs, the console reports that dbt is using the incremental run warehouse. It isn't actually so. During these operations, only the target warehouse is activated. ### snowflake_utils.clone_schema ([source](macros/clone_schema.sql)) This macro clones the source schema into the destination schema. diff --git a/macros/warehouse_size.sql b/macros/warehouse_size.sql index 1f27e26..d97eb90 100644 --- a/macros/warehouse_size.sql +++ b/macros/warehouse_size.sql @@ -1,39 +1,38 @@ {% macro warehouse_size() %} - {% if execute and model.config.materialized == 'incremental' %} + {% set initial_wh = var('snowflake_utils:initial_run_warehouse', none) %} + {% set full_wh = var('snowflake_utils:full_refresh_run_warehouse', none) %} + {% set inc_wh = var('snowflake_utils:incremental_run_warehouse', none) %} - {% set relation = adapter.get_relation(this.database, this.schema, this.table) %} + {% set relation = adapter.get_relation(this.database, this.schema, this.table) %} - {% set initial_wh = var('snowflake_utils:initial_run_warehouse', none) %} - {% set full_wh = var('snowflake_utils:full_refresh_run_warehouse', none) %} - {% set inc_wh = var('snowflake_utils:incremental_run_warehouse', none) %} - - {#-- use alternative warehouse if initial run #} - {% if relation is none and initial_wh is not none %} - {{ dbt_utils.log_info("Initial Run - Using alternative warehouse " ~ initial_wh | upper) }} - {% do return(initial_wh) %} - - {#-- use alternative warehouse if full-refresh run #} - {% elif flags.FULL_REFRESH and full_wh is not none %} + {#-- use alternative warehouse if full-refresh run #} + {# if relation is not none and flags.FULL_REFRESH and full_wh is not none #} + {% if flags.FULL_REFRESH and full_wh is not none %} + {% if execute %} {{ dbt_utils.log_info("Full Refresh Run - Using alternative warehouse " ~ full_wh | upper) }} - {% do return(full_wh) %} - - {#-- use target warehouse if variable not configured for a condition #} - {% else %} - {{ dbt_utils.log_info("Using target warehouse " ~ target.warehouse | upper) }} - {% do return(target.warehouse) %} + {% endif %} + {% do return(full_wh) %} + {#-- use alternative warehouse if incremental run #} + {% elif relation is not none and flags.FULL_REFRESH == False and inc_wh is not none %} + {% if execute %} + {{ dbt_utils.log_info("Incremental Run - Using alternative warehouse " ~ inc_wh | upper) }} {% endif %} + {% do return(inc_wh) %} - {#-- use target warehouse if model is not incremental #} - {% elif execute and model.config.materialized != 'incremental' %} - {{ dbt_utils.log_info("Using target warehouse " ~ target.warehouse | upper) }} - {% do return(target.warehouse) %} + {#-- use alternative warehouse if initial run #} + {% elif relation is none and initial_wh is not none %} + {% if execute %} + {{ dbt_utils.log_info("Initial Run - Using alternative warehouse " ~ initial_wh | upper) }} + {% endif %} + {% do return(initial_wh) %} - {#-- use target warehouse for parsing #} + {#-- use target warehouse if variable not configured for a condition #} {% else %} + {{ dbt_utils.log_info("Using target warehouse " ~ target.warehouse | upper) }} {% do return(target.warehouse) %} {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/packages.yml b/packages.yml index f8950f1..31c7c49 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - package: dbt-labs/dbt_utils - version: ">=0.1.25" + version: ">=0.7.0"