From a1f33db1806a24993df071859497f918d2c6767e Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 23 Jul 2024 14:21:25 -0700 Subject: [PATCH] Adap 821/unit tests infer wrong datatype for None values in fixtures (#885) * Add test to confirm upstream handling * Add changelog. * Add another test. * standardize names to reflect python objects. * revert dev requirements. * Add the macro for the row checking. * Test for user facing exception thrown in base adapter with new redshift macor. * revert the dev requirements --------- Co-authored-by: Mila Page Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../unreleased/Fixes-20240625-170324.yaml | 7 ++ .../redshift/macros/adapters/unit_testing.sql | 11 +++ .../adapter/unit_testing/fixtures.py | 73 +++++++++++++++++++ .../adapter/unit_testing/test_unit_testing.py | 64 ++++++++++++++++ 4 files changed, 155 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240625-170324.yaml create mode 100644 dbt/include/redshift/macros/adapters/unit_testing.sql create mode 100644 tests/functional/adapter/unit_testing/fixtures.py diff --git a/.changes/unreleased/Fixes-20240625-170324.yaml b/.changes/unreleased/Fixes-20240625-170324.yaml new file mode 100644 index 000000000..316a92b95 --- /dev/null +++ b/.changes/unreleased/Fixes-20240625-170324.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: 'Handle unit test fixtures where typing goes wrong from first value in column + being Null. ' +time: 2024-06-25T17:03:24.73937-07:00 +custom: + Author: versusfacit + Issue: "821" diff --git a/dbt/include/redshift/macros/adapters/unit_testing.sql b/dbt/include/redshift/macros/adapters/unit_testing.sql new file mode 100644 index 000000000..5463f4e2b --- /dev/null +++ b/dbt/include/redshift/macros/adapters/unit_testing.sql @@ -0,0 +1,11 @@ +{%- macro redshift__validate_fixture_rows(rows, row_number) -%} + {%- if rows is not none and rows|length > 0 -%} + {%- set row = rows[0] -%} + {%- for key, value in row.items() -%} + {%- if value is none -%} + {%- set fixture_name = "expected output" if model.resource_type == 'unit_test' else ("'" ~ model.name ~ "'") -%} + {{ exceptions.raise_compiler_error("Unit test fixture " ~ fixture_name ~ " in " ~ model.name ~ " does not have any row free of null values, which may cause type mismatch errors during unit test execution.") }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} diff --git a/tests/functional/adapter/unit_testing/fixtures.py b/tests/functional/adapter/unit_testing/fixtures.py new file mode 100644 index 000000000..36212dff3 --- /dev/null +++ b/tests/functional/adapter/unit_testing/fixtures.py @@ -0,0 +1,73 @@ +model_none_value_base = """ +{{ config(materialized="table") }} + +select 1 as id, 'a' as col1 +""" + +model_none_value_model = """ +{{config(materialized="table")}} + +select * from {{ ref('none_value_base') }} +""" + + +test_none_column_value_doesnt_throw_error_csv = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + format: csv + rows: | + id,col1 + ,d + ,e + 6,f + + expect: + format: csv + rows: | + id,col1 + ,d + ,e + 6,f +""" + +test_none_column_value_doesnt_throw_error_dct = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + rows: + - { "id": , "col1": "d"} + - { "id": , "col1": "e"} + - { "id": 6, "col1": "f"} + + expect: + rows: + - {id: , "col1": "d"} + - {id: , "col1": "e"} + - {id: 6, "col1": "f"} +""" + +test_none_column_value_will_throw_error = """ +unit_tests: + - name: test_simple + + model: none_value_model + given: + - input: ref('none_value_base') + rows: + - { "id": , "col1": "d"} + - { "id": , "col1": "e"} + - { "id": 6, "col1": } + + expect: + rows: + - {id: , "col1": "d"} + - {id: , "col1": "e"} + - {id: 6, "col1": } +""" diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index 4d89c4b08..27ed54cb6 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -1,7 +1,21 @@ import pytest + +from dbt.artifacts.schemas.results import RunStatus +from dbt.tests.fixtures.project import write_project_files +from dbt.tests.util import run_dbt + from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput +from tests.functional.adapter.unit_testing.fixtures import ( + model_none_value_base, + model_none_value_model, + test_none_column_value_doesnt_throw_error_csv, + test_none_column_value_doesnt_throw_error_dct, + test_none_column_value_will_throw_error, +) + +from dbt_common.exceptions import CompilationError class TestRedshiftUnitTestingTypes(BaseUnitTestingTypes): @@ -34,6 +48,56 @@ def data_types(self): ] +class RedshiftUnitTestingNone: + def test_nones_handled_dict(self, project): + run_dbt(["build"]) + + +class TestRedshiftUnitTestCsvNone(RedshiftUnitTestingNone): + @pytest.fixture(scope="class") + def models(self): + return { + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + "__properties.yml": test_none_column_value_doesnt_throw_error_csv, + } + + +class TestRedshiftUnitTestDictNone(RedshiftUnitTestingNone): + @pytest.fixture(scope="class") + def models(self): + return { + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + "__properties.yml": test_none_column_value_doesnt_throw_error_dct, + } + + +class TestRedshiftUnitTestingTooManyNonesFails: + @pytest.fixture(scope="class") + def models(self): + return { + "__properties.yml": test_none_column_value_will_throw_error, + "none_value_base.sql": model_none_value_base, + "none_value_model.sql": model_none_value_model, + } + + def test_invalid_input(self, project): + """This is a user-facing exception, so we can't pytest.raise(CompilationError)""" + + def _find_first_error(items): + return next((item for item in items if item.status == RunStatus.Error), None) + + run_result = run_dbt(["build"], expect_pass=False) + first_item = _find_first_error(run_result) + + assert first_item is not None + assert ( + "does not have any row free of null values, which may cause type mismatch errors during unit test execution" + in str(first_item.message) + ) + + class TestRedshiftUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): pass