Skip to content

Commit

Permalink
Fix 1.8 imports and adapter handling, remove sqlite 1.8 testing
Browse files Browse the repository at this point in the history
  • Loading branch information
BAntonellini committed Jun 3, 2024
1 parent 4d0fbc7 commit b46fff6
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 25 deletions.
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ everything: .venv-dbt10/bin/python .venv-dbt11/bin/python .venv-dbt12/bin/python
.venv-dbt17/bin/pip install "dbt-core>=1.7.0,<1.8.0"; \
done

# SQLITE adapter is not supported in dbt 1.8+ yet: https://github.com/codeforkjeff/dbt-sqlite/issues
ADAPTERS = duckdb

.venv-dbt18/bin/python:
python -m venv .venv-dbt18
.venv-dbt18/bin/pip install --upgrade wheel setuptools pip
Expand All @@ -80,6 +83,7 @@ everything: .venv-dbt10/bin/python .venv-dbt11/bin/python .venv-dbt12/bin/python
.venv-dbt18/bin/pip install "dbt-core>=1.8.0,<1.9.0"; \
.venv-dbt18/bin/pip install "dbt-$$adapter"; \
done
.venv-dbt18/bin/pip install --force-reinstall dbt-adapters dbt-common;

clean:
rm -rf .venv-dbt10 .venv-dbt11 .venv-dbt12 .venv-dbt13 .venv-dbt14 .venv-dbt15 .venv-dbt16 .venv-dbt17 .venv-dbt18
Expand Down
36 changes: 29 additions & 7 deletions src/dbt_core_interface/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
from io import BytesIO
from json import dumps as json_dumps
from json import loads as json_lds
from multiprocessing import get_context
from pathlib import Path
from tempfile import NamedTemporaryFile
from traceback import format_exc, print_exc
Expand Down Expand Up @@ -88,7 +89,12 @@

# We maintain the smallest possible surface area of dbt imports
from dbt.adapters.factory import get_adapter_class_by_name
from dbt.clients.system import make_directory
try:
# dbt >= 1.8
from dbt_common.clients.system import make_directory
except ImportError:
# dbt < 1.8
from dbt.clients.system import make_directory
from dbt.config.runtime import RuntimeConfig
from dbt.flags import set_from_args
from dbt.node_types import NodeType
Expand All @@ -112,9 +118,16 @@
# These imports are only used for type checking
from agate import Table # type: ignore # No stubs for agate
from dbt.adapters.base import BaseAdapter, BaseRelation # type: ignore
from dbt.contracts.connection import AdapterResponse
try:
# dbt >= 1.8
from dbt.adapters.contracts.connection import AdapterResponse
from dbt_common.semver import VersionSpecifier

except ImportError:
# dbt < 1.8
from dbt.contracts.connection import AdapterResponse
from dbt.semver import VersionSpecifier
from dbt.contracts.results import ExecutionResult, RunExecutionResult
from dbt.semver import VersionSpecifier
from dbt.task.runnable import ManifestTask

try:
Expand All @@ -128,9 +141,11 @@
from agate import Table, Number, Text, Column

try:
from dbt.clients.agate_helper import Integer
# dbt >= 1.8
from dbt_common.clients.agate_helper import Integer
except ImportError:
from dbt.clients.agate_helper import Number as Integer
# dbt < 1.8
from dbt.clients.agate_helper import Integer
# dbt-core-interface is designed for non-standard use. There is no
# reason to track usage of this package.
disable_tracking()
Expand Down Expand Up @@ -266,8 +281,10 @@ class DbtConfiguration:
use_experimental_parser: bool = False
static_parser: bool = False
partial_parse: bool = False
# A required attribute for dbt, not used by our interface
# required attributes for dbt, not used by our interface
dependencies: List[str] = field(default_factory=list)
REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES: bool = field(default_factory=bool)
which: str = "blah"

def __post_init__(self) -> None:
"""Post init hook to set single_threaded and remove target if not provided."""
Expand Down Expand Up @@ -474,7 +491,12 @@ def initialize_adapter(self) -> None:
LOGGER.debug(f"Failed to cleanup adapter connections: {e}")
# The adapter.setter verifies connection, resets TTL, and updates adapter ref on config
# this is thread safe by virtue of the adapter_mutex on the adapter.setter
self.adapter = self.get_adapter_cls()(self.config)
if (__dbt_major_version__, __dbt_minor_version__) < (1, 8):
self.adapter = self.get_adapter_cls()(self.config)
else:
# from dbt 1.8 adapter decoupling onwwards,
# instantiating an Adapter requires a multiprocessing context.
self.adapter = self.get_adapter_cls()(self.config, get_context("spawn"))

@property
def adapter(self) -> "BaseAdapter":
Expand Down
36 changes: 18 additions & 18 deletions tests/test_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ def test_list():
from dbt_core_interface.project import DbtProject

project = DbtProject(
project_dir="demo_sqlite",
profiles_dir="demo_sqlite",
project_dir="demo_duckdb",
profiles_dir="demo_duckdb",
target="dev",
)
nodes = project.list("*")
Expand Down Expand Up @@ -58,23 +58,23 @@ def test_server():
client = TestApp(app.default)

SIMULATED_CLIENTS = 50 # noqa: N806
DUCKDB_PROJECTS = ( # noqa: N806
[
"j_shop_1_duckdb",
"j_shop_2_duckdb",
"h_niceserver_1_duckdb",
"h_niceserver_2_duckdb",
]
if (__dbt_major_version__, __dbt_minor_version__) < (1, 4)
else [] # DuckDB adapter is not supported in dbt 1.4+ yet
)
SQLITE_PROJECTS = [ # noqa: N806
"j_shop_1_sqlite",
"j_shop_2_sqlite",
"j_shop_3_sqlite",
"j_shop_4_sqlite",
"h_niceserver_1_sqlite",
DUCKDB_PROJECTS = [ # noqa: N806
"j_shop_1_duckdb",
"j_shop_2_duckdb",
"h_niceserver_1_duckdb",
"h_niceserver_2_duckdb",
]
SQLITE_PROJECTS = (
[ # noqa: N806
"j_shop_1_sqlite",
"j_shop_2_sqlite",
"j_shop_3_sqlite",
"j_shop_4_sqlite",
"h_niceserver_1_sqlite",
]
if (__dbt_major_version__, __dbt_minor_version__) < (1, 8)
else []
) # SQLITE adapter is not supported in dbt 1.8+ yet: https://github.com/codeforkjeff/dbt-sqlite/issues
PROJECTS = DUCKDB_PROJECTS + SQLITE_PROJECTS # noqa: N806

for proj in SQLITE_PROJECTS:
Expand Down

0 comments on commit b46fff6

Please sign in to comment.