Merge pull request #15 from JenspederM/fix/avoid-spark-on-bundle

Fix/avoid spark on bundle

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kedro-databricks"
-version = "0.1.4"
+version = "0.1.5"
 description = "A plugin to run Kedro pipelines on Databricks."
 authors = [{ name = "Jens Peder Meldgaard", email = "[email protected]" }]
 dependencies = ["kedro>=0.19.0", "mergedeep>=1.3.4", "tomlkit>=0.13.0"]

scripts/mkdev.sh

@@ -10,15 +10,22 @@ if [ -z "$1" ]; then
 fi
 
 if test -d "$CUR_PATH/$1"; then
-  echo "Directory $1 already exists. Removing it."
+  echo "Directory $1 already exists."
+  read -p "Do you want to remove is? (y/n)?" choice
+  case "$choice" in
+    y|Y ) echo "yes";;
+    n|N ) echo "no"; exit 0;;
+    * ) echo "invalid"; exit 1;;
+  esac
   rm -rf "$CUR_PATH/$1"
 fi
 
 # Build package
 rye build
 
 # Create a new project
-kedro new --starter=databricks-iris --name="$1"
+kedro new --starter="https://github.com/JenspederM/kedro-starters" --directory="databricks-iris" --checkout="feat/align-databricks-iris-with-kedro-databricks" --name="$1"
+# kedro new --starter=databricks-iris --name="$1"
 
 # Databricks needs Java
 echo "java openjdk-21" >> "$CUR_PATH/$1/.tool-versions"

src/kedro_databricks/plugin.py

@@ -3,9 +3,8 @@
 
 import click
 import yaml
-from kedro.config import MissingConfigException
+from kedro.config import AbstractConfigLoader, MissingConfigException
 from kedro.framework.cli.utils import ENV_HELP
-from kedro.framework.context import KedroContext
 from kedro.framework.project import pipelines
 from kedro.framework.session import KedroSession
 from kedro.framework.startup import ProjectMetadata
@@ -35,10 +34,11 @@ def databricks_commands():
     pass
 
 
-def _load_config(context: KedroContext) -> dict[str, Any]:
-    log = logging.getLogger(context._package_name)
+def _load_config(
+    config_loader: AbstractConfigLoader, package_name: str
+) -> dict[str, Any]:
+    log = logging.getLogger(package_name)
     # Backwards compatibility for ConfigLoader that does not support `config_patterns`
-    config_loader = context.config_loader
     if not hasattr(config_loader, "config_patterns"):
         return config_loader.get("databricks*", "databricks/**")  # pragma: no cover
 
@@ -93,8 +93,10 @@ def bundle(
         conf_dir.mkdir(parents=True)
 
     with KedroSession.create(project_path=metadata.project_path, env=env) as session:
-        context = session.load_context()
-        resource_overrides = _load_config(context)
+        resource_overrides = _load_config(
+            config_loader=session._get_config_loader(),
+            package_name=session._package_name,
+        )
 
     if default.startswith("_"):
         raise ValueError(