Merge remote-tracking branch 'upstream/main' into map_batches

.github/dependabot.yml

@@ -5,8 +5,8 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      # Check for updates to GitHub Actions every week
-      interval: "weekly"
+      # Check for updates to GitHub Actions every month
+      interval: "monthly"
     commit-message:
       prefix: "skip changelog" # So this PR will not be added to release-drafter
       include: "scope" # List of the updated dependencies in the commit will be added

.github/workflows/downstream_tests.yml

@@ -1,6 +1,6 @@
 name: Test Downstream Libraries
 
-on:
+on: 
   pull_request:
   push:
     branches: [main]
@@ -55,7 +55,7 @@ jobs:
       matrix:
         python-version: ["3.12"]
         os: [ubuntu-latest]
-        dependencies: ["core", "core,optional"]
+        dependencies: ["core,optional"]
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -73,19 +73,27 @@ jobs:
         run: |
             git clone https://github.com/marimo-team/marimo.git --depth=1
             cd marimo
+            uv venv -p 3.12
             git log
       - name: install-basics
         run: uv pip install --upgrade tox virtualenv setuptools hatch --system
       - name: install-marimo-dev
         run: |
             cd marimo
-            uv pip install -e ".[dev]" --system
+            . .venv/bin/activate
+            uv pip install -e ".[dev]"
+            which python
       - name: install-narwhals-dev
         run: |
-            uv pip uninstall narwhals --system
-            uv pip install -e . --system
+            cd marimo
+            . .venv/bin/activate
+            uv pip uninstall narwhals
+            uv pip install -e ./..
       - name: show-deps
-        run: uv pip freeze
+        run: |
+            cd marimo
+            . .venv/bin/activate
+            uv pip freeze
       - name: Create assets directory, copy over index.html
         continue-on-error: true
         run: |
@@ -96,12 +104,13 @@ jobs:
         if: ${{ matrix.dependencies == 'core,optional' }}
         run: |
             cd marimo
-            hatch run +py=${{ matrix.python-version }} test-optional:test-narwhals
+            . .venv/bin/activate
+            # make sure that we use the .venv when running tests, so that
+            # the local narwhals install is picked up
+            sed -i '/^\[tool.hatch.envs.default\]/a path = ".venv"' pyproject.toml
+            hatch run python -c "import narwhals; print(narwhals.__file__)"
+            hatch run test-optional:test-narwhals
         timeout-minutes: 15
-      - name: Run typechecks
-        run: |
-            cd marimo
-            hatch run typecheck:check
 
   scikit-lego:
     strategy:
@@ -221,3 +230,60 @@ jobs:
         run: |
             cd tubular
             pytest tests --config-file=pyproject.toml
+  
+  # vegafusion:
+  #    env:
+  #      UV_SYSTEM_PYTHON: true
+
+  #    strategy:
+  #      matrix:
+  #        python-version: ["3.11"]
+  #        os: [ubuntu-latest]
+
+  #    runs-on: ${{ matrix.os }}
+  #    steps:
+  #       - uses: actions/checkout@v4
+  #       - uses: actions/setup-python@v5
+  #         with:
+  #           python-version: ${{ matrix.python-version }}
+  #       - name: Install uv
+  #         uses: astral-sh/setup-uv@v3
+  #         with:
+  #           enable-cache: "true"
+  #           cache-suffix: ${{ matrix.python-version }}
+  #           cache-dependency-glob: "**requirements*.txt"
+  #       - name: clone-vegafusion
+  #         run: |
+  #             git clone --single-branch -b v2 https://github.com/vega/vegafusion.git
+  #             cd vegafusion
+  #             git log
+  #       - name: Cache rust dependencies
+  #         uses: Swatinem/rust-cache@v2
+  #         with:
+  #           workspaces: vegafusion
+  #       - name: Build wheels
+  #         uses: PyO3/maturin-action@v1
+  #         with:
+  #           command: build
+  #           manylinux: 2014
+  #           rust-toolchain: stable
+  #           args: --release -m vegafusion/vegafusion-python/Cargo.toml --features=protobuf-src --strip
+  #       - name: Install wheels
+  #         working-directory: vegafusion/target/wheels/
+  #         run: |
+  #           ls -la
+  #           python -m pip install vegafusion-*manylinux*.whl
+
+  #           # Optional dependencies
+  #           python -m pip install pyarrow pandas polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" jupytext voila anywidget ipywidgets chromedriver-binary-auto
+
+  #           # Test dependencies
+  #           python -m pip install pytest altair vega-datasets scikit-image jupytext voila ipykernel anywidget ipywidgets selenium flaky tenacity chromedriver-binary-auto 
+  #       - name: Test lazy imports
+  #         working-directory: vegafusion/vegafusion-python/
+  #         run: python checks/check_lazy_imports.py
+  #       - name: Test vegafusion
+  #         working-directory: vegafusion/vegafusion-python/
+  #         env:
+  #           VEGAFUSION_TEST_HEADLESS: 1
+  #         run: pytest

.github/workflows/publish_to_pypi.yml

@@ -2,6 +2,7 @@ name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
 
 on: push
 
+
 jobs:
   build:
     name: Build distribution 📦

.pre-commit-config.yaml

@@ -1,15 +1,17 @@
+ci:
+  autoupdate_schedule: monthly
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: 'v0.7.0'
+  rev: 'v0.7.1'
   hooks:
     # Run the formatter.
     - id: ruff-format
     # Run the linter.
     - id: ruff
       args: [--fix]
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: 'v1.12.1'
+  rev: 'v1.13.0'
   hooks:
     - id: mypy
       additional_dependencies: ['polars==1.4.1', 'pytest==8.3.2']
@@ -40,7 +42,7 @@ repos:
   hooks:
     - id: nbstripout
 - repo: https://github.com/adamchainz/blacken-docs
-  rev: "1.19.0"  # replace with latest tag on GitHub
+  rev: "1.19.1"  # replace with latest tag on GitHub
   hooks:
   -   id: blacken-docs
       args: [--skip-errors]

README.md

@@ -45,6 +45,8 @@ Join the party!
 - [Hamilton](https://github.com/DAGWorks-Inc/hamilton/tree/main/examples/narwhals)
 - [marimo](https://github.com/marimo-team/marimo)
 - [pymarginaleffects](https://github.com/vincentarelbundock/pymarginaleffects)
+- [py-shiny](https://github.com/posit-dev/py-shiny)
+- [rio](https://github.com/rio-labs/rio)
 - [scikit-lego](https://github.com/koaning/scikit-lego)
 - [scikit-playtime](https://github.com/koaning/scikit-playtime)
 - [timebasedcv](https://github.com/FBruzzesi/timebasedcv)

docs/api-reference/dependencies.md

@@ -15,6 +15,7 @@
         - is_cudf_series
         - is_dask_dataframe
         - is_ibis_table
+        - is_into_dataframe
         - is_into_series
         - is_modin_dataframe
         - is_modin_index

docs/backcompat.md

@@ -91,3 +91,54 @@ Here are exceptions to our backwards compatibility policy:
   expressions, or pandas were to remove support for categorical data. At that point, we might
   need to rethink Narwhals. However, we expect such radical changes to be exceedingly unlikely.
 - we may consider making some type hints more precise.
+
+In general, decision are driven by use-cases, and we conduct a search of public GitHub repositories
+before making any change.
+
+## Breaking changes carried out so far
+
+### After `stable.v1`
+
+- Since Narwhals 1.13.0, the `strict` parameter in `from_native`, `to_native`, and `narwhalify`
+    has been deprecated in favour of `pass_through`. This is because several users expressed
+    confusion/surprise over what `strict=False` did.
+    ```python
+    # v1 syntax:
+    nw.from_native(df, strict=False)
+
+    # main namespace (and, when we get there, v2) syntax:
+    nw.from_native(df, pass_through=True)
+    ```
+    If you are using Narwhals>=1.13.0, then we recommend using `pass_through`, as that
+    works consistently across namespaces.
+
+    In the future:
+
+    - in the main Narwhals namespace, `strict` will be removed in favour of `pass_through`
+    - in `stable.v1`, we will keep both `strict` and `pass_through`
+
+- Since Narwhals 1.9.0, `Datetime` and `Duration` dtypes hash using both `time_unit` and
+    `time_zone`.
+    The effect of this can be seen when placing these dtypes in sets:
+
+    ```python exec="1" source="above" session="backcompat"
+    import narwhals.stable.v1 as nw_v1
+    import narwhals as nw
+
+    # v1 behaviour:
+    assert nw_v1.Datetime("us") in {nw_v1.Datetime}
+
+    # main namespace (and, when we get there, v2) behaviour:
+    assert nw.Datetime("us") not in {nw.Datetime}
+    assert nw.Datetime("us") in {nw.Datetime("us")}
+    ```
+
+    To check if a dtype is a datetime (regardless of `time_unit` or `time_zone`)
+    we recommend using `==` instead, as that works consistenty
+    across namespaces:
+
+    ```python exec="1" source="above" session="backcompat"
+    # Recommended
+    assert nw.Datetime("us") == nw.Datetime
+    assert nw_v1.Datetime("us") == nw_v1.Datetime
+    ```

docs/extending.md

@@ -69,6 +69,7 @@ libraries which have interchange only support can access the following methods a
 converting to Narwhals DataFrame:
 
 - `.schema`, hence column names via `.schema.names()` and column types via `.schema.dtypes()`
+- `.columns`
 - `.to_pandas()` and `.to_arrow()`, for converting to Pandas and Arrow, respectively.
 - `.select(names)` (Ibis and DuckDB), where `names` is a list of (string) column names. This is useful for
   selecting columns before converting to another library.

docs/installation.md

@@ -29,7 +29,7 @@ To verify the installation, start the Python REPL and execute:
 ```python
 >>> import narwhals
 >>> narwhals.__version__
-'1.11.1'
+'1.13.1'
 ```
 If you see the version number, then the installation was successful!
 

mkdocs.yml

@@ -78,9 +78,7 @@ theme:
       toggle:
         icon: material/brightness-4
         name: Switch to system preference
-extra_css:
-  - https://unpkg.com/katex@0/dist/katex.min.css
-  - css/mkdocstrings.css
+
 
 plugins:
 - search
@@ -91,6 +89,8 @@ plugins:
         - https://installer.readthedocs.io/en/stable/objects.inv
         rendering:
           show_signature_annotations: true
+        options:
+          members_order: alphabetical
 
 hooks:
 - utils/generate_backend_completeness.py

narwhals/__init__.py

@@ -67,7 +67,7 @@
 from narwhals.utils import maybe_reset_index
 from narwhals.utils import maybe_set_index
 
-__version__ = "1.11.1"
+__version__ = "1.13.1"
 
 __all__ = [
     "dependencies",

narwhals/_arrow/series.py

@@ -12,6 +12,7 @@
 from narwhals._arrow.utils import floordiv_compat
 from narwhals._arrow.utils import narwhals_to_native_dtype
 from narwhals._arrow.utils import native_to_narwhals_dtype
+from narwhals._arrow.utils import parse_datetime_format
 from narwhals._arrow.utils import validate_column_comparand
 from narwhals.utils import Implementation
 from narwhals.utils import generate_temporary_column_name
@@ -1115,8 +1116,7 @@ def to_datetime(self: Self, format: str | None) -> ArrowSeries:  # noqa: A002
         import pyarrow.compute as pc  # ignore-banned-import()
 
         if format is None:
-            msg = "`format` is required for pyarrow backend."
-            raise ValueError(msg)
+            format = parse_datetime_format(self._arrow_series._native_series)
 
         return self._arrow_series._from_native_series(
             pc.strptime(self._arrow_series._native_series, format=format, unit="us")