Regression when using xarray to plot DataArrays

[3tilley]

Hello,

I apologise if this is caused by an underlying change in holoviews, but I'm using the hvplot interface and can't see an easy way of achieving what I was able to before using holoview explicitly as I'm using other hvplot kwargs.

Also, I'm unable to demonstrate this in a command line script, it has to be in a jupyter notebook cell which I understand to be due to some magic about detecting whether there is a printable environment.

Previous working example

hvplot=0.10.0
holoviews=1.20.0
xarray=2024.11.0

import hvplot.xarray
import xarray as xr

da = xr.DataArray([[0.5, 0.2], [-0.1, 0.5]], dims=["A", "B"], name="C")

da.hvplot.heatmap(x="A", y="B", C="C")

I printed out some information about the object and the index in holoviews/element/util.py for this working example:

<class 'pandas.core.frame.DataFrame'>
   A  B    C
0  0  0  0.5
1  0  1  0.2
2  1  0 -0.1
3  1  1  0.5
:Dataset   [A,B]   (C)
Index(['A', 'B', 'C'], dtype='object')

After upgrading to 0.11.1

hvplot=0.11.1
holoviews=1.20.0
xarray=2024.11.0

~/.venv/lib/python3.12/site-packages/holoviews/element/util.py in ?(self, obj)
    207         print(type(obj.data))
    208         print(obj.data)
    209         print(obj)
    210         print(obj.data.columns)
--> 211         df = obj.data.set_index(index_cols).groupby(index_cols, **groupby_kwargs).first()
    212         label = 'unique' if len(df) == len(obj) else 'non-unique'
    213         levels = self._get_coords(obj)
    214         index = pd.MultiIndex.from_product(levels, names=df.index.names)

~/.venv/lib/python3.12/site-packages/pandas/core/frame.py in ?(self, keys, drop, append, inplace, verify_integrity)
   6118                     if not found:
   6119                         missing.append(col)
   6120 
   6121         if missing:
-> 6122             raise KeyError(f"None of {missing} are in the columns")
   6123 
   6124         if inplace:
   6125             frame = self

KeyError: "None of ['A', 'B'] are in the columns"

The same code gives me the error below. You'll see my amateur debugging lines showing that the format of the underlying index appears to have changed:

<class 'pandas.core.frame.DataFrame'>
       C
A B     
0 0  0.5
  1  0.2
1 0 -0.1
  1  0.5
:Dataset   [A,B]   (C)
Index(['C'], dtype='object')
<class 'pandas.core.frame.DataFrame'>
       C
A B     
0 0  0.5
  1  0.2
1 0 -0.1
  1  0.5
:Dataset   [A,B]   (C)
Index(['C'], dtype='object')

Full stacktrace

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/.venv/lib/python3.12/site-packages/holoviews/core/dimension.py in ?(self, include, exclude)
   1271         object using any display hooks registered on Store.display
   1272         hooks.  The output of all registered display_hooks is then
   1273         combined and returned.
   1274         """
-> 1275         return Store.render(self)

~/.venv/lib/python3.12/site-packages/holoviews/core/options.py in ?(cls, obj)
   1419                     break
   1420 
   1421         data, metadata = {}, {}
   1422         for hook in hooks:
-> 1423             ret = hook(obj)
   1424             if ret is None:
   1425                 continue
   1426             d, md = ret

~/.venv/lib/python3.12/site-packages/holoviews/ipython/display_hooks.py in ?(obj)
    283     # If pretty printing is off, return None (fallback to next display format)
    284     ip = get_ipython()
    285     if not ip.display_formatter.formatters['text/plain'].pprint:
    286         return None
--> 287     return display(obj, raw_output=True)

~/.venv/lib/python3.12/site-packages/holoviews/ipython/display_hooks.py in ?(obj, raw_output, **kwargs)
    251         with option_state(obj):
    252             output = grid_display(obj)
    253     elif isinstance(obj, (CompositeOverlay, ViewableElement)):
    254         with option_state(obj):
--> 255             output = element_display(obj)
    256     elif isinstance(obj, (Layout, NdLayout, AdjointLayout)):
    257         with option_state(obj):
    258             output = layout_display(obj)

~/.venv/lib/python3.12/site-packages/holoviews/ipython/display_hooks.py in ?(element)
    176             msg =  '<i> [Call holoviews.ipython.show_traceback() for details]</i>'
    177             return {'text/html': "<b>{name}</b>{msg}<br>{message}".format(msg=msg, **info)}, {}
    178 
    179         except Exception:
--> 180             raise

~/.venv/lib/python3.12/site-packages/holoviews/ipython/display_hooks.py in ?(element, max_frames)
    191     backend = Store.current_backend
    192     if type(element) not in Store.registry[backend]:
    193         return None
    194 
--> 195     return render(element)

~/.venv/lib/python3.12/site-packages/holoviews/ipython/display_hooks.py in ?(obj, **kwargs)
     72     # Drop back to png if pdf selected, notebook PDF rendering is buggy
     73     if renderer.fig == 'pdf':
     74         renderer = renderer.instance(fig='png')
     75 
---> 76     return renderer.components(obj, **kwargs)

~/.venv/lib/python3.12/site-packages/holoviews/plotting/renderer.py in ?(self, obj, fmt, comm, **kwargs)
    392         if config.comms == 'colab':
    393             load_notebook(config.inline)
    394         embed = (not (dynamic or streams or self.widget_mode == 'live') or config.embed)
    395         if embed or config.comms == 'default':
--> 396             return self._render_panel(plot, embed, comm)
    397         return self._render_ipywidget(plot)

~/.venv/lib/python3.12/site-packages/holoviews/plotting/renderer.py in ?(self, plot, embed, comm)
    399     def _render_panel(self, plot, embed=False, comm=True):
    400         comm = self.comm_manager.get_server_comm() if comm else None
    401         doc = Document()
    402         with config.set(embed=embed):
--> 403             model = plot.layout._render_model(doc, comm)
    404         if embed:
    405             return render_model(model, comm)
    406         ref = model.ref['id']

~/.venv/lib/python3.12/site-packages/panel/viewable.py in ?(self, doc, comm)
    736         if doc is None:
    737             doc = Document()
    738         if comm is None:
    739             comm = state._comm_manager.get_server_comm()
--> 740         model = self.get_root(doc, comm)
    741 
    742         if self._design and self._design.theme.bokeh_theme:
    743             doc.theme = self._design.theme.bokeh_theme

~/.venv/lib/python3.12/site-packages/panel/layout/base.py in ?(self, doc, comm, preprocess)
    312     def get_root(
    313         self, doc: Optional[Document] = None, comm: Optional[Comm] = None,
    314         preprocess: bool = True
    315     ) -> Model:
--> 316         root = super().get_root(doc, comm, preprocess)
    317         # ALERT: Find a better way to handle this
    318         if hasattr(root, 'styles') and 'overflow-x' in root.styles:
    319             del root.styles['overflow-x']

~/.venv/lib/python3.12/site-packages/panel/viewable.py in ?(self, doc, comm, preprocess)
    667         doc = create_doc_if_none_exists(doc)
    668         if self._design and (comm or (state._is_pyodide and not doc.session_context)):
    669             wrapper = self._design._wrapper(self)
    670             if wrapper is self:
--> 671                 root = self._get_model(doc, comm=comm)
    672                 if preprocess:
    673                     self._preprocess(root)
    674             else:

~/.venv/lib/python3.12/site-packages/panel/layout/base.py in ?(self, doc, root, parent, comm)
    178             raise ValueError(f'{type(self).__name__} did not define a _bokeh_model.')
    179         model = self._bokeh_model()
    180         root = root or model
    181         self._models[root.ref['id']] = (model, parent)
--> 182         objects, _ = self._get_objects(model, [], doc, root, comm)
    183         props = self._get_properties(doc)
    184         props[self._property_mapping['objects']] = objects
    185         props.update(self._compute_sizing_mode(objects, props))

~/.venv/lib/python3.12/site-packages/panel/layout/base.py in ?(self, model, old_objects, doc, root, comm)
    161                 old_models.append(child)
    162             else:
    163                 try:
    164                     child = pane._get_model(doc, root, model, comm)
--> 165                 except RerenderError as e:
    166                     if e.layout is not None and e.layout is not self:
    167                         raise e
    168                     e.layout = None

~/.venv/lib/python3.12/site-packages/panel/pane/holoviews.py in ?(self, doc, root, parent, comm)
    434             self._restore_plot = None
    435         elif isinstance(self.object, Plot):
    436             plot = self.object
    437         else:
--> 438             plot = self._render(doc, comm, root)
    439 
    440         plot.pane = self
    441         backend = plot.renderer.backend

~/.venv/lib/python3.12/site-packages/panel/pane/holoviews.py in ?(self, doc, comm, root)
    528             kwargs['root'] = root
    529             if comm:
    530                 kwargs['comm'] = comm
    531 
--> 532         return renderer.get_plot(self.object, **kwargs)

~/.venv/lib/python3.12/site-packages/holoviews/plotting/bokeh/renderer.py in ?(self_or_cls, obj, doc, renderer, **kwargs)
     64         Given a HoloViews Viewable return a corresponding plot instance.
     65         Allows supplying a document attach the plot to, useful when
     66         combining the bokeh model with another plot.
     67         """
---> 68         plot = super().get_plot(obj, doc, renderer, **kwargs)
     69         if plot.document is None:
     70             plot.document = Document() if self_or_cls.notebook_context else curdoc()
     71         if self_or_cls.theme:

~/.venv/lib/python3.12/site-packages/holoviews/plotting/renderer.py in ?(self_or_cls, obj, doc, renderer, comm, **kwargs)
    235                                                    **plot_opts)
    236             defaults = [kd.default for kd in plot.dimensions]
    237             init_key = tuple(v if d is None else d for v, d in
    238                              zip(plot.keys[0], defaults))
--> 239             plot.update(init_key)
    240         else:
    241             plot = obj
    242 

~/.venv/lib/python3.12/site-packages/holoviews/plotting/plot.py in ?(self, key)
    954     def update(self, key):
    955         if len(self) == 1 and key in (0, self.keys[0]) and not self.drawn:
--> 956             return self.initialize_plot()
    957         item = self.__getitem__(key)
    958         self.traverse(lambda x: setattr(x, '_updated', True))
    959         return item

~/.venv/lib/python3.12/site-packages/holoviews/plotting/bokeh/element.py in ?(self, ranges, plot, plots, source)
   2160             element = [el for el in self.hmap.data.values() if el][-1]
   2161         else:
   2162             element = self.hmap.last
   2163         key = util.wrap_tuple(self.hmap.last_key)
-> 2164         ranges = self.compute_ranges(self.hmap, key, ranges)
   2165         self.current_ranges = ranges
   2166         self.current_frame = element
   2167         self.current_key = key

~/.venv/lib/python3.12/site-packages/holoviews/plotting/plot.py in ?(self, obj, key, ranges)
    611             # Only compute ranges if not axiswise on a composite plot
    612             # or not framewise on a Overlay or ElementPlot
    613             if (not (axiswise and not isinstance(obj, HoloMap)) or
    614                 (not framewise and isinstance(obj, HoloMap))):
--> 615                 self._compute_group_range(group, elements, ranges, framewise,
    616                                           axiswise, robust, self.top_level,
    617                                           prev_frame)
    618         self.ranges.update(ranges)

~/.venv/lib/python3.12/site-packages/holoviews/plotting/plot.py in ?(cls, group, elements, ranges, framewise, axiswise, robust, top_level, prev_frame)
    721                 elif el_dim.values:
    722                     ds = Dataset(el_dim.values, el_dim)
    723                     data_range = ds.range(el_dim, dimension_range=False)
    724                 else:
--> 725                     data_range = el.range(el_dim, dimension_range=False)
    726 
    727                 data_ranges[(el, el_dim)] = data_range
    728                 if dtype is not None and dtype.kind in 'uif' and robust:

~/.venv/lib/python3.12/site-packages/holoviews/core/data/__init__.py in ?(*args, **kwargs)
    226                                     output_type=type(result),
    227                                 )
    228             finally:
    229                 if not in_method:
--> 230                     inst._in_method = False
    231             return result

~/.venv/lib/python3.12/site-packages/holoviews/element/raster.py in ?(self, dim, data_range, dimension_range)
    922                     drange = self.gridded.range(dim, data_range, dimension_range)
    923             except Exception:
    924                 drange = None
    925             finally:
--> 926                 self.gridded._binned = False
    927             if drange is not None:
    928                 return drange
    929         return super().range(dim, data_range, dimension_range)

~/.venv/lib/python3.12/site-packages/holoviews/element/raster.py in ?(self)
    889     @property
    890     def gridded(self):
    891         if self._gridded is None:
--> 892             self._gridded = categorical_aggregate2d(self)
    893         return self._gridded

~/.venv/lib/python3.12/site-packages/param/parameterized.py in ?(class_, *args, **params)
   4464     def __new__(class_,*args,**params):
   4465         # Create and __call__() an instance of this class.
   4466         inst = class_.instance()
   4467         inst.param._set_name(class_.__name__)
-> 4468         return inst.__call__(*args,**params)

~/.venv/lib/python3.12/site-packages/holoviews/core/operation.py in ?(self, element, **kwargs)
    216             kwargs['streams'] = self.p.streams
    217         kwargs['per_element'] = self._per_element
    218         kwargs['link_dataset'] = self._propagate_dataset
    219         kwargs['link_inputs'] = self.p.link_inputs
--> 220         return element.apply(self, **kwargs)

~/.venv/lib/python3.12/site-packages/holoviews/core/accessors.py in ?(*args, **kwargs)
     74                                 ],
     75                                 output_type=type(result),
     76                             )
     77             finally:
---> 78                 if not in_method:
     79                     inst._obj._in_method = False
     80 
     81             return result

~/.venv/lib/python3.12/site-packages/holoviews/core/accessors.py in ?(self, apply_function, streams, link_inputs, link_dataset, dynamic, per_element, **kwargs)
    199         elif applies:
    200             inner_kwargs = util.resolve_dependent_kwargs(kwargs)
    201             if hasattr(apply_function, 'dynamic'):
    202                 inner_kwargs['dynamic'] = False
--> 203             new_obj = apply_function(self._obj, **inner_kwargs)
    204             if (link_dataset and isinstance(self._obj, Dataset) and
    205                 isinstance(new_obj, Dataset) and new_obj._dataset is None):
    206                 new_obj._dataset = self._obj.dataset

~/.venv/lib/python3.12/site-packages/holoviews/core/operation.py in ?(self, element, **kwargs)
    210                 return element.clone([(k, self._apply(el, key=k))
    211                                       for k, el in element.items()])
    212             elif ((self._per_element and isinstance(element, Element)) or
    213                   (not self._per_element and isinstance(element, ViewableElement))):
--> 214                 return self._apply(element)
    215         elif 'streams' not in kwargs:
    216             kwargs['streams'] = self.p.streams
    217         kwargs['per_element'] = self._per_element

~/.venv/lib/python3.12/site-packages/holoviews/core/operation.py in ?(self, element, key)
    137         if hasattr(element, '_in_method'):
    138             in_method = element._in_method
    139             if not in_method:
    140                 element._in_method = True
--> 141         ret = self._process(element, key)
    142         if hasattr(element, '_in_method') and not in_method:
    143             element._in_method = in_method
    144 

~/.venv/lib/python3.12/site-packages/holoviews/element/util.py in ?(self, obj, key)
    233             raise ValueError("Must have at two dimensions to aggregate over"
    234                              "and one value dimension to aggregate on.")
    235 
    236         obj = Dataset(obj, datatype=['dataframe'])
--> 237         return self._aggregate_dataset_pandas(obj)

~/.venv/lib/python3.12/site-packages/holoviews/element/util.py in ?(self, obj)
    207         print(type(obj.data))
    208         print(obj.data)
    209         print(obj)
    210         print(obj.data.columns)
--> 211         df = obj.data.set_index(index_cols).groupby(index_cols, **groupby_kwargs).first()
    212         label = 'unique' if len(df) == len(obj) else 'non-unique'
    213         levels = self._get_coords(obj)
    214         index = pd.MultiIndex.from_product(levels, names=df.index.names)

~/.venv/lib/python3.12/site-packages/pandas/core/frame.py in ?(self, keys, drop, append, inplace, verify_integrity)
   6118                     if not found:
   6119                         missing.append(col)
   6120 
   6121         if missing:
-> 6122             raise KeyError(f"None of {missing} are in the columns")
   6123 
   6124         if inplace:
   6125             frame = self

KeyError: "None of ['A', 'B'] are in the columns"

It looks like I can just add a reduce_function, but this is a bit unsatisfying and will hide errors about a non-unique index da.hvplot.heatmap(x="A", y="B", C="C", reduce_function=sum)

I think this is a valid usecase, and it's certainly surprising to xarray users where this dense format is typical (rather than a non-complete hierarchical index).

Was there a change that may have affected this behaviour?

Additionally, is there a way to make this fail in a quarto notebook? The exception seems to get weirdly swallowed, and then printed in place of the heatmap.

• I may be interested in making a pull request to address this

Thanks!