NetCDF IO cleanup, including in-memory reads/writes with netCDF4 (#10624)

* Rewrite DataTree.to_netcdf and support netCDF4 in-memory

This PR includes a handful of significant changes:

1. It refactors the internal structure of `DataTree.to_netcdf()` and
   `DataTree.to_zarr()` to use lower level interfaces, rather than
   calling `Dataset` methods. This allows for properly supporting
   `compute=False` (and likely various other improvements).
2. Reading and writing in-memory data with netCDF4-python is now
   supported, including DataTree.
3. The `engine` argument in `DataTree.to_netcdf()` is now set
   consistently with `Dataset.to_netcdf()`, preferring `netcdf4` to
   `h5netcdf`.
3. Calling `Dataset.to_netcdf()` without a target now always returns a
   `memoryview` object, *including* in the case where `engine='scipy'`
   is used (which currently returns `bytes`). This is a breaking change,
   rather than merely issuing a warning as is done in #10571. I believe
   it probably makes sense to do as a this breaking change because (1)
   it offers significant performance benefits, (2) the default behavior
   without specifying an engine will already change (because `netcdf4`
   is preferred to the `scipy` backend) and (3) restoring previous
   behavior is easy (by wrapping the memoryview with `bytes()`).

Author: shoyer

doc/whats-new.rst

@@ -13,6 +13,10 @@ v2025.09.1 (unreleased)
 New Features
 ~~~~~~~~~~~~
 
+- ``engine='netcdf4'`` now supports reading and writing in-memory netCDF files.
+  All of Xarray's netCDF backends now support in-memory reads and writes
+  (:pull:`10624`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
 
 Breaking changes
 ~~~~~~~~~~~~~~~~
@@ -22,13 +26,28 @@ Breaking changes
   dataset in-place. (:issue:`10167`)
   By `Maximilian Roos <https://github.com/max-sixty>`_.
 
+- The default ``engine`` when reading/writing netCDF files in-memory is now
+  netCDF4, consistent with Xarray's default ``engine`` when read/writing netCDF
+  files to disk (:pull:`10624`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
+
 Deprecations
 ~~~~~~~~~~~~
 
 
 Bug fixes
 ~~~~~~~~~
 
+- Xarray objects opened from file-like objects with ``engine='h5netcdf'`` can
+  now be pickled, as long as the underlying file-like object also supports
+  pickle (:issue:`10712`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
+
+- Closing Xarray objects opened from file-like objects with ```engine='scipy'``
+  no longer closes the underlying file, consistent with the h5netcdf backend
+  (:pull:`10624`).
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
+
 - Fix the ``align_chunks`` parameter on the :py:meth:`~xarray.Dataset.to_zarr` method, it was not being
   passed to the underlying :py:meth:`~xarray.backends.api` method (:issue:`10501`, :pull:`10516`).
 - Fix error when encoding an empty :py:class:`numpy.datetime64` array

xarray/backends/api.py

@@ -99,7 +99,7 @@
 
 def get_default_netcdf_write_engine(
     format: T_NetcdfTypes | None,
-    to_fileobject_or_memoryview: bool,
+    to_fileobject: bool,
 ) -> Literal["netcdf4", "h5netcdf", "scipy"]:
     """Return the default netCDF library to use for writing a netCDF file."""
     module_names = {
@@ -118,7 +118,7 @@ def get_default_netcdf_write_engine(
         else:
             raise ValueError(f"unexpected {format=}")
 
-    if to_fileobject_or_memoryview:
+    if to_fileobject:
         candidates.remove("netcdf4")
 
     for engine in candidates:
@@ -545,14 +545,12 @@ def open_dataset(
     cache: bool | None = None,
     decode_cf: bool | None = None,
     mask_and_scale: bool | Mapping[str, bool] | None = None,
-    decode_times: bool
-    | CFDatetimeCoder
-    | Mapping[str, bool | CFDatetimeCoder]
-    | None = None,
-    decode_timedelta: bool
-    | CFTimedeltaCoder
-    | Mapping[str, bool | CFTimedeltaCoder]
-    | None = None,
+    decode_times: (
+        bool | CFDatetimeCoder | Mapping[str, bool | CFDatetimeCoder] | None
+    ) = None,
+    decode_timedelta: (
+        bool | CFTimedeltaCoder | Mapping[str, bool | CFTimedeltaCoder] | None
+    ) = None,
     use_cftime: bool | Mapping[str, bool] | None = None,
     concat_characters: bool | Mapping[str, bool] | None = None,
     decode_coords: Literal["coordinates", "all"] | bool | None = None,
@@ -788,10 +786,9 @@ def open_dataarray(
     cache: bool | None = None,
     decode_cf: bool | None = None,
     mask_and_scale: bool | None = None,
-    decode_times: bool
-    | CFDatetimeCoder
-    | Mapping[str, bool | CFDatetimeCoder]
-    | None = None,
+    decode_times: (
+        bool | CFDatetimeCoder | Mapping[str, bool | CFDatetimeCoder] | None
+    ) = None,
     decode_timedelta: bool | CFTimedeltaCoder | None = None,
     use_cftime: bool | None = None,
     concat_characters: bool | None = None,
@@ -1018,14 +1015,12 @@ def open_datatree(
     cache: bool | None = None,
     decode_cf: bool | None = None,
     mask_and_scale: bool | Mapping[str, bool] | None = None,
-    decode_times: bool
-    | CFDatetimeCoder
-    | Mapping[str, bool | CFDatetimeCoder]
-    | None = None,
-    decode_timedelta: bool
-    | CFTimedeltaCoder
-    | Mapping[str, bool | CFTimedeltaCoder]
-    | None = None,
+    decode_times: (
+        bool | CFDatetimeCoder | Mapping[str, bool | CFDatetimeCoder] | None
+    ) = None,
+    decode_timedelta: (
+        bool | CFTimedeltaCoder | Mapping[str, bool | CFTimedeltaCoder] | None
+    ) = None,
     use_cftime: bool | Mapping[str, bool] | None = None,
     concat_characters: bool | Mapping[str, bool] | None = None,
     decode_coords: Literal["coordinates", "all"] | bool | None = None,
@@ -1260,14 +1255,12 @@ def open_groups(
     cache: bool | None = None,
     decode_cf: bool | None = None,
     mask_and_scale: bool | Mapping[str, bool] | None = None,
-    decode_times: bool
-    | CFDatetimeCoder
-    | Mapping[str, bool | CFDatetimeCoder]
-    | None = None,
-    decode_timedelta: bool
-    | CFTimedeltaCoder
-    | Mapping[str, bool | CFTimedeltaCoder]
-    | None = None,
+    decode_times: (
+        bool | CFDatetimeCoder | Mapping[str, bool | CFDatetimeCoder] | None
+    ) = None,
+    decode_timedelta: (
+        bool | CFTimedeltaCoder | Mapping[str, bool | CFTimedeltaCoder] | None
+    ) = None,
     use_cftime: bool | Mapping[str, bool] | None = None,
     concat_characters: bool | Mapping[str, bool] | None = None,
     decode_coords: Literal["coordinates", "all"] | bool | None = None,
@@ -1523,10 +1516,9 @@ def _remove_path(
 
 
 def open_mfdataset(
-    paths: str
-    | os.PathLike
-    | ReadBuffer
-    | NestedSequence[str | os.PathLike | ReadBuffer],
+    paths: (
+        str | os.PathLike | ReadBuffer | NestedSequence[str | os.PathLike | ReadBuffer]
+    ),
     chunks: T_Chunks = None,
     concat_dim: (
         str
@@ -1540,10 +1532,9 @@ def open_mfdataset(
     compat: CompatOptions | CombineKwargDefault = _COMPAT_DEFAULT,
     preprocess: Callable[[Dataset], Dataset] | None = None,
     engine: T_Engine = None,
-    data_vars: Literal["all", "minimal", "different"]
-    | None
-    | list[str]
-    | CombineKwargDefault = _DATA_VARS_DEFAULT,
+    data_vars: (
+        Literal["all", "minimal", "different"] | None | list[str] | CombineKwargDefault
+    ) = _DATA_VARS_DEFAULT,
     coords=_COORDS_DEFAULT,
     combine: Literal["by_coords", "nested"] = "by_coords",
     parallel: bool = False,
@@ -2068,8 +2059,8 @@ def to_netcdf(
     path_or_file = _normalize_path(path_or_file)
 
     if engine is None:
-        to_fileobject_or_memoryview = not isinstance(path_or_file, str)
-        engine = get_default_netcdf_write_engine(format, to_fileobject_or_memoryview)
+        to_fileobject = isinstance(path_or_file, IOBase)
+        engine = get_default_netcdf_write_engine(format, to_fileobject)
 
     # validate Dataset keys, DataArray names, and attr keys/values
     _validate_dataset_names(dataset)