from_zarr code

local/from_zarr.ipynb

@@ -0,0 +1,161 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "240d8ca9-3dd4-4df6-8fa7-2ac9c4a697cf",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from dask.array import from_zarr\n",
+    "\n",
+    "zar_file = \"s3://coiled-datasets/synthetic-data/array-random-390KB.zarr\"\n",
+    "x = from_zarr(zar_file)\n",
+    "\n",
+    "print(x)\n",
+    "# dask.array<from-zarr, shape=(100, 100, 5), dtype=float64, chunksize=(10, 10, 1), chunktype=numpy.ndarray>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e43b65f6-17a1-439d-bd1e-ff6e9bf4c7b6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dask.array import from_zarr\n",
+    "\n",
+    "x = from_zarr(\"group.zarr\", component=\"group_foo/sample_array\")\n",
+    "print(x)\n",
+    "# dask.array<from-zarr, shape=(10, 10), dtype=int32, chunksize=(2, 2), chunktype=numpy.ndarray>\n",
+    "\n",
+    "product = x @ x.T\n",
+    "print(f\"Number of tasks: {len((product).dask)}\")\n",
+    "# Number of tasks: 376"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6fac7a32-f4be-45f7-a822-3734604894eb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dask.array import from_zarr\n",
+    "\n",
+    "x = from_zarr(\"group.zarr\", component=\"group_foo/sample_array\", chunks=(2, 10))\n",
+    "print(x)\n",
+    "# dask.array<from-zarr, shape=(10, 10), dtype=int32, chunksize=(2, 2), chunktype=numpy.ndarray>\n",
+    "\n",
+    "product = x @ x.T\n",
+    "print(f\"Number of tasks: {len((product).dask)}\")\n",
+    "# Number of tasks: 61"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "971518f6-025a-4fbf-81df-9b5b957c0500",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dask.array import from_zarr\n",
+    "\n",
+    "x = from_zarr(\"group.zarr\", component=\"group_foo/sample_array\")\n",
+    "print(x)\n",
+    "# dask.array<from-zarr, shape=(10, 10), dtype=int32, chunksize=(2, 2), chunktype=numpy.ndarray>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3083b34f-1f20-4524-b71b-caef7f3f9a51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dask.array import from_zarr\n",
+    "\n",
+    "x = from_zarr(\"sample.zarr\")\n",
+    "print(x.dask)\n",
+    "# HighLevelGraph with 2 layers.\n",
+    "# <dask.highlevelgraph.HighLevelGraph object at 0x1877b8f40>\n",
+    "#  0. original-from-zarr-aa89d568fa5d6a465403e4a4457083a9\n",
+    "#  1. from-zarr-aa89d568fa5d6a465403e4a4457083a9"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "76a334c4-af0c-4bd5-a33b-e3081c3d9f22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dask.array import from_zarr\n",
+    "\n",
+    "x = from_zarr(\"sample.zarr\", inline_array=True)\n",
+    "print(x.dask)\n",
+    "# HighLevelGraph with 1 layers.\n",
+    "# <dask.highlevelgraph.HighLevelGraph object at 0x1877ecac0>\n",
+    "#  0. from-zarr-dd5aae068dabbf1fea826d7abea7b6f8"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6e51e2cc-6254-4d16-af9b-ddd0c2e4ba73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import zarr\n",
+    "from dask.array import from_zarr, stack\n",
+    "from matplotlib.pyplot import imshow\n",
+    "\n",
+    "# load the list of components inside the zarr\n",
+    "zarr_file = \"weather_group.zarr\"\n",
+    "list_components = list(zarr.open(zarr_file, mode=\"r\")[\"temperature\"])\n",
+    "\n",
+    "# create a list of dask arrays\n",
+    "arrays = [from_zarr(zarr_file, component=f\"temperature/{c}\") for c in list_components]\n",
+    "print(len(arrays))\n",
+    "\n",
+    "# stack the arrays into a single array\n",
+    "temperature = stack(arrays, axis=0)\n",
+    "\n",
+    "print(temperature)\n",
+    "# dask.array<stack, shape=(31, 5760, 11520), dtype=float64, chunksize=(1, 500, 500), chunktype=numpy.ndarray>\n",
+    "\n",
+    "# calculate average temperature (at a single location across all dates)\n",
+    "result = temperature.mean(axis=0)\n",
+    "\n",
+    "print(result)\n",
+    "# dask.array<mean_agg-aggregate, shape=(5760, 11520), dtype=float64, chunksize=(500, 500), chunktype=numpy.ndarray>\n",
+    "\n",
+    "# visualize the result\n",
+    "imshow(result, cmap=\"RdBu_r\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}