[WIP] Fixing download issue for full dataset

ice_station_zebra/data_processors/cli.py

@@ -1,6 +1,7 @@
 import logging
 
 import typer
+from typing import Annotated
 from omegaconf import DictConfig
 
 from ice_station_zebra.cli import hydra_adaptor
@@ -16,13 +17,17 @@
 
 @datasets_cli.command("create")
 @hydra_adaptor
-def create(config: DictConfig) -> None:
+def create(config: DictConfig,
+            overwrite: Annotated[
+                bool, typer.Option(help="Specify whether to overwrite existing datasets")
+           ] = False,
+           ) -> None:
     """Create all datasets."""
     register_filters()
     factory = ZebraDataProcessorFactory(config)
     for dataset in factory.datasets:
         logger.info("Working on %s.", dataset.name)
-        dataset.create()
+        dataset.create(overwrite=overwrite)
 
 
 @datasets_cli.command("inspect")

ice_station_zebra/data_processors/zebra_data_processor.py

@@ -30,19 +30,24 @@ def __init__(
         self.config: DictConfig = OmegaConf.to_object(config["datasets"][name])  # type: ignore[assignment]
         self.preprocessor = cls_preprocessor(self.config)
 
-    def create(self) -> None:
+    def create(self, overwrite) -> None:
         """Ensure that a single Anemoi dataset exists."""
-        try:
-            self.inspect()
-            logger.info(
-                "Dataset %s already exists at %s, no need to download.",
-                self.name,
-                self.path_dataset,
-            )
-        except (AttributeError, FileNotFoundError, PathNotFoundError):
-            logger.info("Dataset %s not found at %s.", self.name, self.path_dataset)
+        if (overwrite):
+            logger.info("Overwrite set to true, redownloading %s to %s", self.name, self.path_dataset)
             shutil.rmtree(self.path_dataset, ignore_errors=True)
             self.download()
+        else:
+            try:
+                self.inspect()
+                logger.info(
+                    "Dataset %s already exists at %s, no need to download.",
+                    self.name,
+                    self.path_dataset,
+                )
+            except (AttributeError, FileNotFoundError, PathNotFoundError):
+                logger.info("Dataset %s not found at %s.", self.name, self.path_dataset)
+                shutil.rmtree(self.path_dataset, ignore_errors=True)
+                self.download()
 
     def download(self) -> None:
         """Download a single Anemoi dataset."""