diff --git a/src/huggingface_inference_toolkit/diffusers_utils.py b/src/huggingface_inference_toolkit/diffusers_utils.py
index 9f893a81..61886659 100644
--- a/src/huggingface_inference_toolkit/diffusers_utils.py
+++ b/src/huggingface_inference_toolkit/diffusers_utils.py
@@ -64,10 +64,15 @@ def __call__(
             logger.warning("Sending num_images_per_prompt > 1 to pipeline is not supported. Using default value 1.")
 
         if "target_size" in kwargs:
-            kwargs["height"] = kwargs["target_size"].pop("height", None)
-            kwargs["width"] = kwargs["target_size"].pop("width", None)
+            kwargs["height"] = kwargs["target_size"].pop("height")
+            kwargs["width"] = kwargs["target_size"].pop("width")
             kwargs.pop("target_size")
 
+        if kwargs.get("height") != kwargs.get("width"):
+            raise ValueError(
+                f"Provided `height={kwargs.get('height')}` and `width={kwargs.get('width')}`, but either both must have a value or both must be None (or not provided)."
+            )
+
         if "output_type" in kwargs and kwargs["output_type"] != "pil":
             kwargs.pop("output_type")
             logger.warning("The `output_type` cannot be modified, and PIL will be used by default instead.")