From b2f4a124c9829ff7e823c321e4bdc9b46ef01610 Mon Sep 17 00:00:00 2001
From: Shirin Yamani <yamani.shirin@ucalgary.ca>
Date: Mon, 10 Jun 2024 13:17:22 -0600
Subject: [PATCH] additional load_in_4bit removed

---
 examples/guanaco_generate.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/guanaco_generate.py b/examples/guanaco_generate.py
index 35147a00..d4492007 100644
--- a/examples/guanaco_generate.py
+++ b/examples/guanaco_generate.py
@@ -41,7 +41,8 @@ def get_last_checkpoint(checkpoint_dir):
     model_name_or_path,
     torch_dtype=torch.bfloat16,
     device_map={"": 0},
-    load_in_4bit=True,
+    #load_in_4bit is already set in config, it can't be twice!
+    #load_in_4bit=True,
     quantization_config=BitsAndBytesConfig(
         load_in_4bit=True,
         bnb_4bit_compute_dtype=torch.bfloat16,