Skip to content

SpinQuant online Transform added #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

SpinQuant online Transform added #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d94687e
3232
kimjoohyungsd Jun 20, 2025
60a90d9
2025-06-28
kimjoohyungsd Jun 28, 2025
ea33187
2025-07-02
kimjoohyungsd Jul 2, 2025
2c32dfc
2025-07-02
kimjoohyungsd Jul 2, 2025
54e85b1
2025-07-04
kimjoohyungsd Jul 4, 2025
4453c2b
2025-09-01
kimjoohyungsd Sep 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,10 +193,10 @@ if (LLAMA_BUILD_COMMON)
add_subdirectory(common)
endif()

if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
include(CTest)
add_subdirectory(tests)
endif()
# if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
# include(CTest)
# add_subdirectory(tests)
# endif()

if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_EXAMPLES)
add_subdirectory(examples)
Expand Down
16 changes: 15 additions & 1 deletion common/arg.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2274,6 +2274,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.use_mmap = false;
}
).set_env("LLAMA_ARG_NO_MMAP"));
add_opt(common_arg(
{"--use-R3"},
"apply R3 Online Hadamard Transform during computational process mentioned in Spinquant paper",
[](common_params & params) {
params.online_R3 = true;
}
));
add_opt(common_arg(
{"--use-R4"},
"apply R4 Online Hadamard Transform during computational process mentioned in Spinquant paper",
[](common_params & params) {
params.online_R4 = true;
}
));
add_opt(common_arg(
{"--numa"}, "TYPE",
"attempt optimizations that help on some NUMA systems\n"
Expand Down Expand Up @@ -3225,7 +3239,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.vocoder.use_guide_tokens = true;
}
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
add_opt(common_arg(
{"--tts-speaker-file"}, "FNAME",
"speaker file path for audio generation",
[](common_params & params, const std::string & value) {
Expand Down
3 changes: 3 additions & 0 deletions common/common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1101,6 +1101,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
mparams.use_mmap = params.use_mmap;
mparams.use_mlock = params.use_mlock;
mparams.check_tensors = params.check_tensors;
mparams.online_R4 = params.online_R4;

if (params.kv_overrides.empty()) {
mparams.kv_overrides = NULL;
Expand Down Expand Up @@ -1152,6 +1153,8 @@ struct llama_context_params common_context_params_to_llama(const common_params &
cparams.op_offload = !params.no_op_offload;
cparams.swa_full = params.swa_full;

cparams.online_R3 = params.online_R3;// online x` transform을 llama_context_params에도 설정을 한다

cparams.type_k = params.cache_type_k;
cparams.type_v = params.cache_type_v;

Expand Down
3 changes: 3 additions & 0 deletions common/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,9 @@ struct common_params {

bool single_turn = false; // single turn chat conversation

bool online_R3 = false; //Online에서 R3 Hadamard Transform을 활용하는지
bool online_R4 = false; //Online에서 R4 Hadamard Transform을 활용하는지

ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K
ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V

Expand Down
Loading