Skip to content

fix(gemma4): load mixed-quant GGUF checkpoints #494

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Poyo-Yon wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(gemma4): load mixed-quant GGUF checkpoints #494

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions python/freetoken/engine/engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,7 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:
hybrid_max_fetch=config.moe_hybrid_max_fetch,
layout=layout,
max_slots=max_slots,
gguf_quant_types=banks.gguf_quant_types,
)
# before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set
cache.cpu_layer_ids = cpu_layer_ids
Expand Down
19 changes: 19 additions & 0 deletions python/freetoken/kernel/csrc/gguf/gguf_kernel.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 3:
Expand All @@ -581,6 +582,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 6:
Expand All @@ -594,6 +596,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 7:
Expand All @@ -607,6 +610,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 8:
Expand All @@ -620,6 +624,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 10:
Expand All @@ -633,6 +638,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 11:
Expand All @@ -646,6 +652,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 12:
Expand All @@ -659,6 +666,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 13:
Expand All @@ -672,6 +680,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 14:
Expand All @@ -685,6 +694,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 16:
Expand All @@ -698,6 +708,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 17:
Expand All @@ -711,6 +722,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 18:
Expand All @@ -724,6 +736,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 19:
Expand All @@ -737,6 +750,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 20:
Expand All @@ -750,6 +764,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 21:
Expand All @@ -763,6 +778,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 22:
Expand All @@ -776,6 +792,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 23:
Expand All @@ -789,6 +806,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
case 29:
Expand All @@ -802,6 +820,7 @@ torch::Tensor ggml_moe_a8_vec(
col,
row,
quant_X.stride(0),
W.stride(0),
stream);
break;
}
Expand Down
Loading