diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py index 3773d86d96..bf7e75b6e8 100644 --- a/python/freetoken/kernel/aot_models.py +++ b/python/freetoken/kernel/aot_models.py @@ -142,7 +142,8 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int moe_intermediate_size=768, expert_formats=("bf16",), aliases=("Qwen/Qwen3-30B-A3B-Thinking-2507",), - arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM"), # Qwen3-VL-30B-A3B: same text tower + # Qwen3-VL-30B-A3B: same text tower + arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM", "Qwen3VLMoeGGUFForCausalLM"), ), AotModel( name="Qwen/Qwen3.5-35B-A3B", @@ -395,7 +396,8 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int architecture="Qwen3ForCausalLM", hidden_size=4096, kv_groups=((8, 128),), - arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM"), # Qwen3-VL-8B: same text tower + # Qwen3-VL: same text tower + arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM", "Qwen3VLGGUFForCausalLM"), ), AotModel( name="mistralai/Mistral-7B-Instruct-v0.3", diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py index eb859abe4a..61198ead29 100644 --- a/python/freetoken/models/gguf/config.py +++ b/python/freetoken/models/gguf/config.py @@ -35,6 +35,8 @@ "qwen3moe": "Qwen3MoeGGUFForCausalLM", "qwen35": "Qwen3_5GGUFForCausalLM", "qwen35moe": "Qwen3_5MoeGGUFForCausalLM", + "qwen3vl": "Qwen3VLGGUFForCausalLM", + "qwen3vlmoe": "Qwen3VLMoeGGUFForCausalLM", "qwen4exp": "Qwen4ExpGGUFForCausalLM", } # (GGUF architecture, mmproj projector) -> registry key, for text GGUFs whose release is told apart only by its mmproj diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.py index 5d0ff489ce..a9c77d7999 100644 --- a/python/freetoken/models/gguf/tokenizer.py +++ b/python/freetoken/models/gguf/tokenizer.py @@ -99,6 +99,12 @@ def load_gguf_tokenizer(model_path: str): pre_tokenizers.Split(Regex(GGUF_PRE_TOKENIZER_SPLITS["llama3"]), behavior="isolated"), pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False), ]) + elif pre in ("qwen2", "qwen35"): + # every Qwen release's tokenizer.json applies NFC (Unicode Normalization Form C) before the + # byte-level BPE, but a GGUF carries no normalizer; without it decomposed text differs + from tokenizers import normalizers + + fast.normalizer = normalizers.NFC() elif pre == "joyai-llm": from tokenizers import Regex, pre_tokenizers diff --git a/python/freetoken/models/qwen3_vl/__init__.py b/python/freetoken/models/qwen3_vl/__init__.py index 62640d4394..2b311ba5dd 100644 --- a/python/freetoken/models/qwen3_vl/__init__.py +++ b/python/freetoken/models/qwen3_vl/__init__.py @@ -1,4 +1,5 @@ from .config import VisionConfig, parse_config, parse_vision_config +from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf from .model import ( Qwen3VLForConditionalGeneration, Qwen3VLMoeForConditionalGeneration, @@ -13,10 +14,13 @@ "Qwen3VLVisionModel", "VisionConfig", "deepstack_add", + "iter_gguf_weights", "iter_vision_weights", "iter_weights", "iter_weights_parallel", "parse_config", + "parse_gguf_config", "parse_vision_config", "rename_vl_prefix", + "to_gguf", ] diff --git a/python/freetoken/models/qwen3_vl/gguf.py b/python/freetoken/models/qwen3_vl/gguf.py index 42c40c2eea..52f1edd7c7 100644 --- a/python/freetoken/models/qwen3_vl/gguf.py +++ b/python/freetoken/models/qwen3_vl/gguf.py @@ -1,4 +1,6 @@ -"""The Qwen VL vision tower from a llama.cpp ``qwen3vl_merger`` mmproj: the HF config keys, the image processor config, and the tower weights. +"""Qwen3-VL GGUFs: the text trunks of llama.cpp ``qwen3vl`` / ``qwen3vlmoe`` files, and the Qwen VL vision tower from a ``qwen3vl_merger`` mmproj (the HF config keys, the image processor config, and the tower weights). + +The text trunk runs without its vision side: no tower and plain rope instead of interleaved mrope. llama.cpp's IMROPE keeps the standard full-spectrum theta (ggml.h: the dim index is not reset per section), so equal (t, h, w) text positions reduce to NeoX rope and q/k carry no permute, like the ``qwen3`` / ``qwen3moe`` GGUFs. The routed experts stack as ``ffn_*_exps`` because the converter splits HF's packed ``gate_up_proj`` / ``down_proj`` experts (conversion/qwen3vl.py). llama.cpp splits the Conv3d patch embedding into one tensor per temporal slice (``v.patch_embd.weight`` and ``.weight.1``); everything else is stored in HF layout. """ @@ -6,12 +8,25 @@ from __future__ import annotations import re -from typing import Any, Iterator +from dataclasses import replace +from types import SimpleNamespace +from typing import Any, TYPE_CHECKING, Iterator import torch from freetoken.models.gguf.mmproj import Mmproj, mmproj_tensor -from freetoken.models.gguf.reader import gguf_tensor_index, load_gguf_metadata +from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, load_gguf_metadata +from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrix + +from .config import parse_config + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + from freetoken.models.gguf.config import GgufConfigShim + +# ---------------------------------------------------------------------------------------------- +# Vision tower (a qwen3vl_merger mmproj) +# ---------------------------------------------------------------------------------------------- _PROJECTOR = "qwen3vl_merger" _IMAGE_TOKEN = "<|image_pad|>" @@ -111,4 +126,155 @@ def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]: yield f"visual.{_name(gguf_name)}", mmproj_tensor(info) -__all__ = ["gguf_mm_config", "iter_mmproj_weights"] +# ---------------------------------------------------------------------------------------------- +# Text trunk (a qwen3vl / qwen3vlmoe GGUF) +# ---------------------------------------------------------------------------------------------- + +_LAYER_LEAVES = { + "self_attn.q_proj": "attn_q", + "self_attn.k_proj": "attn_k", + "self_attn.v_proj": "attn_v", + "self_attn.o_proj": "attn_output", + "mlp.gate_proj": "ffn_gate", + "mlp.up_proj": "ffn_up", + "mlp.down_proj": "ffn_down", + "mlp.gate": "ffn_gate_inp", + "mlp.experts.0.gate_proj": "ffn_gate_exps", + "mlp.experts.0.up_proj": "ffn_up_exps", + "mlp.experts.0.down_proj": "ffn_down_exps", +} +_QKV_LEAVES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj") +_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"} +_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$") + + +def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]: + """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for.""" + if name in _TOP_LEAVES: + return (_TOP_LEAVES[name],) + m = _LAYER_RE.match(name) + if m is None: + return () + blk, leaf = f"blk.{m.group(1)}", m.group(2) + if leaf in _QKV_LEAVES and f"{blk}.attn_qkv.weight" in tensors: + return (f"{blk}.attn_qkv",) + if leaf in ("mlp.experts.0.gate_proj", "mlp.experts.0.up_proj") and f"{blk}.ffn_gate_up_exps.weight" in tensors: + return (f"{blk}.ffn_gate_up_exps",) + tensor = _LAYER_LEAVES.get(leaf) + return (f"{blk}.{tensor}",) if tensor is not None else () + + +def _check_supported(shim: GgufConfigShim) -> None: + if any(k.startswith("prism.hadamard.") for k in shim.metadata): + raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported") + check_scales(shim.model_path) + + +def _hf_config(shim: GgufConfigShim) -> SimpleNamespace: + """The HF config fields ``parse_config`` reads, from the GGUF metadata.""" + md = shim.metadata + arch = shim.model_type # the GGUF architecture: qwen3vl or qwen3vlmoe + moe = arch.endswith("moe") + + def kv(key: str): + return md[f"{arch}.{key}"] + + _check_supported(shim) + rope = {"rope_type": "default", "rope_theta": float(kv("rope.freq_base"))} + scaling = md.get(f"{arch}.rope.scaling.type", "none") + if scaling == "yarn": + rope.update( + rope_type="yarn", + factor=float(kv("rope.scaling.factor")), + original_max_position_embeddings=int(kv("rope.scaling.original_context_length")), + ) + elif scaling != "none": + raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} is not supported") + key_length = md.get(f"{arch}.attention.key_length") + text = SimpleNamespace( + model_type="qwen3_moe" if moe else "qwen3", + architectures=list(shim.architectures), + hidden_act="silu", + hidden_size=int(kv("embedding_length")), + num_hidden_layers=int(kv("block_count")), + num_attention_heads=int(kv("attention.head_count")), + num_key_value_heads=int(kv("attention.head_count_kv")), + head_dim=int(key_length) if key_length is not None else None, + rope_scaling=None if scaling == "none" else rope, + rope_parameters=rope, + max_position_embeddings=int(kv("context_length")), + rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")), + vocab_size=int(shim.vocab_size), + tie_word_embeddings=bool(shim.tie_word_embeddings), + intermediate_size=int(kv("feed_forward_length")), + ) + if moe: + text.num_experts = int(kv("expert_count")) + text.num_experts_per_tok = int(kv("expert_used_count")) + text.moe_intermediate_size = int(kv("expert_feed_forward_length")) + # llama.cpp always renormalizes the top-k weights of a qwen3moe-style router + text.norm_topk_prob = True + else: + text.num_experts = 0 + # text only: no vision_config, so the wrapper builds no tower and keeps 1-D rope + return SimpleNamespace( + text_config=text, + model_type="qwen3_vl", + architectures=list(shim.architectures), + image_token_id=None, + vision_config=None, + tie_word_embeddings=bool(shim.tie_word_embeddings), + ) + + +def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig: + config = parse_config(_hf_config(shim)) + return replace(config, expert_quant="gguf") if config.num_experts else config + + +def iter_gguf_weights( + model_path: str, + device: torch.device, + *, + include_moe_experts: bool, + include_non_moe: bool, +) -> Iterator[tuple[str, torch.Tensor]]: + """The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``).""" + if not include_non_moe: + return + from freetoken.utils import cached_load_hf_config + + text = _hf_config(cached_load_hf_config(model_path)).text_config + index = gguf_tensor_index(model_path) + hidden = text.hidden_size + + def t(name: str) -> GgufTensorInfo: + return index[name if name in index else f"{name}.weight"] + + yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden) + if not text.tie_word_embeddings: + yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden) + yield "model.norm.weight", float_tensor(t("output_norm")) + + for layer in range(text.num_hidden_layers): + blk, pre = f"blk.{layer}", f"model.layers.{layer}" + yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm")) + yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm")) + sa = f"{pre}.self_attn" + # convert_hf_to_gguf.py --fuse-qkv stores q, k and v as one attn_qkv matrix + qkv = [t(f"{blk}.attn_qkv")] if f"{blk}.attn_qkv.weight" in index else [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")] + yield from fused(f"{sa}.qkv_proj", qkv, [matrix(p) for p in qkv], hidden) + yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm")) + yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm")) + out = t(f"{blk}.attn_output") + yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0])) + mlp = f"{pre}.mlp" + if text.num_experts: + yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp")) + else: + gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down")) + yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden) + yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0])) + + +__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"] diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py index bbe34f0674..ad8ade862c 100644 --- a/python/freetoken/models/register.py +++ b/python/freetoken/models/register.py @@ -162,6 +162,21 @@ class ModelSpec: encoders=_QWEN_VL_ENCODERS, packed_modules_mapping=_DENSE_PACKED + _STACKED_EXPERTS_PACKED, ), + # llama.cpp qwen3vl / qwen3vlmoe GGUF (text only): the same model classes with every tensor in its packed ggml type + "Qwen3VLGGUFForCausalLM": ModelSpec( + "freetoken.models.qwen3_vl", + "Qwen3VLForConditionalGeneration", + parse_config="parse_gguf_config", + iter_weights="iter_gguf_weights", + packed_modules_mapping=_DENSE_PACKED, + ), + "Qwen3VLMoeGGUFForCausalLM": ModelSpec( + "freetoken.models.qwen3_vl", + "Qwen3VLMoeForConditionalGeneration", + parse_config="parse_gguf_config", + iter_weights="iter_gguf_weights", + packed_modules_mapping=_DENSE_PACKED + _EXPERTS_PACKED, + ), "MiniMaxM2ForCausalLM": ModelSpec( "freetoken.models.minimax_m2", "MiniMaxM2ForCausalLM",