feat(gguf): serve qwen3vl and qwen3vlmoe gguf checkpoints text only

python/freetoken/kernel/aot_models.py

@@ -142,7 +142,8 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
         moe_intermediate_size=768,
         expert_formats=("bf16",),
         aliases=("Qwen/Qwen3-30B-A3B-Thinking-2507",),
-        arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM"),  # Qwen3-VL-30B-A3B: same text tower
+        # Qwen3-VL-30B-A3B: same text tower
+        arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM", "Qwen3VLMoeGGUFForCausalLM"),
     ),
     AotModel(
         name="Qwen/Qwen3.5-35B-A3B",
@@ -395,7 +396,8 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
         architecture="Qwen3ForCausalLM",
         hidden_size=4096,
         kv_groups=((8, 128),),
-        arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM"),  # Qwen3-VL-8B: same text tower
+        # Qwen3-VL: same text tower
+        arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM", "Qwen3VLGGUFForCausalLM"),
     ),
     AotModel(
         name="mistralai/Mistral-7B-Instruct-v0.3",

python/freetoken/models/gguf/config.py

@@ -35,6 +35,8 @@
     "qwen3moe": "Qwen3MoeGGUFForCausalLM",
     "qwen35": "Qwen3_5GGUFForCausalLM",
     "qwen35moe": "Qwen3_5MoeGGUFForCausalLM",
+    "qwen3vl": "Qwen3VLGGUFForCausalLM",
+    "qwen3vlmoe": "Qwen3VLMoeGGUFForCausalLM",
     "qwen4exp": "Qwen4ExpGGUFForCausalLM",
 }
 # (GGUF architecture, mmproj projector) -> registry key, for text GGUFs whose release is told apart only by its mmproj

python/freetoken/models/gguf/tokenizer.py

@@ -99,6 +99,12 @@ def load_gguf_tokenizer(model_path: str):
             pre_tokenizers.Split(Regex(GGUF_PRE_TOKENIZER_SPLITS["llama3"]), behavior="isolated"),
             pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
         ])
+    elif pre in ("qwen2", "qwen35"):
+        # every Qwen release's tokenizer.json applies NFC (Unicode Normalization Form C) before the
+        # byte-level BPE, but a GGUF carries no normalizer; without it decomposed text differs
+        from tokenizers import normalizers
+
+        fast.normalizer = normalizers.NFC()
     elif pre == "joyai-llm":
         from tokenizers import Regex, pre_tokenizers
 

python/freetoken/models/qwen3_vl/__init__.py

@@ -1,4 +1,5 @@
 from .config import VisionConfig, parse_config, parse_vision_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
 from .model import (
     Qwen3VLForConditionalGeneration,
     Qwen3VLMoeForConditionalGeneration,
@@ -13,10 +14,13 @@
     "Qwen3VLVisionModel",
     "VisionConfig",
     "deepstack_add",
+    "iter_gguf_weights",
     "iter_vision_weights",
     "iter_weights",
     "iter_weights_parallel",
     "parse_config",
+    "parse_gguf_config",
     "parse_vision_config",
     "rename_vl_prefix",
+    "to_gguf",
 ]

python/freetoken/models/qwen3_vl/gguf.py

@@ -1,17 +1,32 @@
-"""The Qwen VL vision tower from a llama.cpp ``qwen3vl_merger`` mmproj: the HF config keys, the image processor config, and the tower weights.
+"""Qwen3-VL GGUFs: the text trunks of llama.cpp ``qwen3vl`` / ``qwen3vlmoe`` files, and the Qwen VL vision tower from a ``qwen3vl_merger`` mmproj (the HF config keys, the image processor config, and the tower weights).
+
+The text trunk runs without its vision side: no tower and plain rope instead of interleaved mrope. llama.cpp's IMROPE keeps the standard full-spectrum theta (ggml.h: the dim index is not reset per section), so equal (t, h, w) text positions reduce to NeoX rope and q/k carry no permute, like the ``qwen3`` / ``qwen3moe`` GGUFs. The routed experts stack as ``ffn_*_exps`` because the converter splits HF's packed ``gate_up_proj`` / ``down_proj`` experts (conversion/qwen3vl.py).
 
 llama.cpp splits the Conv3d patch embedding into one tensor per temporal slice (``v.patch_embd.weight`` and ``.weight.1``); everything else is stored in HF layout.
 """
 
 from __future__ import annotations
 
 import re
-from typing import Any, Iterator
+from dataclasses import replace
+from types import SimpleNamespace
+from typing import Any, TYPE_CHECKING, Iterator
 
 import torch
 
 from freetoken.models.gguf.mmproj import Mmproj, mmproj_tensor
-from freetoken.models.gguf.reader import gguf_tensor_index, load_gguf_metadata
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, load_gguf_metadata
+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrix
+
+from .config import parse_config
+
+if TYPE_CHECKING:
+    from freetoken.models.config import ModelConfig
+    from freetoken.models.gguf.config import GgufConfigShim
+
+# ----------------------------------------------------------------------------------------------
+# Vision tower (a qwen3vl_merger mmproj)
+# ----------------------------------------------------------------------------------------------
 
 _PROJECTOR = "qwen3vl_merger"
 _IMAGE_TOKEN = "<|image_pad|>"
@@ -111,4 +126,155 @@ def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]:
             yield f"visual.{_name(gguf_name)}", mmproj_tensor(info)
 
 
-__all__ = ["gguf_mm_config", "iter_mmproj_weights"]
+# ----------------------------------------------------------------------------------------------
+# Text trunk (a qwen3vl / qwen3vlmoe GGUF)
+# ----------------------------------------------------------------------------------------------
+
+_LAYER_LEAVES = {
+    "self_attn.q_proj": "attn_q",
+    "self_attn.k_proj": "attn_k",
+    "self_attn.v_proj": "attn_v",
+    "self_attn.o_proj": "attn_output",
+    "mlp.gate_proj": "ffn_gate",
+    "mlp.up_proj": "ffn_up",
+    "mlp.down_proj": "ffn_down",
+    "mlp.gate": "ffn_gate_inp",
+    "mlp.experts.0.gate_proj": "ffn_gate_exps",
+    "mlp.experts.0.up_proj": "ffn_up_exps",
+    "mlp.experts.0.down_proj": "ffn_down_exps",
+}
+_QKV_LEAVES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj")
+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}
+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+    """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+    if name in _TOP_LEAVES:
+        return (_TOP_LEAVES[name],)
+    m = _LAYER_RE.match(name)
+    if m is None:
+        return ()
+    blk, leaf = f"blk.{m.group(1)}", m.group(2)
+    if leaf in _QKV_LEAVES and f"{blk}.attn_qkv.weight" in tensors:
+        return (f"{blk}.attn_qkv",)
+    if leaf in ("mlp.experts.0.gate_proj", "mlp.experts.0.up_proj") and f"{blk}.ffn_gate_up_exps.weight" in tensors:
+        return (f"{blk}.ffn_gate_up_exps",)
+    tensor = _LAYER_LEAVES.get(leaf)
+    return (f"{blk}.{tensor}",) if tensor is not None else ()
+
+
+def _check_supported(shim: GgufConfigShim) -> None:
+    if any(k.startswith("prism.hadamard.") for k in shim.metadata):
+        raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")
+    check_scales(shim.model_path)
+
+
+def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
+    """The HF config fields ``parse_config`` reads, from the GGUF metadata."""
+    md = shim.metadata
+    arch = shim.model_type  # the GGUF architecture: qwen3vl or qwen3vlmoe
+    moe = arch.endswith("moe")
+
+    def kv(key: str):
+        return md[f"{arch}.{key}"]
+
+    _check_supported(shim)
+    rope = {"rope_type": "default", "rope_theta": float(kv("rope.freq_base"))}
+    scaling = md.get(f"{arch}.rope.scaling.type", "none")
+    if scaling == "yarn":
+        rope.update(
+            rope_type="yarn",
+            factor=float(kv("rope.scaling.factor")),
+            original_max_position_embeddings=int(kv("rope.scaling.original_context_length")),
+        )
+    elif scaling != "none":
+        raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} is not supported")
+    key_length = md.get(f"{arch}.attention.key_length")
+    text = SimpleNamespace(
+        model_type="qwen3_moe" if moe else "qwen3",
+        architectures=list(shim.architectures),
+        hidden_act="silu",
+        hidden_size=int(kv("embedding_length")),
+        num_hidden_layers=int(kv("block_count")),
+        num_attention_heads=int(kv("attention.head_count")),
+        num_key_value_heads=int(kv("attention.head_count_kv")),
+        head_dim=int(key_length) if key_length is not None else None,
+        rope_scaling=None if scaling == "none" else rope,
+        rope_parameters=rope,
+        max_position_embeddings=int(kv("context_length")),
+        rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+        vocab_size=int(shim.vocab_size),
+        tie_word_embeddings=bool(shim.tie_word_embeddings),
+        intermediate_size=int(kv("feed_forward_length")),
+    )
+    if moe:
+        text.num_experts = int(kv("expert_count"))
+        text.num_experts_per_tok = int(kv("expert_used_count"))
+        text.moe_intermediate_size = int(kv("expert_feed_forward_length"))
+        # llama.cpp always renormalizes the top-k weights of a qwen3moe-style router
+        text.norm_topk_prob = True
+    else:
+        text.num_experts = 0
+    # text only: no vision_config, so the wrapper builds no tower and keeps 1-D rope
+    return SimpleNamespace(
+        text_config=text,
+        model_type="qwen3_vl",
+        architectures=list(shim.architectures),
+        image_token_id=None,
+        vision_config=None,
+        tie_word_embeddings=bool(shim.tie_word_embeddings),
+    )
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+    config = parse_config(_hf_config(shim))
+    return replace(config, expert_quant="gguf") if config.num_experts else config
+
+
+def iter_gguf_weights(
+    model_path: str,
+    device: torch.device,
+    *,
+    include_moe_experts: bool,
+    include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+    """The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""
+    if not include_non_moe:
+        return
+    from freetoken.utils import cached_load_hf_config
+
+    text = _hf_config(cached_load_hf_config(model_path)).text_config
+    index = gguf_tensor_index(model_path)
+    hidden = text.hidden_size
+
+    def t(name: str) -> GgufTensorInfo:
+        return index[name if name in index else f"{name}.weight"]
+
+    yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
+    if not text.tie_word_embeddings:
+        yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
+    yield "model.norm.weight", float_tensor(t("output_norm"))
+
+    for layer in range(text.num_hidden_layers):
+        blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+        yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))
+        yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm"))
+        sa = f"{pre}.self_attn"
+        # convert_hf_to_gguf.py --fuse-qkv stores q, k and v as one attn_qkv matrix
+        qkv = [t(f"{blk}.attn_qkv")] if f"{blk}.attn_qkv.weight" in index else [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]
+        yield from fused(f"{sa}.qkv_proj", qkv, [matrix(p) for p in qkv], hidden)
+        yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm"))
+        yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm"))
+        out = t(f"{blk}.attn_output")
+        yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
+        mlp = f"{pre}.mlp"
+        if text.num_experts:
+            yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))
+        else:
+            gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))
+            yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)
+            yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0]))
+
+
+__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"]

python/freetoken/models/register.py

@@ -162,6 +162,21 @@ class ModelSpec:
         encoders=_QWEN_VL_ENCODERS,
         packed_modules_mapping=_DENSE_PACKED + _STACKED_EXPERTS_PACKED,
     ),
+    # llama.cpp qwen3vl / qwen3vlmoe GGUF (text only): the same model classes with every tensor in its packed ggml type
+    "Qwen3VLGGUFForCausalLM": ModelSpec(
+        "freetoken.models.qwen3_vl",
+        "Qwen3VLForConditionalGeneration",
+        parse_config="parse_gguf_config",
+        iter_weights="iter_gguf_weights",
+        packed_modules_mapping=_DENSE_PACKED,
+    ),
+    "Qwen3VLMoeGGUFForCausalLM": ModelSpec(
+        "freetoken.models.qwen3_vl",
+        "Qwen3VLMoeForConditionalGeneration",
+        parse_config="parse_gguf_config",
+        iter_weights="iter_gguf_weights",
+        packed_modules_mapping=_DENSE_PACKED + _EXPERTS_PACKED,
+    ),
     "MiniMaxM2ForCausalLM": ModelSpec(
         "freetoken.models.minimax_m2",
         "MiniMaxM2ForCausalLM",