From 8d4819c3be6f13bf6f4beb56a71de633efe5203f Mon Sep 17 00:00:00 2001 From: Mukkesh Ganesh Date: Thu, 22 Oct 2026 21:22:16 -0700 Subject: [PATCH] feat(gguf): serve image input on glm5-next ggufs from a glm5v mmproj --- python/freetoken/models/glm5_next/__init__.py | 4 +- python/freetoken/models/glm5_next/gguf.py | 150 +++++++++++++++++- python/freetoken/models/register.py | 8 +- 3 files changed, 153 insertions(+), 9 deletions(-) diff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.py index 0c548da7f6..2c23510740 100644 --- a/python/freetoken/models/glm5_next/__init__.py +++ b/python/freetoken/models/glm5_next/__init__.py @@ -1,5 +1,5 @@ from .config import VisionConfig, parse_config, parse_vision_config -from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf +from .gguf import gguf_mm_config, iter_gguf_weights, iter_mmproj_weights, parse_gguf_config, to_gguf from .model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration from .vision import Glm5NextVisionModel from .weight import iter_expert_pieces, iter_vision_weights, iter_weights, nvfp4_expert_spec @@ -13,6 +13,8 @@ "parse_config", "parse_gguf_config", "parse_vision_config", + "gguf_mm_config", + "iter_mmproj_weights", "iter_vision_weights", "iter_gguf_weights", "iter_weights", diff --git a/python/freetoken/models/glm5_next/gguf.py b/python/freetoken/models/glm5_next/gguf.py index 73eb0a82ce..c918225396 100644 --- a/python/freetoken/models/glm5_next/gguf.py +++ b/python/freetoken/models/glm5_next/gguf.py @@ -6,6 +6,12 @@ clamped SwiGLU), so ``parse_gguf_config`` derives a text-config-shaped namespace from the GGUF KV metadata and reuses ``glm5_next.config.parse_config`` wholesale. +A ``glm5v`` mmproj beside the text file adds the vision tower: its metadata becomes +the HF ``vision_config`` keys and the image processor config, and its tensors load +under ``visual.`` as the HF checkpoints do. The llama.cpp converter splits the Conv3d +patch embedding into two temporal halves (``v.patch_embd.weight`` and ``.weight.1``), +stacked back at load; every other tower tensor is stored in HF layout. + The llama.cpp converter (unsloth's glm5-next writer; tensor names identical across the ``glm5-next`` and the older ``glm5next`` arch spellings) stores KDA's ``A_log`` pre-transformed as ``-exp(A_log)`` -- inverted at load -- and splits the MLA @@ -22,13 +28,19 @@ import re from dataclasses import replace from types import SimpleNamespace -from typing import TYPE_CHECKING, Iterator +from typing import TYPE_CHECKING, Any, Iterator import numpy as np import torch from freetoken.layers.quantization.ggml_types import is_quantized -from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, gguf_tensor_names +from freetoken.models.gguf.mmproj import Mmproj, find_mmproj, log_mmproj, mmproj_tensor +from freetoken.models.gguf.reader import ( + GgufTensorInfo, + gguf_tensor_index, + gguf_tensor_names, + load_gguf_metadata, +) from freetoken.models.gguf.tensors import check_scales, fused, matrix from .config import parse_config @@ -227,13 +239,136 @@ def need(key: str): text_config=text, model_type="glm5_next", architectures=list(shim.architectures), - vision_config=None, - image_token_id=None, + vision_config=getattr(shim, "vision_config", None), + image_token_id=getattr(shim, "image_token_id", None), ) config = parse_config(hf_like) return replace(config, expert_quant="gguf") +# ---------------------------------------------------------------------------------------------- +# Vision tower (a glm5v mmproj) +# ---------------------------------------------------------------------------------------------- + +_PROJECTOR = "glm5v" +_IMAGE_TOKEN = "<|image|>" +_PATCH = ("v.patch_embd.weight", "v.patch_embd.weight.1") + +# tower / merger tensors in HF layout already; only the patch embedding is split +_TOP = { + "v.patch_embd.bias": "patch_embed.proj.bias", + "v.post_ln.weight": "post_layernorm.weight", + "mm.patch_merger.weight": "downsample.weight", + "mm.patch_merger.bias": "downsample.bias", + "mm.model.fc.weight": "merger.proj.weight", + "mm.post_norm.weight": "merger.post_projection_norm.weight", + "mm.post_norm.bias": "merger.post_projection_norm.bias", + "mm.gate.weight": "merger.gate_proj.weight", + "mm.up.weight": "merger.up_proj.weight", + "mm.down.weight": "merger.down_proj.weight", +} +_BLOCK = { + "attn_qkv": "attn.qkv", + "attn_out": "attn.proj", + "attn_q_norm": "attn.q_norm", + "attn_k_norm": "attn.k_norm", + "ffn_gate": "mlp.gate_proj", + "ffn_up": "mlp.up_proj", + "ffn_down": "mlp.down_proj", + "ln1": "norm1", + "ln2": "norm2", +} +_BLOCK_RE = re.compile(r"^v\.blk\.(\d+)\.(\w+)\.(weight|bias)$") + + +def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int) -> tuple[dict[str, Any], dict[str, Any]]: + """The HF config.json keys and the preprocessor_config.json of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``.""" + if mmproj.projector != _PROJECTOR: + raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; GLM-5.3-Flash needs {_PROJECTOR!r}") + md, index = load_gguf_metadata(mmproj.path), gguf_tensor_index(mmproj.path) + # the converter writes use_silu=True only for the silu tower; a gelu tower carries + # use_gelu or neither key, and the model has no gelu epilogue + if md.get("clip.use_silu") is not True: + raise NotImplementedError(f"mmproj {mmproj.path}: only the silu tower is served") + + def kv(key: str): + return md[f"clip.vision.{key}"] + + arch = text_md["general.architecture"] + text_hidden = int(text_md[f"{arch}.embedding_length"]) + if int(kv("projection_dim")) != text_hidden: + raise ValueError(f"mmproj {mmproj.path} projects to {int(kv('projection_dim'))} but the text model is {text_hidden} wide") + tokens = text_md["tokenizer.ggml.tokens"] + if _IMAGE_TOKEN not in tokens: + raise ValueError(f"the text GGUF tokenizer has no {_IMAGE_TOKEN} token") + # the converter only splits a temporal-2 patch embedding, so a glm5v file always + # carries both halves; one without the second is not this tower + if _PATCH[1] not in index: + raise ValueError(f"mmproj {mmproj.path}: the second patch-embedding half {_PATCH[1]!r} is missing") + temporal = 2 + patch, merge = int(kv("patch_size")), int(kv("spatial_merge_size")) + # llama.cpp stores the canvas budget in single-frame pixels; the processor counts + # soft tokens, one per (patch * merge) square of pixels per frame + per_token = patch * patch * merge * merge + budget = {} + for key in ("image_min_pixels", "image_max_pixels"): + pixels = int(kv(key)) + if pixels % per_token: + raise ValueError(f"mmproj {mmproj.path}: {key} {pixels} is not a whole number of {per_token}-pixel tokens") + budget[key] = pixels // per_token + vision_config = { + "hidden_act": "silu", + "hidden_size": int(kv("embedding_length")), + "depth": int(kv("block_count")), + "num_heads": int(kv("attention.head_count")), + "intermediate_size": int(kv("feed_forward_length")), + # ggml order [in, out]: the gate's out features are the merger's intermediate width + "projection_intermediate_size": int(index["mm.gate.weight"].ne[1]), + "out_hidden_size": text_hidden, + # ggml order [kw, kh, in, out] + "in_channels": int(index[_PATCH[0]].ne[2]), + "patch_size": patch, + "temporal_patch_size": temporal, + "spatial_merge_size": merge, + "rms_norm_eps": float(kv("attention.layer_norm_epsilon")), + "swiglu_limit": float(kv("swiglu_clamp")), + "attention_bias": "v.blk.0.attn_qkv.bias" in index, + } + hf_fields = { + "vision_config": vision_config, + "image_token_id": tokens.index(_IMAGE_TOKEN), + } + preprocessor = { + "image_processor_type": "Glm5NextImageProcessor", + "patch_size": patch, + "temporal_patch_size": temporal, + "merge_size": merge, + "image_mean": [float(v) for v in kv("image_mean")], + "image_std": [float(v) for v in kv("image_std")], + "min_image_tokens": budget["image_min_pixels"], + "max_image_tokens": budget["image_max_pixels"], + } + return hf_fields, preprocessor + + +def _vision_name(gguf_name: str) -> str: + if gguf_name in _TOP: + return _TOP[gguf_name] + if (m := _BLOCK_RE.match(gguf_name)) and m[2] in _BLOCK: + return f"blocks.{m[1]}.{_BLOCK[m[2]]}.{m[3]}" + raise ValueError(f"unexpected tensor {gguf_name!r} in a {_PROJECTOR} mmproj") + + +def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]: + """The tower state dict under ``visual.``, named as the HF checkpoints load.""" + index = gguf_tensor_index(mmproj.path) + slices = [mmproj_tensor(index[n]) for n in _PATCH] + yield "visual.patch_embed.proj.weight", torch.stack(slices, dim=2) + for gguf_name, info in index.items(): + if gguf_name not in _PATCH: + yield f"visual.{_vision_name(gguf_name)}", mmproj_tensor(info) + + # ---------------------------------------------------------------------------------------------- # Weights # ---------------------------------------------------------------------------------------------- @@ -279,6 +414,8 @@ def iter_gguf_weights( from freetoken.utils import cached_load_hf_config config = parse_gguf_config(cached_load_hf_config(model_path)) + vision = include_vision and config.is_multimodal + log_mmproj(model_path, vision) args = config.glm5_args index = gguf_tensor_index(model_path) hidden = config.hidden_size @@ -363,5 +500,8 @@ def bf16(name: str) -> torch.Tensor: info = t(f"{blk}.{src}") yield from fused(f"{mlp}.shared_experts.{proj}", [info], [matrix(info)], k) + if vision: + yield from iter_mmproj_weights(find_mmproj(model_path)) + -__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"] +__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"] diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py index ad8ade862c..d78a85486f 100644 --- a/python/freetoken/models/register.py +++ b/python/freetoken/models/register.py @@ -468,15 +468,17 @@ class ModelSpec: "Glm5NextForCausalLM", packed_modules_mapping=_GLM5_NEXT_PACKED, ), - # llama.cpp glm5-next GGUF (unsloth GLM-5.3-Flash-GGUF; text only): the same model - # classes with every tensor in its packed ggml type + # llama.cpp glm5-next GGUF (unsloth GLM-5.3-Flash-GGUF): the same model classes + # with every tensor in its packed ggml type; a glm5v mmproj adds the vision tower "Glm5NextGGUFForCausalLM": ModelSpec( "freetoken.models.glm5_next", - "Glm5NextForCausalLM", + "Glm5NextForConditionalGeneration", parse_config="parse_gguf_config", iter_weights="iter_gguf_weights", checkpoint_roots=_LANGUAGE_MODEL_ROOT, packed_modules_mapping=_GLM5_NEXT_PACKED, + mm_processor=_GLM5_NEXT_PROCESSOR, + encoders=_GLM5_NEXT_ENCODERS, # the MLA kv_b ships as two per-head tensors (attn_k_b transposed + attn_v_b), quantized in # dynamic quants (unsloth UD); the reader dequantizes and fuses them into the dense module unquantized_modules=("*.self_attn.kv_b_proj",),