From a6904053bfaa8204c3c9c0bb4f8caf00bfb4219d Mon Sep 17 00:00:00 2001 From: Ryan Liu Date: Wed, 28 Oct 2026 03:25:07 +0000 Subject: [PATCH] Fall back to the MaxText tokenizer in the logit checker's golden mode # Description Root cause: #d81f8fab1 ("add Qwen3 VL tests, update logit checker, and golden logit generator") moved the `AutoTokenizer.from_pretrained` call in `forward_pass_logit_checker` out of the `--run_hf_model` branch so the multimodal golden-logits path could use it. Since then every golden-logits run must load an HF tokenizer, although that mode only uses it to print the top-k tokens. #1fb4be117 later turned the load into a loop over `--hf_model_path`, `tokenizer_path` and `HF_IDS[model_name]`, which repaired the models that have an `HF_IDS` entry, but models whose `tokenizer_path` is a SentencePiece file and that are absent from `HF_IDS` (gemma-2b, gemma-7b, llama2-7b) still cannot satisfy any candidate and die before comparing anything: OSError: It looks like the config file at '.../tokenizers/tokenizer.gemma' is not a valid JSON file. In `tests/end_to_end/tpu/{gemma/2b,llama2/7b}` this stayed hidden behind earlier failures in the same scripts, most recently the `generate_param_only_checkpoint` PRNG mismatch (b/576849994, fixed in #), which runs first; this change is only reachable once that one is in. When every `AutoTokenizer` candidate fails and `--run_hf_model` is not set, build the tokenizer with MaxText's own `build_tokenizer` (the same `tokenizer_path`/`tokenizer_type` the model was trained with). Golden-logits mode only calls `decode` on it, so the top-k printout now decodes a one-element list, which HF and MaxText tokenizers both accept, and the llama3.1/mixtral `pad_token` assignment is guarded with `hasattr`, since MaxText tokenizers have no `eos_token`. `--run_hf_model` is excluded on purpose: it tokenizes the prompts for both models and needs a real HF tokenizer, and there `--hf_model_path` always loads first. Adding `HF_IDS` entries instead was rejected: `pyconfig` uses that dict to default `tokenizer_path`, so it would switch every `tokenizer_path`-less command for those models to a gated HF download. # Tests - `pylint` 10.00/10, `pyink --check` clean. - `build_tokenizer(...).decode([id])` returns the expected text for both SentencePiece files in the test image (`tokenizer.gemma`, `tokenizer.llama2`); HF `PreTrainedTokenizerBase.decode` accepts `int | list[int]`. - E2E on v5p-8 via the `maxtext_end_to_end` DAG with a test image built from this branch: pending (fill in workload links for gemma-2b and llama2-7b). # Checklist Before submitting this PR, please make sure (put X in square brackets): - [X] I have performed a self-review of my code. For an optional AI review, add the `gemini-review` label. - [X] I have necessary comments in my code, particularly in hard-to-understand areas. - [X] I have run end-to-end tests and provided workload links above if applicable. - [X] I have made or will make corresponding changes to the doc if needed, including adding new documentation pages to the relevant Table of Contents (toctree directive) as explained in [our documentation](https://maxtext.readthedocs.io/en/latest/development.html#adding-new-documentation-files). --- tests/utils/forward_pass_logit_checker.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/utils/forward_pass_logit_checker.py b/tests/utils/forward_pass_logit_checker.py index baf3d9c6df..29e41c801b 100644 --- a/tests/utils/forward_pass_logit_checker.py +++ b/tests/utils/forward_pass_logit_checker.py @@ -81,6 +81,7 @@ import jax import jax.numpy as jnp from maxtext.configs import pyconfig +from maxtext.input_pipeline.tokenizer import build_tokenizer from maxtext.utils.globals import MAXTEXT_TEST_ASSETS_ROOT, HF_IDS from maxtext.checkpoint_conversion.utils.hf_utils import convert_jax_weight_to_torch from maxtext.common.common_types import DECODING_ACTIVE_SEQUENCE_INDICATOR, MODEL_MODE_TRAIN @@ -147,7 +148,8 @@ def get_top_k_tokens_scores(logits_tensor, tokenizer_instance, k=10, description for i in range(k): tok_id = topk_results.indices[i].item() score = topk_results.values[i].item() - tok = tokenizer_instance.decode(tok_id) + # A list of ids is what both HF tokenizers and MaxText's own tokenizers decode. + tok = tokenizer_instance.decode([tok_id]) collected_tokens.append({"id": int(tok_id), "token": tok.strip(), "score": float(score)}) tokens.append({"id": int(tok_id), "token": tok.strip(), "score": float(score)}) @@ -383,10 +385,19 @@ def main(config, test_args): # pylint: disable=W0621 last_exception = e max_logging.log(f"Failed to load tokenizer from {path}: {e}") + if tokenizer is None and not test_args.run_hf_model: + # Golden-logits mode only calls `decode` on the tokenizer (to print top-k tokens), but models + # such as gemma-2b and llama2-7b ship a SentencePiece file that AutoTokenizer cannot open and + # have no HF_IDS fallback. Use MaxText's own loader for them. --run_hf_model is excluded: it + # tokenizes the prompts for both models and needs a real HF tokenizer. + max_logging.log(f"Falling back to the MaxText {config.tokenizer_type} tokenizer at {config.tokenizer_path}.") + tokenizer = build_tokenizer(config.tokenizer_path, config.tokenizer_type, False, False, hf_token) + if tokenizer is None: raise last_exception - if config.model_name.startswith(("llama3.1", "mixtral")): + # MaxText's own tokenizers have no pad/eos token attributes; only HF tokenizers need this. + if config.model_name.startswith(("llama3.1", "mixtral")) and hasattr(tokenizer, "eos_token"): tokenizer.pad_token = tokenizer.eos_token max_logging.log("Initializing MaxText model")