FEAT: Add PuzzledConverter (word-puzzle jailbreak from arXiv:2508.01306)

doc/bibliography.md

@@ -5,6 +5,6 @@ All academic papers, research blogs, and technical reports referenced throughout
 :::{dropdown} Citation Keys
 :class: hidden-citations
 
-[@aakanksha2024multilingual; @adversaai2023universal; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @atr2026; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @embracethered2024unicode; @embracethered2025sneakybits; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kingma2014adam; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pfohl2024equitymedqa; @promptfoo2025ccp; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
+[@aakanksha2024multilingual; @adversaai2023universal; @ahn2025puzzled; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @atr2026; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @embracethered2024unicode; @embracethered2025sneakybits; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kingma2014adam; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pfohl2024equitymedqa; @promptfoo2025ccp; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
 
 :::

doc/code/converters/1_text_to_text_converters.ipynb

@@ -290,6 +290,7 @@
     "    InsertPunctuationConverter,\n",
     "    LeetspeakConverter,\n",
     "    MathObfuscationConverter,\n",
+    "    PuzzledConverter,\n",
     "    RandomCapitalLettersConverter,\n",
     "    RepeatTokenConverter,\n",
     "    StringJoinConverter,\n",
@@ -359,7 +360,11 @@
     "\n",
     "# CodeChameleon [@lv2024codechameleon] encrypts and wraps in code\n",
     "code_chameleon = CodeChameleonConverter(encrypt_type=\"reverse\")\n",
-    "print(\"CodeChameleon:\", await code_chameleon.convert_async(prompt=prompt))  # type: ignore"
+    "print(\"CodeChameleon:\", await code_chameleon.convert_async(prompt=prompt))  # type: ignore\n",
+    "\n",
+    "# PUZZLED [@ahn2025puzzled] hides sensitive words in a word puzzle the target must solve\n",
+    "puzzled = PuzzledConverter(puzzle_type=\"word_search\", seed=1)\n",
+    "print(\"Puzzled:\", await puzzled.convert_async(prompt=prompt))  # type: ignore"
    ]
   },
   {

doc/code/converters/1_text_to_text_converters.py

@@ -101,6 +101,7 @@
     InsertPunctuationConverter,
     LeetspeakConverter,
     MathObfuscationConverter,
+    PuzzledConverter,
     RandomCapitalLettersConverter,
     RepeatTokenConverter,
     StringJoinConverter,
@@ -172,6 +173,10 @@
 code_chameleon = CodeChameleonConverter(encrypt_type="reverse")
 print("CodeChameleon:", await code_chameleon.convert_async(prompt=prompt))  # type: ignore
 
+# PUZZLED [@ahn2025puzzled] hides sensitive words in a word puzzle the target must solve
+puzzled = PuzzledConverter(puzzle_type="word_search", seed=1)
+print("Puzzled:", await puzzled.convert_async(prompt=prompt))  # type: ignore
+
 # %% [markdown]
 # ### 1.3 Text Manipulation Converters
 #

doc/references.bib

@@ -353,6 +353,14 @@ @article{lv2024codechameleon
   url       = {https://arxiv.org/abs/2402.16717},
 }
 
+@article{ahn2025puzzled,
+  title     = {{PUZZLED}: Jailbreaking {LLMs} through Word-Based Puzzles},
+  author    = {Yelim Ahn and Jaejin Lee},
+  journal   = {arXiv preprint arXiv:2508.01306},
+  year      = {2025},
+  url       = {https://arxiv.org/abs/2508.01306},
+}
+
 @article{zeng2024persuasion,
   title     = {How Johnny Can Persuade {LLMs} to Jailbreak Them: Rethinking Persuasion to Challenge {AI} Safety by Humanizing {LLMs}},
   author    = {Yi Zeng and Hongpeng Lin and Jingwen Zhang and Diyi Yang and Ruoxi Jia and Weiyan Shi},

pyrit/converter/__init__.py

@@ -64,6 +64,7 @@
 from pyrit.converter.pdf_converter import PDFConverter
 from pyrit.converter.persuasion_converter import PersuasionConverter
 from pyrit.converter.policy_puppetry_converter import PolicyPuppetryConverter, PolicyPuppetryTemplate
+from pyrit.converter.puzzled import PuzzledConverter
 from pyrit.converter.qr_code_converter import QRCodeConverter
 from pyrit.converter.random_capital_letters_converter import RandomCapitalLettersConverter
 from pyrit.converter.random_translation_converter import RandomTranslationConverter
@@ -208,6 +209,7 @@ def __getattr__(name: str) -> object:
     "PositionSelectionStrategy",
     "Converter",
     "ProportionSelectionStrategy",
+    "PuzzledConverter",
     "QRCodeConverter",
     "ROT13Converter",
     "RandomCapitalLettersConverter",

pyrit/converter/puzzled/__init__.py

@@ -0,0 +1,36 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+The PUZZLED jailbreak technique (arXiv:2508.01306): a converter that hides a prompt's
+sensitive words inside a word puzzle, plus the keyword-masking and puzzle-building blocks
+it is assembled from.
+"""
+
+from pyrit.converter.puzzled.keyword_masker import (
+    MaskedWord,
+    MaskResult,
+    mask_count_for_length,
+    mask_prompt,
+)
+from pyrit.converter.puzzled.puzzle_builders import (
+    PuzzleType,
+    build_anagram,
+    build_crossword,
+    build_word_search,
+    crossword_symbol_map,
+)
+from pyrit.converter.puzzled.puzzled_converter import PuzzledConverter
+
+__all__ = [
+    "MaskResult",
+    "MaskedWord",
+    "PuzzleType",
+    "PuzzledConverter",
+    "build_anagram",
+    "build_crossword",
+    "build_word_search",
+    "crossword_symbol_map",
+    "mask_count_for_length",
+    "mask_prompt",
+]

pyrit/converter/puzzled/keyword_masker.py

@@ -0,0 +1,279 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+Keyword selection and masking for the PUZZLED jailbreak technique
+(Ahn & Lee, "PUZZLED: Jailbreaking LLMs through Word-Based Puzzles", arXiv:2508.01306).
+
+The masker chooses the most sensitive words in a prompt, replaces each with an indexed
+``[WORD1]`` placeholder, and records a deterministic clue (length and part of speech) for
+every masked word. Part-of-speech tagging uses spaCy when its ``en_core_web_sm`` model is
+installed and degrades to a length-based heuristic when it is not, mirroring the graceful
+fallback already used in ``pyrit/executor/benchmark/fairness_bias.py``.
+"""
+
+import logging
+import re
+from dataclasses import dataclass
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Part-of-speech label used when spaCy is unavailable or the word was caller-supplied.
+_GENERIC_POS = "word"
+
+# spaCy coarse tags we treat as maskable, mapped to the clue label we expose.
+_MASKABLE_POS = {"NOUN": "noun", "PROPN": "noun", "VERB": "verb"}
+
+# Harm-related words the PUZZLED paper masks first, so the puzzle hides the part of the
+# instruction that carries the harmful intent. Seeded from the paper's Table 4 examples
+# (Ahn & Lee, arXiv:2508.01306); callers can extend or override via ``essential_words``.
+_ESSENTIAL_HARM_WORDS = frozenset(
+    {
+        "exploit",
+        "malware",
+        "firearm",
+        "hacking",
+        "suicide",
+        "propaganda",
+        "cyberbullying",
+        "extortion",
+        "hate",
+        "misinformation",
+        "hijack",
+        "manipulation",
+        "ransomware",
+        "sabotage",
+        "terrorism",
+        "stalk",
+        "smuggle",
+        "harassment",
+        "phishing",
+        "abuse",
+    }
+)
+
+# Contextual "amplifier" words masked after the essential list when more masks are needed.
+_SUPPLEMENTARY_WORDS = frozenset(
+    {
+        "identity",
+        "encryption",
+        "financial",
+        "insider",
+        "passport",
+        "passwords",
+        "private",
+        "psychological",
+        "software",
+        "tactics",
+        "targets",
+        "reputation",
+        "redirects",
+        "device",
+        "accessing",
+        "credit",
+        "database",
+        "voting",
+        "medical",
+        "witness",
+    }
+)
+
+# Module-level spaCy pipeline, loaded once on first use.
+_nlp = None
+_nlp_loaded = False
+
+
+@dataclass(frozen=True)
+class MaskedWord:
+    """A single masked keyword and the metadata needed to build its puzzle clue."""
+
+    text: str
+    placeholder: str
+    pos: str
+
+    @property
+    def clue(self) -> str:
+        """The deterministic clue for this word, e.g. ``"9-letter noun"``."""
+        return f"{len(self.text)}-letter {self.pos}"
+
+
+@dataclass(frozen=True)
+class MaskResult:
+    """The outcome of masking a prompt."""
+
+    masked_prompt: str
+    masked_words: list[MaskedWord]
+
+
+def mask_count_for_length(token_count: int) -> int:
+    """
+    Return how many words to mask for a prompt of ``token_count`` whitespace tokens.
+
+    The thresholds follow the PUZZLED paper: longer instructions hide more words.
+
+    Args:
+        token_count (int): Number of whitespace-separated tokens in the prompt.
+
+    Returns:
+        int: The target number of words to mask.
+    """
+    if token_count <= 10:
+        return 3
+    if token_count <= 15:
+        return 4
+    if token_count <= 20:
+        return 5
+    return 6
+
+
+def _get_nlp() -> Any:
+    """
+    Load and cache the spaCy pipeline.
+
+    Returns:
+        Any: The loaded spaCy ``Language`` pipeline, or ``None`` if spaCy or its
+            ``en_core_web_sm`` model is unavailable.
+    """
+    global _nlp, _nlp_loaded
+    if _nlp_loaded:
+        return _nlp
+    _nlp_loaded = True
+    try:
+        import spacy  # type: ignore[ty:unresolved-import]
+
+        _nlp = spacy.load("en_core_web_sm")
+    except Exception:
+        logger.info("spaCy model 'en_core_web_sm' unavailable; using length-based keyword selection instead.")
+        _nlp = None
+    return _nlp
+
+
+def _pos_lookup(prompt: str) -> dict[str, str]:
+    """
+    Map each alphabetic noun/verb (lowercased) in the prompt to its clue label.
+
+    Returns an empty mapping when spaCy is unavailable.
+
+    Args:
+        prompt (str): The prompt to tag.
+
+    Returns:
+        dict[str, str]: Lowercased word to part-of-speech label ("noun" or "verb").
+    """
+    nlp = _get_nlp()
+    if nlp is None:
+        return {}
+    lookup: dict[str, str] = {}
+    for token in nlp(prompt):
+        if token.is_alpha and token.pos_ in _MASKABLE_POS:
+            # Keep the first tag seen for a given surface form.
+            lookup.setdefault(token.text.lower(), _MASKABLE_POS[token.pos_])
+    return lookup
+
+
+def _rank_candidates(
+    prompt: str,
+    pos_lookup: dict[str, str],
+    essential_words: list[str] | None,
+) -> list[str]:
+    """
+    Order candidate words for masking by priority.
+
+    Priority follows the PUZZLED paper: caller-supplied essential words first, then the
+    built-in essential harm words (``_ESSENTIAL_HARM_WORDS``), then the supplementary
+    amplifier words (``_SUPPLEMENTARY_WORDS``), then nouns/verbs (when spaCy is available),
+    then any remaining words. Within each tier, longer words rank higher because they carry
+    more of the instruction's meaning and make harder puzzles. Ties break alphabetically for
+    determinism.
+
+    Args:
+        prompt (str): The prompt being masked.
+        pos_lookup (dict[str, str]): Noun/verb tags from ``_pos_lookup``.
+        essential_words (list[str] | None): Sensitive words to prefer, if any.
+
+    Returns:
+        list[str]: Distinct candidate words (as they appear in the prompt) in priority order.
+    """
+    tokens = re.findall(r"[A-Za-z]+", prompt)
+    seen: set[str] = set()
+    unique: list[str] = []
+    for token in tokens:
+        key = token.lower()
+        if key not in seen:
+            seen.add(key)
+            unique.append(token)
+
+    essential_lower = {w.lower() for w in (essential_words or [])}
+
+    def tier(word: str) -> int:
+        key = word.lower()
+        if key in essential_lower:
+            return 0
+        if key in _ESSENTIAL_HARM_WORDS:
+            return 1
+        if key in _SUPPLEMENTARY_WORDS:
+            return 2
+        if key in pos_lookup:
+            return 3
+        return 4
+
+    return sorted(unique, key=lambda w: (tier(w), -len(w), w.lower()))
+
+
+def mask_prompt(
+    prompt: str,
+    *,
+    num_to_mask: int | None = None,
+    essential_words: list[str] | None = None,
+) -> MaskResult:
+    """
+    Replace the most sensitive words in ``prompt`` with indexed placeholders.
+
+    Words are chosen by ``_rank_candidates``, then the placeholders are numbered by
+    the order the chosen words appear in the prompt, so ``[WORD1]`` is always the leftmost
+    masked word. Only the first occurrence of each chosen word is masked.
+
+    Args:
+        prompt (str): The instruction to mask.
+        num_to_mask (int | None): How many words to mask. Defaults to the length-based rule.
+        essential_words (list[str] | None): Sensitive words to prefer when selecting.
+
+    Returns:
+        MaskResult: The masked prompt and the ordered list of masked words.
+
+    Raises:
+        ValueError: If ``prompt`` contains no maskable words.
+    """
+    token_count = len(prompt.split())
+    target = num_to_mask if num_to_mask is not None else mask_count_for_length(token_count)
+
+    pos_lookup = _pos_lookup(prompt)
+    ranked = _rank_candidates(prompt, pos_lookup, essential_words)
+    if not ranked:
+        raise ValueError("The prompt has no maskable words.")
+
+    chosen = ranked[: max(0, target)]
+
+    # Find the first occurrence span of each chosen word, then order by position so the
+    # placeholder numbering reads left to right.
+    spans: list[tuple[int, int, str]] = []
+    for word in chosen:
+        match = re.search(rf"\b{re.escape(word)}\b", prompt)
+        if match:
+            spans.append((match.start(), match.end(), word))
+    spans.sort()
+
+    masked_words: list[MaskedWord] = []
+    pieces: list[str] = []
+    cursor = 0
+    for index, (start, end, word) in enumerate(spans, start=1):
+        placeholder = f"[WORD{index}]"
+        pos = pos_lookup.get(word.lower(), _GENERIC_POS)
+        masked_words.append(MaskedWord(text=word, placeholder=placeholder, pos=pos))
+        pieces.append(prompt[cursor:start])
+        pieces.append(placeholder)
+        cursor = end
+    pieces.append(prompt[cursor:])
+
+    return MaskResult(masked_prompt="".join(pieces), masked_words=masked_words)