From 0cd414f76c0ecf08bc3c27ad597addccde42b77a Mon Sep 17 00:00:00 2001 From: Greta Date: Mon, 5 Oct 2026 15:39:03 +0100 Subject: [PATCH 01/10] Extend Qiskit IVR example Signed-off-by: Greta --- .../qiskit_code_validation/README.md | 145 +++++++++++++++++- .../qiskit_code_validation.py | 35 ++++- .../validation_helpers.py | 115 +++++++++++++- 3 files changed, 288 insertions(+), 7 deletions(-) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md b/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md index fa952030d9..a9a3b30f91 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md @@ -1,14 +1,16 @@ # Qiskit Code Validation with Instruct-Validate-Repair -This example demonstrates using Mellea's Instruct-Validate-Repair (IVR) pattern to generate Qiskit quantum computing code that automatically passes `flake8-qiskit-migration` validation rules (QKT rules). +This example demonstrates using Mellea's Instruct-Validate-Repair (IVR) pattern to generate Qiskit quantum computing code that automatically passes `flake8-qiskit-migration` validation rules (QKT rules), and, optionally, the problem's own test suite and the [LintQ](https://github.com/sola-st/LintQ) static analyzer for quantum programs. ## What This Example Does Takes a prompt containing deprecated Qiskit code and: 1. Generates corrected code using the LLM -2. Validates the output against QKT rules +2. Validates the output against QKT rules, the test suite and LintQ 3. Automatically repairs the code if validation fails (up to 10 attempts) +`validation_helpers.py` also provides test-suite and LintQ validators for benchmark problems (Qiskit HumanEval, QuantumKatas); see [Validating with Test Suites and LintQ](#validating-with-test-suites-and-lintq). + ## Quick Start ```bash @@ -64,7 +66,7 @@ Benchmarks on `mistral-small-3.2-24b-qiskit` model: ``` qiskit_code_validation/ -├── qiskit_code_validation.py # Main example +├── qiskit_code_validation.py # Main example (single prompt) ├── validation_helpers.py # Validation utilities └── README.md # This file ``` @@ -73,6 +75,14 @@ qiskit_code_validation/ - `extract_code_from_markdown()`: Extracts code from markdown blocks - `validate_qiskit_migration()`: Validates against QKT rules - `validate_input_code()`: Pre-validates input prompts +- `validate_correctness()`: Runs code against a benchmark problem's own test suite +- `validate_lintq()`: Runs the LintQ CodeQL queries over the code + +All validators return `(is_valid, error_message)`, so the message is fed straight back to the model as the repair reason. + +### Adding LintQ to the Single-Prompt Example + +Set `use_lintq = True` in `test_qiskit_code_validation()` to also reject code flagged by LintQ. This needs the [LintQ setup](#lintq-setup) described below. ## Trying Different Prompts @@ -199,6 +209,123 @@ qc.measure_all() **Note**: The exact output may vary depending on the model and its interpretation of the prompt. +## Validating with Test Suites and LintQ + +Passing the QKT rules only shows that the code uses current Qiskit APIs, not that it solves the task or that it is free of quantum-specific bugs. For benchmark problems, `validate_correctness()` and `validate_lintq()` add two more checks: + +| Validator | What it checks | Repair reason fed back to the model | +|-----------|----------------|-------------------------------------| +| **QKT rules** | `flake8-qiskit-migration`, as in the single-prompt example | The QKT rule messages | +| **Test suite** | Executes the code against the problem's own tests: `check(candidate)` for Qiskit HumanEval, `test_()` for QuantumKatas (30s timeout) | The assertion message or full traceback | +| **LintQ** | Runs the [LintQ](https://github.com/sola-st/LintQ) CodeQL queries (double measurement, operations after measurement, unused qubits, ...) | `[rule-id] message` per finding | + +Both plug into the same `generate_validated_qiskit_code()` loop through its `extra_requirements` parameter. Running LintQ only once the tests pass avoids linting code that does not run, and saves time: each LintQ run builds a CodeQL database (around 20s). + +```python +from validation_helpers import ( + extract_code_from_markdown, + validate_correctness, + validate_lintq, +) + +from mellea.stdlib.requirements import req, simple_validate + + +def tests_and_lintq(problem: dict): + """Pass the problem's tests, then LintQ.""" + + def check(output: str) -> tuple[bool, str]: + code = extract_code_from_markdown(output) + passed, reason = validate_correctness(problem, code) + return validate_lintq(code) if passed else (passed, reason) + + return req( + "The code must pass the problem's test suite and raise no LintQ warnings", + validation_fn=simple_validate(check), + ) + + +code, success, attempts = generate_validated_qiskit_code( + m, problem["prompt"], strategy, extra_requirements=[tests_and_lintq(problem)] +) +``` + +> ⚠️ **Security**: the test-suite validator runs `exec()` on model-generated code and on the dataset's test code, with a timeout but **not** in a sandbox. Run it only on datasets you trust, on a machine where running arbitrary Python is acceptable. + +### Setup + +The benchmark dependencies, the datasets and LintQ are **not** part of this repository. Install and download them anywhere you like; the paths below are only examples. + +#### Python Dependencies + +The benchmark tests import Qiskit and a few plotting and graph libraries. Install them in the project environment: + +```bash +uv pip install flake8-qiskit-migration "qiskit>=2.1.0,<2.5.0" qiskit-aer qiskit-ibm-runtime qiskit-ibm-transpiler matplotlib networkx pylatexenc seaborn +``` + +#### Datasets + +The benchmarks are published by Qiskit on Hugging Face, with the same fields (`task_id`, `prompt`, `canonical_solution`, `test`, `entry_point`); each row is one `problem` for `validate_correctness()`. + +| Benchmark | Tasks | Prompt style | License | +|-----------|-------|--------------|---------| +| [Qiskit HumanEval hard](https://huggingface.co/datasets/Qiskit/qiskit_humaneval_hard) | 151 | Natural-language instruction: the model writes the whole function | Apache-2.0 | +| [Qiskit HumanEval](https://huggingface.co/datasets/Qiskit/qiskit_humaneval) | 151 | Code stub with a docstring | Apache-2.0 | +| [Qiskit QuantumKatas](https://huggingface.co/datasets/Qiskit/Qiskit-QuantumKatas) | 350 | Task description plus a function stub | **CC BY-NC-SA 4.0 (non-commercial)** | + +```bash +mkdir -p data +# Qiskit HumanEval hard (and, the same way, Qiskit/qiskit_humaneval) +uv run --with datasets python -c "from datasets import load_dataset; load_dataset('Qiskit/qiskit_humaneval_hard', split='test').to_json('data/qiskit_humaneval_hard.jsonl', force_ascii=False)" +# Qiskit QuantumKatas +curl -L -o data/qiskit_quantumkatas.jsonl https://huggingface.co/datasets/Qiskit/Qiskit-QuantumKatas/resolve/main/qiskit_quantumkatas.jsonl +``` + +Check each dataset's license before redistributing it, or any results generated from it: results generated from QuantumKatas fall under CC BY-NC-SA 4.0. + +Some Qiskit HumanEval tasks call IBM Quantum services: they can only pass with a saved IBM Quantum account (`QiskitRuntimeService.save_account(...)`); without one, even their canonical solutions fail. The remaining canonical solutions, and all 350 QuantumKatas ones, pass the test-suite validator. + +#### LintQ Setup + +LintQ is a set of [CodeQL](https://codeql.github.com/) queries, so it needs the CodeQL CLI and a checkout of the LintQ repository: + +1. Install the CodeQL CLI and put `codeql` on your `PATH`, e.g. `brew install codeql` on macOS, or download the CodeQL bundle from the [codeql-action releases](https://github.com/github/codeql-action/releases). Check it with `codeql version`. +2. Clone LintQ and install its query pack dependencies: + + ```bash + git clone --depth 1 https://github.com/sola-st/LintQ.git + (cd LintQ/qlint/codeql/src && codeql pack install) + ``` + +3. Point `LINTQ_DIR` at the checkout: + + ```bash + export LINTQ_DIR="$(pwd)/LintQ" + ``` + + +### Benchmark Results + +All runs use `MultiTurnStrategy` with a loop budget of 3 (the first attempt plus up to 2 repairs). + +#### gpt-oss-120b (test suite + LintQ) + +These runs validated the test suite and LintQ. + +| Metric | QHE hard: first pass | QHE hard: post repair | QuantumKatas: first pass | QuantumKatas: post repair | +|--------|------|------|------|------| +| Tasks | 151 | 151 | 350 | 350 | +| Passed both validators | 24/151 (16%) | **70/151 (46%)** | 25/350 (7%) | **163/350 (47%)** | +| Failed | 127/151 (84%) | 81/151 (54%) | 325/350 (93%) | 187/350 (53%) | +| Failed the test suite | 113 | 77 | 297 | 148 | +| Flagged by LintQ | 14 | 4 | 28 | 39 | +| LintQ warnings raised | 17 | 4 | 30 | 47 | +| Fixed by the repair loop | — | 46 | — | 138 | + +The repair loop roughly triples the Qiskit HumanEval hard pass rate and multiplies the QuantumKatas one by more than six. On QuantumKatas, LintQ flags more tasks after repair than before: a task that fails its tests is never linted, and many programs only start running (and so get linted) after repair. + + ## Changing the Model To try a different model, edit the `model_id` variable in the `test_qiskit_code_validation()` function: @@ -257,6 +384,18 @@ If using a general-purpose model, it may not have enough Qiskit knowledge to pas - Setting `system_prompt = QISKIT_SYSTEM_PROMPT` to guide the model toward modern Qiskit APIs - Using simpler prompts +### LintQ Not Configured +``` +KeyError: 'LINTQ_DIR' +``` +**Solution**: Follow [LintQ Setup](#lintq-setup) and `export LINTQ_DIR=/path/to/LintQ`. + +### CodeQL Errors +``` +subprocess.CalledProcessError: Command '['codeql', 'database', ...]' returned non-zero exit status +``` +**Solution**: Check `codeql version` works, and that `codeql pack install` was run in `$LINTQ_DIR/qlint/codeql/src`. + ### Import Error: flake8-qiskit-migration ``` ModuleNotFoundError: No module named 'flake8_qiskit_migration' diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py index 21310ae6b9..f95524e249 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py @@ -28,7 +28,11 @@ import time -from validation_helpers import validate_qiskit_migration +from validation_helpers import ( + extract_code_from_markdown, + validate_lintq, + validate_qiskit_migration, +) from mellea import MelleaSession, start_session from mellea.backends import ModelOption @@ -184,6 +188,22 @@ def test_qiskit_code_validation() -> None: # RepairTemplateStrategy: Adds validation failure reasons to the instruction and retries use_multiturn_strategy = False + # Setting use_lintq to True also rejects code flagged by the LintQ CodeQL queries. + # Requires the codeql CLI and LINTQ_DIR (see README.md → LintQ setup) + use_lintq = True + extra_requirements = ( + [ + req( + "Code must raise no LintQ warnings", + validation_fn=simple_validate( + lambda output: validate_lintq(extract_code_from_markdown(output)) + ), + ) + ] + if use_lintq + else None + ) + # Initialize the required context ctx = ChatContext() if use_multiturn_strategy else SimpleContext() if use_multiturn_strategy: @@ -202,7 +222,11 @@ def test_qiskit_code_validation() -> None: start_time = time.time() code, success, attempts = generate_validated_qiskit_code( - m, prompt, strategy, system_prompt=system_prompt + m, + prompt, + strategy, + system_prompt=system_prompt, + extra_requirements=extra_requirements, ) elapsed = time.time() - start_time @@ -211,11 +235,16 @@ def test_qiskit_code_validation() -> None: print("======================\n") if success: - print("✓ Code passes Qiskit migration validation") + if use_lintq: + print("✓ Code passes Qiskit migration validation and it is code smells free") + else: + print("✓ Code passes Qiskit migration validation") else: _, error_msg = validate_qiskit_migration(code) print("✗ Validation errors:") print(error_msg) + if use_lintq: + print(validate_lintq(extract_code_from_markdown(code))[1]) if __name__ == "__main__": diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py index f66baca4cc..fc1334816d 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py @@ -2,11 +2,20 @@ """Helper functions for Qiskit code validation. This module provides utilities for extracting code from markdown and validating -Qiskit code against migration rules using the flake8-qiskit-migration plugin. +Qiskit code against migration rules using the flake8-qiskit-migration plugin, +against a benchmark problem's test suite, and against LintQ. """ import ast +import ctypes +import json +import os import re +import subprocess +import tempfile +import threading +import traceback +from typing import Any try: from flake8_qiskit_migration.plugin import Plugin @@ -108,3 +117,107 @@ def validate_input_code(prompt: str) -> tuple[bool, str]: return False, error_msg return True, "" + + +def validate_correctness(problem: dict[str, Any], code: str) -> tuple[bool, str]: + """Run `code` against a benchmark problem's own test suite. + + Adapted from the Qiskit HumanEval test harness (Copyright IBM, Apache-2.0). + Supports Qiskit HumanEval (`check(candidate)`) and QuantumKatas + (`test_()`). Warning: this `exec()`s untrusted code; it is not a sandbox. + + Args: + problem: Problem dict with `prompt`, `test` and `entry_point` keys + code: Candidate solution + + Returns: + Tuple of (is_valid, error_message) + """ + # Code-stub prompts are prepended; natural-language prompts (QHE hard) are not. + prompt = problem["prompt"] + is_stub = prompt.lstrip().startswith(("from ", "import ", "def ", "#")) + entry = problem["entry_point"] + program = f"{prompt if is_stub else ''}\n{code}\n{problem['test']}" + + # Mellea validates on a worker thread, so the timeout is raised via the C API. + tid = ctypes.c_ulong(threading.get_ident()) + timer = threading.Timer( + 30, + ctypes.pythonapi.PyThreadState_SetAsyncExc, + (tid, ctypes.py_object(TimeoutError)), + ) + cwd = os.getcwd() + os.environ["MPLBACKEND"] = "Agg" + with tempfile.TemporaryDirectory() as tmp: + os.chdir(tmp) # some tasks write files + timer.start() + try: + ns: dict[str, Any] = {} + exec(program, ns) + if entry not in ns: + return False, f"Entry point `{entry}` not defined" + test = ns.get("check") or ns.get(f"test_{entry}") + if test is None: + return False, f"No check() or test_{entry}() in the test code" + if "check" in ns: + test(ns[entry]) + else: + test() + return True, "" + except TimeoutError: + print("Validation failed: Execution timed out after 30s") + return False, "Execution timed out after 30s" + except AssertionError as e: + print(f"Validation failed: Test assertion failed: {e}") + return False, f"Test assertion failed: {e}" + except Exception as e: + print(f"{type(e).__name__}: {e}\n{traceback.format_exc()}") + return False, f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + finally: + timer.cancel() + os.chdir(cwd) + + +def validate_lintq(code: str) -> tuple[bool, str]: + """Run the LintQ CodeQL queries over `code`. + + Requires the `codeql` CLI and `LINTQ_DIR` pointing at a LintQ checkout (see README.md). + + Args: + code: Python source to analyse + + Returns: + Tuple of (is_valid, error_message), one `[rule-id] message` line per finding + """ + lintq = os.environ["LINTQ_DIR"] + with tempfile.TemporaryDirectory() as tmp: + os.makedirs(f"{tmp}/src") + with open(f"{tmp}/src/candidate.py", "w") as f: + f.write(code) + for cmd in ( + [ + "database", + "create", + f"{tmp}/db", + "--language=python", + f"--source-root={tmp}/src", + ], + [ + "database", + "analyze", + f"{tmp}/db", + f"{lintq}/LintQ-all.qls", + "--format=sarifv2.1.0", + f"--output={tmp}/out.sarif", + f"--additional-packs={lintq}/qlint/codeql/src:{lintq}/qlint/codeql/lib", + ], + ): + subprocess.run(["codeql", *cmd], check=True, capture_output=True) + with open(f"{tmp}/out.sarif") as f: + results = [r for run in json.load(f)["runs"] for r in run["results"]] + + if not results: + return True, "" + findings = "\n".join(f"[{r['ruleId']}] {r['message']['text']}" for r in results) + print(f"Validation failed: LintQ warnings:\n{findings}") + return False, f"LintQ warnings:\n{findings}" From 0f8070e2e953716f4181669e525a99d40da24673 Mon Sep 17 00:00:00 2001 From: Greta Date: Fri, 9 Oct 2026 10:35:34 +0100 Subject: [PATCH 02/10] Fix validate_correctness against answer defined check() and sys.exit Signed-off-by: Greta --- .../validation_helpers.py | 49 +++++++++++++++---- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py index fc1334816d..b8128304f6 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py @@ -7,6 +7,7 @@ """ import ast +import builtins import ctypes import json import os @@ -15,6 +16,7 @@ import tempfile import threading import traceback +from collections.abc import Iterator from typing import Any try: @@ -119,6 +121,23 @@ def validate_input_code(prompt: str) -> tuple[bool, str]: return True, "" +class _Timeout(BaseException): + """Raised into the validating thread when the code runs too long. + + A `BaseException`, unlike `TimeoutError`, so generated code that catches + `Exception` cannot swallow it. + """ + + +# Builtins the executed code does not get: they could escape or stall the check +# (defense in depth, not a sandbox). +_SAFE_BUILTINS = { + k: v + for k, v in vars(builtins).items() + if k not in {"exec", "eval", "compile", "breakpoint", "input"} +} + + def validate_correctness(problem: dict[str, Any], code: str) -> tuple[bool, str]: """Run `code` against a benchmark problem's own test suite. @@ -139,12 +158,23 @@ def validate_correctness(problem: dict[str, Any], code: str) -> tuple[bool, str] entry = problem["entry_point"] program = f"{prompt if is_stub else ''}\n{code}\n{problem['test']}" + # Call the test function the test code defines, so a candidate that defines its + # own `check()` cannot stand in for the real test. + test_fns = { + node.name + for node in ast.parse(problem["test"]).body + if isinstance(node, ast.FunctionDef) + } + test_name = "check" if "check" in test_fns else f"test_{entry}" + if test_name not in test_fns: + return False, f"No check() or test_{entry}() in the test code" + # Mellea validates on a worker thread, so the timeout is raised via the C API. tid = ctypes.c_ulong(threading.get_ident()) timer = threading.Timer( 30, ctypes.pythonapi.PyThreadState_SetAsyncExc, - (tid, ctypes.py_object(TimeoutError)), + (tid, ctypes.py_object(_Timeout)), ) cwd = os.getcwd() os.environ["MPLBACKEND"] = "Agg" @@ -152,21 +182,22 @@ def validate_correctness(problem: dict[str, Any], code: str) -> tuple[bool, str] os.chdir(tmp) # some tasks write files timer.start() try: - ns: dict[str, Any] = {} + ns: dict[str, Any] = {"__builtins__": _SAFE_BUILTINS} exec(program, ns) if entry not in ns: return False, f"Entry point `{entry}` not defined" - test = ns.get("check") or ns.get(f"test_{entry}") - if test is None: - return False, f"No check() or test_{entry}() in the test code" - if "check" in ns: - test(ns[entry]) + if test_name == "check": + ns["check"](ns[entry]) else: - test() + ns[test_name]() return True, "" - except TimeoutError: + except _Timeout: print("Validation failed: Execution timed out after 30s") return False, "Execution timed out after 30s" + except SystemExit as e: + # Not an Exception: left uncaught, it would stop the repair loop. + print(f"Validation failed: the code called sys.exit({e.code!r})") + return False, f"The code called sys.exit({e.code!r}) instead of returning" except AssertionError as e: print(f"Validation failed: Test assertion failed: {e}") return False, f"Test assertion failed: {e}" From d3cf27e6117578203246c301fa46ca5f774f1c0b Mon Sep 17 00:00:00 2001 From: Greta Date: Wed, 14 Oct 2026 11:25:12 +0100 Subject: [PATCH 03/10] Start with use_lintq = False Signed-off-by: Greta --- .../qiskit_code_validation/qiskit_code_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py index f95524e249..d055fc93d4 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py @@ -190,7 +190,7 @@ def test_qiskit_code_validation() -> None: # Setting use_lintq to True also rejects code flagged by the LintQ CodeQL queries. # Requires the codeql CLI and LINTQ_DIR (see README.md → LintQ setup) - use_lintq = True + use_lintq = False extra_requirements = ( [ req( From 36c82c999dbbb10fa06d5659b480f59ced8a7c47 Mon Sep 17 00:00:00 2001 From: Greta Dolcetti <53076361+gretadolcetti@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:16:52 +0100 Subject: [PATCH 04/10] Update docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py Co-authored-by: Alex Bozarth Signed-off-by: Greta Dolcetti <53076361+gretadolcetti@users.noreply.github.com> --- .../qiskit_code_validation/qiskit_code_validation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py index d055fc93d4..898343a67b 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py @@ -236,7 +236,9 @@ def test_qiskit_code_validation() -> None: if success: if use_lintq: - print("✓ Code passes Qiskit migration validation and it is code smells free") + print( + "✓ Code passes Qiskit migration validation and it is code smells free" + ) else: print("✓ Code passes Qiskit migration validation") else: From 2f5a50f3332e418d629369db4dd14fd3d80dc4ca Mon Sep 17 00:00:00 2001 From: Greta Dolcetti <53076361+gretadolcetti@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:17:04 +0100 Subject: [PATCH 05/10] Update docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py Co-authored-by: Alex Bozarth Signed-off-by: Greta Dolcetti <53076361+gretadolcetti@users.noreply.github.com> --- .../qiskit_code_validation/validation_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py index b8128304f6..01bec61e06 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py @@ -16,7 +16,6 @@ import tempfile import threading import traceback -from collections.abc import Iterator from typing import Any try: From 87f59dc0f4a48907e579e561efe9ecdf4a9c48e9 Mon Sep 17 00:00:00 2001 From: Greta Date: Fri, 16 Oct 2026 10:00:16 +0100 Subject: [PATCH 06/10] Avoid recompuring LintQ and better error print Signed-off-by: Greta --- .../qiskit_code_validation.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py index 898343a67b..631ec9ee24 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py @@ -191,13 +191,20 @@ def test_qiskit_code_validation() -> None: # Setting use_lintq to True also rejects code flagged by the LintQ CodeQL queries. # Requires the codeql CLI and LINTQ_DIR (see README.md → LintQ setup) use_lintq = False + + # LintQ report of each generation, so the final one is not recomputed below. + lintq_reports: list[str] = [] + + def lintq_check(output: str) -> tuple[bool, str]: + passed, report = validate_lintq(extract_code_from_markdown(output)) + lintq_reports.append(report) + return passed, report + extra_requirements = ( [ req( "Code must raise no LintQ warnings", - validation_fn=simple_validate( - lambda output: validate_lintq(extract_code_from_markdown(output)) - ), + validation_fn=simple_validate(lintq_check), ) ] if use_lintq @@ -243,10 +250,10 @@ def test_qiskit_code_validation() -> None: print("✓ Code passes Qiskit migration validation") else: _, error_msg = validate_qiskit_migration(code) + # The returned code is the last generation, so its LintQ report is the last one. + errors = [error_msg, lintq_reports[-1] if lintq_reports else ""] print("✗ Validation errors:") - print(error_msg) - if use_lintq: - print(validate_lintq(extract_code_from_markdown(code))[1]) + print("\n".join(e for e in errors if e)) if __name__ == "__main__": From 56a20a003b67ae5cbe9ddf5a7993f91c80e8c348 Mon Sep 17 00:00:00 2001 From: Greta Date: Fri, 16 Oct 2026 10:00:56 +0100 Subject: [PATCH 07/10] Restore MPLBACKEND env var after validation Signed-off-by: Greta --- .../qiskit_code_validation/validation_helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py index 01bec61e06..0577a5a445 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py @@ -206,6 +206,10 @@ def validate_correctness(problem: dict[str, Any], code: str) -> tuple[bool, str] finally: timer.cancel() os.chdir(cwd) + if mpl_backend is None: + os.environ.pop("MPLBACKEND", None) + else: + os.environ["MPLBACKEND"] = mpl_backend def validate_lintq(code: str) -> tuple[bool, str]: From 58cf1d6e4fec280eef71ea2185a0babe4f21298b Mon Sep 17 00:00:00 2001 From: Greta Date: Fri, 16 Oct 2026 10:01:37 +0100 Subject: [PATCH 08/10] Refactor test runner and document timeout Signed-off-by: Greta --- .../qiskit_code_validation/README.md | 6 +- .../validation_helpers.py | 93 ++++++++++++------- 2 files changed, 63 insertions(+), 36 deletions(-) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md b/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md index a9a3b30f91..e0105cacdb 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md @@ -216,7 +216,7 @@ Passing the QKT rules only shows that the code uses current Qiskit APIs, not tha | Validator | What it checks | Repair reason fed back to the model | |-----------|----------------|-------------------------------------| | **QKT rules** | `flake8-qiskit-migration`, as in the single-prompt example | The QKT rule messages | -| **Test suite** | Executes the code against the problem's own tests: `check(candidate)` for Qiskit HumanEval, `test_()` for QuantumKatas (30s timeout) | The assertion message or full traceback | +| **Test suite** | Executes the code against the problem's own tests: `check(candidate)` for Qiskit HumanEval, `test_()` for QuantumKatas (best-effort 30s time limit, see below) | The assertion message or full traceback | | **LintQ** | Runs the [LintQ](https://github.com/sola-st/LintQ) CodeQL queries (double measurement, operations after measurement, unused qubits, ...) | `[rule-id] message` per finding | Both plug into the same `generate_validated_qiskit_code()` loop through its `extra_requirements` parameter. Running LintQ only once the tests pass avoids linting code that does not run, and saves time: each LintQ run builds a CodeQL database (around 20s). @@ -250,7 +250,9 @@ code, success, attempts = generate_validated_qiskit_code( ) ``` -> ⚠️ **Security**: the test-suite validator runs `exec()` on model-generated code and on the dataset's test code, with a timeout but **not** in a sandbox. Run it only on datasets you trust, on a machine where running arbitrary Python is acceptable. +> ⚠️ **Security**: the test-suite validator runs `exec()` on model-generated code and on the dataset's test code, in the current process and **not** in a sandbox. Run it only on datasets you trust, on a machine where running arbitrary Python is acceptable. +> +> The 30s time limit is best effort: it is only checked between Python bytecodes, so code blocked inside a C call (a simulator run, `time.sleep`) is stopped only once that call returns, and a bare `except:` in the generated code can swallow it. ### Setup diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py index 0577a5a445..70ceb30ee4 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py @@ -120,6 +120,10 @@ def validate_input_code(prompt: str) -> tuple[bool, str]: return True, "" +# Time limit for running a candidate against its tests. +TIMEOUT_SECONDS = 30 + + class _Timeout(BaseException): """Raised into the validating thread when the code runs too long. @@ -137,12 +141,61 @@ class _Timeout(BaseException): } +def _run_tests(program: str, entry: str, test_name: str) -> tuple[bool, str]: + """Execute `program` and call its test function, stopping it after the time limit.""" + # Mellea validates on a worker thread, so the timeout is raised via the C API. + tid = ctypes.c_ulong(threading.get_ident()) + timer = threading.Timer( + TIMEOUT_SECONDS, + ctypes.pythonapi.PyThreadState_SetAsyncExc, + (tid, ctypes.py_object(_Timeout)), + ) + try: + try: + timer.start() + ns: dict[str, Any] = {"__builtins__": _SAFE_BUILTINS} + exec(program, ns) + if entry not in ns: + return False, f"Entry point `{entry}` not defined" + if test_name == "check": + ns["check"](ns[entry]) + else: + ns[test_name]() + return True, "" + finally: + timer.cancel() + timer.join() + # cancel() cannot recall a _Timeout the timer already raised into this + # thread: clear it, so it cannot surface later inside Mellea. If it is + # delivered first, the except below still catches it. + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + except _Timeout: + print(f"Validation failed: Execution timed out after {TIMEOUT_SECONDS}s") + return False, f"Execution timed out after {TIMEOUT_SECONDS}s" + except SystemExit as e: + # Not an Exception: left uncaught, it would stop the repair loop. + print(f"Validation failed: the code called sys.exit({e.code!r})") + return False, f"The code called sys.exit({e.code!r}) instead of returning" + except AssertionError as e: + print(f"Validation failed: Test assertion failed: {e}") + return False, f"Test assertion failed: {e}" + except Exception as e: + print(f"{type(e).__name__}: {e}\n{traceback.format_exc()}") + return False, f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + + def validate_correctness(problem: dict[str, Any], code: str) -> tuple[bool, str]: """Run `code` against a benchmark problem's own test suite. Adapted from the Qiskit HumanEval test harness (Copyright IBM, Apache-2.0). Supports Qiskit HumanEval (`check(candidate)`) and QuantumKatas - (`test_()`). Warning: this `exec()`s untrusted code; it is not a sandbox. + (`test_()`). + + Warning: this `exec()`s untrusted code in the current process; it is not a + sandbox. The time limit is best effort: it is only checked between Python + bytecodes, so code blocked inside a C call (a simulator run, `time.sleep`) is + stopped only once that call returns, and a bare `except:` in the code can + swallow it. Args: problem: Problem dict with `prompt`, `test` and `entry_point` keys @@ -168,43 +221,15 @@ def validate_correctness(problem: dict[str, Any], code: str) -> tuple[bool, str] if test_name not in test_fns: return False, f"No check() or test_{entry}() in the test code" - # Mellea validates on a worker thread, so the timeout is raised via the C API. - tid = ctypes.c_ulong(threading.get_ident()) - timer = threading.Timer( - 30, - ctypes.pythonapi.PyThreadState_SetAsyncExc, - (tid, ctypes.py_object(_Timeout)), - ) - cwd = os.getcwd() + # Run in a scratch directory (some tasks write files) with a non-interactive + # matplotlib backend, and restore both afterwards. + cwd, mpl_backend = os.getcwd(), os.environ.get("MPLBACKEND") os.environ["MPLBACKEND"] = "Agg" with tempfile.TemporaryDirectory() as tmp: - os.chdir(tmp) # some tasks write files - timer.start() + os.chdir(tmp) try: - ns: dict[str, Any] = {"__builtins__": _SAFE_BUILTINS} - exec(program, ns) - if entry not in ns: - return False, f"Entry point `{entry}` not defined" - if test_name == "check": - ns["check"](ns[entry]) - else: - ns[test_name]() - return True, "" - except _Timeout: - print("Validation failed: Execution timed out after 30s") - return False, "Execution timed out after 30s" - except SystemExit as e: - # Not an Exception: left uncaught, it would stop the repair loop. - print(f"Validation failed: the code called sys.exit({e.code!r})") - return False, f"The code called sys.exit({e.code!r}) instead of returning" - except AssertionError as e: - print(f"Validation failed: Test assertion failed: {e}") - return False, f"Test assertion failed: {e}" - except Exception as e: - print(f"{type(e).__name__}: {e}\n{traceback.format_exc()}") - return False, f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + return _run_tests(program, entry, test_name) finally: - timer.cancel() os.chdir(cwd) if mpl_backend is None: os.environ.pop("MPLBACKEND", None) From c4d02d3088a203608f40cdcc8c89ee473f9e023c Mon Sep 17 00:00:00 2001 From: Greta Dolcetti <53076361+gretadolcetti@users.noreply.github.com> Date: Fri, 23 Oct 2026 08:47:52 +0100 Subject: [PATCH 09/10] refactor: drop LintQ from Qiskit IVR example Signed-off-by: Greta Dolcetti <53076361+gretadolcetti@users.noreply.github.com> --- .../qiskit_code_validation/README.md | 85 +++---------------- .../qiskit_code_validation.py | 46 +--------- .../validation_helpers.py | 48 +---------- 3 files changed, 17 insertions(+), 162 deletions(-) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md b/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md index e0105cacdb..59087b3641 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md @@ -1,15 +1,15 @@ # Qiskit Code Validation with Instruct-Validate-Repair -This example demonstrates using Mellea's Instruct-Validate-Repair (IVR) pattern to generate Qiskit quantum computing code that automatically passes `flake8-qiskit-migration` validation rules (QKT rules), and, optionally, the problem's own test suite and the [LintQ](https://github.com/sola-st/LintQ) static analyzer for quantum programs. +This example demonstrates using Mellea's Instruct-Validate-Repair (IVR) pattern to generate Qiskit quantum computing code that automatically passes `flake8-qiskit-migration` validation rules (QKT rules), and, optionally, the problem's own test suite. ## What This Example Does Takes a prompt containing deprecated Qiskit code and: 1. Generates corrected code using the LLM -2. Validates the output against QKT rules, the test suite and LintQ +2. Validates the output against QKT rules 3. Automatically repairs the code if validation fails (up to 10 attempts) -`validation_helpers.py` also provides test-suite and LintQ validators for benchmark problems (Qiskit HumanEval, QuantumKatas); see [Validating with Test Suites and LintQ](#validating-with-test-suites-and-lintq). +`validation_helpers.py` also provides a test-suite validator for benchmark problems (Qiskit HumanEval, QuantumKatas); see [Validating with Test Suites](#validating-with-test-suites). ## Quick Start @@ -76,14 +76,9 @@ qiskit_code_validation/ - `validate_qiskit_migration()`: Validates against QKT rules - `validate_input_code()`: Pre-validates input prompts - `validate_correctness()`: Runs code against a benchmark problem's own test suite -- `validate_lintq()`: Runs the LintQ CodeQL queries over the code All validators return `(is_valid, error_message)`, so the message is fed straight back to the model as the repair reason. -### Adding LintQ to the Single-Prompt Example - -Set `use_lintq = True` in `test_qiskit_code_validation()` to also reject code flagged by LintQ. This needs the [LintQ setup](#lintq-setup) described below. - ## Trying Different Prompts To try different prompts, edit the `prompt` variable in `test_qiskit_code_validation()` function. Here are some examples you can copy/paste: @@ -209,44 +204,40 @@ qc.measure_all() **Note**: The exact output may vary depending on the model and its interpretation of the prompt. -## Validating with Test Suites and LintQ +## Validating with Test Suites -Passing the QKT rules only shows that the code uses current Qiskit APIs, not that it solves the task or that it is free of quantum-specific bugs. For benchmark problems, `validate_correctness()` and `validate_lintq()` add two more checks: +Passing the QKT rules only shows that the code uses current Qiskit APIs, not that it solves the task. For benchmark problems, `validate_correctness()` adds a second check: | Validator | What it checks | Repair reason fed back to the model | |-----------|----------------|-------------------------------------| | **QKT rules** | `flake8-qiskit-migration`, as in the single-prompt example | The QKT rule messages | | **Test suite** | Executes the code against the problem's own tests: `check(candidate)` for Qiskit HumanEval, `test_()` for QuantumKatas (best-effort 30s time limit, see below) | The assertion message or full traceback | -| **LintQ** | Runs the [LintQ](https://github.com/sola-st/LintQ) CodeQL queries (double measurement, operations after measurement, unused qubits, ...) | `[rule-id] message` per finding | -Both plug into the same `generate_validated_qiskit_code()` loop through its `extra_requirements` parameter. Running LintQ only once the tests pass avoids linting code that does not run, and saves time: each LintQ run builds a CodeQL database (around 20s). +It plugs into the same `generate_validated_qiskit_code()` loop through its `extra_requirements` parameter. ```python from validation_helpers import ( extract_code_from_markdown, validate_correctness, - validate_lintq, ) from mellea.stdlib.requirements import req, simple_validate -def tests_and_lintq(problem: dict): - """Pass the problem's tests, then LintQ.""" +def passes_tests(problem: dict): + """Pass the problem's own test suite.""" def check(output: str) -> tuple[bool, str]: - code = extract_code_from_markdown(output) - passed, reason = validate_correctness(problem, code) - return validate_lintq(code) if passed else (passed, reason) + return validate_correctness(problem, extract_code_from_markdown(output)) return req( - "The code must pass the problem's test suite and raise no LintQ warnings", + "The code must pass the problem's test suite", validation_fn=simple_validate(check), ) code, success, attempts = generate_validated_qiskit_code( - m, problem["prompt"], strategy, extra_requirements=[tests_and_lintq(problem)] + m, problem["prompt"], strategy, extra_requirements=[passes_tests(problem)] ) ``` @@ -256,7 +247,7 @@ code, success, attempts = generate_validated_qiskit_code( ### Setup -The benchmark dependencies, the datasets and LintQ are **not** part of this repository. Install and download them anywhere you like; the paths below are only examples. +The benchmark dependencies and the datasets are **not** part of this repository. Install and download them anywhere you like; the paths below are only examples. #### Python Dependencies @@ -288,46 +279,6 @@ Check each dataset's license before redistributing it, or any results generated Some Qiskit HumanEval tasks call IBM Quantum services: they can only pass with a saved IBM Quantum account (`QiskitRuntimeService.save_account(...)`); without one, even their canonical solutions fail. The remaining canonical solutions, and all 350 QuantumKatas ones, pass the test-suite validator. -#### LintQ Setup - -LintQ is a set of [CodeQL](https://codeql.github.com/) queries, so it needs the CodeQL CLI and a checkout of the LintQ repository: - -1. Install the CodeQL CLI and put `codeql` on your `PATH`, e.g. `brew install codeql` on macOS, or download the CodeQL bundle from the [codeql-action releases](https://github.com/github/codeql-action/releases). Check it with `codeql version`. -2. Clone LintQ and install its query pack dependencies: - - ```bash - git clone --depth 1 https://github.com/sola-st/LintQ.git - (cd LintQ/qlint/codeql/src && codeql pack install) - ``` - -3. Point `LINTQ_DIR` at the checkout: - - ```bash - export LINTQ_DIR="$(pwd)/LintQ" - ``` - - -### Benchmark Results - -All runs use `MultiTurnStrategy` with a loop budget of 3 (the first attempt plus up to 2 repairs). - -#### gpt-oss-120b (test suite + LintQ) - -These runs validated the test suite and LintQ. - -| Metric | QHE hard: first pass | QHE hard: post repair | QuantumKatas: first pass | QuantumKatas: post repair | -|--------|------|------|------|------| -| Tasks | 151 | 151 | 350 | 350 | -| Passed both validators | 24/151 (16%) | **70/151 (46%)** | 25/350 (7%) | **163/350 (47%)** | -| Failed | 127/151 (84%) | 81/151 (54%) | 325/350 (93%) | 187/350 (53%) | -| Failed the test suite | 113 | 77 | 297 | 148 | -| Flagged by LintQ | 14 | 4 | 28 | 39 | -| LintQ warnings raised | 17 | 4 | 30 | 47 | -| Fixed by the repair loop | — | 46 | — | 138 | - -The repair loop roughly triples the Qiskit HumanEval hard pass rate and multiplies the QuantumKatas one by more than six. On QuantumKatas, LintQ flags more tasks after repair than before: a task that fails its tests is never linted, and many programs only start running (and so get linted) after repair. - - ## Changing the Model To try a different model, edit the `model_id` variable in the `test_qiskit_code_validation()` function: @@ -386,18 +337,6 @@ If using a general-purpose model, it may not have enough Qiskit knowledge to pas - Setting `system_prompt = QISKIT_SYSTEM_PROMPT` to guide the model toward modern Qiskit APIs - Using simpler prompts -### LintQ Not Configured -``` -KeyError: 'LINTQ_DIR' -``` -**Solution**: Follow [LintQ Setup](#lintq-setup) and `export LINTQ_DIR=/path/to/LintQ`. - -### CodeQL Errors -``` -subprocess.CalledProcessError: Command '['codeql', 'database', ...]' returned non-zero exit status -``` -**Solution**: Check `codeql version` works, and that `codeql pack install` was run in `$LINTQ_DIR/qlint/codeql/src`. - ### Import Error: flake8-qiskit-migration ``` ModuleNotFoundError: No module named 'flake8_qiskit_migration' diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py index 631ec9ee24..21310ae6b9 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/qiskit_code_validation.py @@ -28,11 +28,7 @@ import time -from validation_helpers import ( - extract_code_from_markdown, - validate_lintq, - validate_qiskit_migration, -) +from validation_helpers import validate_qiskit_migration from mellea import MelleaSession, start_session from mellea.backends import ModelOption @@ -188,29 +184,6 @@ def test_qiskit_code_validation() -> None: # RepairTemplateStrategy: Adds validation failure reasons to the instruction and retries use_multiturn_strategy = False - # Setting use_lintq to True also rejects code flagged by the LintQ CodeQL queries. - # Requires the codeql CLI and LINTQ_DIR (see README.md → LintQ setup) - use_lintq = False - - # LintQ report of each generation, so the final one is not recomputed below. - lintq_reports: list[str] = [] - - def lintq_check(output: str) -> tuple[bool, str]: - passed, report = validate_lintq(extract_code_from_markdown(output)) - lintq_reports.append(report) - return passed, report - - extra_requirements = ( - [ - req( - "Code must raise no LintQ warnings", - validation_fn=simple_validate(lintq_check), - ) - ] - if use_lintq - else None - ) - # Initialize the required context ctx = ChatContext() if use_multiturn_strategy else SimpleContext() if use_multiturn_strategy: @@ -229,11 +202,7 @@ def lintq_check(output: str) -> tuple[bool, str]: start_time = time.time() code, success, attempts = generate_validated_qiskit_code( - m, - prompt, - strategy, - system_prompt=system_prompt, - extra_requirements=extra_requirements, + m, prompt, strategy, system_prompt=system_prompt ) elapsed = time.time() - start_time @@ -242,18 +211,11 @@ def lintq_check(output: str) -> tuple[bool, str]: print("======================\n") if success: - if use_lintq: - print( - "✓ Code passes Qiskit migration validation and it is code smells free" - ) - else: - print("✓ Code passes Qiskit migration validation") + print("✓ Code passes Qiskit migration validation") else: _, error_msg = validate_qiskit_migration(code) - # The returned code is the last generation, so its LintQ report is the last one. - errors = [error_msg, lintq_reports[-1] if lintq_reports else ""] print("✗ Validation errors:") - print("\n".join(e for e in errors if e)) + print(error_msg) if __name__ == "__main__": diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py index 70ceb30ee4..520e16b819 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py @@ -3,16 +3,14 @@ This module provides utilities for extracting code from markdown and validating Qiskit code against migration rules using the flake8-qiskit-migration plugin, -against a benchmark problem's test suite, and against LintQ. +and against a benchmark problem's own test suite. """ import ast import builtins import ctypes -import json import os import re -import subprocess import tempfile import threading import traceback @@ -236,47 +234,3 @@ def validate_correctness(problem: dict[str, Any], code: str) -> tuple[bool, str] else: os.environ["MPLBACKEND"] = mpl_backend - -def validate_lintq(code: str) -> tuple[bool, str]: - """Run the LintQ CodeQL queries over `code`. - - Requires the `codeql` CLI and `LINTQ_DIR` pointing at a LintQ checkout (see README.md). - - Args: - code: Python source to analyse - - Returns: - Tuple of (is_valid, error_message), one `[rule-id] message` line per finding - """ - lintq = os.environ["LINTQ_DIR"] - with tempfile.TemporaryDirectory() as tmp: - os.makedirs(f"{tmp}/src") - with open(f"{tmp}/src/candidate.py", "w") as f: - f.write(code) - for cmd in ( - [ - "database", - "create", - f"{tmp}/db", - "--language=python", - f"--source-root={tmp}/src", - ], - [ - "database", - "analyze", - f"{tmp}/db", - f"{lintq}/LintQ-all.qls", - "--format=sarifv2.1.0", - f"--output={tmp}/out.sarif", - f"--additional-packs={lintq}/qlint/codeql/src:{lintq}/qlint/codeql/lib", - ], - ): - subprocess.run(["codeql", *cmd], check=True, capture_output=True) - with open(f"{tmp}/out.sarif") as f: - results = [r for run in json.load(f)["runs"] for r in run["results"]] - - if not results: - return True, "" - findings = "\n".join(f"[{r['ruleId']}] {r['message']['text']}" for r in results) - print(f"Validation failed: LintQ warnings:\n{findings}") - return False, f"LintQ warnings:\n{findings}" From 01d56f1dbe0530e521f8c6dc0749029403ce9294 Mon Sep 17 00:00:00 2001 From: Greta Dolcetti <53076361+gretadolcetti@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:24:50 +0100 Subject: [PATCH 10/10] docs: make test-suite snippet runnable in Qiskit IVR README Signed-off-by: Greta Dolcetti <53076361+gretadolcetti@users.noreply.github.com> --- .../qiskit_code_validation/README.md | 14 +++++++------- .../qiskit_code_validation/validation_helpers.py | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md b/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md index 59087b3641..ce43fe38f1 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/README.md @@ -206,16 +206,13 @@ qc.measure_all() ## Validating with Test Suites -Passing the QKT rules only shows that the code uses current Qiskit APIs, not that it solves the task. For benchmark problems, `validate_correctness()` adds a second check: +Passing the QKT rules only shows that the code uses current Qiskit APIs, not that it solves the task. For benchmark problems, `validate_correctness()` runs the code against the problem's own tests (`check(candidate)` for Qiskit HumanEval, `test_()` for QuantumKatas, with a best-effort 30s time limit) and feeds the assertion message or traceback back to the model as the repair reason. -| Validator | What it checks | Repair reason fed back to the model | -|-----------|----------------|-------------------------------------| -| **QKT rules** | `flake8-qiskit-migration`, as in the single-prompt example | The QKT rule messages | -| **Test suite** | Executes the code against the problem's own tests: `check(candidate)` for Qiskit HumanEval, `test_()` for QuantumKatas (best-effort 30s time limit, see below) | The assertion message or full traceback | - -It plugs into the same `generate_validated_qiskit_code()` loop through its `extra_requirements` parameter. +To try it, drop the snippet below into `qiskit_code_validation.py`: the imports and `passes_tests()` at the top, the last two statements in `test_qiskit_code_validation()` in place of the existing `generate_validated_qiskit_code()` call, so they reuse its `m` and `strategy`. It reads a dataset downloaded as in [Setup](#setup). ```python +import json + from validation_helpers import ( extract_code_from_markdown, validate_correctness, @@ -236,6 +233,9 @@ def passes_tests(problem: dict): ) +# One dataset row = one problem dict +problem = json.loads(next(open("data/qiskit_humaneval_hard.jsonl"))) + code, success, attempts = generate_validated_qiskit_code( m, problem["prompt"], strategy, extra_requirements=[passes_tests(problem)] ) diff --git a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py index 520e16b819..9582cfb6bc 100644 --- a/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py +++ b/docs/examples/instruct_validate_repair/qiskit_code_validation/validation_helpers.py @@ -130,8 +130,9 @@ class _Timeout(BaseException): """ -# Builtins the executed code does not get: they could escape or stall the check -# (defense in depth, not a sandbox). +# Builtins the executed code does not get: they could escape or stall the check. +# This is defense in depth, not a sandbox: `__import__` is kept for `import` +# statements, so the executed code can still use any module. _SAFE_BUILTINS = { k: v for k, v in vars(builtins).items()