From 414df5f563b00d72954c88529d8e4c446bfb5a75 Mon Sep 17 00:00:00 2001 From: jacobdenobel Date: Thu, 5 Nov 2026 17:55:22 +0100 Subject: [PATCH 1/5] E03 step 1/3: suite registry with JSON export Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/python-publish.yml | 14 ++ CHANGELOG.md | 13 ++ WP_NOTES.md | 23 +++ ioh/__init__.py | 9 + ioh/__init__.pyi | 1 + ioh/suite_registry.py | 236 +++++++++++++++++++++++++++ tests/python/test_registry.py | 163 ++++++++++++++++++ 7 files changed, 459 insertions(+) create mode 100644 WP_NOTES.md create mode 100644 ioh/suite_registry.py create mode 100644 tests/python/test_registry.py diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 2437c004..da86d6bf 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -78,6 +78,9 @@ jobs: build_sdist: name: Build source distribution runs-on: ubuntu-latest + permissions: + # to attach the suite registry to a release + contents: write steps: - uses: actions/checkout@v7 with: @@ -95,6 +98,17 @@ jobs: run: | python -m pip install dist/*.tar.gz python tests/python/test_golden.py + - name: Write the suite registry + run: python -m ioh.suite_registry -o registry.json + - name: Attach the suite registry to the release + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ github.token }} + run: gh release upload "${{ github.event.release.tag_name }}" registry.json --clobber + - uses: actions/upload-artifact@v7 + with: + name: suite-registry + path: registry.json - uses: actions/upload-artifact@v7 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index a8543103..0b3e2e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## Unreleased + +### Added + +* `ioh.registry()` returns a machine-readable registry of the suites (BBOB, SBOX, PBO, Submodular and the two star + discrepancy suites): the problems with their ids and names, whether each is maximised, its bounds and known optimum + (and whether the optimum depends on the instance or the dimension), the instance and dimension ranges, and the + conventions a leaderboard needs: standard budget, standard instances and dimensions, target bounds and the scoring + method (AOCC for every suite; `ioh.suite_registry.SCORING_OVERRIDES` sets another per problem). `python -m + ioh.suite_registry -o registry.json` writes it as JSON; the release workflow attaches that file to each release. + The budget and target conventions are provisional. A test keeps the registry consistent with the problem classes and + `include/ioh/suite.hpp`. + ## 0.4.0 The first release since 0.3.22. Parts of the Python and C++ APIs changed: the diff --git a/WP_NOTES.md b/WP_NOTES.md new file mode 100644 index 00000000..6dc891da --- /dev/null +++ b/WP_NOTES.md @@ -0,0 +1,23 @@ +# WP E03: Suite registry export +Branch: wp/E03-suite-registry +Status: in progress + +## Done +- step 1/3: `ioh/suite_registry.py` with `registry()` (also `ioh.registry()`), `scoring()`, `python -m ioh.suite_registry -o FILE` JSON export; the release workflow (build_sdist job) writes registry.json, uploads it as an artifact and attaches it to the release. +- step 2/3: leaderboard fields per suite: standard budget, standard instances and dimensions, target bounds, scoring (AOCC; per-problem overrides via `SCORING_OVERRIDES`, `problems[].scoring`). +- step 3/3: pending (tests, see below) + +## Next step +Step 3: tests/python/test_registry.py. + +## Decisions +- Structure of the problems (ids, names, direction, bounds, optima) is read live from the installed package, so it cannot drift. The conventions are a table in `SUITE_CONVENTIONS`. +- The module is `ioh.suite_registry`; `ioh.registry()` is a lazy wrapper (a module named `registry` would clash with the function). +- Suites listed: BBOB, SBOX, PBO, Submodular, RealStarDiscrepancy, IntegerStarDiscrepancy. The generic `Real` and `Integer` suites (all problems) are not benchmark suites and are left out. CEC problems are not in a suite class, so not included. +- The ranges of instances/dimensions are checked against include/ioh/suite.hpp by the test (the C++ only asserts them). +- Unknown optima (graph, star discrepancy: C++ reports inf) are `null` with `known_optimum: false`. +- No new dependency. Python module built in `.venv` (pip install --no-build-isolation .), tests run from /tmp (`python tests/python/test_registry.py`) so the installed package is used. + +## Open questions / Blocked +- PROVISIONAL conventions, to be confirmed by the leaderboard owner (W13/D13): budgets (BBOB/SBOX 10000*D, PBO 100*D, others null), standard instances (1-5) and dimensions, target bounds (BBOB/SBOX 1e-8..1e2 precision, log; PBO linear from 0, upper null; Submodular and star discrepancy unset). +- The release attach step (`gh release upload`) is untested; needs a real release. diff --git a/ioh/__init__.py b/ioh/__init__.py index 326e457c..f2183f22 100644 --- a/ioh/__init__.py +++ b/ioh/__init__.py @@ -259,6 +259,15 @@ def wrap_problem( return get_problem(name, instance, dimension, problem_class) +def registry() -> dict: + '''The registry of all suites: their problems (ids, names, direction, bounds, optima) and the conventions of a + leaderboard (limits, standard budget, standard instances, target bounds and scoring method). A JSON compatible + dict, see ioh.suite_registry.registry for its structure; `python -m ioh.suite_registry` writes it as JSON. + ''' + from .suite_registry import registry as _registry + return _registry() + + def get_problem_id(problem_name: str, problem_class: ProblemClass) -> int: '''Get the problem id corresponding to a problem name of a given type diff --git a/ioh/__init__.pyi b/ioh/__init__.pyi index 09f407e4..f71a482f 100644 --- a/ioh/__init__.pyi +++ b/ioh/__init__.pyi @@ -29,6 +29,7 @@ class ProblemClass(enum.Enum): def get_problem(fid: int | str, instance: int = 1, dimension: int = 5, problem_class: ProblemClass = ...) -> ProblemType: ... def wrap_problem(function: typing.Callable[[numpy.typing.NDArray], float], name: str = None, problem_class: ProblemClass = ..., dimension: int = 5, instance: int = 1, optimization_type: OptimizationType = ..., lb: VariableType = 0, ub: VariableType = 1, transform_variables: typing.Callable[[numpy.typing.NDArray, int], numpy.typing.NDArray | typing.Sequence[VariableType]] = None, transform_objectives: typing.Callable[[float, int], float] = None, calculate_objective: typing.Callable[[int, int], IntegerSolution | RealSolution] = None, constraints: list[IntegerConstraint | RealConstraint] = None) -> ProblemType: ... +def registry() -> dict: ... def get_problem_id(problem_name: str, problem_class: ProblemClass) -> int: ... class Experiment: diff --git a/ioh/suite_registry.py b/ioh/suite_registry.py new file mode 100644 index 00000000..37d256c9 --- /dev/null +++ b/ioh/suite_registry.py @@ -0,0 +1,236 @@ +'''Machine-readable registry of the suites of IOHexperimenter. + +``ioh.registry()`` returns a dict (JSON compatible) with every suite, its problems and the conventions a +leaderboard needs. The structure of the problems (ids, names, direction, bounds, optima) is read from the problem +classes of the installed package, so it cannot drift from the C++ code. The conventions (limits of instances and +dimensions, standard budget, standard instances, target bounds and scoring method) are listed in ``SUITE_CONVENTIONS`` +below. ``python -m ioh.suite_registry [-o FILE]`` writes the registry as JSON (the release workflow attaches it to each +release). + +Scoring is configuration, not code: every suite has a scoring method (AOCC to start with), and a problem can override +it in ``SCORING_OVERRIDES``. ``scoring(suite, problem_id)`` gives the method that applies. +''' + +import argparse +import copy +import json +import math +import sys +import typing +from importlib import metadata + +from . import ProblemClass, get_problem, OptimizationType + +SCHEMA_VERSION = 1 + +#: The scoring method of every suite. AOCC: the area over the convergence curve. +DEFAULT_SCORING: typing.Dict[str, typing.Any] = {"method": "AOCC", "parameters": {}} + +#: Scoring per problem, ``{suite name: {problem id: {"method": ..., "parameters": {...}}}}``; a problem that is not +#: listed uses the scoring of its suite. +SCORING_OVERRIDES: typing.Dict[str, typing.Dict[int, typing.Dict[str, typing.Any]]] = {} + +_LOG_PRECISION = {"scale": "log", "relative_to_optimum": True, "lower": 1e-8, "upper": 1e2} +_UNKNOWN_TARGETS = {"scale": "linear", "relative_to_optimum": False, "lower": None, "upper": None} + +#: Per suite: the class of its problems, the largest instance and dimension its C++ class accepts (these are checked +#: against include/ioh/suite.hpp by the tests), and the conventions of a leaderboard. A budget is +#: ``base + per_dimension * dimension`` evaluations. ``target_bounds`` are the range of target values that scoring +#: considers: ``relative_to_optimum`` ones are precisions (distance to the optimum), the others are function values. +#: ``None`` means that the convention is not settled for that suite. +SUITE_CONVENTIONS: typing.Dict[str, typing.Dict[str, typing.Any]] = { + "BBOB": { + "problem_class": ProblemClass.BBOB, + "instances": {"min": 1, "max": 100}, + "dimensions": {"min": 2, "max": 100}, + "standard_instances": [1, 2, 3, 4, 5], + "standard_dimensions": [2, 3, 5, 10, 20, 40], + "budget": {"base": 0, "per_dimension": 10000}, + "target_bounds": _LOG_PRECISION, + }, + "SBOX": { + "problem_class": ProblemClass.SBOX, + "instances": {"min": 1, "max": 100}, + "dimensions": {"min": 2, "max": 100}, + "standard_instances": [1, 2, 3, 4, 5], + "standard_dimensions": [2, 3, 5, 10, 20, 40], + "budget": {"base": 0, "per_dimension": 10000}, + "target_bounds": _LOG_PRECISION, + }, + "PBO": { + "problem_class": ProblemClass.PBO, + "instances": {"min": 1, "max": 100}, + "dimensions": {"min": 1, "max": 20000}, + "standard_instances": [1, 2, 3, 4, 5], + "standard_dimensions": [16, 64, 100], + "budget": {"base": 0, "per_dimension": 100}, + "target_bounds": {"scale": "linear", "relative_to_optimum": True, "lower": 0.0, "upper": None}, + }, + "Submodular": { + "problem_class": ProblemClass.GRAPH, + "instances": {"min": 1, "max": 1}, + "dimensions": {"min": 1, "max": 1}, + "standard_instances": [1], + "standard_dimensions": [1], + "budget": None, + "target_bounds": _UNKNOWN_TARGETS, + }, + "RealStarDiscrepancy": { + "problem_class": ProblemClass.STAR_REAL, + "instances": {"min": 1, "max": 10000}, + "dimensions": {"min": 1, "max": 10000}, + "standard_instances": [1], + "standard_dimensions": [5], + "budget": None, + "target_bounds": _UNKNOWN_TARGETS, + }, + "IntegerStarDiscrepancy": { + "problem_class": ProblemClass.STAR_INTEGER, + "instances": {"min": 1, "max": 10000}, + "dimensions": {"min": 1, "max": 10000}, + "standard_instances": [1], + "standard_dimensions": [5], + "budget": None, + "target_bounds": _UNKNOWN_TARGETS, + }, +} + +#: Dimensions of problems that need a perfect square (checked by get_problem) +_SQUARE_DIMENSION = {"PBO": {21, 23}} + + +def _number(value: float) -> typing.Optional[float]: + '''A finite number, or None for NaN and infinity (an optimum that is not known)''' + value = float(value) + return value if math.isfinite(value) else None + + +def _optimum(problem_class: ProblemClass, fid: int, instance: int, dimension: int): + return _number(get_problem(fid, instance, dimension, problem_class).optimum.y) + + +def _problem_entry( + suite_name: str, problem_class: ProblemClass, fid: int, name: str, conventions: dict +) -> typing.Dict[str, typing.Any]: + dims = conventions["dimensions"] + reference_dimension = conventions["standard_dimensions"][0] + if fid in _SQUARE_DIMENSION.get(suite_name, ()): + reference_dimension = 16 + problem = get_problem(fid, 1, reference_dimension, problem_class) + meta = problem.meta_data + # the graph problems ignore the requested dimension: each has its own + fixed_dimension = int(meta.n_variables) if suite_name == "Submodular" else None + other_dimension = min(reference_dimension + 1, dims["max"]) + + y = _number(problem.optimum.y) + # does the optimal value change with the instance or the dimension? Compared on a second instance (when the + # suite has one) and a second dimension + instance_dependent = False + dimension_dependent = False + if y is not None: + if conventions["instances"]["max"] > 1: + instance_dependent = _optimum(problem_class, fid, 2, reference_dimension) != y + if fixed_dimension is None and fid not in _SQUARE_DIMENSION.get(suite_name, ()): + dimension_dependent = _optimum(problem_class, fid, 1, other_dimension) != y + + return { + "id": fid, + "name": name, + "optimization_type": "MAX" if meta.optimization_type == OptimizationType.MAX else "MIN", + "maximization": meta.optimization_type == OptimizationType.MAX, + "bounds": {"lower": float(problem.bounds.lb[0]), "upper": float(problem.bounds.ub[0])}, + "fixed_dimension": fixed_dimension, + "dimension_requirement": "perfect_square" if fid in _SQUARE_DIMENSION.get(suite_name, ()) else None, + "known_optimum": y is not None, + "optimum": { + "instance": 1, + "dimension": reference_dimension if fixed_dimension is None else fixed_dimension, + "value": y, + "depends_on_instance": instance_dependent, + "depends_on_dimension": dimension_dependent, + }, + # None: the scoring of the suite applies + "scoring": copy.deepcopy(SCORING_OVERRIDES.get(suite_name, {}).get(fid)), + } + + +def _build() -> typing.Dict[str, typing.Any]: + suites = {} + for name, conventions in SUITE_CONVENTIONS.items(): + problem_class = conventions["problem_class"] + entry = { + "name": name, + "problem_class": problem_class.name, + "variable_type": "real" if problem_class.is_real() else "integer", + "instances": dict(conventions["instances"]), + "dimensions": dict(conventions["dimensions"]), + "standard_instances": list(conventions["standard_instances"]), + "standard_dimensions": list(conventions["standard_dimensions"]), + "budget": copy.deepcopy(conventions["budget"]), + "target_bounds": copy.deepcopy(conventions["target_bounds"]), + "scoring": copy.deepcopy(DEFAULT_SCORING), + "problems": [ + _problem_entry(name, problem_class, fid, pname, conventions) + for fid, pname in sorted(problem_class.problems.items()) + ], + } + suites[name] = entry + try: + version = metadata.version("ioh") + except metadata.PackageNotFoundError: + version = None + return {"schema_version": SCHEMA_VERSION, "ioh_version": version, "suites": suites} + + +_cache: typing.Optional[typing.Dict[str, typing.Any]] = None + + +def registry() -> typing.Dict[str, typing.Any]: + '''The registry of all suites, a JSON compatible dict (a copy: change it freely). + + ``registry()["suites"][name]`` holds, per suite: the ``problem_class``, the ``instances`` and ``dimensions`` + ranges that the suite accepts, ``standard_instances`` and ``standard_dimensions``, the standard ``budget`` + (``base + per_dimension * dimension`` evaluations, None when not settled), ``target_bounds``, the ``scoring`` + method and the list of ``problems``. A problem has its ``id``, ``name``, ``optimization_type`` ("MIN" or "MAX"), + ``maximization``, ``bounds``, ``fixed_dimension`` (for problems that ignore the requested dimension), + ``known_optimum``, ``optimum`` (the value of the optimum of instance 1 in the reference dimension, and whether it + depends on the instance or the dimension) and a scoring override (``scoring``, None for the suite's). + ''' + global _cache + if _cache is None: + _cache = _build() + return copy.deepcopy(_cache) + + +def scoring(suite_name: str, problem_id: typing.Optional[int] = None) -> typing.Dict[str, typing.Any]: + '''The scoring method for a suite, or for one of its problems: its own override when it has one, else the + suite's.''' + entry = registry()["suites"][suite_name] + if problem_id is not None: + for problem in entry["problems"]: + if problem["id"] == problem_id: + return problem["scoring"] or entry["scoring"] + raise KeyError(f"{suite_name} has no problem with id {problem_id}") + return entry["scoring"] + + +def to_json(indent: typing.Optional[int] = 2) -> str: + '''The registry as JSON text''' + return json.dumps(registry(), indent=indent, allow_nan=False) + "\n" + + +def main(argv: typing.Optional[typing.Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(prog="python -m ioh.suite_registry", description="Write the registry as JSON.") + parser.add_argument("-o", "--output", help="the file to write (default: standard output)") + args = parser.parse_args(argv) + text = to_json() + if args.output: + with open(args.output, "w", encoding="utf-8", newline="\n") as f: + f.write(text) + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/python/test_registry.py b/tests/python/test_registry.py new file mode 100644 index 00000000..4d69305b --- /dev/null +++ b/tests/python/test_registry.py @@ -0,0 +1,163 @@ +import json +import os +import re +import subprocess +import sys +import unittest + +import ioh +from ioh import suite_registry + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +SUITE_HPP = os.path.join(ROOT, "include", "ioh", "suite.hpp") + + +def suite_headers(): + '''name -> (max_instance, max_dimension) of the suite classes in include/ioh/suite.hpp''' + with open(SUITE_HPP, encoding="utf-8") as f: + text = f.read() + result = {} + for match in re.finditer(r'struct (\w+) final : Suite<.*?registry_name = "(\w+)".*?(?=\n (?://!|struct)|\Z)', + text, re.S): + body = match.group(0) + limits = re.search(r"registry_name,\s*(\d+),\s*(\d+),", body) + result[match.group(2)] = tuple(int(g) for g in limits.groups()) if limits else None + return result + + +class TestRegistry(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.registry = ioh.registry() + cls.suites = cls.registry["suites"] + + def test_structure(self): + self.assertEqual(self.registry["schema_version"], suite_registry.SCHEMA_VERSION) + self.assertEqual( + set(self.suites), + {"BBOB", "SBOX", "PBO", "Submodular", "RealStarDiscrepancy", "IntegerStarDiscrepancy"}, + ) + for name, entry in self.suites.items(): + self.assertEqual(entry["name"], name) + self.assertTrue(hasattr(ioh.suite, name), name) + for key in ("instances", "dimensions", "standard_instances", "standard_dimensions", "budget", + "target_bounds", "scoring", "problems"): + self.assertIn(key, entry) + + def test_problems_match_the_problem_classes(self): + for name, entry in self.suites.items(): + problem_class = ioh.ProblemClass[entry["problem_class"]] + listed = {p["id"]: p["name"] for p in entry["problems"]} + self.assertEqual(listed, dict(problem_class.problems), name) + self.assertEqual(entry["variable_type"], "real" if problem_class.is_real() else "integer") + + def test_problem_fields_match_the_problems(self): + for name, entry in self.suites.items(): + problem_class = ioh.ProblemClass[entry["problem_class"]] + for p in entry["problems"]: + dimension = p["optimum"]["dimension"] + problem = ioh.get_problem(p["id"], p["optimum"]["instance"], dimension, problem_class) + self.assertEqual(problem.meta_data.name, p["name"]) + self.assertEqual(problem.meta_data.problem_id, p["id"]) + self.assertEqual( + problem.meta_data.optimization_type == ioh.OptimizationType.MAX, p["maximization"]) + self.assertEqual(p["optimization_type"], "MAX" if p["maximization"] else "MIN") + self.assertEqual(float(problem.bounds.lb[0]), p["bounds"]["lower"]) + self.assertEqual(float(problem.bounds.ub[0]), p["bounds"]["upper"]) + self.assertEqual(p["known_optimum"], p["optimum"]["value"] is not None) + if p["known_optimum"]: + self.assertEqual(float(problem.optimum.y), p["optimum"]["value"]) + if p["fixed_dimension"] is not None: + self.assertEqual(problem.meta_data.n_variables, p["fixed_dimension"]) + + def test_known_values(self): + bbob = {p["name"]: p for p in self.suites["BBOB"]["problems"]} + self.assertEqual(len(bbob), 24) + self.assertFalse(bbob["Sphere"]["maximization"]) + self.assertTrue(bbob["Sphere"]["optimum"]["depends_on_instance"]) + self.assertFalse(bbob["Sphere"]["optimum"]["depends_on_dimension"]) + pbo = {p["name"]: p for p in self.suites["PBO"]["problems"]} + self.assertTrue(pbo["OneMax"]["maximization"]) + self.assertEqual(pbo["OneMax"]["optimum"]["value"], 16.0) + self.assertEqual(pbo["IsingTriangular"]["dimension_requirement"], "perfect_square") + graph = self.suites["Submodular"]["problems"][0] + self.assertEqual(graph["fixed_dimension"], 800) + self.assertFalse(graph["known_optimum"]) + + def test_conventions_are_usable(self): + for name, entry in self.suites.items(): + for key, standard in (("instances", "standard_instances"), ("dimensions", "standard_dimensions")): + limits = entry[key] + self.assertLessEqual(limits["min"], limits["max"]) + self.assertTrue(entry[standard], name) + for value in entry[standard]: + self.assertTrue(limits["min"] <= value <= limits["max"], (name, standard, value)) + budget = entry["budget"] + if budget is not None: + self.assertGreater(budget["base"] + budget["per_dimension"], 0) + bounds = entry["target_bounds"] + self.assertIn(bounds["scale"], ("log", "linear")) + if bounds["lower"] is not None and bounds["upper"] is not None: + self.assertLess(bounds["lower"], bounds["upper"]) + # the standard instances and dimensions create the suite's problems + first = entry["problems"][0]["id"] + s = getattr(ioh.suite, name)([first], entry["standard_instances"][:1], entry["standard_dimensions"][:1]) + self.assertEqual(len(list(s)), 1) + + def test_limits_match_the_cpp_suites(self): + if not os.path.exists(SUITE_HPP): + self.skipTest("include/ioh/suite.hpp is not available") + limits = suite_headers() + for name in ("Real", "Integer"): + limits.pop(name) + self.assertEqual(set(limits), set(self.suites)) + for name, entry in self.suites.items(): + max_instance, max_dimension = limits[name] + self.assertEqual(entry["instances"]["max"], max_instance, name) + if name != "Submodular": # ignores its dimensions: its limit is a fixed dimension per problem + self.assertEqual(entry["dimensions"]["max"], max_dimension, name) + + def test_default_dimensions_are_in_range(self): + for name, entry in self.suites.items(): + s = getattr(ioh.suite, name)([entry["problems"][0]["id"]]) + for instance in s.instances: + self.assertTrue(entry["instances"]["min"] <= instance <= entry["instances"]["max"]) + for dimension in s.dimensions: + self.assertTrue(entry["dimensions"]["min"] <= dimension <= entry["dimensions"]["max"]) + + def test_scoring_is_aocc_by_default(self): + for name, entry in self.suites.items(): + self.assertEqual(entry["scoring"]["method"], "AOCC") + self.assertEqual(ioh.suite_registry.scoring(name), entry["scoring"]) + for p in entry["problems"]: + self.assertIsNone(p["scoring"]) + self.assertEqual(suite_registry.scoring("BBOB", 3)["method"], "AOCC") + with self.assertRaises(KeyError): + suite_registry.scoring("BBOB", 99) + + def test_scoring_overrides(self): + override = {"method": "ECDF", "parameters": {"targets": 51}} + try: + suite_registry.SCORING_OVERRIDES["BBOB"] = {2: override} + suite_registry._cache = None + self.assertEqual(suite_registry.scoring("BBOB", 2), override) + self.assertEqual(suite_registry.scoring("BBOB", 1)["method"], "AOCC") + self.assertEqual(suite_registry.scoring("BBOB")["method"], "AOCC") + self.assertEqual(suite_registry.scoring("PBO", 2)["method"], "AOCC") + finally: + suite_registry.SCORING_OVERRIDES.clear() + suite_registry._cache = None + + def test_registry_is_a_copy(self): + ioh.registry()["suites"]["BBOB"]["problems"].clear() + self.assertEqual(len(ioh.registry()["suites"]["BBOB"]["problems"]), 24) + + def test_json(self): + self.assertEqual(json.loads(suite_registry.to_json()), self.registry) + process = subprocess.run([sys.executable, "-m", "ioh.suite_registry"], capture_output=True, text=True) + self.assertEqual(process.returncode, 0, process.stderr) + self.assertEqual(json.loads(process.stdout), self.registry) + + +if __name__ == "__main__": + unittest.main() From 4afa0affeb5e99467171f18b534bdef6e0231559 Mon Sep 17 00:00:00 2001 From: jacobdenobel Date: Thu, 5 Nov 2026 17:55:26 +0100 Subject: [PATCH 2/5] E03 step 3/3: notes, ready for review Co-Authored-By: Claude Sonnet 5.5 --- WP_NOTES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WP_NOTES.md b/WP_NOTES.md index 6dc891da..b4f17d5c 100644 --- a/WP_NOTES.md +++ b/WP_NOTES.md @@ -1,14 +1,14 @@ # WP E03: Suite registry export Branch: wp/E03-suite-registry -Status: in progress +Status: ready for review ## Done - step 1/3: `ioh/suite_registry.py` with `registry()` (also `ioh.registry()`), `scoring()`, `python -m ioh.suite_registry -o FILE` JSON export; the release workflow (build_sdist job) writes registry.json, uploads it as an artifact and attaches it to the release. - step 2/3: leaderboard fields per suite: standard budget, standard instances and dimensions, target bounds, scoring (AOCC; per-problem overrides via `SCORING_OVERRIDES`, `problems[].scoring`). -- step 3/3: pending (tests, see below) +- step 3/3: tests/python/test_registry.py (11 tests: consistency with problem classes, suite.hpp limits, usable conventions, scoring overrides, JSON and CLI), CHANGELOG entry under Unreleased. Steps 2 and 3 landed in the same commit as step 1 (414df5f5). ## Next step -Step 3: tests/python/test_registry.py. +Nothing: review. Run: cd /tmp && python /path/to/tests/python/test_registry.py with the package installed. ## Decisions - Structure of the problems (ids, names, direction, bounds, optima) is read live from the installed package, so it cannot drift. The conventions are a table in `SUITE_CONVENTIONS`. From 6a52a7328c0d64709d8e229dac15c9b4454a27ac Mon Sep 17 00:00:00 2001 From: jacobdenobel Date: Fri, 6 Nov 2026 11:56:36 +0100 Subject: [PATCH 3/5] E03: remove WP_NOTES.md, ready for review Co-Authored-By: Claude Opus 5.5 --- WP_NOTES.md | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 WP_NOTES.md diff --git a/WP_NOTES.md b/WP_NOTES.md deleted file mode 100644 index b4f17d5c..00000000 --- a/WP_NOTES.md +++ /dev/null @@ -1,23 +0,0 @@ -# WP E03: Suite registry export -Branch: wp/E03-suite-registry -Status: ready for review - -## Done -- step 1/3: `ioh/suite_registry.py` with `registry()` (also `ioh.registry()`), `scoring()`, `python -m ioh.suite_registry -o FILE` JSON export; the release workflow (build_sdist job) writes registry.json, uploads it as an artifact and attaches it to the release. -- step 2/3: leaderboard fields per suite: standard budget, standard instances and dimensions, target bounds, scoring (AOCC; per-problem overrides via `SCORING_OVERRIDES`, `problems[].scoring`). -- step 3/3: tests/python/test_registry.py (11 tests: consistency with problem classes, suite.hpp limits, usable conventions, scoring overrides, JSON and CLI), CHANGELOG entry under Unreleased. Steps 2 and 3 landed in the same commit as step 1 (414df5f5). - -## Next step -Nothing: review. Run: cd /tmp && python /path/to/tests/python/test_registry.py with the package installed. - -## Decisions -- Structure of the problems (ids, names, direction, bounds, optima) is read live from the installed package, so it cannot drift. The conventions are a table in `SUITE_CONVENTIONS`. -- The module is `ioh.suite_registry`; `ioh.registry()` is a lazy wrapper (a module named `registry` would clash with the function). -- Suites listed: BBOB, SBOX, PBO, Submodular, RealStarDiscrepancy, IntegerStarDiscrepancy. The generic `Real` and `Integer` suites (all problems) are not benchmark suites and are left out. CEC problems are not in a suite class, so not included. -- The ranges of instances/dimensions are checked against include/ioh/suite.hpp by the test (the C++ only asserts them). -- Unknown optima (graph, star discrepancy: C++ reports inf) are `null` with `known_optimum: false`. -- No new dependency. Python module built in `.venv` (pip install --no-build-isolation .), tests run from /tmp (`python tests/python/test_registry.py`) so the installed package is used. - -## Open questions / Blocked -- PROVISIONAL conventions, to be confirmed by the leaderboard owner (W13/D13): budgets (BBOB/SBOX 10000*D, PBO 100*D, others null), standard instances (1-5) and dimensions, target bounds (BBOB/SBOX 1e-8..1e2 precision, log; PBO linear from 0, upper null; Submodular and star discrepancy unset). -- The release attach step (`gh release upload`) is untested; needs a real release. From e1a5253a5ead271eae6bd9ee03ecb2dbab89365b Mon Sep 17 00:00:00 2001 From: jacobdenobel Date: Fri, 6 Nov 2026 12:15:09 +0100 Subject: [PATCH 4/5] E03: write the registry from outside the repository root in the sdist job Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/python-publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index da86d6bf..923055a6 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -99,7 +99,10 @@ jobs: python -m pip install dist/*.tar.gz python tests/python/test_golden.py - name: Write the suite registry - run: python -m ioh.suite_registry -o registry.json + # from another directory: in the repository root Python would import the unbuilt ioh/ folder + run: | + cd "$RUNNER_TEMP" + python -m ioh.suite_registry -o "$GITHUB_WORKSPACE/registry.json" - name: Attach the suite registry to the release if: github.event_name == 'release' env: From a8875f48e9e11b9267c873ffcfd051f24eb483a4 Mon Sep 17 00:00:00 2001 From: jacobdenobel Date: Fri, 6 Nov 2026 12:46:30 +0100 Subject: [PATCH 5/5] E03: Codacy style fixes in the registry module and its test Co-Authored-By: Claude Sonnet 5.5 --- ioh/suite_registry.py | 18 ++++++++++-------- tests/python/test_registry.py | 9 +++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/ioh/suite_registry.py b/ioh/suite_registry.py index 37d256c9..0ea9b3ae 100644 --- a/ioh/suite_registry.py +++ b/ioh/suite_registry.py @@ -1,4 +1,4 @@ -'''Machine-readable registry of the suites of IOHexperimenter. +"""Machine-readable registry of the suites of IOHexperimenter. ``ioh.registry()`` returns a dict (JSON compatible) with every suite, its problems and the conventions a leaderboard needs. The structure of the problems (ids, names, direction, bounds, optima) is read from the problem @@ -9,7 +9,7 @@ Scoring is configuration, not code: every suite has a scoring method (AOCC to start with), and a problem can override it in ``SCORING_OVERRIDES``. ``scoring(suite, problem_id)`` gives the method that applies. -''' +""" import argparse import copy @@ -100,7 +100,7 @@ def _number(value: float) -> typing.Optional[float]: - '''A finite number, or None for NaN and infinity (an optimum that is not known)''' + """A finite number, or None for NaN and infinity (an optimum that is not known).""" value = float(value) return value if math.isfinite(value) else None @@ -186,7 +186,7 @@ def _build() -> typing.Dict[str, typing.Any]: def registry() -> typing.Dict[str, typing.Any]: - '''The registry of all suites, a JSON compatible dict (a copy: change it freely). + """The registry of all suites, a JSON compatible dict (a copy: change it freely). ``registry()["suites"][name]`` holds, per suite: the ``problem_class``, the ``instances`` and ``dimensions`` ranges that the suite accepts, ``standard_instances`` and ``standard_dimensions``, the standard ``budget`` @@ -195,7 +195,7 @@ def registry() -> typing.Dict[str, typing.Any]: ``maximization``, ``bounds``, ``fixed_dimension`` (for problems that ignore the requested dimension), ``known_optimum``, ``optimum`` (the value of the optimum of instance 1 in the reference dimension, and whether it depends on the instance or the dimension) and a scoring override (``scoring``, None for the suite's). - ''' + """ global _cache if _cache is None: _cache = _build() @@ -203,8 +203,10 @@ def registry() -> typing.Dict[str, typing.Any]: def scoring(suite_name: str, problem_id: typing.Optional[int] = None) -> typing.Dict[str, typing.Any]: - '''The scoring method for a suite, or for one of its problems: its own override when it has one, else the - suite's.''' + """The scoring method for a suite, or for one of its problems. + + A problem's own override applies when it has one, else the suite's. + """ entry = registry()["suites"][suite_name] if problem_id is not None: for problem in entry["problems"]: @@ -215,7 +217,7 @@ def scoring(suite_name: str, problem_id: typing.Optional[int] = None) -> typing. def to_json(indent: typing.Optional[int] = 2) -> str: - '''The registry as JSON text''' + """The registry as JSON text.""" return json.dumps(registry(), indent=indent, allow_nan=False) + "\n" diff --git a/tests/python/test_registry.py b/tests/python/test_registry.py index 4d69305b..69adc7e6 100644 --- a/tests/python/test_registry.py +++ b/tests/python/test_registry.py @@ -1,7 +1,7 @@ import json import os import re -import subprocess +import subprocess # nosec B404: only to run this interpreter on its own module import sys import unittest @@ -13,7 +13,7 @@ def suite_headers(): - '''name -> (max_instance, max_dimension) of the suite classes in include/ioh/suite.hpp''' + """name -> (max_instance, max_dimension) of the suite classes in include/ioh/suite.hpp.""" with open(SUITE_HPP, encoding="utf-8") as f: text = f.read() result = {} @@ -52,7 +52,7 @@ def test_problems_match_the_problem_classes(self): self.assertEqual(entry["variable_type"], "real" if problem_class.is_real() else "integer") def test_problem_fields_match_the_problems(self): - for name, entry in self.suites.items(): + for entry in self.suites.values(): problem_class = ioh.ProblemClass[entry["problem_class"]] for p in entry["problems"]: dimension = p["optimum"]["dimension"] @@ -154,7 +154,8 @@ def test_registry_is_a_copy(self): def test_json(self): self.assertEqual(json.loads(suite_registry.to_json()), self.registry) - process = subprocess.run([sys.executable, "-m", "ioh.suite_registry"], capture_output=True, text=True) + process = subprocess.run( # nosec B603: this interpreter, with fixed arguments + [sys.executable, "-m", "ioh.suite_registry"], capture_output=True, text=True) self.assertEqual(process.returncode, 0, process.stderr) self.assertEqual(json.loads(process.stdout), self.registry)