diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c2415b1..0a3efc01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +### Added + +* `FlatFile` and `Analyzer` (C++ `set_full_precision(true)`, Python `full_precision = True`) can write values with + round-trip precision (`%.17g`) instead of `%.10f`. With `%.10f`, values below 5e-11 are written as 0, which hides + targets such as 1e-8 to 1e-10 relative to the optimum. The option also applies to the x columns, and it is off by + default: existing output does not change. Set it before attaching the first problem. The Analyzer records it in the + JSON metadata as `"precision": "%.17g"`; the key is absent for the default precision, and readers that do not know + it can ignore it. The header and the number of columns are the same. + ### Fixed * `FlatFile` data rows had no separator between the last meta data column (`run`) and the first logged property, so diff --git a/include/ioh/logger/analyzer.hpp b/include/ioh/logger/analyzer.hpp index 7b7af0f8..745700ee 100644 --- a/include/ioh/logger/analyzer.hpp +++ b/include/ioh/logger/analyzer.hpp @@ -188,6 +188,8 @@ namespace ioh::logger const std::vector attribute_names; //! Scenarios std::vector dims; + //! Printf-style precision of the values in the data files ("" for the legacy "%.10f", not written) + const std::string precision; /** * @brief Construct a new Experiment Info object @@ -199,16 +201,18 @@ namespace ioh::logger * @param run_attribute_names Run attributes * @param attribute_names attributes names * @param dims Scenarios + * @param precision Precision of the values in the data files, written as "precision" when not empty */ ExperimentInfo(const std::string &suite, const problem::MetaData &problem, const AlgorithmInfo &algorithm, const std::vector> &attributes = {}, const std::vector &run_attribute_names = {}, const std::vector &attribute_names = {}, - const std::vector &dims = {}) : + const std::vector &dims = {}, const std::string &precision = "") : suite(suite), problem(problem), algorithm(algorithm), attributes(attributes), - run_attribute_names(run_attribute_names), attribute_names(attribute_names), dims(dims) + run_attribute_names(run_attribute_names), attribute_names(attribute_names), dims(dims), + precision(precision) { } @@ -217,10 +221,11 @@ namespace ioh::logger return fmt::format( "{{\n\t\"version\": \"{}\", \n\t\"suite\": \"{}\", \n\t\"function_id\": {}, " "\n\t\"function_name\": \"{}\", " - "\n\t\"maximization\": {}, " + "\n\t\"maximization\": {}, {}" "\n\t\"algorithm\": {{{}}},{}{}{}\n\t\"scenarios\": [\n\t\t{{{}}}\n\t]\n}}\n", PROJECT_VER, suite, problem.problem_id, problem.name, - (problem.optimization_type == common::OptimizationType::MAX), algorithm, + (problem.optimization_type == common::OptimizationType::MAX), + precision.empty() ? "" : fmt::format("\n\t\"precision\": \"{}\", ", precision), algorithm, !attributes.empty() ? fmt::format("\n\t\"experiment_attributes\": [{{{}}}],", fmt::join(attributes, "}, {")) : "", @@ -570,7 +575,8 @@ namespace ioh::logger experiments_.insert({current_filename_, {suite_, problem, algorithm_, common::as_vector(attributes_.experiment), - common::keys(attributes_.run), attribute_names()}}); + common::keys(attributes_.run), attribute_names(), {}, + full_precision_ ? "%.17g" : ""}}); } //! Gets called after the last evaluation of a run diff --git a/include/ioh/logger/flatfile.hpp b/include/ioh/logger/flatfile.hpp index d520c084..6098ef8f 100644 --- a/include/ioh/logger/flatfile.hpp +++ b/include/ioh/logger/flatfile.hpp @@ -60,6 +60,9 @@ namespace ioh::logger //! Output stream std::ofstream out_; + //! Write doubles with round-trip precision ("{:.17g}") instead of "{:.10f}"? + bool full_precision_ = false; + //! Current run @@ -173,16 +176,37 @@ namespace ioh::logger IOH_DBG(xdebug, "print watched properties") for (auto p = properties_vector_.begin(); p != properties_vector_.end();){ - out_ << p->get().call_to_string(log_info, nan_) << (++p != properties_vector_.end() ? sep_ : ""); + const Property &prop = p->get(); + if (full_precision_ && prop.format() == DEFAULT_DOUBLE_FORMAT) + { + const auto value = prop(log_info); + out_ << (value ? fmt::format(FULL_PRECISION_DOUBLE_FORMAT, *value) : nan_); + } + else + out_ << prop.call_to_string(log_info, nan_); + out_ << (++p != properties_vector_.end() ? sep_ : ""); } if (store_positions_) - out_ << sep_ << format("{:f}", fmt::join(log_info.x, sep_)); + out_ << sep_ << (full_precision_ ? format("{:.17g}", fmt::join(log_info.x, sep_)) + : format("{:f}", fmt::join(log_info.x, sep_))); out_ << eol_; out_.flush(); } + /** Write values with round-trip precision (`{:.17g}`) instead of `{:.10f}`. + * + * Applies to the properties that use the default double format (DEFAULT_DOUBLE_FORMAT) and to the x + * positions. Properties with an explicit format, such as evaluations, keep it. Default: false, which keeps + * the output of earlier versions, where values below 5e-11 are written as 0. Set it before the first + * problem is attached. + */ + void set_full_precision(const bool full_precision) { full_precision_ = full_precision; } + + //! Whether values are written with round-trip precision. + bool full_precision() const { return full_precision_; } + //! Accessor for output directory virtual fs::path output_directory() const { return output_directory_; } diff --git a/include/ioh/logger/properties.hpp b/include/ioh/logger/properties.hpp index 546835ba..39b21e98 100644 --- a/include/ioh/logger/properties.hpp +++ b/include/ioh/logger/properties.hpp @@ -12,6 +12,9 @@ namespace ioh //! Default format for storing doubles inline std::string DEFAULT_DOUBLE_FORMAT = "{:.10f}"; + + //! Format for storing doubles with round-trip precision (used by FlatFile::set_full_precision) + inline const std::string FULL_PRECISION_DOUBLE_FORMAT = "{:.17g}"; /** @defgroup Properties Properties * Accessors to values that are to be logged. diff --git a/ioh/iohcpp/logger/__init__.pyi b/ioh/iohcpp/logger/__init__.pyi index d6d0c717..470a8c3e 100644 --- a/ioh/iohcpp/logger/__init__.pyi +++ b/ioh/iohcpp/logger/__init__.pyi @@ -89,6 +89,13 @@ class FlatFile(AbstractLogger): @property def filename(self) -> str: ... + @property + def full_precision(self) -> bool: + """Write values with round-trip precision (%.17g) instead of %.10f. Set it before the first problem is attached.""" + + @full_precision.setter + def full_precision(self, arg: bool, /) -> None: ... + @property def output_directory(self) -> str: ... @@ -170,6 +177,13 @@ class Analyzer(AbstractLogger): Boolean value which indicates whether to store the x-positions in the file """ + @property + def full_precision(self) -> bool: + """Write values with round-trip precision (%.17g) instead of %.10f. Set it before the first problem is attached.""" + + @full_precision.setter + def full_precision(self, arg: bool, /) -> None: ... + def add_experiment_attribute(self, arg0: str, arg1: str, /) -> None: ... def set_experiment_attributes(self, arg: Mapping[str, str], /) -> None: ... diff --git a/ioh/src/logger.cpp b/ioh/src/logger.cpp index a4e8425d..8c61393d 100644 --- a/ioh/src/logger.cpp +++ b/ioh/src/logger.cpp @@ -206,6 +206,10 @@ namespace ioh_python::logger_cpp ) .def_prop_ro("filename", &FlatFile::filename) + .def_prop_rw( + "full_precision", [](const PyFlatFile &f) { return f.full_precision(); }, + [](PyFlatFile &f, const bool v) { f.set_full_precision(v); }, + "Write values with round-trip precision (%.17g) instead of %.10f. Set it before the first problem is attached.") .def_prop_ro("output_directory", [](PyFlatFile &f) { return fs::absolute(f.output_directory()).generic_string(); }) // bound to FlatFile's own methods, so that super() in a Python override runs them once (a virtual call @@ -262,6 +266,10 @@ namespace ioh_python::logger_cpp store_positions: bool = false Boolean value which indicates whether to store the x-positions in the file )pbdoc") + .def_prop_rw( + "full_precision", [](const PyAnalyzerType &a) { return a.full_precision(); }, + [](PyAnalyzerType &a, const bool v) { a.set_full_precision(v); }, + "Write values with round-trip precision (%.17g) instead of %.10f. Set it before the first problem is attached.") .def("add_experiment_attribute", &PyAnalyzerType::add_experiment_attribute) .def("set_experiment_attributes", &PyAnalyzerType::set_experiment_attributes) diff --git a/tests/cpp/logger/test_flatfile.cpp b/tests/cpp/logger/test_flatfile.cpp index f3987d07..22cc0e5e 100644 --- a/tests/cpp/logger/test_flatfile.cpp +++ b/tests/cpp/logger/test_flatfile.cpp @@ -1,6 +1,7 @@ #include "../utils.hpp" #include "ioh/logger/flatfile.hpp" +#include "ioh/logger/analyzer.hpp" #include "ioh/problem/bbob/sphere.hpp" #include "ioh/problem/bbob/attractive_sector.hpp" @@ -121,3 +122,129 @@ TEST_F(BaseTest, logger_flatfile_separator_without_meta_data) check_flatfile("\t", false, false); check_flatfile(",", false, true); } + +namespace +{ + //! A property with a value below the resolution of "{:.10f}" and a fractional part that needs 17 digits + struct TinyValue : logger::Property + { + explicit TinyValue(const std::string &name = "tiny") : Property(name) {} + std::optional operator()(const logger::Info &) const override { return 3e-11; } + }; + + struct ThirdValue : logger::Property + { + explicit ThirdValue(const std::string &name = "third") : Property(name) {} + std::optional operator()(const logger::Info &) const override { return 1.0 / 3.0; } + }; + + std::vector split_fields(const std::string &line) + { + std::istringstream ss(line); + std::vector fields; + for (std::string f; ss >> f;) + fields.push_back(f); + return fields; + } + + std::vector read_lines(const fs::path &file) + { + std::ifstream in(file); + std::vector lines; + for (std::string l; std::getline(in, l);) + lines.push_back(l); + return lines; + } +} // namespace + +TEST_F(BaseTest, flatfile_full_precision) +{ + TinyValue tiny; + ThirdValue third; + watch::Evaluations evaluations; + const std::array files = {"IOH_default_precision.dat", "IOH_full_precision.dat"}; + for (size_t i = 0; i < 2; ++i) + { + auto p = problem::bbob::Sphere(1, 2); + logger::FlatFile logger({trigger::always}, {evaluations, tiny, third}, files[i], ".", " ", "", "None", "\n", + false, true, {}); + EXPECT_FALSE(logger.full_precision()); + logger.set_full_precision(i == 1); + p.attach_logger(logger); + p(std::vector{0.1, 0.2}); + p.detach_logger(); + } + + const auto def = read_lines(files[0]); + const auto full = read_lines(files[1]); + ASSERT_EQ(def.size(), 2u); + ASSERT_EQ(full.size(), 2u); + // The header, and the number of columns, do not change + EXPECT_EQ(def[0], full[0]); + EXPECT_EQ(split_fields(def[1]).size(), split_fields(def[0]).size()); + EXPECT_EQ(split_fields(full[1]).size(), split_fields(full[0]).size()); + + // Today: below 5e-11 is written as 0, and 1/3 is cut after 10 digits + const auto d = split_fields(def[1]), f = split_fields(full[1]); + const size_t n = d.size(); + EXPECT_EQ(d[n - 5], "1"); + EXPECT_EQ(std::stod(d[n - 4]), 0.0); + EXPECT_NE(std::stod(d[n - 3]), 1.0 / 3.0); + + + // With the option: the exact doubles come back + EXPECT_EQ(std::stod(f[n - 4]), 3e-11); + EXPECT_EQ(std::stod(f[n - 3]), 1.0 / 3.0); + // x positions: round-trip as well + EXPECT_EQ(std::stod(f[n - 2]), 0.1); + EXPECT_EQ(std::stod(f[n - 1]), 0.2); + // The evaluation counter keeps its own format + EXPECT_EQ(f[n - 5], "1"); + + for (const auto &file : files) + fs::remove(file); +} + +TEST_F(BaseTest, analyzer_full_precision) +{ + TinyValue tiny; + std::vector data_lines[2]; + std::string json[2]; + for (size_t i = 0; i < 2; ++i) + { + fs::path out; + { + auto p = problem::bbob::Sphere(1, 2); + logger::Analyzer logger({trigger::always}, {tiny}, fs::current_path(), + i == 0 ? "ioh_prec_default" : "ioh_prec_full"); + logger.set_full_precision(i == 1); + out = logger.output_directory(); + p.attach_logger(logger); + p(std::vector{0.1, 0.2}); + p.detach_logger(); + } + data_lines[i] = read_lines(out / "data_f1_Sphere" / "IOHprofiler_f1_DIM2.dat"); + json[i] = ioh::common::file::as_string(out / "IOHprofiler_f1_Sphere.json"); + fs::remove_all(out); + } + + // The header and the column count do not change, and every line still parses + ASSERT_EQ(data_lines[0].size(), 2u); + ASSERT_EQ(data_lines[1].size(), 2u); + EXPECT_EQ(data_lines[0][0], data_lines[1][0]); + const size_t ncol = split_fields(data_lines[0][0]).size(); + EXPECT_EQ(ncol, 3u); + for (size_t i = 0; i < 2; ++i) + { + const auto fields = split_fields(data_lines[i][1]); + ASSERT_EQ(fields.size(), ncol); + for (const auto &f : fields) + EXPECT_NO_THROW(std::stod(f)); + } + EXPECT_EQ(std::stod(split_fields(data_lines[0][1])[2]), 0.0); + EXPECT_EQ(std::stod(split_fields(data_lines[1][1])[2]), 3e-11); + + // The precision is recorded in the JSON metadata, as an extra key, only when the option is on + EXPECT_EQ(json[0].find("precision"), std::string::npos); + EXPECT_NE(json[1].find("\"precision\": \"%.17g\""), std::string::npos); +} diff --git a/tests/python/test_logger.py b/tests/python/test_logger.py index 57ba2024..487dda7b 100644 --- a/tests/python/test_logger.py +++ b/tests/python/test_logger.py @@ -1,3 +1,4 @@ +import gc import json import os import pickle @@ -63,6 +64,54 @@ def test_analyzer(self): for i in range(5): self.assertEqual(i*10, float(next(f).split()[-1])) + def test_full_precision(self): + import tempfile + + class Tiny: + value = 3e-11 + + rows = {} + for full in (False, True): + with tempfile.TemporaryDirectory() as tmp: + pr = ioh.get_problem("Sphere", 1, 2) + l = ioh.logger.Analyzer([ioh.logger.trigger.ALWAYS], root=tmp, folder_name="d") + self.assertFalse(l.full_precision) + l.full_precision = full + l.watch(Tiny(), ["value"]) + pr.attach_logger(l) + pr([0.1, 0.2]) + l.close() + with open(os.path.join(tmp, "d", "data_f1_Sphere", "IOHprofiler_f1_DIM2.dat")) as f: + header, row = f.read().splitlines()[:2] + with open(os.path.join(tmp, "d", "IOHprofiler_f1_Sphere.json")) as f: + meta = json.load(f) + self.assertEqual(len(header.split()), len(row.split())) + rows[full] = (header, float(row.split()[-1]), meta) + self.assertEqual(rows[False][0], rows[True][0]) + self.assertEqual(rows[False][1], 0.0) + self.assertEqual(rows[True][1], 3e-11) + self.assertNotIn("precision", rows[False][2]) + self.assertEqual(rows[True][2]["precision"], "%.17g") + + def test_flatfile_full_precision(self): + pr = ioh.get_problem("Sphere", 1, 2) + class Tiny: + value = 3e-11 + l = ioh.logger.FlatFile([ioh.logger.trigger.ALWAYS], [], filename="IOH_fp.dat", separator=" ") + l.full_precision = True + self.assertTrue(l.full_precision) + l.watch(Tiny(), ["value"]) + pr.attach_logger(l) + pr([0.1, 0.2]) + with open("IOH_fp.dat") as f: + lines = f.read().splitlines() + # FlatFile has no close(): release the file (the problem holds the logger) before removing it, for Windows + pr.detach_logger() + del l, pr + gc.collect() + os.remove("IOH_fp.dat") + self.assertEqual(float(lines[-1].split()[-1]), 3e-11) + def test_triggers(self): always = ioh.logger.trigger.Always() on_improvement = ioh.logger.trigger.OnImprovement()