E05: Full-precision output

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## Unreleased
 
+### Added
+
+* `FlatFile` and `Analyzer` (C++ `set_full_precision(true)`, Python `full_precision = True`) can write values with
+  round-trip precision (`%.17g`) instead of `%.10f`. With `%.10f`, values below 5e-11 are written as 0, which hides
+  targets such as 1e-8 to 1e-10 relative to the optimum. The option also applies to the x columns, and it is off by
+  default: existing output does not change. Set it before attaching the first problem. The Analyzer records it in the
+  JSON metadata as `"precision": "%.17g"`; the key is absent for the default precision, and readers that do not know
+  it can ignore it. The header and the number of columns are the same.
+
 ### Fixed
 
 * `FlatFile` data rows had no separator between the last meta data column (`run`) and the first logged property, so

include/ioh/logger/analyzer.hpp

@@ -188,6 +188,8 @@ namespace ioh::logger
                 const std::vector<std::string> attribute_names;
                 //! Scenarios
                 std::vector<ScenarioInfo> dims;
+                //! Printf-style precision of the values in the data files ("" for the legacy "%.10f", not written)
+                const std::string precision;
 
                 /**
                  * @brief Construct a new Experiment Info object
@@ -199,16 +201,18 @@ namespace ioh::logger
                  * @param run_attribute_names Run attributes
                  * @param attribute_names attributes names
                  * @param dims Scenarios
+                 * @param precision Precision of the values in the data files, written as "precision" when not empty
                  */
                 ExperimentInfo(const std::string &suite, const problem::MetaData &problem,
                                const AlgorithmInfo &algorithm,
                                const std::vector<Attribute<std::string>> &attributes = {},
                                const std::vector<std::string> &run_attribute_names = {},
                                const std::vector<std::string> &attribute_names = {},
-                               const std::vector<ScenarioInfo> &dims = {}) :
+                               const std::vector<ScenarioInfo> &dims = {}, const std::string &precision = "") :
                     suite(suite),
                     problem(problem), algorithm(algorithm), attributes(attributes),
-                    run_attribute_names(run_attribute_names), attribute_names(attribute_names), dims(dims)
+                    run_attribute_names(run_attribute_names), attribute_names(attribute_names), dims(dims),
+                    precision(precision)
                 {
                 }
 
@@ -217,10 +221,11 @@ namespace ioh::logger
                     return fmt::format(
                         "{{\n\t\"version\": \"{}\", \n\t\"suite\": \"{}\", \n\t\"function_id\": {}, "
                         "\n\t\"function_name\": \"{}\", "
-                        "\n\t\"maximization\": {}, "
+                        "\n\t\"maximization\": {}, {}"
                         "\n\t\"algorithm\": {{{}}},{}{}{}\n\t\"scenarios\": [\n\t\t{{{}}}\n\t]\n}}\n",
                         PROJECT_VER, suite, problem.problem_id, problem.name,
-                        (problem.optimization_type == common::OptimizationType::MAX), algorithm,
+                        (problem.optimization_type == common::OptimizationType::MAX),
+                        precision.empty() ? "" : fmt::format("\n\t\"precision\": \"{}\", ", precision), algorithm,
                         !attributes.empty()
                             ? fmt::format("\n\t\"experiment_attributes\": [{{{}}}],", fmt::join(attributes, "}, {"))
                             : "",
@@ -570,7 +575,8 @@ namespace ioh::logger
                         experiments_.insert({current_filename_,
                                              {suite_, problem, algorithm_,
                                               common::as_vector<str, str, sAttr>(attributes_.experiment),
-                                              common::keys(attributes_.run), attribute_names()}});
+                                              common::keys(attributes_.run), attribute_names(), {},
+                                              full_precision_ ? "%.17g" : ""}});
                 }
 
                 //! Gets called after the last evaluation of a run

include/ioh/logger/flatfile.hpp

@@ -60,6 +60,9 @@ namespace ioh::logger
         //! Output stream
         std::ofstream out_;
 
+        //! Write doubles with round-trip precision ("{:.17g}") instead of "{:.10f}"?
+        bool full_precision_ = false;
+
 
 
         //! Current run
@@ -173,16 +176,37 @@ namespace ioh::logger
             IOH_DBG(xdebug, "print watched properties")
 
             for (auto p = properties_vector_.begin(); p != properties_vector_.end();){
-                out_ << p->get().call_to_string(log_info, nan_) << (++p != properties_vector_.end() ? sep_ : "");
+                const Property &prop = p->get();
+                if (full_precision_ && prop.format() == DEFAULT_DOUBLE_FORMAT)
+                {
+                    const auto value = prop(log_info);
+                    out_ << (value ? fmt::format(FULL_PRECISION_DOUBLE_FORMAT, *value) : nan_);
+                }
+                else
+                    out_ << prop.call_to_string(log_info, nan_);
+                out_ << (++p != properties_vector_.end() ? sep_ : "");
             }
 
             if (store_positions_)
-                out_ << sep_ << format("{:f}", fmt::join(log_info.x, sep_));
+                out_ << sep_ << (full_precision_ ? format("{:.17g}", fmt::join(log_info.x, sep_))
+                                                      : format("{:f}", fmt::join(log_info.x, sep_)));
 
             out_ << eol_;
             out_.flush();
         }
 
+        /** Write values with round-trip precision (`{:.17g}`) instead of `{:.10f}`.
+         *
+         * Applies to the properties that use the default double format (DEFAULT_DOUBLE_FORMAT) and to the x
+         * positions. Properties with an explicit format, such as evaluations, keep it. Default: false, which keeps
+         * the output of earlier versions, where values below 5e-11 are written as 0. Set it before the first
+         * problem is attached.
+         */
+        void set_full_precision(const bool full_precision) { full_precision_ = full_precision; }
+
+        //! Whether values are written with round-trip precision.
+        bool full_precision() const { return full_precision_; }
+
         //! Accessor for output directory
         virtual fs::path output_directory() const { return output_directory_; }
 

include/ioh/logger/properties.hpp

@@ -12,6 +12,9 @@ namespace ioh
 
         //! Default format for storing doubles
         inline std::string DEFAULT_DOUBLE_FORMAT = "{:.10f}";
+
+        //! Format for storing doubles with round-trip precision (used by FlatFile::set_full_precision)
+        inline const std::string FULL_PRECISION_DOUBLE_FORMAT = "{:.17g}";
 
         /** @defgroup Properties Properties
          * Accessors to values that are to be logged.

ioh/iohcpp/logger/__init__.pyi

@@ -89,6 +89,13 @@ class FlatFile(AbstractLogger):
     @property
     def filename(self) -> str: ...
 
+    @property
+    def full_precision(self) -> bool:
+        """Write values with round-trip precision (%.17g) instead of %.10f. Set it before the first problem is attached."""
+
+    @full_precision.setter
+    def full_precision(self, arg: bool, /) -> None: ...
+
     @property
     def output_directory(self) -> str: ...
 
@@ -170,6 +177,13 @@ class Analyzer(AbstractLogger):
             Boolean value which indicates whether to store the x-positions in the file
         """
 
+    @property
+    def full_precision(self) -> bool:
+        """Write values with round-trip precision (%.17g) instead of %.10f. Set it before the first problem is attached."""
+
+    @full_precision.setter
+    def full_precision(self, arg: bool, /) -> None: ...
+
     def add_experiment_attribute(self, arg0: str, arg1: str, /) -> None: ...
 
     def set_experiment_attributes(self, arg: Mapping[str, str], /) -> None: ...

ioh/src/logger.cpp

@@ -206,6 +206,10 @@ namespace ioh_python::logger_cpp
 
                  )
             .def_prop_ro("filename", &FlatFile::filename)
+            .def_prop_rw(
+                "full_precision", [](const PyFlatFile &f) { return f.full_precision(); },
+                [](PyFlatFile &f, const bool v) { f.set_full_precision(v); },
+                "Write values with round-trip precision (%.17g) instead of %.10f. Set it before the first problem is attached.")
             .def_prop_ro("output_directory",
                          [](PyFlatFile &f) { return fs::absolute(f.output_directory()).generic_string(); })
             // bound to FlatFile's own methods, so that super() in a Python override runs them once (a virtual call
@@ -262,6 +266,10 @@ namespace ioh_python::logger_cpp
                 store_positions: bool = false
                     Boolean value which indicates whether to store the x-positions in the file
             )pbdoc")
+            .def_prop_rw(
+                "full_precision", [](const PyAnalyzerType &a) { return a.full_precision(); },
+                [](PyAnalyzerType &a, const bool v) { a.set_full_precision(v); },
+                "Write values with round-trip precision (%.17g) instead of %.10f. Set it before the first problem is attached.")
             .def("add_experiment_attribute", &PyAnalyzerType::add_experiment_attribute)
             .def("set_experiment_attributes", &PyAnalyzerType::set_experiment_attributes)
 

tests/cpp/logger/test_flatfile.cpp

@@ -1,6 +1,7 @@
 #include "../utils.hpp"
 
 #include "ioh/logger/flatfile.hpp"
+#include "ioh/logger/analyzer.hpp"
 #include "ioh/problem/bbob/sphere.hpp"
 #include "ioh/problem/bbob/attractive_sector.hpp"
 
@@ -121,3 +122,129 @@ TEST_F(BaseTest, logger_flatfile_separator_without_meta_data)
     check_flatfile("\t", false, false);
     check_flatfile(",", false, true);
 }
+
+namespace
+{
+    //! A property with a value below the resolution of "{:.10f}" and a fractional part that needs 17 digits
+    struct TinyValue : logger::Property
+    {
+        explicit TinyValue(const std::string &name = "tiny") : Property(name) {}
+        std::optional<double> operator()(const logger::Info &) const override { return 3e-11; }
+    };
+
+    struct ThirdValue : logger::Property
+    {
+        explicit ThirdValue(const std::string &name = "third") : Property(name) {}
+        std::optional<double> operator()(const logger::Info &) const override { return 1.0 / 3.0; }
+    };
+
+    std::vector<std::string> split_fields(const std::string &line)
+    {
+        std::istringstream ss(line);
+        std::vector<std::string> fields;
+        for (std::string f; ss >> f;)
+            fields.push_back(f);
+        return fields;
+    }
+
+    std::vector<std::string> read_lines(const fs::path &file)
+    {
+        std::ifstream in(file);
+        std::vector<std::string> lines;
+        for (std::string l; std::getline(in, l);)
+            lines.push_back(l);
+        return lines;
+    }
+} // namespace
+
+TEST_F(BaseTest, flatfile_full_precision)
+{
+    TinyValue tiny;
+    ThirdValue third;
+    watch::Evaluations evaluations;
+    const std::array<std::string, 2> files = {"IOH_default_precision.dat", "IOH_full_precision.dat"};
+    for (size_t i = 0; i < 2; ++i)
+    {
+        auto p = problem::bbob::Sphere(1, 2);
+        logger::FlatFile logger({trigger::always}, {evaluations, tiny, third}, files[i], ".", " ", "", "None", "\n",
+                                false, true, {});
+        EXPECT_FALSE(logger.full_precision());
+        logger.set_full_precision(i == 1);
+        p.attach_logger(logger);
+        p(std::vector<double>{0.1, 0.2});
+        p.detach_logger();
+    }
+
+    const auto def = read_lines(files[0]);
+    const auto full = read_lines(files[1]);
+    ASSERT_EQ(def.size(), 2u);
+    ASSERT_EQ(full.size(), 2u);
+    // The header, and the number of columns, do not change
+    EXPECT_EQ(def[0], full[0]);
+    EXPECT_EQ(split_fields(def[1]).size(), split_fields(def[0]).size());
+    EXPECT_EQ(split_fields(full[1]).size(), split_fields(full[0]).size());
+
+    // Today: below 5e-11 is written as 0, and 1/3 is cut after 10 digits
+    const auto d = split_fields(def[1]), f = split_fields(full[1]);
+    const size_t n = d.size();
+    EXPECT_EQ(d[n - 5], "1");
+    EXPECT_EQ(std::stod(d[n - 4]), 0.0);
+    EXPECT_NE(std::stod(d[n - 3]), 1.0 / 3.0);
+
+
+    // With the option: the exact doubles come back
+    EXPECT_EQ(std::stod(f[n - 4]), 3e-11);
+    EXPECT_EQ(std::stod(f[n - 3]), 1.0 / 3.0);
+    // x positions: round-trip as well
+    EXPECT_EQ(std::stod(f[n - 2]), 0.1);
+    EXPECT_EQ(std::stod(f[n - 1]), 0.2);
+    // The evaluation counter keeps its own format
+    EXPECT_EQ(f[n - 5], "1");
+
+    for (const auto &file : files)
+        fs::remove(file);
+}
+
+TEST_F(BaseTest, analyzer_full_precision)
+{
+    TinyValue tiny;
+    std::vector<std::string> data_lines[2];
+    std::string json[2];
+    for (size_t i = 0; i < 2; ++i)
+    {
+        fs::path out;
+        {
+            auto p = problem::bbob::Sphere(1, 2);
+            logger::Analyzer logger({trigger::always}, {tiny}, fs::current_path(),
+                                    i == 0 ? "ioh_prec_default" : "ioh_prec_full");
+            logger.set_full_precision(i == 1);
+            out = logger.output_directory();
+            p.attach_logger(logger);
+            p(std::vector<double>{0.1, 0.2});
+            p.detach_logger();
+        }
+        data_lines[i] = read_lines(out / "data_f1_Sphere" / "IOHprofiler_f1_DIM2.dat");
+        json[i] = ioh::common::file::as_string(out / "IOHprofiler_f1_Sphere.json");
+        fs::remove_all(out);
+    }
+
+    // The header and the column count do not change, and every line still parses
+    ASSERT_EQ(data_lines[0].size(), 2u);
+    ASSERT_EQ(data_lines[1].size(), 2u);
+    EXPECT_EQ(data_lines[0][0], data_lines[1][0]);
+    const size_t ncol = split_fields(data_lines[0][0]).size();
+    EXPECT_EQ(ncol, 3u);
+    for (size_t i = 0; i < 2; ++i)
+    {
+        const auto fields = split_fields(data_lines[i][1]);
+        ASSERT_EQ(fields.size(), ncol);
+        for (const auto &f : fields)
+            EXPECT_NO_THROW(std::stod(f));
+    }
+    EXPECT_EQ(std::stod(split_fields(data_lines[0][1])[2]), 0.0);
+    EXPECT_EQ(std::stod(split_fields(data_lines[1][1])[2]), 3e-11);
+
+    // The precision is recorded in the JSON metadata, as an extra key, only when the option is on
+    EXPECT_EQ(json[0].find("precision"), std::string::npos);
+    EXPECT_NE(json[1].find("\"precision\": \"%.17g\""), std::string::npos);
+}

tests/python/test_logger.py

@@ -1,3 +1,4 @@
+import gc
 import json
 import os
 import pickle
@@ -63,6 +64,54 @@ def test_analyzer(self):
             for i in range(5):
                 self.assertEqual(i*10, float(next(f).split()[-1]))
 
+    def test_full_precision(self):
+        import tempfile
+
+        class Tiny:
+            value = 3e-11
+
+        rows = {}
+        for full in (False, True):
+            with tempfile.TemporaryDirectory() as tmp:
+                pr = ioh.get_problem("Sphere", 1, 2)
+                l = ioh.logger.Analyzer([ioh.logger.trigger.ALWAYS], root=tmp, folder_name="d")
+                self.assertFalse(l.full_precision)
+                l.full_precision = full
+                l.watch(Tiny(), ["value"])
+                pr.attach_logger(l)
+                pr([0.1, 0.2])
+                l.close()
+                with open(os.path.join(tmp, "d", "data_f1_Sphere", "IOHprofiler_f1_DIM2.dat")) as f:
+                    header, row = f.read().splitlines()[:2]
+                with open(os.path.join(tmp, "d", "IOHprofiler_f1_Sphere.json")) as f:
+                    meta = json.load(f)
+                self.assertEqual(len(header.split()), len(row.split()))
+                rows[full] = (header, float(row.split()[-1]), meta)
+        self.assertEqual(rows[False][0], rows[True][0])
+        self.assertEqual(rows[False][1], 0.0)
+        self.assertEqual(rows[True][1], 3e-11)
+        self.assertNotIn("precision", rows[False][2])
+        self.assertEqual(rows[True][2]["precision"], "%.17g")
+
+    def test_flatfile_full_precision(self):
+        pr = ioh.get_problem("Sphere", 1, 2)
+        class Tiny:
+            value = 3e-11
+        l = ioh.logger.FlatFile([ioh.logger.trigger.ALWAYS], [], filename="IOH_fp.dat", separator=" ")
+        l.full_precision = True
+        self.assertTrue(l.full_precision)
+        l.watch(Tiny(), ["value"])
+        pr.attach_logger(l)
+        pr([0.1, 0.2])
+        with open("IOH_fp.dat") as f:
+            lines = f.read().splitlines()
+        # FlatFile has no close(): release the file (the problem holds the logger) before removing it, for Windows
+        pr.detach_logger()
+        del l, pr
+        gc.collect()
+        os.remove("IOH_fp.dat")
+        self.assertEqual(float(lines[-1].split()[-1]), 3e-11)
+
     def test_triggers(self):
         always = ioh.logger.trigger.Always()
         on_improvement = ioh.logger.trigger.OnImprovement()