diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c2415b1..021f818e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +### Added + +* The `AdversarialAttack` problems: black-box adversarial attacks on a bundled MNIST (784 variables) and CIFAR-10 + (3072 variables) classifier, as a many-dimensional benchmark. The objective, maximized, is + `-log softmax(model(x + delta))[y] - alpha * ||delta||_2` for a fixed, correctly classified image `x` with label `y` + and a perturbation `delta` in `[-eps, eps]^d`. The 12 problems have ids 5001-5006 (MNIST) and 5011-5016 (CIFAR-10), + and are created by name (`AdversarialMNIST5001`, ...) or id, in `ioh.ProblemClass.ADVERSARIAL`, in the + `AdversarialAttack` suite (suite id 5) and in the factory of all real problems. The `instance` selects the base image + (`(instance - 1) mod 100`) and so changes the whole landscape; `n_variables` is ignored. The network is a dependency + free C++ implementation, whose weights (float16) and 100 images per dataset are new data files: `static/` grows by + about 11 MB. Reference: . +* Because the ids 5001-5016 are taken, a real problem that is registered at run time (`wrap_problem`, `register_problem`) + gets the next free id after them, 5017, instead of 1121. + ### Fixed * `FlatFile` data rows had no separator between the last meta data column (`run`) and the first logged property, so diff --git a/CMakeLists.txt b/CMakeLists.txt index c66a82cf..c916d3f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,7 @@ file(GLOB_RECURSE HEADERS include/ioh *.hpp) # problems and suites (src/registry), so that only these files instantiate every problem class add_library(ioh STATIC src/data.cpp + src/registry/adversarial.cpp src/registry/bbob.cpp src/registry/cec.cpp src/registry/dynamic_bin_val.cpp diff --git a/README.md b/README.md index 8081d13e..50c4b5ac 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ * BBOB (Single Objective Noiseless) (COCO) * SBOX-COST (COCO) * StarDiscrepancy +* AdversarialAttack (black-box adversarial attacks on MNIST and CIFAR-10 classifiers) * PBO * Submodular Graph Problems * CEC 2013 Special Session and Competition on Niching Methods for Multimodal Function Optimization diff --git a/doc/python/source/python/adversarial.rst b/doc/python/source/python/adversarial.rst new file mode 100644 index 00000000..e003d20f --- /dev/null +++ b/doc/python/source/python/adversarial.rst @@ -0,0 +1,9 @@ +Adversarial Attack +================== +Black-box adversarial attacks on image classifiers, as a many-dimensional benchmark + +.. automodapi:: ioh.iohcpp.problem + :inherited-members: + :no-main-docstr: + :no-heading: + :include: AdversarialAttack diff --git a/doc/python/source/python/problem.rst b/doc/python/source/python/problem.rst index b85ee87f..26f3031a 100644 --- a/doc/python/source/python/problem.rst +++ b/doc/python/source/python/problem.rst @@ -20,6 +20,7 @@ Problem Types pbo.rst submodular.rst star_discrepancy.rst + adversarial.rst wmodel.rst cec2013.rst cec2022.rst \ No newline at end of file diff --git a/doc/python/source/upgrading.md b/doc/python/source/upgrading.md index d0365901..612056d3 100644 --- a/doc/python/source/upgrading.md +++ b/doc/python/source/upgrading.md @@ -133,7 +133,7 @@ existing ids and returns them. A suite declares `registry_id` and `registry_name` and is registered with `register_suite()`; derive from `Suite<...>` instead of `RealSuite` or `IntegerSuite` (which registered a suite under its class name with the next free -id). The built-in suites have fixed ids: Real 1, BBOB 2, SBOX 3, RealStarDiscrepancy 4; Integer 1, PBO 2, Submodular 3, +id). The built-in suites have fixed ids: Real 1, BBOB 2, SBOX 3, RealStarDiscrepancy 4, AdversarialAttack 5; Integer 1, PBO 2, Submodular 3, IntegerStarDiscrepancy 4. ### Removed and renamed diff --git a/include/ioh/common/builtins.hpp b/include/ioh/common/builtins.hpp index 0a4530bf..79057158 100644 --- a/include/ioh/common/builtins.hpp +++ b/include/ioh/common/builtins.hpp @@ -27,6 +27,11 @@ namespace ioh::problem struct GraphProblem; } + namespace adversarial + { + class AdversarialAttack; + } + namespace star_discrepancy::real { class StarDiscrepancy; @@ -58,7 +63,7 @@ namespace ioh::common //! @name The factories with built-in entries ///@{ - //! All real single-objective problems: BBOB, CEC2013, CEC2022, real star discrepancy + //! All real single-objective problems: BBOB, CEC2013, CEC2022, real star discrepancy, adversarial attack void add_builtins(Factory, int, int> &); //! All integer single-objective problems: PBO, graph, integer star discrepancy, DynamicBinVal @@ -71,10 +76,11 @@ namespace ioh::common void add_builtins(Factory &); void add_builtins(Factory &); void add_builtins(Factory &); + void add_builtins(Factory &); void add_builtins(Factory &); void add_builtins(Factory &); - //! The real suites: Real, BBOB, SBOX, RealStarDiscrepancy + //! The real suites: Real, BBOB, SBOX, RealStarDiscrepancy, AdversarialAttack void add_builtins(Factory>, std::vector, std::vector, std::vector> &); diff --git a/include/ioh/problem.hpp b/include/ioh/problem.hpp index 04c94d13..e918dd48 100644 --- a/include/ioh/problem.hpp +++ b/include/ioh/problem.hpp @@ -19,3 +19,5 @@ #include "problem/star_discrepancy/real.hpp" #include "problem/star_discrepancy/integer.hpp" +#include "problem/adversarial/adversarial_attack.hpp" + diff --git a/include/ioh/problem/adversarial/adversarial_attack.hpp b/include/ioh/problem/adversarial/adversarial_attack.hpp new file mode 100644 index 00000000..ea6e5259 --- /dev/null +++ b/include/ioh/problem/adversarial/adversarial_attack.hpp @@ -0,0 +1,191 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "ioh/common/format.hpp" +#include "ioh/problem/single.hpp" +#include "network.hpp" + +namespace ioh::problem +{ + namespace adversarial + { + /** + * @brief Numerically stable log(softmax(logits)[label]) = logits[label] - logsumexp(logits). + */ + inline double log_softmax_at(const std::vector &logits, const int label) + { + const float max_logit = *std::max_element(logits.begin(), logits.end()); + double sum_exp = 0.0; + for (const auto v : logits) + sum_exp += std::exp(static_cast(v) - static_cast(max_logit)); + const double log_sum_exp = static_cast(max_logit) + std::log(sum_exp); + return static_cast(logits[static_cast(label)]) - log_sum_exp; + } + + /** + * @brief Black-box adversarial-attack objective on a fixed image classifier, + * proposed as a many-dimensional global optimization benchmark (amhe-attack, + * https://github.com/wz7475/amhe-attack). For a fixed base image `x` (from a + * bundled 100-image bank per dataset) with true label `y`, and a search vector + * `delta` (the perturbation) bounded in [-eps, eps]^d: + * + * f(delta) = -log softmax(model(x + delta))[y] - alpha * ||delta||_2 + * + * maximized. `x + delta` is NOT clamped to [0,1] by default (matching the + * reference implementation's main experimental path in amheattack/attack.py); + * pass `clamp_input=true` for the amheattack/stochastic_attack.py semantics. + * + * Unlike most IOH problems, `instance` selects which of the 100 bundled base + * images is attacked -- it changes the entire landscape, not just an affine + * transformation of one fixed function. There is no known optimum, so + * `optimum()` is left at its default (NaN). + */ + class AdversarialAttack : public RealSingleObjective + { + public: + //! The registry entries (see registry_entries_of): 12 (dataset, eps, alpha) presets with ids 5001-5006 + //! (MNIST) and 5011-5016 (CIFAR-10). Listing them loads no data: the network and the image bank are + //! loaded when the first problem of a dataset is created. + static std::vector> registry_entries(); + + private: + std::shared_ptr net_; + std::shared_ptr bank_; + std::string dataset_; + double eps_; + double alpha_; + int image_index_; + int label_; + bool clamp_input_; + + // Per-object scratch buffers: NOT shared between instances, so one + // AdversarialAttack object must not be evaluated concurrently from + // multiple threads (see tests/cpp/problem/test_threaded.cpp). + mutable std::vector input_; + mutable std::vector logits_; + mutable std::vector scratch_a_; + mutable std::vector scratch_b_; + + [[nodiscard]] static int dimension_for(const std::string &dataset) + { + return static_cast(get_network(dataset)->input_size()); + } + + protected: + double evaluate(const std::vector &x) override + { + const auto &base = bank_->images[static_cast(image_index_)]; + double sq = 0.0; + for (size_t i = 0; i < x.size(); ++i) + { + const float v = base[i] + static_cast(x[i]); + input_[i] = clamp_input_ ? std::min(1.0f, std::max(0.0f, v)) : v; + sq += x[i] * x[i]; + } + + net_->forward(input_, logits_, scratch_a_, scratch_b_); + const double log_p_true = log_softmax_at(logits_, label_); + return -log_p_true - alpha_ * std::sqrt(sq); + } + + public: + /** + * @brief Construct a new AdversarialAttack object. + * + * @param problem_id the id of the problem (one per (dataset, eps, alpha) preset) + * @param instance the instance id: selects the base image, 1-indexed into the + * 100-image bank as `(instance - 1) mod 100` + * @param n_variables ignored: dimension is fixed by `dataset` (784 for mnist, + * 3072 for cifar10) and derived from the loaded network + * @param name the name of the problem + * @param dataset "mnist" or "cifar10" + * @param eps the perturbation bound: search space is [-eps, eps]^d + * @param alpha weight of the L2 perturbation-size penalty + * @param clamp_input whether to clamp x + delta to [0, 1] before classification + */ + AdversarialAttack(const int problem_id, const int instance, const int n_variables, + const std::string &name, const std::string &dataset, const double eps, + const double alpha, const bool clamp_input = false) : + RealSingleObjective(MetaData(problem_id, instance, name, dimension_for(dataset), + common::OptimizationType::MAX), + Bounds(dimension_for(dataset), -eps, eps)), + net_(get_network(dataset)), bank_(get_images(dataset)), dataset_(dataset), eps_(eps), alpha_(alpha), + image_index_(static_cast((static_cast(instance - 1)) % bank_->images.size())), + label_(bank_->labels[static_cast(image_index_)]), clamp_input_(clamp_input), + input_(net_->input_size()), scratch_a_(net_->scratch_size()), scratch_b_(net_->scratch_size()) + { + static_cast(n_variables); // dimension is derived from `dataset`, not user-supplied + // Negative weight, matching star_discrepancy: this is a MAX problem, so a hard + // bounds violation must push y towards -infinity, not +infinity. + enforce_bounds(-std::numeric_limits::infinity(), constraint::Enforced::HARD); + } + + [[nodiscard]] const std::string &dataset() const { return dataset_; } + [[nodiscard]] double eps() const { return eps_; } + [[nodiscard]] double alpha() const { return alpha_; } + [[nodiscard]] int image_index() const { return image_index_; } + [[nodiscard]] int true_label() const { return label_; } + [[nodiscard]] bool clamp_input() const { return clamp_input_; } + + //! Softmax probabilities from the most recent `evaluate` call. + [[nodiscard]] std::vector probabilities() const + { + const float max_logit = *std::max_element(logits_.begin(), logits_.end()); + std::vector probs(logits_.size()); + double sum = 0.0; + for (size_t i = 0; i < logits_.size(); ++i) + { + probs[i] = std::exp(static_cast(logits_[i] - max_logit)); + sum += probs[i]; + } + for (auto &p : probs) + p /= sum; + return probs; + } + + //! argmax of the most recent `evaluate` call's logits; != true_label() means the attack succeeded. + [[nodiscard]] int predicted_label() const + { + return static_cast(std::max_element(logits_.begin(), logits_.end()) - logits_.begin()); + } + }; + } // namespace adversarial + + inline std::vector> + adversarial::AdversarialAttack::registry_entries() + { + struct Preset + { + int id; + const char *dataset; + double eps; + double alpha; + }; + + const std::vector presets = { + {5001, "mnist", 0.1, 0.0}, {5002, "mnist", 0.3, 0.0}, {5003, "mnist", 1.0, 0.0}, + {5004, "mnist", 0.1, 0.05}, {5005, "mnist", 0.3, 0.05}, {5006, "mnist", 1.0, 0.05}, + {5011, "cifar10", 0.1, 0.0}, {5012, "cifar10", 0.3, 0.0}, {5013, "cifar10", 1.0, 0.0}, + {5014, "cifar10", 0.1, 0.05}, {5015, "cifar10", 0.3, 0.05}, {5016, "cifar10", 1.0, 0.05}, + }; + + std::vector> entries; + for (const auto &preset : presets) + { + const std::string dataset = preset.dataset; + const auto name = fmt::format("Adversarial{}{}", dataset == "mnist" ? "MNIST" : "CIFAR10", preset.id); + entries.push_back({preset.id, name, + [preset, dataset, name](const int instance, const int n_variables) { + return std::make_shared(preset.id, instance, n_variables, name, + dataset, preset.eps, preset.alpha); + }}); + } + return entries; + } +} // namespace ioh::problem diff --git a/include/ioh/problem/adversarial/network.hpp b/include/ioh/problem/adversarial/network.hpp new file mode 100644 index 00000000..0cd0ee5d --- /dev/null +++ b/include/ioh/problem/adversarial/network.hpp @@ -0,0 +1,324 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "tensor_file.hpp" + +namespace ioh::problem::adversarial +{ + //! A single 2D convolution layer (stride 1, symmetric padding), weight shape [out,in,kh,kw]. + struct ConvLayer + { + Tensor weight; + Tensor bias; + size_t padding{}; + }; + + //! A single dense (fully-connected) layer, weight shape [out,in]. + struct DenseLayer + { + Tensor weight; + Tensor bias; + }; + + /** + * @brief A small, fixed, dependency-free feed-forward CNN: alternating + * Conv2d+ReLU (some followed by 2x2 max pooling), flatten, then + * Linear+ReLU, Linear. Reproduces the eval-mode forward pass of the + * torch reference models in amheattack/models.py (dropout is the + * identity in eval mode, so it is omitted here entirely). + * + * Immutable and safe to share (via shared_ptr) between AdversarialAttack + * instances of the same dataset -- only the caller-provided scratch + * buffers in `forward` carry per-evaluation state. + */ + struct Network + { + std::vector convs; + //! Indices (into `convs`) of conv layers immediately followed by a 2x2 max pool. + std::vector pool_after; + std::vector dense; + size_t c{}, h{}, w{}; + + [[nodiscard]] size_t input_size() const { return c * h * w; } + [[nodiscard]] size_t n_classes() const { return dense.empty() ? 0 : dense.back().bias.numel(); } + + //! Upper bound on the largest activation buffer needed anywhere in the forward pass. + [[nodiscard]] size_t scratch_size() const + { + size_t cur_c = c, cur_h = h, cur_w = w; + size_t max_elems = cur_c * cur_h * cur_w; + for (size_t i = 0; i < convs.size(); ++i) + { + cur_c = convs[i].weight.shape[0]; + max_elems = std::max(max_elems, cur_c * cur_h * cur_w); + if (pool_after[i]) + { + cur_h /= 2; + cur_w /= 2; + } + } + for (const auto &d : dense) + max_elems = std::max(max_elems, d.weight.shape[0]); + return max_elems; + } + + /** + * @brief Runs the forward pass. `input` must have `input_size()` elements + * in NCHW (C-outer, W-innermost) layout, un-normalized in [0, 1]-ish range + * (no clamping is performed here). `logits` is resized to `n_classes()`. + * `scratch_a`/`scratch_b` are ping-ponged activation buffers, resized to + * `scratch_size()` on first use by the caller. + */ + void forward(const std::vector &input, std::vector &logits, std::vector &scratch_a, + std::vector &scratch_b) const + { + const float *cur = input.data(); + size_t cur_c = c, cur_h = h, cur_w = w; + float *buf_a = scratch_a.data(); + float *buf_b = scratch_b.data(); + + for (size_t i = 0; i < convs.size(); ++i) + { + const auto &layer = convs[i]; + const size_t out_c = layer.weight.shape[0]; + const size_t in_c = layer.weight.shape[1]; + const size_t kh = layer.weight.shape[2]; + const size_t kw = layer.weight.shape[3]; + if (in_c != cur_c) + throw std::runtime_error("adversarial::Network: channel mismatch in conv layer"); + + float *out = buf_a; + conv2d_relu(cur, cur_c, cur_h, cur_w, layer.weight.data.data(), layer.bias.data.data(), out_c, kh, kw, + layer.padding, out); + cur = out; + cur_c = out_c; + std::swap(buf_a, buf_b); + + if (pool_after[i]) + { + float *pooled = buf_a; + maxpool2(cur, cur_c, cur_h, cur_w, pooled); + cur = pooled; + cur_h /= 2; + cur_w /= 2; + std::swap(buf_a, buf_b); + } + } + + const size_t flat = cur_c * cur_h * cur_w; + std::vector flat_copy; + const float *dense_in = cur; + if (dense_in == input.data()) + { + // No conv layers at all: copy input so we never read/write the same buffer. + flat_copy.assign(input.begin(), input.end()); + dense_in = flat_copy.data(); + } + + for (size_t i = 0; i < dense.size(); ++i) + { + const bool is_last = (i + 1 == dense.size()); + const auto &layer = dense[i]; + const size_t out_n = layer.weight.shape[0]; + const size_t in_n = layer.weight.shape[1]; + if (in_n != (i == 0 ? flat : dense[i - 1].weight.shape[0])) + throw std::runtime_error("adversarial::Network: dimension mismatch in dense layer"); + + float *out = is_last ? nullptr : buf_a; + if (is_last) + { + logits.resize(out_n); + out = logits.data(); + } + linear(dense_in, in_n, layer.weight.data.data(), layer.bias.data.data(), out_n, !is_last, out); + dense_in = out; + if (!is_last) + std::swap(buf_a, buf_b); + } + } + + private: + //! Direct (non-im2col) convolution with fused ReLU, symmetric zero padding, stride 1. + static void conv2d_relu(const float *in, size_t in_c, size_t in_h, size_t in_w, const float *weight, + const float *bias, size_t out_c, size_t kh, size_t kw, size_t pad, float *out) + { + for (size_t oc = 0; oc < out_c; ++oc) + { + const float *w_oc = weight + oc * in_c * kh * kw; + float *out_oc = out + oc * in_h * in_w; + for (size_t oy = 0; oy < in_h; ++oy) + { + for (size_t ox = 0; ox < in_w; ++ox) + { + float acc = bias[oc]; + for (size_t ic = 0; ic < in_c; ++ic) + { + const float *in_ic = in + ic * in_h * in_w; + const float *w_ic = w_oc + ic * kh * kw; + for (size_t ky = 0; ky < kh; ++ky) + { + const long iy = static_cast(oy + ky) - static_cast(pad); + if (iy < 0 || iy >= static_cast(in_h)) + continue; + const float *in_row = in_ic + static_cast(iy) * in_w; + const float *w_row = w_ic + ky * kw; + for (size_t kx = 0; kx < kw; ++kx) + { + const long ix = static_cast(ox + kx) - static_cast(pad); + if (ix < 0 || ix >= static_cast(in_w)) + continue; + acc += in_row[static_cast(ix)] * w_row[kx]; + } + } + } + out_oc[oy * in_w + ox] = std::max(0.0f, acc); + } + } + } + } + + //! 2x2 max pooling, stride 2, no padding (input dims assumed even). + static void maxpool2(const float *in, size_t c, size_t h, size_t w, float *out) + { + const size_t oh = h / 2, ow = w / 2; + for (size_t ch = 0; ch < c; ++ch) + { + const float *in_ch = in + ch * h * w; + float *out_ch = out + ch * oh * ow; + for (size_t oy = 0; oy < oh; ++oy) + { + for (size_t ox = 0; ox < ow; ++ox) + { + const size_t iy = oy * 2, ix = ox * 2; + const float v00 = in_ch[iy * w + ix]; + const float v01 = in_ch[iy * w + ix + 1]; + const float v10 = in_ch[(iy + 1) * w + ix]; + const float v11 = in_ch[(iy + 1) * w + ix + 1]; + out_ch[oy * ow + ox] = std::max(std::max(v00, v01), std::max(v10, v11)); + } + } + } + } + + //! Dense layer: out = weight @ in + bias, optionally fused with ReLU. + static void linear(const float *in, size_t in_n, const float *weight, const float *bias, size_t out_n, + bool relu, float *out) + { + for (size_t o = 0; o < out_n; ++o) + { + const float *w_row = weight + o * in_n; + float acc = bias[o]; + for (size_t i = 0; i < in_n; ++i) + acc += w_row[i] * in[i]; + out[o] = relu ? std::max(0.0f, acc) : acc; + } + } + }; + + //! A bank of fixed base images (in [0, 1], NCHW layout) and their ground-truth labels. + struct ImageBank + { + std::vector> images; + std::vector labels; + }; + + /** + * @brief Builds the MNIST or CIFAR-10 network from its exported weight file. + * Architecture is implied by the tensor names/shapes found in the file. + */ + inline Network load_network(const std::string &dataset) + { + const auto file = dataset == "mnist" ? "adversarial/mnist_cnn.ioh" : "adversarial/cifar_cnn.ioh"; + const auto tensors = read_tensor_file(ioh::data::file(file)); + + Network net; + std::vector conv_names; + std::vector dense_names; + std::vector pool_after; + + if (dataset == "mnist") + { + net.c = 1; + net.h = 28; + net.w = 28; + conv_names = {"conv1", "conv2"}; + pool_after = {true, true}; + dense_names = {"fc1", "fc2"}; + } + else if (dataset == "cifar10") + { + net.c = 3; + net.h = 32; + net.w = 32; + conv_names = {"conv1", "conv2", "conv3", "conv4"}; + pool_after = {false, true, false, true}; + dense_names = {"fc1", "fc2"}; + } + else + { + throw std::runtime_error(fmt::format("adversarial::load_network: unknown dataset '{}'", dataset)); + } + + for (size_t i = 0; i < conv_names.size(); ++i) + { + ConvLayer layer; + layer.weight = tensors.at(conv_names[i] + ".weight"); + layer.bias = tensors.at(conv_names[i] + ".bias"); + layer.padding = (layer.weight.shape[2] - 1) / 2; // "same" padding, matches models.py + net.convs.push_back(std::move(layer)); + } + net.pool_after = pool_after; + + for (const auto &name : dense_names) + { + DenseLayer layer; + layer.weight = tensors.at(name + ".weight"); + layer.bias = tensors.at(name + ".bias"); + net.dense.push_back(std::move(layer)); + } + + return net; + } + + //! Loads the fixed 100-image bank (uint8 -> [0,1] float) for `dataset`. + inline ImageBank load_images(const std::string &dataset) + { + const auto file = dataset == "mnist" ? "adversarial/mnist_images.ioh" : "adversarial/cifar_images.ioh"; + const auto tensors = read_tensor_file(ioh::data::file(file)); + + const auto &images_t = tensors.at("images"); // [n, c, h, w], values already dequantized 0..255 + const auto &labels_t = tensors.at("labels"); // [n] + + const size_t n = images_t.shape[0]; + const size_t per_image = images_t.numel() / n; + + ImageBank bank; + bank.images.resize(n); + bank.labels.resize(n); + for (size_t i = 0; i < n; ++i) + { + bank.images[i].resize(per_image); + for (size_t j = 0; j < per_image; ++j) + bank.images[i][j] = images_t.data[i * per_image + j] / 255.0f; + bank.labels[i] = static_cast(labels_t.data[i]); + } + return bank; + } + + //! The network for `dataset` ("mnist" or "cifar10"), loaded on first use and shared (see ioh::data::cached) + inline std::shared_ptr get_network(const std::string &dataset) + { + return ioh::data::cached("adversarial/network/" + dataset, [&] { return load_network(dataset); }); + } + + //! The image bank for `dataset`, loaded on first use and shared (see ioh::data::cached) + inline std::shared_ptr get_images(const std::string &dataset) + { + return ioh::data::cached("adversarial/images/" + dataset, [&] { return load_images(dataset); }); + } +} // namespace ioh::problem::adversarial diff --git a/include/ioh/problem/adversarial/tensor_file.hpp b/include/ioh/problem/adversarial/tensor_file.hpp new file mode 100644 index 00000000..f13a3586 --- /dev/null +++ b/include/ioh/problem/adversarial/tensor_file.hpp @@ -0,0 +1,176 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "ioh/common/data.hpp" +#include "ioh/common/format.hpp" + +namespace ioh::problem::adversarial +{ + /** + * @brief A dense, row-major float32 tensor, dequantized on load. + */ + struct Tensor + { + std::vector shape; + std::vector data; + + [[nodiscard]] size_t numel() const + { + return std::accumulate(shape.begin(), shape.end(), static_cast(1), std::multiplies<>()); + } + }; + + //! name -> Tensor + using TensorMap = std::map; + + namespace detail + { + //! IEEE-754 binary16 -> binary32, handling subnormals, inf and nan. + inline float half_to_float(const uint16_t h) + { + const uint32_t sign = static_cast(h & 0x8000u) << 16; + uint32_t exponent = (h >> 10) & 0x1F; + uint32_t mantissa = h & 0x3FF; + uint32_t bits; + + if (exponent == 0) + { + if (mantissa == 0) + { + // signed zero + bits = sign; + } + else + { + // subnormal half -> normalize into a float32 + exponent = 127 - 15 + 1; + while ((mantissa & 0x400) == 0) + { + mantissa <<= 1; + --exponent; + } + mantissa &= 0x3FF; + bits = sign | (exponent << 23) | (mantissa << 13); + } + } + else if (exponent == 0x1F) + { + // inf / nan + bits = sign | 0x7F800000u | (mantissa << 13); + } + else + { + bits = sign | ((exponent - 15 + 127) << 23) | (mantissa << 13); + } + + float result; + std::memcpy(&result, &bits, sizeof(float)); + return result; + } + + inline bool is_little_endian() + { + const uint32_t probe = 1; + uint8_t first_byte; + std::memcpy(&first_byte, &probe, 1); + return first_byte == 1; + } + + template + inline T read_pod(std::istream &is) + { + T value{}; + is.read(reinterpret_cast(&value), sizeof(T)); + if (!is) + throw std::runtime_error("adversarial::read_tensor_file: unexpected end of file"); + return value; + } + } // namespace detail + + /** + * @brief Reads the little-endian "IOHNET01" tensor container used by the + * adversarial-attack suite (weights and image banks), dequantizing + * float16/uint8 payloads to float32 in the returned tensors. + * + * Container layout (no padding): + * magic char[8] "IOHNET01" + * n_tensors uint32 + * per tensor: name_len uint32, name char[name_len], + * ndim uint32, dims uint32[ndim], + * dtype uint32 (0=f16, 1=f32, 2=u8), + * data (row-major) + */ + inline TensorMap read_tensor_file(const fs::path &path) + { + if (!detail::is_little_endian()) + throw std::runtime_error("adversarial::read_tensor_file: big-endian platforms are not supported"); + + std::ifstream is(path, std::ios::binary); + if (!is) + throw std::runtime_error(fmt::format("adversarial::read_tensor_file: could not open {}", path.string())); + + char magic[8]; + is.read(magic, 8); + if (!is || std::memcmp(magic, "IOHNET01", 8) != 0) + throw std::runtime_error(fmt::format("adversarial::read_tensor_file: bad magic in {}", path.string())); + + const auto n_tensors = detail::read_pod(is); + TensorMap result; + + for (uint32_t t = 0; t < n_tensors; ++t) + { + const auto name_len = detail::read_pod(is); + std::string name(name_len, '\0'); + is.read(name.data(), static_cast(name_len)); + + const auto ndim = detail::read_pod(is); + Tensor tensor; + tensor.shape.resize(ndim); + for (uint32_t d = 0; d < ndim; ++d) + tensor.shape[d] = detail::read_pod(is); + + const auto dtype = detail::read_pod(is); + const auto n = tensor.numel(); + tensor.data.resize(n); + + if (dtype == 0) // float16 + { + std::vector raw(n); + is.read(reinterpret_cast(raw.data()), static_cast(n * sizeof(uint16_t))); + for (size_t i = 0; i < n; ++i) + tensor.data[i] = detail::half_to_float(raw[i]); + } + else if (dtype == 1) // float32 + { + is.read(reinterpret_cast(tensor.data.data()), static_cast(n * sizeof(float))); + } + else if (dtype == 2) // uint8 + { + std::vector raw(n); + is.read(reinterpret_cast(raw.data()), static_cast(n)); + for (size_t i = 0; i < n; ++i) + tensor.data[i] = static_cast(raw[i]); + } + else + { + throw std::runtime_error( + fmt::format("adversarial::read_tensor_file: unknown dtype {} in {}", dtype, path.string())); + } + + if (!is) + throw std::runtime_error( + fmt::format("adversarial::read_tensor_file: truncated tensor '{}' in {}", name, path.string())); + + result.emplace(std::move(name), std::move(tensor)); + } + + return result; + } +} // namespace ioh::problem::adversarial diff --git a/include/ioh/suite.hpp b/include/ioh/suite.hpp index 95ca0bc1..b38dd6c1 100644 --- a/include/ioh/suite.hpp +++ b/include/ioh/suite.hpp @@ -364,6 +364,30 @@ namespace ioh::suite } }; + //! Black-box adversarial-attack suite: fixed (dataset, eps, alpha) presets as problem ids, the base image as + //! instance, and a dimension that is fixed per dataset + struct AdversarialAttack final : Suite + { + //! The id in the suite registry + static constexpr int registry_id = 5; + //! The name in the suite registry, and of the suite + static constexpr const char *registry_name = "AdversarialAttack"; + + /** + * @brief Construct a new AdversarialAttack suite object + * + * @param problem_ids List of problem ids + * @param instances List of problem instances, i.e. base-image indices (defaults to the first) + * @param dimensions ignored: the dimension is fixed per problem id's dataset (784 or 3072) + */ + AdversarialAttack(const std::vector &problem_ids = {}, const std::vector &instances = {1}, + const std::vector &dimensions = {1}) : + Suite(problem_ids, instances, dimensions, registry_name, 100, 1, + problem::ProblemFactoryType::instance()) + { + } + }; + //! PBO suite struct PBO final : Suite { diff --git a/ioh/__init__.py b/ioh/__init__.py index 326e457c..b801700e 100644 --- a/ioh/__init__.py +++ b/ioh/__init__.py @@ -100,6 +100,8 @@ class ProblemClass(enum.Enum): CEC2013 = "CEC2013" CEC2022 = "CEC2022" + ADVERSARIAL = "AdversarialAttack" + def is_real(self): return self in ( ProblemClass.CEC2013, @@ -108,6 +110,7 @@ def is_real(self): ProblemClass.BBOB, ProblemClass.SBOX, ProblemClass.STAR_REAL, + ProblemClass.ADVERSARIAL, ) def is_single_objective(self): diff --git a/ioh/__init__.pyi b/ioh/__init__.pyi index 5f8c3d29..5eb75c89 100644 --- a/ioh/__init__.pyi +++ b/ioh/__init__.pyi @@ -22,6 +22,7 @@ class ProblemClass(enum.Enum): STAR_INTEGER = 'IntegerStarDiscrepancy' CEC2013 = 'CEC2013' CEC2022 = 'CEC2022' + ADVERSARIAL = 'AdversarialAttack' def is_real(self): ... def is_single_objective(self): ... @property diff --git a/ioh/iohcpp/problem/__init__.pyi b/ioh/iohcpp/problem/__init__.pyi index 3ebd048d..5e936372 100644 --- a/ioh/iohcpp/problem/__init__.pyi +++ b/ioh/iohcpp/problem/__init__.pyi @@ -1594,6 +1594,106 @@ class IntegerStarDiscrepancy(IntegerSingleObjective): the dimensionality of the search space """ +class AdversarialAttack(RealSingleObjective): + """ + Black-box adversarial-attack problems, a many-dimensional benchmark: search for a bounded perturbation + of a fixed, correctly classified image that minimizes the confidence of a classifier in the true label, + penalized by the L2 norm of the perturbation. + + f(delta) = -log softmax(model(x + delta))[y] - alpha * ||delta||_2, maximized. + + The problems have ids 5001-5006 (MNIST, 784 variables) and 5011-5016 (CIFAR-10, 3072 variables). + `instance` selects which of 100 bundled base images is attacked: unlike most problems, it changes the + entire landscape, not an affine transformation of one function. `x + delta` is not clamped to [0, 1] by + default. The dimension is fixed by the dataset: `n_variables` is ignored. + + Reference + --------- + https://github.com/wz7475/amhe-attack + """ + + def __init__(self, dataset: str = 'mnist', instance: int = 1, eps: float = 0.3, alpha: float = 0.0, clamp_input: bool = False) -> None: + """ + Parameters + ---------- + dataset: str + "mnist" (784 variables) or "cifar10" (3072 variables) + instance: int = 1 + Selects the base image: `(instance - 1) mod 100` in the bundled image bank + eps: float = 0.3 + The perturbation bound: the search space is [-eps, eps]^d + alpha: float = 0.0 + The weight of the L2 penalty on the size of the perturbation + clamp_input: bool = False + Whether to clamp x + delta to [0, 1] before classification + """ + + @overload + @staticmethod + def create(problem_name: str, instance_id: int, dimension: int) -> AdversarialAttack: + """ + Create a problem instance + + Parameters + ---------- + problem_name: str + a string indicating the problem name. + instance_id: int + an integer identifier of the problem instance + dimension: int + the dimensionality of the search space + """ + + @overload + @staticmethod + def create(problem_id: int, instance_id: int, dimension: int) -> AdversarialAttack: + """ + Create a problem instance + + Parameters + ---------- + problem_id: int + a number indicating the problem numeric identifier. + instance_id: int + an integer identifier of the problem instance + dimension: int + the dimensionality of the search space + """ + + @property + def dataset(self) -> str: + """The dataset: "mnist" or "cifar10\"""" + + @property + def eps(self) -> float: + """The perturbation bound""" + + @property + def alpha(self) -> float: + """The weight of the L2 penalty""" + + @property + def image_index(self) -> int: + """The index of the base image in the image bank""" + + @property + def true_label(self) -> int: + """The true label of the base image""" + + @property + def clamp_input(self) -> bool: + """Whether x + delta is clamped to [0, 1]""" + + def predicted_label(self) -> int: + """ + The class with the highest logit in the most recent evaluation: not true_label when the attack succeeded + """ + + def probabilities(self) -> Annotated[NDArray[numpy.float64], dict(shape=(None,))]: + """ + The softmax probabilities of the most recent evaluation, as a new numpy array + """ + class ManyAffine(RealSingleObjective): @overload def __init__(self, instance: int, n_variables: int) -> None: ... diff --git a/ioh/iohcpp/suite.pyi b/ioh/iohcpp/suite.pyi index 76302ba9..93936323 100644 --- a/ioh/iohcpp/suite.pyi +++ b/ioh/iohcpp/suite.pyi @@ -122,6 +122,24 @@ class RealStarDiscrepancy(RealBase): def __init__(self, problem_ids: Sequence[int], instances: Sequence[int] = [1], dimensions: Sequence[int] = [5]) -> None: ... +class AdversarialAttack(RealBase): + """ + Suite with AdversarialAttack functions included in IOH. + + This class is an iterator for problems, you can define a set of problems to iterate over. + + Parameters + ---------- + problem_ids: list[int] + A list of problems ids to include in this instantiation of the suite + instances: list[int] + A list of problem instances to include in this instantiation of the suite + dimensions: list[int] + A list of problem dimensions to include in this instantiation of the suite + """ + + def __init__(self, problem_ids: Sequence[int], instances: Sequence[int] = [1], dimensions: Sequence[int] = [1]) -> None: ... + class IntegerBase: """IntegerBase suite""" diff --git a/ioh/src/common.hpp b/ioh/src/common.hpp index 5a03d755..b83830a4 100644 --- a/ioh/src/common.hpp +++ b/ioh/src/common.hpp @@ -384,6 +384,7 @@ void define_cec2013_problems(nb::module_ &m); void define_cec2022_problems(nb::module_ &m); void define_dynamic_bin_val_problem(nb::module_ &m); void define_star_discrepancy_problems(nb::module_ &m); +void define_adversarial_problems(nb::module_ &m); void define_many_affine(nb::module_ &m); void define_funnel(nb::module_ &m); void define_pbo_problems(nb::module_ &m); diff --git a/ioh/src/problem.cpp b/ioh/src/problem.cpp index 3929ef89..99dc8ebb 100644 --- a/ioh/src/problem.cpp +++ b/ioh/src/problem.cpp @@ -399,6 +399,7 @@ void define_problem(nb::module_ &m) define_wmodels(m); define_submodular_problems(m); define_star_discrepancy_problems(m); + define_adversarial_problems(m); define_many_affine(m); define_funnel(m); diff --git a/ioh/src/real.cpp b/ioh/src/real.cpp index 75172efb..7261cd9a 100644 --- a/ioh/src/real.cpp +++ b/ioh/src/real.cpp @@ -342,6 +342,65 @@ void define_star_discrepancy_problems(nb::module_ &m) "IntegerStarDiscrepancy"); } +void define_adversarial_problems(nb::module_ &m) +{ + using ioh::problem::adversarial::AdversarialAttack; + + nb::class_ c(m, "AdversarialAttack", R"pbdoc( + Black-box adversarial-attack problems, a many-dimensional benchmark: search for a bounded perturbation + of a fixed, correctly classified image that minimizes the confidence of a classifier in the true label, + penalized by the L2 norm of the perturbation. + + f(delta) = -log softmax(model(x + delta))[y] - alpha * ||delta||_2, maximized. + + The problems have ids 5001-5006 (MNIST, 784 variables) and 5011-5016 (CIFAR-10, 3072 variables). + `instance` selects which of 100 bundled base images is attacked: unlike most problems, it changes the + entire landscape, not an affine transformation of one function. `x + delta` is not clamped to [0, 1] by + default. The dimension is fixed by the dataset: `n_variables` is ignored. + + Reference + --------- + https://github.com/wz7475/amhe-attack + )pbdoc"); + ioh_python::real::define_factory(c, ioh_python::real::cec_create_name_doc, ioh_python::real::cec_create_id_doc); + + c.def( + "__init__", + [](AdversarialAttack *self, const std::string &dataset, const int instance, const double eps, + const double alpha, const bool clamp_input) { + const std::string name = fmt::format("AdversarialAttack{}", dataset == "mnist" ? "MNIST" : "CIFAR10"); + new (self) AdversarialAttack(0, instance, 0, name, dataset, eps, alpha, clamp_input); + }, + nb::arg("dataset") = "mnist", nb::arg("instance") = 1, nb::arg("eps") = 0.3, nb::arg("alpha") = 0.0, + nb::arg("clamp_input") = false, + R"pbdoc( + Parameters + ---------- + dataset: str + "mnist" (784 variables) or "cifar10" (3072 variables) + instance: int = 1 + Selects the base image: `(instance - 1) mod 100` in the bundled image bank + eps: float = 0.3 + The perturbation bound: the search space is [-eps, eps]^d + alpha: float = 0.0 + The weight of the L2 penalty on the size of the perturbation + clamp_input: bool = False + Whether to clamp x + delta to [0, 1] before classification + )pbdoc") + .def_prop_ro("dataset", &AdversarialAttack::dataset, "The dataset: \"mnist\" or \"cifar10\"") + .def_prop_ro("eps", &AdversarialAttack::eps, "The perturbation bound") + .def_prop_ro("alpha", &AdversarialAttack::alpha, "The weight of the L2 penalty") + .def_prop_ro("image_index", &AdversarialAttack::image_index, "The index of the base image in the image bank") + .def_prop_ro("true_label", &AdversarialAttack::true_label, "The true label of the base image") + .def_prop_ro("clamp_input", &AdversarialAttack::clamp_input, "Whether x + delta is clamped to [0, 1]") + .def("predicted_label", &AdversarialAttack::predicted_label, + "The class with the highest logit in the most recent evaluation: not true_label when the attack " + "succeeded") + .def( + "probabilities", [](const AdversarialAttack &self) { return ioh_python::make_array(self.probabilities()); }, + "The softmax probabilities of the most recent evaluation, as a new numpy array"); +} + void define_many_affine(nb::module_ &m) { using ioh::problem::bbob::ManyAffine; diff --git a/ioh/src/real_suite.cpp b/ioh/src/real_suite.cpp index 72cb0d3d..0264188f 100644 --- a/ioh/src/real_suite.cpp +++ b/ioh/src/real_suite.cpp @@ -72,4 +72,5 @@ void define_real_suite(nb::module_ &m) define_suite(m, "BBOB"); define_suite(m, "SBOX"); define_suite(m, "RealStarDiscrepancy"); + define_suite(m, "AdversarialAttack", 1); } diff --git a/src/data_manifest.inc b/src/data_manifest.inc index 8d590add..e14f1956 100644 --- a/src/data_manifest.inc +++ b/src/data_manifest.inc @@ -1,8 +1,13 @@ // Generated by tools/gen_manifest.py from static/manifest.json: do not edit -const char *const generated_data_version = "e7c83044fb1018af"; +const char *const generated_data_version = "1786becf6d8a4d73"; const char *const generated_files[] = { + "adversarial/PROVENANCE.txt", + "adversarial/cifar_cnn.ioh", + "adversarial/cifar_images.ioh", + "adversarial/mnist_cnn.ioh", + "adversarial/mnist_images.ioh", "cec_transformations/2013/CF3_M_D10.dat", "cec_transformations/2013/CF3_M_D2.dat", "cec_transformations/2013/CF3_M_D20.dat", diff --git a/src/registry/adversarial.cpp b/src/registry/adversarial.cpp new file mode 100644 index 00000000..1408d25c --- /dev/null +++ b/src/registry/adversarial.cpp @@ -0,0 +1,10 @@ +// Built-in entries of the adversarial attack factory +#include "families.hpp" + +namespace ioh::common +{ + void add_builtins(Factory &factory) + { + registry::add_adversarial(factory); + } +} // namespace ioh::common diff --git a/src/registry/families.hpp b/src/registry/families.hpp index 14413503..fccc60fc 100644 --- a/src/registry/families.hpp +++ b/src/registry/families.hpp @@ -149,6 +149,13 @@ namespace ioh::registry add(factory, type_list{}); } + //! The 12 adversarial attack problems (ids 5001-5006 and 5011-5016) + template + void add_adversarial(common::Factory &factory) + { + add(factory, type_list{}); + } + //! The 30 integer star discrepancy problems (ids 30-59) template void add_star_discrepancy_integer(common::Factory &factory) diff --git a/src/registry/single_objective.cpp b/src/registry/single_objective.cpp index fa239049..9b69968b 100644 --- a/src/registry/single_objective.cpp +++ b/src/registry/single_objective.cpp @@ -9,6 +9,7 @@ namespace ioh::common registry::add_cec2013(factory); registry::add_cec2022(factory); registry::add_star_discrepancy_real(factory); + registry::add_adversarial(factory); } void add_builtins(Factory &factory) diff --git a/src/registry/suites.cpp b/src/registry/suites.cpp index dff77d8f..41164a26 100644 --- a/src/registry/suites.cpp +++ b/src/registry/suites.cpp @@ -1,5 +1,5 @@ // Built-in entries of the suite factories, with the ids and names the suite classes declare (registry_id, -// registry_name): Real 1, BBOB 2, SBOX 3, RealStarDiscrepancy 4; Integer 1, PBO 2, Submodular 3, +// registry_name): Real 1, BBOB 2, SBOX 3, RealStarDiscrepancy 4, AdversarialAttack 5; Integer 1, PBO 2, Submodular 3, // IntegerStarDiscrepancy 4 #include "families.hpp" @@ -9,7 +9,7 @@ namespace ioh::common std::vector> &factory) { registry::add_suites(factory); + suite::RealStarDiscrepancy, suite::AdversarialAttack>(factory); } void add_builtins(Factory, std::vector, std::vector, diff --git a/static.tar.gz b/static.tar.gz index bf5dd844..77c4126d 100644 Binary files a/static.tar.gz and b/static.tar.gz differ diff --git a/static/adversarial/PROVENANCE.txt b/static/adversarial/PROVENANCE.txt new file mode 100644 index 00000000..548b3c40 --- /dev/null +++ b/static/adversarial/PROVENANCE.txt @@ -0,0 +1,13 @@ +torch version: 2.14.0+cpu + +MNIST state_dict sha256: 10c3c57b10bc9e9dd55de5a9234fe9a2b1ce9c40e11b3d70d2c58c44993b45ef +MNIST fp32 test accuracy: 0.9787 +MNIST fp16-roundtrip test accuracy: 0.9787 (delta=+0.0000) + +CIFAR10 state_dict sha256: 5017f5465ac526f59ac7e34c32ba4f28a07240fbe4341e774f7ba79fd8991888 +CIFAR10 fp32 test accuracy: 0.7173 +CIFAR10 fp16-roundtrip test accuracy: 0.7173 (delta=+0.0000) + +Image bank: first 100 correctly-classified test-set images, natural +(shuffle=False) order, transforms.ToTensor() only (no normalization). +Golden file: adversarial_reference.json, generated by this script. diff --git a/static/adversarial/cifar_cnn.ioh b/static/adversarial/cifar_cnn.ioh new file mode 100644 index 00000000..d97e1295 Binary files /dev/null and b/static/adversarial/cifar_cnn.ioh differ diff --git a/static/adversarial/cifar_images.ioh b/static/adversarial/cifar_images.ioh new file mode 100644 index 00000000..75b0395f Binary files /dev/null and b/static/adversarial/cifar_images.ioh differ diff --git a/static/adversarial/mnist_cnn.ioh b/static/adversarial/mnist_cnn.ioh new file mode 100644 index 00000000..d8ddc603 Binary files /dev/null and b/static/adversarial/mnist_cnn.ioh differ diff --git a/static/adversarial/mnist_images.ioh b/static/adversarial/mnist_images.ioh new file mode 100644 index 00000000..bf5b490e Binary files /dev/null and b/static/adversarial/mnist_images.ioh differ diff --git a/static/manifest.json b/static/manifest.json index 222b2da2..8b79d110 100644 --- a/static/manifest.json +++ b/static/manifest.json @@ -1,7 +1,32 @@ { "format": 1, - "data_version": "e7c83044fb1018af", + "data_version": "1786becf6d8a4d73", "files": [ + { + "path": "adversarial/PROVENANCE.txt", + "size": 609, + "sha256": "20052f3df07a713fc4fb10a91f7fe0967b90200d7ba68df2a4e81ce2a5bec42b" + }, + { + "path": "adversarial/cifar_cnn.ioh", + "size": 4720660, + "sha256": "c3c067240d042c0fed88b8a1a7f3de01698af416465613bc864a02b3a8b42a22" + }, + { + "path": "adversarial/cifar_images.ioh", + "size": 307368, + "sha256": "ceaad0063366683e10c41d40a685a35bd8b7c1de46c49ed1db0c587124b326cb" + }, + { + "path": "adversarial/mnist_cnn.ioh", + "size": 6549520, + "sha256": "64259ac5b5ba19e3b7f750ed8cb2f6d7ca57f7192ca05336a69259fe296972fa" + }, + { + "path": "adversarial/mnist_images.ioh", + "size": 78568, + "sha256": "3e2fbe8cbcf991ea8f38d928b11987f556df5fd822d9774ddb397b842243c0d9" + }, { "path": "cec_transformations/2013/CF3_M_D10.dat", "size": 26100, diff --git a/tests/cpp/problem/test_adversarial.cpp b/tests/cpp/problem/test_adversarial.cpp new file mode 100644 index 00000000..f47552cb --- /dev/null +++ b/tests/cpp/problem/test_adversarial.cpp @@ -0,0 +1,112 @@ +#include "../utils.hpp" + +#include "ioh/suite.hpp" +#include "ioh/problem/adversarial/adversarial_attack.hpp" + +#include +#include +#include +#include +#include + +namespace +{ + //! Reproduces the LCG used by amhe-attack/scripts/export_ioh_assets.py to + //! generate the perturbation vectors behind tests/data/adversarial.in, so this + //! test needs to store only (fid, iid, seed, y) rather than full vectors. + std::vector lcg_perturbation(uint32_t seed, size_t dim, double eps) + { + uint32_t s = seed & 0x7FFFFFFFu; + std::vector x(dim); + for (size_t i = 0; i < dim; ++i) + { + s = (1103515245u * s + 12345u) & 0x7FFFFFFFu; + x[i] = eps * (2.0 * static_cast(s) / 2147483648.0 - 1.0); + } + return x; + } +} // namespace + +TEST_F(BaseTest, test_adversarial_factory_ids) +{ + const auto &factory = ioh::problem::ProblemRegistry::instance(); + const auto ids = factory.ids(); + EXPECT_EQ(ids.size(), 12u); + for (const int id : {5001, 5002, 5003, 5004, 5005, 5006, 5011, 5012, 5013, 5014, 5015, 5016}) + EXPECT_NE(std::find(ids.begin(), ids.end(), id), ids.end()) << "missing preset id " << id; +} + +TEST_F(BaseTest, test_adversarial_dimension_and_bounds) +{ + const auto &factory = ioh::problem::ProblemRegistry::instance(); + + auto mnist = factory.create(5001, 1, 1); // n_variables is ignored + EXPECT_EQ(mnist->meta_data().n_variables, 784); + EXPECT_EQ(mnist->bounds().lb.at(0), -0.1); + EXPECT_EQ(mnist->bounds().ub.at(0), 0.1); + EXPECT_EQ(mnist->meta_data().optimization_type.type(), ioh::common::OptimizationType::MAX); + + auto cifar = factory.create(5011, 1, 1); + EXPECT_EQ(cifar->meta_data().n_variables, 3072); +} + +TEST_F(BaseTest, test_adversarial_shared_network) +{ + const auto &factory = ioh::problem::ProblemRegistry::instance(); + + auto p1 = factory.create(5001, 1, 1); + auto p2 = factory.create(5001, 2, 1); + // Different base images (different instances)... + EXPECT_NE(dynamic_cast(*p1).true_label(), + dynamic_cast(*p2).true_label()); +} + +TEST_F(BaseTest, test_adversarial_regression) +{ + const auto &factory = ioh::problem::ProblemRegistry::instance(); + + // fid -> (eps), needed to regenerate the perturbation for each row. + const std::map eps_of_fid = { + {5001, 0.1}, {5002, 0.3}, {5003, 1.0}, {5004, 0.1}, {5005, 0.3}, {5006, 1.0}, + {5011, 0.1}, {5012, 0.3}, {5013, 1.0}, {5014, 0.1}, {5015, 0.3}, {5016, 1.0}, + }; + + const auto path = test_data_file("adversarial.in"); + std::ifstream is(path); + ASSERT_TRUE(is.is_open()); + + int fid, iid, seed; + double y; + size_t n_rows = 0; + while (is >> fid >> iid >> seed >> y) + { + ++n_rows; + auto problem = factory.create(fid, iid, 1); + const auto dim = static_cast(problem->meta_data().n_variables); + // Matches amhe-attack/scripts/export_ioh_assets.py: lcg_perturbation(seed=seed*1000+img_idx, ...) + const auto composite_seed = static_cast(seed) * 1000u + static_cast(iid - 1); + const auto x = lcg_perturbation(composite_seed, dim, eps_of_fid.at(fid)); + const auto got = (*problem)(x); + // Reference values come from a from-scratch fp32 reimplementation of the CNN + // forward pass, evaluated against the torch reference (see + // amhe-attack/scripts/export_ioh_assets.py). Plain-loop accumulation order + // here differs from torch's oneDNN dispatch, so the two diverge slightly more + // as inputs grow past the model's [0, 1] training range (large eps); a mixed + // absolute/relative tolerance avoids flaking on those extreme presets while + // still catching real regressions (a genuine bug fails by orders of magnitude). + const double err = std::abs(got - y); + const double rel_err = err / std::max(1.0, std::abs(y)); + EXPECT_TRUE(err <= 1e-4 || rel_err <= 5e-3) + << "fid=" << fid << " iid=" << iid << " seed=" << seed << " expected=" << y << " got=" << got; + } + EXPECT_EQ(n_rows, 180u) << "tests/data/adversarial.in row count changed unexpectedly"; +} + +TEST_F(BaseTest, test_adversarial_suite) +{ + using namespace ioh::suite; + AdversarialAttack suite({5001}, {1, 2, 3}, {1}); + EXPECT_EQ(suite.size(), 3u); + for (auto &problem : suite) + EXPECT_EQ(problem->meta_data().n_variables, 784); +} diff --git a/tests/data/adversarial.in b/tests/data/adversarial.in new file mode 100644 index 00000000..456a4016 --- /dev/null +++ b/tests/data/adversarial.in @@ -0,0 +1,180 @@ +5001 1 0 4.649922647530387e-05 +5004 1 0 -0.08049245879891533 +5002 1 0 0.0001425405952315433 +5005 1 0 -0.24147433348094027 +5003 1 0 0.043626541037379285 +5006 1 0 -0.7617630392165269 +5001 1 1 7.275620304294706e-05 +5004 1 1 -0.07886036111618139 +5002 1 1 0.0010527255265890374 +5005 1 1 -0.23574662643108396 +5003 1 1 3.835304086714754 +5006 1 1 3.0459729135225104 +5001 1 2 5.122052266368467e-05 +5004 1 2 -0.08058802576058502 +5002 1 2 0.00018766617040790232 +5005 1 2 -0.24173007267933813 +5003 1 2 0.24108992848827498 +5006 1 2 -0.5653025343442117 +5001 2 0 0.0009563760040102387 +5004 2 0 -0.0784088321313318 +5002 2 0 0.00032429258211324726 +5005 2 0 -0.23777133182391288 +5003 2 0 1.1907579036999516e-05 +5006 2 0 -0.7936401737743835 +5001 2 1 0.0015110500515557105 +5004 2 1 -0.07817520880104892 +5002 2 1 0.0011090387867890173 +5005 2 1 -0.2379497377710248 +5003 2 1 0.004775222742752657 +5006 2 1 -0.7920873657832934 +5001 2 2 0.0017079375554367503 +5004 2 2 -0.07761045352582731 +5002 2 2 0.0013984644372320597 +5005 2 2 -0.2365567088065601 +5003 2 2 0.004206246737874561 +5006 2 2 -0.7889776640747661 +5001 3 0 0.002039398199017506 +5004 3 0 -0.08027612113369885 +5002 3 0 0.012590354513038377 +5005 3 0 -0.23435620348511063 +5003 3 0 5.812279203448297 +5006 3 0 4.989124010121133 +5001 3 1 0.002617720171744227 +5004 3 1 -0.07917226904663258 +5002 3 1 0.015848545654293573 +5005 3 1 -0.2295214220008368 +5003 3 1 3.5635482045538702 +5006 3 1 2.7456483123701023 +5001 3 2 0.0018747123929210362 +5004 3 2 -0.0783850532318632 +5002 3 2 0.007055174369734443 +5005 3 2 -0.23372412250461827 +5003 3 2 2.8912381604009254 +5006 3 2 2.088640504153083 +5001 4 0 0.00010433063354538973 +5004 4 0 -0.08230233834875073 +5002 4 0 0.00024298777620548637 +5005 4 0 -0.24697701917068282 +5003 4 0 1.2888779367339502 +5006 4 0 0.4648112469109891 +5001 4 1 0.00011811397476970046 +5004 4 1 -0.08004922354936354 +5002 4 1 0.0002258006489110071 +5005 4 1 -0.24027621192348866 +5003 4 1 0.023919172423459627 +5006 4 1 -0.7777542028178727 +5001 4 2 9.673413725969676e-05 +5004 4 2 -0.08047302542938614 +5002 4 2 0.00015977780997635104 +5005 4 2 -0.24154950088996113 +5003 4 2 0.45603989786830923 +5006 4 2 -0.34965769779814915 +5001 5 0 0.000987567936640419 +5004 5 0 -0.07879686056602755 +5002 5 0 0.0013265499598537297 +5005 5 0 -0.2380267355481501 +5003 5 0 0.348704964990301 +5006 5 0 -0.4491393200363786 +5001 5 1 0.0010689978237524178 +5004 5 1 -0.07897272831891855 +5002 5 1 0.0019287805961060872 +5005 5 1 -0.23819639783190677 +5003 5 1 3.936546524810296 +5006 5 1 3.1361292633835864 +5001 5 2 0.0007851066817923801 +5004 5 2 -0.07987888956357372 +5002 5 2 0.000739386617313687 +5005 5 2 -0.2412526021187846 +5003 5 2 4.031732054963482 +5006 5 2 3.2250920925098208 +5011 1 0 0.5220043888423358 +5014 1 0 0.3630356637484662 +5012 1 0 0.6908968291731339 +5015 1 0 0.21399065389152522 +5013 1 0 6.461863431821211 +5016 1 0 4.872176180882516 +5011 1 1 0.8913361617566834 +5014 1 1 0.7318624269601322 +5012 1 1 1.0278549218592115 +5015 1 1 0.5494337174695578 +5013 1 1 11.86144993260867 +5016 1 1 10.266712584643159 +5011 1 2 0.5600997230661277 +5014 1 2 0.40013043731789355 +5012 1 2 1.2576140630992714 +5015 1 2 0.777706205854569 +5013 1 2 11.676893043960682 +5016 1 2 10.07720018647834 +5011 2 0 0.9707938553925811 +5014 2 0 0.8111250088540937 +5012 2 0 6.005605026329107 +5015 2 0 5.526598486713644 +5013 2 0 22.85092138565052 +5016 2 0 21.254232920265643 +5011 2 1 1.5187666219348426 +5014 2 1 1.3590613757447558 +5012 2 1 9.09237575186782 +5015 2 1 8.613260013297559 +5013 2 1 23.92980844273502 +5016 2 1 22.33275598083415 +5011 2 2 1.4248094984682442 +5014 2 2 1.2649842736659336 +5012 2 2 8.376184932902934 +5015 2 2 7.896709258496002 +5013 2 2 21.409381474429168 +5016 2 2 19.81112922640606 +5011 3 0 0.12184284950499626 +5014 3 0 -0.039757689049167336 +5012 3 0 1.932272267583568 +5015 3 0 1.4474706519210772 +5013 3 0 10.783686136358083 +5016 3 0 9.167680750816448 +5011 3 1 0.12588133761514755 +5014 3 1 -0.03521163133273528 +5012 3 1 1.50544836857071 +5015 3 1 1.0221694617270616 +5013 3 1 10.737730546880156 +5016 3 1 9.126800857401328 +5011 3 2 0.15925587770341068 +5014 3 2 0.0005463824476903689 +5012 3 2 1.048920512171202 +5015 3 2 0.572792026404041 +5013 3 2 4.493860953286836 +5016 3 2 2.9067660007296325 +5011 4 0 0.029577821803774447 +5014 4 0 -0.13217673176530495 +5012 4 0 0.014466336933610388 +5015 4 0 -0.47079732377362776 +5013 4 0 0.0797617470352287 +5016 4 0 -1.5377837886555652 +5011 4 1 0.019183573318588457 +5014 4 1 -0.1399838994600554 +5012 4 1 0.021879779133117047 +5015 4 1 -0.4556226392028145 +5013 4 1 5.434048997200901 +5016 4 1 3.8423742694144627 +5011 4 2 0.018107199577348996 +5014 4 2 -0.1422637625695695 +5012 4 2 0.016085236395928532 +5015 4 2 -0.46502765004482677 +5013 4 2 0.07387150668882146 +5016 4 2 -1.529838114780363 +5011 5 0 0.010384142020619144 +5014 5 0 -0.1502853133235717 +5012 5 0 0.0018213885092664426 +5015 5 0 -0.480186977523306 +5013 5 0 3.0612097354017527 +5016 5 0 1.4545151819598445 +5011 5 1 0.005952479095163499 +5014 5 1 -0.1532355721779082 +5012 5 1 0.0005145338632732833 +5015 5 1 -0.4770496199559418 +5013 5 1 4.993748753279473 +5016 5 1 3.4018682405487555 +5011 5 2 0.0009338172179228177 +5014 5 2 -0.16048372705600258 +5012 5 2 0.0001038343981728782 +5015 5 2 -0.48414879842360325 +5013 5 2 1.1860034700536817 +5016 5 2 -0.42817197268557217 diff --git a/tests/data/golden/behaviour.json b/tests/data/golden/behaviour.json index f0d6b080..e638dc46 100644 --- a/tests/data/golden/behaviour.json +++ b/tests/data/golden/behaviour.json @@ -1,4 +1,28 @@ { +"ADVERSARIAL/5001/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -78.4}, "meta": {"instance": 1, "n_variables": 784, "name": "AdversarialMNIST5001", "optimization_type": "MAX", "problem_id": 5001}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 78.4}, "x_sums": [-0.940472970167777, 0.33786211272710587, 2.825173208430673], "y": [5.8789908349510256e-05, 5.40620356428434e-05, 4.3948474388599834e-05]}, +"ADVERSARIAL/5001/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -78.4}, "meta": {"instance": 2, "n_variables": 784, "name": "AdversarialMNIST5001", "optimization_type": "MAX", "problem_id": 5001}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 78.4}, "x_sums": [-2.500574680236943, 2.7363198687963592, -1.9518810056195741], "y": [0.0016298653925730378, 0.0011552548810342955, 0.001115869538935499]}, +"ADVERSARIAL/5002/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -235.2}, "meta": {"instance": 1, "n_variables": 784, "name": "AdversarialMNIST5002", "optimization_type": "MAX", "problem_id": 5002}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 235.2}, "x_sums": [0.24640389275063473, -0.9095667012316226, -4.360820027244445], "y": [0.0012472891312658163, 0.00044572085170813125, 0.00020735191850640433]}, +"ADVERSARIAL/5002/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -235.2}, "meta": {"instance": 2, "n_variables": 784, "name": "AdversarialMNIST5002", "optimization_type": "MAX", "problem_id": 5002}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 235.2}, "x_sums": [4.010873346546429, -5.8874096236217275, -0.23723918201600247], "y": [0.0019828524697089733, 0.0011593017813353867, 0.0019878765405163534]}, +"ADVERSARIAL/5003/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -784.0}, "meta": {"instance": 1, "n_variables": 784, "name": "AdversarialMNIST5003", "optimization_type": "MAX", "problem_id": 5003}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 784.0}, "x_sums": [2.618000069722973, -21.718268261097258, -20.237557340377546], "y": [0.7373398630018393, 0.20083055775265635, 0.058191545987847704]}, +"ADVERSARIAL/5003/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -784.0}, "meta": {"instance": 2, "n_variables": 784, "name": "AdversarialMNIST5003", "optimization_type": "MAX", "problem_id": 5003}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 784.0}, "x_sums": [18.47604287580987, 1.3762906274290727, -16.939411020022966], "y": [0.010695585152477705, 0.0007196009070504772, 0.001338226284126165]}, +"ADVERSARIAL/5004/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -78.4}, "meta": {"instance": 1, "n_variables": 784, "name": "AdversarialMNIST5004", "optimization_type": "MAX", "problem_id": 5004}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 78.4}, "x_sums": [1.6434037898088243, -2.0446046041989856, -0.48307993441183344], "y": [-0.07990026390923259, -0.08059170715725648, -0.08049057562796198]}, +"ADVERSARIAL/5004/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -78.4}, "meta": {"instance": 2, "n_variables": 784, "name": "AdversarialMNIST5004", "optimization_type": "MAX", "problem_id": 5004}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 78.4}, "x_sums": [0.34536508534628413, -0.8238123962040114, -1.5198737045478687], "y": [-0.07870498845834623, -0.0797064027302188, -0.07952575997029675]}, +"ADVERSARIAL/5005/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -235.2}, "meta": {"instance": 1, "n_variables": 784, "name": "AdversarialMNIST5005", "optimization_type": "MAX", "problem_id": 5005}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 235.2}, "x_sums": [-5.055339933165511, -1.5350719327688522, -1.49027649604912], "y": [-0.2503828935396235, -0.23401178086117946, -0.24193321889178235]}, +"ADVERSARIAL/5005/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -235.2}, "meta": {"instance": 2, "n_variables": 784, "name": "AdversarialMNIST5005", "optimization_type": "MAX", "problem_id": 5005}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 235.2}, "x_sums": [3.6619962025024226, -5.796378782033275, 7.84515348112175], "y": [-0.24059098382307115, -0.2356249443438274, -0.2352116803238986]}, +"ADVERSARIAL/5006/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -784.0}, "meta": {"instance": 1, "n_variables": 784, "name": "AdversarialMNIST5006", "optimization_type": "MAX", "problem_id": 5006}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 784.0}, "x_sums": [21.474368357329787, 19.128771738400538, -14.96209216530264], "y": [2.9046907705864964, -0.4787828612550884, -0.7287943192511821]}, +"ADVERSARIAL/5006/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 784, "sum": -784.0}, "meta": {"instance": 2, "n_variables": 784, "name": "AdversarialMNIST5006", "optimization_type": "MAX", "problem_id": 5006}, "optimum_x": {"len": 784, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 784, "sum": 784.0}, "x_sums": [8.15836764081166, 2.190769610070647, 23.264624014695972], "y": [-0.8188786201987922, -0.7971045147865178, -0.8320369495201064]}, +"ADVERSARIAL/5011/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -307.20000000000005}, "meta": {"instance": 1, "n_variables": 3072, "name": "AdversarialCIFAR105011", "optimization_type": "MAX", "problem_id": 5011}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 307.20000000000005}, "x_sums": [4.2375194933478175, 3.405894377478484, -1.105404709160741], "y": [0.6881894509007562, 0.6727927941609035, 0.5567305370215125]}, +"ADVERSARIAL/5011/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -307.20000000000005}, "meta": {"instance": 2, "n_variables": 3072, "name": "AdversarialCIFAR105011", "optimization_type": "MAX", "problem_id": 5011}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 307.20000000000005}, "x_sums": [-2.985475047454602, 0.13211797791619387, 2.3071348815520594], "y": [1.0124204144656952, 1.1097466071880149, 1.8889662579097513]}, +"ADVERSARIAL/5012/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -921.5999999999999}, "meta": {"instance": 1, "n_variables": 3072, "name": "AdversarialCIFAR105012", "optimization_type": "MAX", "problem_id": 5012}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 921.5999999999999}, "x_sums": [8.390507942148814, 4.5372442814528755, -4.705759384177741], "y": [1.4913643189648198, 0.6799238518443116, 1.1123480011000462]}, +"ADVERSARIAL/5012/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -921.5999999999999}, "meta": {"instance": 2, "n_variables": 3072, "name": "AdversarialCIFAR105012", "optimization_type": "MAX", "problem_id": 5012}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 921.5999999999999}, "x_sums": [-3.0907496133677235, 16.047441069040044, -5.056338077765983], "y": [4.867299573422761, 5.241934084366024, 7.655032921304584]}, +"ADVERSARIAL/5013/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -3072.0}, "meta": {"instance": 1, "n_variables": 3072, "name": "AdversarialCIFAR105013", "optimization_type": "MAX", "problem_id": 5013}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 3072.0}, "x_sums": [-13.357148640665866, 10.470450118463145, 22.604992999645948], "y": [6.028685879688318, 14.16134488358151, 11.166824355493015]}, +"ADVERSARIAL/5013/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -3072.0}, "meta": {"instance": 2, "n_variables": 3072, "name": "AdversarialCIFAR105013", "optimization_type": "MAX", "problem_id": 5013}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 3072.0}, "x_sums": [-4.180053361007484, 8.652110160645732, -34.313262012535105], "y": [18.059750135241764, 18.016512536282683, 18.74386725251916]}, +"ADVERSARIAL/5014/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -307.20000000000005}, "meta": {"instance": 1, "n_variables": 3072, "name": "AdversarialCIFAR105014", "optimization_type": "MAX", "problem_id": 5014}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 307.20000000000005}, "x_sums": [-0.09634753683797558, -6.311101836362762, 1.1563050127678516], "y": [0.4821342399136078, 0.34181957216754577, 0.4770854167881863]}, +"ADVERSARIAL/5014/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -307.20000000000005}, "meta": {"instance": 2, "n_variables": 3072, "name": "AdversarialCIFAR105014", "optimization_type": "MAX", "problem_id": 5014}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 307.20000000000005}, "x_sums": [4.1992763436014044, -0.5535363832147433, -1.426320981230126], "y": [1.2719159938685685, 1.2489633163250242, 1.2668226049203408]}, +"ADVERSARIAL/5015/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -921.5999999999999}, "meta": {"instance": 1, "n_variables": 3072, "name": "AdversarialCIFAR105015", "optimization_type": "MAX", "problem_id": 5015}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 921.5999999999999}, "x_sums": [1.2596913883344, 2.530608508023203, 14.792761886662888], "y": [0.49041669776438956, 1.8410917763706398, 0.9205737448992444]}, +"ADVERSARIAL/5015/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -921.5999999999999}, "meta": {"instance": 2, "n_variables": 3072, "name": "AdversarialCIFAR105015", "optimization_type": "MAX", "problem_id": 5015}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 921.5999999999999}, "x_sums": [-11.145415060844028, 11.242993187504863, -13.362510016493518], "y": [6.109942622066977, 4.935166186106732, 8.880743798710075]}, +"ADVERSARIAL/5016/1/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -3072.0}, "meta": {"instance": 1, "n_variables": 3072, "name": "AdversarialCIFAR105016", "optimization_type": "MAX", "problem_id": 5016}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 3072.0}, "x_sums": [-35.84546756083757, -30.082807108346117, -3.9106987419330617], "y": [13.331752170759248, 9.394941594975519, 8.928409413704538]}, +"ADVERSARIAL/5016/2/1": {"fixed_dimension": false, "fixed_instance": false, "lb": {"len": 3072, "sum": -3072.0}, "meta": {"instance": 2, "n_variables": 3072, "name": "AdversarialCIFAR105016", "optimization_type": "MAX", "problem_id": 5016}, "optimum_x": {"len": 3072, "sum": NaN}, "optimum_y": Infinity, "ub": {"len": 3072, "sum": 3072.0}, "x_sums": [34.82090214206118, -10.334981114677708, -22.053902178533185], "y": [17.491936815167367, 18.967461634226353, 19.981960180417893]}, "BBOB/1/1/10": {"fixed_dimension": false, "fixed_instance": false, "lb": [-5.0, -5.0, -5.0, -5.0, -5.0, -5.0, -5.0, -5.0, -5.0, -5.0], "meta": {"instance": 1, "n_variables": 10, "name": "Sphere", "optimization_type": "MIN", "problem_id": 1}, "optimum_x": [0.2527999999999997, -1.1568, -0.7240000000000002, 1.9264000000000001, -2.6808, 0.4391999999999996, -0.11680000000000001, 0.1200000000000001, -1.6376, -3.0512], "optimum_y": 79.48, "ub": [5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], "x_sums": [11.363264999677718, 14.510679273377999, -11.009461106533502], "y": [217.82955380148985, 154.61635843379582, 175.323612623577], "y_at_optimum": 79.48}, "BBOB/1/1/2": {"fixed_dimension": false, "fixed_instance": false, "lb": [-5.0, -5.0], "meta": {"instance": 1, "n_variables": 2, "name": "Sphere", "optimization_type": "MIN", "problem_id": 1}, "optimum_x": [0.2527999999999997, -1.1568], "optimum_y": 79.48, "ub": [5.0, 5.0], "x_sums": [1.9243673398981471, 1.6941485440689235, -1.3478291445577177], "y": [84.24054406000359, 92.34222119717222, 88.30978629349613], "y_at_optimum": 79.48}, "BBOB/1/2/10": {"fixed_dimension": false, "fixed_instance": false, "lb": [-5.0, -5.0, -5.0, -5.0, -5.0, -5.0, -5.0, -5.0, -5.0, -5.0], "meta": {"instance": 2, "n_variables": 10, "name": "Sphere", "optimization_type": "MIN", "problem_id": 1}, "optimum_x": [-3.8984, -2.8904, -3.8024, 3.9055999999999997, 0.8592000000000004, -2.9048, 3.4448, -0.6000000000000001, -0.8824000000000001, -3.7352], "optimum_y": 394.48, "ub": [5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], "x_sums": [3.8814289633601238, -2.8840389512567874, -19.102863670521813], "y": [510.62970310292997, 643.875285295012, 517.9992281611582], "y_at_optimum": 394.48}, @@ -713,6 +737,7 @@ "direct/WModelLeadingOnes/default": {"fixed_dimension": false, "fixed_instance": false, "lb": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "meta": {"instance": 1, "n_variables": 16, "name": "WModelLeadingOnes", "optimization_type": "MAX", "problem_id": 2}, "optimum_x": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "optimum_y": 16.0, "penalty_sparse": [0.0, 0.0, 0.0], "ub": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "x_sums": [5.0, 6.0, 5.0], "y": [0.0, 0.0, 0.0], "y_at_optimum": 16.0, "y_sparse": [0.0, 0.0, 0.0]}, "direct/WModelOneMax/configured": {"fixed_dimension": false, "fixed_instance": false, "lb": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "meta": {"instance": 2, "n_variables": 16, "name": "WModelOneMax", "optimization_type": "MAX", "problem_id": 2}, "optimum_x": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "optimum_y": 4.0, "penalty_sparse": [0.0, 0.0, 0.0], "ub": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "x_sums": [8.0, 6.0, 10.0], "y": [1345.3217558360295, 1346.8332496768019, 1346.8332496768019], "y_at_optimum": 1345.3217558360295, "y_sparse": [1345.3217558360295, 1345.3217558360295, 1345.3217558360295]}, "direct/WModelOneMax/default": {"fixed_dimension": false, "fixed_instance": false, "lb": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], "meta": {"instance": 1, "n_variables": 16, "name": "WModelOneMax", "optimization_type": "MAX", "problem_id": 2}, "optimum_x": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "optimum_y": 16.0, "penalty_sparse": [0.0, 0.0, 0.0], "ub": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], "x_sums": [10.0, 10.0, 8.0], "y": [10.0, 10.0, 8.0], "y_at_optimum": 16.0, "y_sparse": [1.0, 2.0, 5.0]}, +"suite/AdversarialAttack": {"dimensions": [1], "instances": [1, 2, 3], "len": 6, "name": "AdversarialAttack", "problem_ids": [5001, 5011], "sequence": [[5001, "AdversarialMNIST5001", 1, 784], [5001, "AdversarialMNIST5001", 2, 784], [5001, "AdversarialMNIST5001", 3, 784], [5011, "AdversarialCIFAR105011", 1, 3072], [5011, "AdversarialCIFAR105011", 2, 3072], [5011, "AdversarialCIFAR105011", 3, 3072]]}, "suite/BBOB": {"dimensions": [2, 5], "instances": [1, 2], "len": 12, "name": "BBOB", "problem_ids": [1, 12, 24], "sequence": [[1, "Sphere", 1, 2], [1, "Sphere", 2, 2], [1, "Sphere", 1, 5], [1, "Sphere", 2, 5], [12, "BentCigar", 1, 2], [12, "BentCigar", 2, 2], [12, "BentCigar", 1, 5], [12, "BentCigar", 2, 5], [24, "LunacekBiRastrigin", 1, 2], [24, "LunacekBiRastrigin", 2, 2], [24, "LunacekBiRastrigin", 1, 5], [24, "LunacekBiRastrigin", 2, 5]]}, "suite/Integer": {"dimensions": [16], "instances": [1], "len": 5, "name": "Integer", "problem_ids": [1, 21, 30, 2000, 10001], "sequence": [[1, "OneMax", 1, 16], [21, "IsingTriangular", 1, 16], [30, "UniformStarDiscrepancy10", 1, 16], [2000, "MaxCut2000", 1, 800], [10001, "DynamicBinValUniform", 1, 16]]}, "suite/IntegerStarDiscrepancy": {"dimensions": [3], "instances": [1], "len": 2, "name": "IntegerStarDiscrepancy", "problem_ids": [30, 59], "sequence": [[30, "UniformStarDiscrepancy10", 1, 3], [59, "HaltonStarDiscrepancy1000", 1, 3]]}, diff --git a/tests/data/golden/registry.json b/tests/data/golden/registry.json index 2efdc6d9..7e17052f 100644 --- a/tests/data/golden/registry.json +++ b/tests/data/golden/registry.json @@ -1,5 +1,19 @@ { "problem_classes": { + "ADVERSARIAL": { + "5001": "AdversarialMNIST5001", + "5002": "AdversarialMNIST5002", + "5003": "AdversarialMNIST5003", + "5004": "AdversarialMNIST5004", + "5005": "AdversarialMNIST5005", + "5006": "AdversarialMNIST5006", + "5011": "AdversarialCIFAR105011", + "5012": "AdversarialCIFAR105012", + "5013": "AdversarialCIFAR105013", + "5014": "AdversarialCIFAR105014", + "5015": "AdversarialCIFAR105015", + "5016": "AdversarialCIFAR105016" + }, "BBOB": { "1": "Sphere", "10": "EllipsoidRotated", @@ -382,6 +396,18 @@ "49": "SobolStarDiscrepancy1000", "5": "LinearSlope", "50": "HaltonStarDiscrepancy10", + "5001": "AdversarialMNIST5001", + "5002": "AdversarialMNIST5002", + "5003": "AdversarialMNIST5003", + "5004": "AdversarialMNIST5004", + "5005": "AdversarialMNIST5005", + "5006": "AdversarialMNIST5006", + "5011": "AdversarialCIFAR105011", + "5012": "AdversarialCIFAR105012", + "5013": "AdversarialCIFAR105013", + "5014": "AdversarialCIFAR105014", + "5015": "AdversarialCIFAR105015", + "5016": "AdversarialCIFAR105016", "51": "HaltonStarDiscrepancy25", "52": "HaltonStarDiscrepancy50", "53": "HaltonStarDiscrepancy100", @@ -627,6 +653,20 @@ "8": "OneMaxRuggedness1", "9": "OneMaxRuggedness2" }, + "AdversarialAttack": { + "5001": "AdversarialMNIST5001", + "5002": "AdversarialMNIST5002", + "5003": "AdversarialMNIST5003", + "5004": "AdversarialMNIST5004", + "5005": "AdversarialMNIST5005", + "5006": "AdversarialMNIST5006", + "5011": "AdversarialCIFAR105011", + "5012": "AdversarialCIFAR105012", + "5013": "AdversarialCIFAR105013", + "5014": "AdversarialCIFAR105014", + "5015": "AdversarialCIFAR105015", + "5016": "AdversarialCIFAR105016" + }, "AttractiveSector": { "1": "Sphere", "10": "EllipsoidRotated", @@ -1110,6 +1150,18 @@ "49": "SobolStarDiscrepancy1000", "5": "LinearSlope", "50": "HaltonStarDiscrepancy10", + "5001": "AdversarialMNIST5001", + "5002": "AdversarialMNIST5002", + "5003": "AdversarialMNIST5003", + "5004": "AdversarialMNIST5004", + "5005": "AdversarialMNIST5005", + "5006": "AdversarialMNIST5006", + "5011": "AdversarialCIFAR105011", + "5012": "AdversarialCIFAR105012", + "5013": "AdversarialCIFAR105013", + "5014": "AdversarialCIFAR105014", + "5015": "AdversarialCIFAR105015", + "5016": "AdversarialCIFAR105016", "51": "HaltonStarDiscrepancy25", "52": "HaltonStarDiscrepancy50", "53": "HaltonStarDiscrepancy100", @@ -1198,6 +1250,18 @@ "49": "SobolStarDiscrepancy1000", "5": "LinearSlope", "50": "HaltonStarDiscrepancy10", + "5001": "AdversarialMNIST5001", + "5002": "AdversarialMNIST5002", + "5003": "AdversarialMNIST5003", + "5004": "AdversarialMNIST5004", + "5005": "AdversarialMNIST5005", + "5006": "AdversarialMNIST5006", + "5011": "AdversarialCIFAR105011", + "5012": "AdversarialCIFAR105012", + "5013": "AdversarialCIFAR105013", + "5014": "AdversarialCIFAR105014", + "5015": "AdversarialCIFAR105015", + "5016": "AdversarialCIFAR105016", "51": "HaltonStarDiscrepancy25", "52": "HaltonStarDiscrepancy50", "53": "HaltonStarDiscrepancy100", @@ -1286,6 +1350,18 @@ "49": "SobolStarDiscrepancy1000", "5": "LinearSlope", "50": "HaltonStarDiscrepancy10", + "5001": "AdversarialMNIST5001", + "5002": "AdversarialMNIST5002", + "5003": "AdversarialMNIST5003", + "5004": "AdversarialMNIST5004", + "5005": "AdversarialMNIST5005", + "5006": "AdversarialMNIST5006", + "5011": "AdversarialCIFAR105011", + "5012": "AdversarialCIFAR105012", + "5013": "AdversarialCIFAR105013", + "5014": "AdversarialCIFAR105014", + "5015": "AdversarialCIFAR105015", + "5016": "AdversarialCIFAR105016", "51": "HaltonStarDiscrepancy25", "52": "HaltonStarDiscrepancy50", "53": "HaltonStarDiscrepancy100", @@ -2424,6 +2500,18 @@ "49": "SobolStarDiscrepancy1000", "5": "LinearSlope", "50": "HaltonStarDiscrepancy10", + "5001": "AdversarialMNIST5001", + "5002": "AdversarialMNIST5002", + "5003": "AdversarialMNIST5003", + "5004": "AdversarialMNIST5004", + "5005": "AdversarialMNIST5005", + "5006": "AdversarialMNIST5006", + "5011": "AdversarialCIFAR105011", + "5012": "AdversarialCIFAR105012", + "5013": "AdversarialCIFAR105013", + "5014": "AdversarialCIFAR105014", + "5015": "AdversarialCIFAR105015", + "5016": "AdversarialCIFAR105016", "51": "HaltonStarDiscrepancy25", "52": "HaltonStarDiscrepancy50", "53": "HaltonStarDiscrepancy100", @@ -3176,6 +3264,18 @@ "49": "SobolStarDiscrepancy1000", "5": "LinearSlope", "50": "HaltonStarDiscrepancy10", + "5001": "AdversarialMNIST5001", + "5002": "AdversarialMNIST5002", + "5003": "AdversarialMNIST5003", + "5004": "AdversarialMNIST5004", + "5005": "AdversarialMNIST5005", + "5006": "AdversarialMNIST5006", + "5011": "AdversarialCIFAR105011", + "5012": "AdversarialCIFAR105012", + "5013": "AdversarialCIFAR105013", + "5014": "AdversarialCIFAR105014", + "5015": "AdversarialCIFAR105015", + "5016": "AdversarialCIFAR105016", "51": "HaltonStarDiscrepancy25", "52": "HaltonStarDiscrepancy50", "53": "HaltonStarDiscrepancy100", @@ -3264,6 +3364,18 @@ "49": "SobolStarDiscrepancy1000", "5": "LinearSlope", "50": "HaltonStarDiscrepancy10", + "5001": "AdversarialMNIST5001", + "5002": "AdversarialMNIST5002", + "5003": "AdversarialMNIST5003", + "5004": "AdversarialMNIST5004", + "5005": "AdversarialMNIST5005", + "5006": "AdversarialMNIST5006", + "5011": "AdversarialCIFAR105011", + "5012": "AdversarialCIFAR105012", + "5013": "AdversarialCIFAR105013", + "5014": "AdversarialCIFAR105014", + "5015": "AdversarialCIFAR105015", + "5016": "AdversarialCIFAR105016", "51": "HaltonStarDiscrepancy25", "52": "HaltonStarDiscrepancy50", "53": "HaltonStarDiscrepancy100", @@ -3945,9 +4057,9 @@ "y": 6.0 }, "golden_wrap_real": { - "id": 1121, - "id_after_rewrap": 1121, - "id_by_name": 1121, + "id": 5017, + "id_after_rewrap": 5017, + "id_by_name": 5017, "y": 6.0 } } diff --git a/tests/python/golden.py b/tests/python/golden.py index a0e13feb..7d670344 100644 --- a/tests/python/golden.py +++ b/tests/python/golden.py @@ -34,6 +34,7 @@ "CEC2022": ([10, 20], [1, 2], 3), # transformation data exists for 10 and 20 for every function "STAR_REAL": ([2, 5], [1, 2], 3), "STAR_INTEGER": ([2, 5], [1, 2], 3), + "ADVERSARIAL": ([1], [1, 2], 3), # the dimension is fixed per dataset; the requested one is ignored "GRAPH": ([2], [1], 2), # dimension and instance are ignored (fixed graph instances) } DYNAMIC_BIN_VAL = ([10], [1, 2], 3) @@ -59,6 +60,7 @@ "PBO": ([1, 20, 25], [1, 2], [16]), "Submodular": ([2000, 2100, 2200, 2300], [1], [1]), "IntegerStarDiscrepancy": ([30, 59], [1], [3]), + "AdversarialAttack": ([5001, 5011], [1, 2, 3], [1]), } @@ -281,7 +283,7 @@ def union_pairs(registry): pairs = [] families = registry["problem_classes"] dbv = registry["views"].get("DynamicBinVal", {}) - for union, members in (("REAL", ("BBOB", "CEC2013", "CEC2022", "STAR_REAL")), + for union, members in (("REAL", ("BBOB", "CEC2013", "CEC2022", "STAR_REAL", "ADVERSARIAL")), ("INTEGER", ("PBO", "GRAPH", "STAR_INTEGER"))): for pid, name in families[union].items(): family = next((f for f in members if families[f].get(pid) == name), None) diff --git a/tests/python/test_problem.py b/tests/python/test_problem.py index bc714090..1d706613 100644 --- a/tests/python/test_problem.py +++ b/tests/python/test_problem.py @@ -180,6 +180,33 @@ def test_integer_star_discrepancy(self): self.assertEqual(random([10, 10]), 0) self.assertEqual(random([11, 11]), -float("inf")) + def test_adversarial_attack(self): + mnist = ioh.get_problem(5001, 1, 1, problem_class=ioh.ProblemClass.ADVERSARIAL) + self.assertEqual(mnist.meta_data.n_variables, 784) # n_variables is ignored + self.assertEqual(mnist.meta_data.name, "AdversarialMNIST5001") + self.assertEqual(mnist.meta_data.optimization_type, ioh.OptimizationType.MAX) + self.assertEqual(mnist.bounds.ub[0], 0.1) + self.assertEqual(ioh.get_problem(5011, 1, 1, ioh.ProblemClass.ADVERSARIAL).meta_data.n_variables, 3072) + + # the unperturbed image is correctly classified, with a small loss (the objective is -log p(true label)) + y = mnist(np.zeros(784)) + self.assertGreaterEqual(y, 0.0) + self.assertLess(y, 1.0) + probabilities = mnist.probabilities() + self.assertAlmostEqual(probabilities.sum(), 1.0) + self.assertEqual(mnist.predicted_label(), mnist.true_label) + + # the instance selects the base image, a different landscape + other = ioh.get_problem(5001, 2, 1, ioh.ProblemClass.ADVERSARIAL) + self.assertNotEqual(other.image_index, mnist.image_index) + + direct = ioh.problem.AdversarialAttack("cifar10", instance=1, eps=0.3, alpha=0.05) + self.assertEqual((direct.dataset, direct.eps, direct.alpha), ("cifar10", 0.3, 0.05)) + self.assertEqual(direct.meta_data.n_variables, 3072) + + suite = ioh.suite.AdversarialAttack([5001, 5004], [1, 2]) + self.assertEqual(len(suite), 4) + def test_enforced_bounds(self): p = ioh.get_problem(1, 1, 2) x0 = [10, 1]