Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions RELEASES.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
- Raise errors instead of printing a message and returning `None` or dummy values in `ot.datasets.make_data_classif` (unknown dataset), `ot.stochastic.solve_semi_dual_entropic` (unknown method) and `ot.mapping.nearest_brenier_potential_*` (missing CVXPY), and emit warnings instead of prints in `ot.gaussian.bures_barycenter_fixpoint`, `ot.regpath` and `ot.partial.entropic_partial_wasserstein` (PR #888)
- Fix docstring errors (wrong or duplicated parameters, typos) and fix the file patterns of the ruff hooks in `.pre-commit-config.yaml` (PR #888)
- Fix `arange` in all backends silently ignoring the dtype of `type_as` (and the device in the NumPy, JAX, TensorFlow and CuPy backends), unlike `zeros`, `ones` and `full`. `roll_cols` in `ot.lp.solver_circle` no longer passes a plain Python `int` as `type_as` (PR #903, Issue #864)
- Fix interpreter crashes in the exact OT solvers on degenerate inputs: `ot.emd` and `ot.emd2` reject a cost matrix with an empty dimension with a `ValueError`, marginals with a zero total mass return the exact zero plan with finite dual potentials instead of NaNs, non-finite marginals are rejected, sparse problems without any allowed transport arc report `INFEASIBLE` instead of crashing, and `ot.emd_1d` rejects an empty support with a `ValueError`. The native entry points (`emd_c`, `emd_c_sparse`, `emd_c_lazy`) validate marginal, potential and edge buffers against the problem they solve, and the network simplex pivot rule no longer reads an uninitialised arc index when no entering arc is found (Issue #901)

## 0.9.7.post1

Expand Down
23 changes: 23 additions & 0 deletions ot/lp/EMD_wrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,14 @@ int EMD_wrap(int n1, int n2, double *X, double *Y, double *D, double *G,
}
}

// Empty support: there is no arc to transport mass along. When both sides
// are empty the zero plan is the only feasible plan and is optimal,
// otherwise the problem is infeasible.
if (n == 0 || m == 0) {
*cost = 0.0;
return (n == 0 && m == 0) ? OPTIMAL : INFEASIBLE;
}

// Define graph and solver
std::vector<uint64_t> indI(n), indJ(m);
std::vector<double> weights1(n), weights2(m);
Expand Down Expand Up @@ -345,6 +353,14 @@ int EMD_wrap_sparse(
}
}

// Empty support: see EMD_wrap. No flow can be extracted, so n_flows_out
// is reset before returning the status of the degenerate problem.
if (n == 0 || m == 0) {
*cost = 0.0;
*n_flows_out = 0;
return (n == 0 && m == 0) ? OPTIMAL : INFEASIBLE;
}

std::vector<uint64_t> indI(n); // indI[graph_idx] = original_source_idx
std::vector<uint64_t> indJ(m); // indJ[graph_idx] = original_target_idx
std::vector<double> weights1(n); // Source masses (positive only)
Expand Down Expand Up @@ -391,6 +407,13 @@ int EMD_wrap_sparse(
for (uint64_t k = 0; k < n_edges; k++) {
int64_t src_orig = edge_sources[k];
int64_t tgt_orig = edge_targets[k];
// Reject endpoints outside the marginals: they would index
// source_to_graph / target_to_graph out of bounds.
if (src_orig < 0 || tgt_orig < 0 ||
static_cast<uint64_t>(src_orig) >= static_cast<uint64_t>(n1) ||
static_cast<uint64_t>(tgt_orig) >= static_cast<uint64_t>(n2)) {
return INFEASIBLE;
}
int64_t src = source_to_graph[src_orig];
int64_t tgt = target_to_graph[tgt_orig];

Expand Down
49 changes: 46 additions & 3 deletions ot/lp/_network_simplex.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ def center_ot_dual(alpha0, beta0, a=None, b=None):
since :math:`\sum_i a_i=\sum_j b_j` this can be solved by adding/removing
a constant from both :math:`\alpha_0` and :math:`\beta_0`.

When both marginals have a zero total mass the constant is undefined and the
potentials are returned unchanged.

.. math::
c &= \frac{\beta_0^T \mathbf{b} - \alpha_0^T \mathbf{a}}{\mathbf{1}^T \mathbf{b} + \mathbf{1}^T \mathbf{a}}

Expand Down Expand Up @@ -80,7 +83,9 @@ def center_ot_dual(alpha0, beta0, a=None, b=None):
# compute constant that balances the weighted sums of the duals
ips = nx.sum(b * beta0, axis=0) - nx.sum(a * alpha0, axis=0)
denom = nx.sum(a, axis=0) + nx.sum(b, axis=0)
c = ips / denom
# A zero denominator means both marginals are empty: there is nothing to
# balance, and 0/0 would turn the potentials into NaN.
c = ips / nx.where(denom == 0, denom + 1, denom)

# update duals
alpha = alpha0 + c
Expand Down Expand Up @@ -171,6 +176,32 @@ def estimate_dual_null_weights(alpha0, beta0, a, b, M):
return center_ot_dual(alpha, beta, a, b)


def _check_cost_matrix_shape(n1, n2):
"""Raise for a cost matrix with no row or no column."""
if n1 == 0 or n2 == 0:
raise ValueError(
"The cost matrix must have at least one row and one column, got "
f"shape ({n1}, {n2}). An empty marginal cannot be defaulted to a "
"uniform distribution."
)


def _rescale_target_mass(a, b):
"""Rescale ``b`` to the total mass of ``a``, allowing empty marginals.

A target histogram with a zero total mass cannot be rescaled: it is left
unchanged, so that the solver reports it as infeasible, while the other
histograms of a batch are still solved. When ``a`` has no mass either, the
zero plan is the only feasible plan and is optimal.
"""
mass_a = a.sum(0)
mass_b = b.sum(0, keepdims=True)
if not (np.isfinite(mass_a).all() and np.isfinite(mass_b).all()):
raise ValueError("a and b must have a finite total mass")

return b * mass_a / np.where(mass_b != 0, mass_b, 1.0)


def _compute_active_subset(a, b, M, row_mask, col_mask):
"""Return filtered inputs restricted to the rows/columns with mass."""
need_filter = np.any(~row_mask) or np.any(~col_mask)
Expand Down Expand Up @@ -259,6 +290,10 @@ def emd(

.. note:: An error will be raised if the vectors :math:`\mathbf{a}` and :math:`\mathbf{b}` do not sum to the same value.

.. note:: A cost matrix with no row or no column raises a ``ValueError``.
When both marginals have a zero total mass, the optimal plan is the
zero plan with a zero cost.

Uses the algorithm proposed in :ref:`[1] <references-emd>`.

Parameters
Expand Down Expand Up @@ -388,6 +423,8 @@ def emd(
if n1 is None:
n1, n2 = M.shape

_check_cost_matrix_shape(n1, n2)

if a is None or len(a) == 0:
a = nx.ones((n1,), type_as=type_as) / n1
if b is None or len(b) == 0:
Expand All @@ -414,7 +451,7 @@ def emd(
err_msg="a and b vector must have the same sum",
decimal=6,
)
b = b * a.sum() / b.sum()
b = _rescale_target_mass(a, b)

asel = a != 0
bsel = b != 0
Expand Down Expand Up @@ -567,6 +604,10 @@ def emd2(

.. note:: An error will be raised if the vectors :math:`\mathbf{a}` and :math:`\mathbf{b}` do not sum to the same value.

.. note:: A cost matrix with no row or no column raises a ``ValueError``.
When both marginals have a zero total mass, the optimal plan is the
zero plan with a zero cost.

Uses the algorithm proposed in :ref:`[1] <references-emd2>`.

Parameters
Expand Down Expand Up @@ -698,6 +739,8 @@ def emd2(
if n1 is None:
n1, n2 = M.shape

_check_cost_matrix_shape(n1, n2)

# if empty array given then use uniform distributions
if a is None or len(a) == 0:
a = nx.ones((n1,), type_as=type_as) / n1
Expand Down Expand Up @@ -734,7 +777,7 @@ def emd2(
err_msg="a and b vector must have the same sum",
decimal=6,
)
b = b * a.sum(0) / b.sum(0, keepdims=True)
b = _rescale_target_mass(a, b)

asel = a != 0

Expand Down
59 changes: 59 additions & 0 deletions ot/lp/emd_wrap.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,14 @@ def emd_c(np.ndarray[double, ndim=1, mode="c"] a, np.ndarray[double, ndim=1, mod
cdef int result_code = 0
cdef int nG=0

if (a.shape[0] != n1 and a.shape[0] != 0) or (
b.shape[0] != n2 and b.shape[0] != 0
):
raise ValueError(
f"Shape mismatch: the cost matrix has shape ({n1}, {n2}) but a has "
f"{a.shape[0]} entries and b has {b.shape[0]}"
)

cdef double cost=0
cdef np.ndarray[double, ndim=1, mode="c"] alpha=np.zeros(n1)
cdef np.ndarray[double, ndim=1, mode="c"] beta=np.zeros(n2)
Expand Down Expand Up @@ -124,6 +132,12 @@ def emd_c(np.ndarray[double, ndim=1, mode="c"] a, np.ndarray[double, ndim=1, mod
if alpha_init is not None and beta_init is not None:
alpha_init_c = np.ascontiguousarray(alpha_init, dtype=np.float64)
beta_init_c = np.ascontiguousarray(beta_init, dtype=np.float64)
if alpha_init_c.size != n1 or beta_init_c.size != n2:
raise ValueError(
f"Shape mismatch: the initial potentials have sizes "
f"{alpha_init_c.size} and {beta_init_c.size}, expected {n1} "
f"and {n2}"
)
alpha_init_ptr = <double*> alpha_init_c.data
beta_init_ptr = <double*> beta_init_c.data

Expand Down Expand Up @@ -175,6 +189,12 @@ def emd_1d_sorted(np.ndarray[double, ndim=1, mode="c"] u_weights,
cdef Py_ssize_t n = u_weights.shape[0]
cdef Py_ssize_t m = v_weights.shape[0]

if n == 0 or m == 0:
raise ValueError(
"The 1d EMD problem needs at least one source and one target point, "
f"got {n} and {m}"
)

cdef Py_ssize_t i = 0
cdef double w_i = u_weights[0]
cdef Py_ssize_t j = 0
Expand Down Expand Up @@ -278,10 +298,28 @@ def emd_c_sparse(np.ndarray[double, ndim=1, mode="c"] a,
cdef int n1 = a.shape[0]
cdef int n2 = b.shape[0]
cdef uint64_t n_edges = edge_sources.shape[0]
cdef uint64_t max_source = 0
cdef uint64_t max_target = 0
cdef uint64_t n_flows_out = 0
cdef int result_code = 0
cdef double cost = 0

if edge_targets.shape[0] != n_edges or edge_costs.shape[0] != n_edges:
raise ValueError(
"edge_sources, edge_targets and edge_costs must have the same "
f"length, got {n_edges}, {edge_targets.shape[0]} and "
f"{edge_costs.shape[0]}"
)
if n_edges > 0:
max_source = edge_sources.max()
max_target = edge_targets.max()
if max_source >= <uint64_t>n1 or max_target >= <uint64_t>n2:
raise ValueError(
f"Edge endpoints must be smaller than the marginals: got a "
f"maximal source index of {max_source} (n1={n1}) and a maximal "
f"target index of {max_target} (n2={n2})"
)

# Allocate output arrays (max size = n_edges)
cdef np.ndarray[uint64_t, ndim=1, mode="c"] flow_sources = np.zeros(n_edges, dtype=np.uint64)
cdef np.ndarray[uint64_t, ndim=1, mode="c"] flow_targets = np.zeros(n_edges, dtype=np.uint64)
Expand All @@ -292,6 +330,12 @@ def emd_c_sparse(np.ndarray[double, ndim=1, mode="c"] a,
cdef double* alpha_init_ptr = NULL
cdef double* beta_init_ptr = NULL
if alpha_init is not None and beta_init is not None:
if alpha_init.shape[0] != n1 or beta_init.shape[0] != n2:
raise ValueError(
f"Shape mismatch: the initial potentials have sizes "
f"{alpha_init.shape[0]} and {beta_init.shape[0]}, expected "
f"{n1} and {n2}"
)
alpha_init_ptr = <double*> alpha_init.data
beta_init_ptr = <double*> beta_init.data

Expand Down Expand Up @@ -328,6 +372,15 @@ def emd_c_lazy(np.ndarray[double, ndim=1, mode="c"] a, np.ndarray[double, ndim=1
cdef uint64_t n_flows_out = 0
cdef uint64_t max_flows_out = n1 + n2

if (a.shape[0] != n1 and a.shape[0] != 0) or (
b.shape[0] != n2 and b.shape[0] != 0
):
raise ValueError(
f"Shape mismatch: the coordinates have shapes ({n1}, {dim}) and "
f"({n2}, {coords_b.shape[1]}) but a has {a.shape[0]} entries and b "
f"has {b.shape[0]}"
)

# Validate dimension consistency
if coords_b.shape[1] != dim:
raise ValueError(f"Coordinate dimension mismatch: coords_a has {dim} dimensions but coords_b has {coords_b.shape[1]}")
Expand Down Expand Up @@ -356,6 +409,12 @@ def emd_c_lazy(np.ndarray[double, ndim=1, mode="c"] a, np.ndarray[double, ndim=1
cdef double* alpha_init_ptr = NULL
cdef double* beta_init_ptr = NULL
if alpha_init is not None and beta_init is not None:
if alpha_init.shape[0] != n1 or beta_init.shape[0] != n2:
raise ValueError(
f"Shape mismatch: the initial potentials have sizes "
f"{alpha_init.shape[0]} and {beta_init.shape[0]}, expected "
f"{n1} and {n2}"
)
alpha_init_ptr = <double*> alpha_init.data
beta_init_ptr = <double*> beta_init.data

Expand Down
16 changes: 15 additions & 1 deletion ot/lp/network_simplex_simple.h
Original file line number Diff line number Diff line change
Expand Up @@ -819,6 +819,10 @@ namespace lemon {
cnt = _block_size;
}
}
// No arc with a negative reduced cost was scanned, so
// _in_arc was never assigned and the current flow is
// optimal. Return before dereferencing it.
if (min >= 0) return false;
a=fabs(_pi[_ns.arcSource(_in_arc)])>fabs(_pi[_ns.arcTarget(_in_arc)]) ? fabs(_pi[_ns.arcSource(_in_arc)]):fabs(_pi[_ns.arcTarget(_in_arc)]);
a=a>fabs(getCost(_in_arc))?a:fabs(getCost(_in_arc));
if (min >= -EPSILON*a) return false;
Expand Down Expand Up @@ -928,9 +932,15 @@ namespace lemon {
_D_ptr = D;
_D_n2 = n2;
// Precompute max cost once for reuse in init()
if (_arc_num == 0) {
// No arc means there is no cost to read: D[0] would be out of
// bounds and the scan below would never terminate.
_has_max_cost = false;
return *this;
}
_has_max_cost = true;
_max_cost = D[0];
for (ArcsType i = 1; i != _arc_num; ++i) {
for (ArcsType i = 1; i < _arc_num; ++i) {
if (D[i] > _max_cost) _max_cost = D[i];
}
return *this;
Expand Down Expand Up @@ -1931,6 +1941,10 @@ namespace lemon {
_all_arc_num = f;
}

// Initialize in_arc to a valid value: the pivot rule reads it to
// scale the tolerance even when no entering arc has been found.
in_arc = 0;

return true;
}

Expand Down
8 changes: 8 additions & 0 deletions ot/lp/solver_1d.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,14 @@ def emd_1d(
+ "`['sqeuclidean', 'minkowski', 'cityblock', 'euclidean']`"
)

# an empty support has no transport plan: reject it before the marginal
# defaulting divides by zero
if x_a.shape[0] == 0 or x_b.shape[0] == 0:
raise ValueError(
"The 1d EMD problem needs at least one source and one target point, "
f"got {x_a.shape[0]} and {x_b.shape[0]}"
)

# if empty array given then use uniform distributions
if a is None or a.ndim == 0 or len(a) == 0:
a = nx.ones((x_a.shape[0],), type_as=x_a) / x_a.shape[0]
Expand Down
Loading
Loading