askrene-getroutes adaptive bounds

plugins/askrene/child/mcf.c

@@ -167,6 +167,25 @@
 // cost function arcs.
 static const double CHANNEL_PIVOTS[]={0,0.5,0.8,0.95};
 
+/* MCF preserves flow at intermediate hops, therefore fees do not contribute to
+ * flows or flows costs. We can exceed capacity limits once fees are added
+ * and/or discover very high probability costs triggered by them. To mitigate
+ * this we scale down the min/max limits by this factor assuming a worst case
+ * fee of 1% of the flow amount. This works because multiplying the flow by g
+ * produces the same cost than multiplying the min/max bounds by 1/g.
+ *
+ * We want a new cost function
+ * 	C'(x) = C(x + fees) where x+fees = x*(1+0.01)= x*g
+ *
+ * C'(x) = C(x*g) =
+ * 	case x*g <= a, same as x <= a/g: 0
+ * 	case x*g >= b, same as x >= b/g: infinity
+ * 	case a<=x*g<b, same as a/g<=x<b/g: -log(1-(x*g-a)/(b-a)) = -log(1-(x-a/g)/(b/g-a/g))
+ * */
+/* FIXME: This could prevent us from finding flows that fit tightly through
+ * channel capacities. */
+static const double FLOW_FEE_ADJUSTMENT = 1.01;
+
 static const s64 INFINITE = INT64_MAX;
 static const s64 MU_MAX = 100;
 
@@ -338,9 +357,9 @@ static bool channel_is_available(const struct route_query *rq,
  * @low: the liquidity is known to be greater or equal than "low"
  * @high: the liquidity is known to be less than "high"
  * @amount: how much is required to forward */
-static double pickhardt_richter_probability(struct amount_msat low,
-					    struct amount_msat high,
-					    struct amount_msat amount)
+double pickhardt_richter_probability(struct amount_msat low,
+				     struct amount_msat high,
+				     struct amount_msat amount)
 {
 	struct amount_msat all_states, good_states;
 	if (amount_msat_greater_eq(amount, high))
@@ -369,6 +388,18 @@ static void linearize_channel(const struct pay_parameters *params,
 	if (amount_msat_greater(mincap, maxcap))
 		mincap = maxcap;
 
+	/* Allow space for fees at 1% */
+	if (!amount_msat_scale(&mincap, mincap, 1 / FLOW_FEE_ADJUSTMENT)) {
+		child_log(tmpctx, LOG_UNUSUAL,
+			  "%s: Couldn't scale down mincap=%s by %lf", __func__,
+			  fmt_amount_msat(tmpctx, mincap), FLOW_FEE_ADJUSTMENT);
+	}
+	if (!amount_msat_scale(&maxcap, maxcap, 1 / FLOW_FEE_ADJUSTMENT)) {
+		child_log(tmpctx, LOG_UNUSUAL,
+			  "%s: Couldn't scale down maxcap=%s by %lf", __func__,
+			  fmt_amount_msat(tmpctx, maxcap), FLOW_FEE_ADJUSTMENT);
+	}
+
 	u64 a = amount_msat_ratio_floor(mincap, params->accuracy),
 	    b = 1 + amount_msat_ratio_floor(maxcap, params->accuracy);
 

plugins/askrene/child/mcf.h

@@ -8,6 +8,7 @@
 #include <common/gossmap.h>
 #include <common/jsonrpc_errors.h>
 
+struct flow;
 struct route_query;
 
 /* A wrapper to the min. cost flow solver that actually takes into consideration
@@ -33,4 +34,13 @@ const char *single_path_routes(const tal_t *ctx, struct route_query *rq,
 			       double *probability,
 			       enum jsonrpc_errcode *ecode);
 
+/* The probability of forwarding a payment amount given a high and low liquidity
+ * bounds.
+ * @low: the liquidity is known to be greater or equal than "low"
+ * @high: the liquidity is known to be less than "high"
+ * @amount: how much is required to forward */
+double pickhardt_richter_probability(struct amount_msat low,
+				     struct amount_msat high,
+				     struct amount_msat amount);
+
 #endif /* LIGHTNING_PLUGINS_ASKRENE_CHILD_MCF_H */

plugins/askrene/child/route_query.c

@@ -1,10 +1,18 @@
 #include "config.h"
+#include <ccan/asort/asort.h>
 #include <common/gossmap.h>
+#include <common/utils.h>
 #include <plugins/askrene/child/additional_costs.h>
+#include <plugins/askrene/child/mcf.h>
 #include <plugins/askrene/child/route_query.h>
 #include <plugins/askrene/layer.h>
 #include <plugins/askrene/reserve.h>
 
+/* It could be any number between 0 and 1. It represents the fraction of lower
+ * liquidity bound that we adjust when we find a failure. The smaller it is the
+ * more we trust previous knowledge. Similar to a "learning velocity" for AI. */
+#define ASKRENE_FAILURE_RELAX_FRACTION 0.5
+
 struct amount_msat get_additional_per_htlc_cost(const struct route_query *rq,
 						const struct short_channel_id_dir *scidd)
 {
@@ -16,6 +24,182 @@ struct amount_msat get_additional_per_htlc_cost(const struct route_query *rq,
 		return AMOUNT_MSAT(0);
 }
 
+static int intel_cmp(const struct channel_intel *a,
+		     const struct channel_intel *b, void *unused)
+{
+	const u64 a_time = channel_intel_timestamp(a);
+	const u64 b_time = channel_intel_timestamp(b);
+	if (a_time < b_time)
+		return -1;
+	if (a_time > b_time)
+		return 1;
+	return 0;
+}
+
+/* Bounds in one direction determine the bounds on the other direction. */
+static void reverse_bounds(struct amount_msat *rev_min,
+			   struct amount_msat *rev_max,
+			   struct amount_msat capacity,
+			   struct amount_msat min,
+			   struct amount_msat max)
+{
+	if (!amount_msat_sub(rev_min, capacity, max)) {
+		assert(0);
+	}
+	if (!amount_msat_sub(rev_max, capacity, min)) {
+		assert(0);
+	}
+}
+
+/* When we have been informed of an "unconstrained" flow event. */
+static void bounds_by_unconstrained(struct amount_msat x,
+				    struct amount_msat capacity,
+				    struct amount_msat *min,
+				    struct amount_msat *max,
+				    bool reverse)
+{
+	if(reverse){
+		struct amount_msat rev_min, rev_max;
+		reverse_bounds(&rev_min, &rev_max, capacity, *min, *max);
+		bounds_by_unconstrained(x, capacity, &rev_min, &rev_max, false);
+		reverse_bounds(min, max, capacity, rev_min, rev_max);
+		return;
+	}
+	*min = amount_msat_max(*min, x);
+	*max = amount_msat_max(*max, x);
+}
+
+/* When we have been informed of a "constrained" flow event. */
+static void bounds_by_constrained(struct amount_msat x,
+				  struct amount_msat capacity,
+				  struct amount_msat *min,
+				  struct amount_msat *max,
+				  bool reverse)
+{
+	if(reverse){
+		struct amount_msat rev_min, rev_max;
+		reverse_bounds(&rev_min, &rev_max, capacity, *min, *max);
+		bounds_by_constrained(x, capacity, &rev_min, &rev_max, false);
+		reverse_bounds(min, max, capacity, rev_min, rev_max);
+		return;
+	}
+
+	double prob_fail;
+	struct amount_msat high, amount;
+	if (amount_msat_greater(x, *max)) {
+		/* Trivial case, we were expecting x to fail. */
+	} else if (amount_msat_less(x, *min)) {
+		/* This should have succeeded 100% of the times,
+		 * our knowledge was wrong. */
+		*min = amount_msat_min(*min, x);
+		*max = amount_msat_min(*max, x);
+		if (!amount_msat_scale(min, *min,
+				       1.0 - ASKRENE_FAILURE_RELAX_FRACTION)) {
+			*min = AMOUNT_MSAT(0);
+		}
+	} else {
+		/* We got failure for a quantity between min and
+		 * max bounds. We relax a little the lower bound
+		 * in relation to the probability of this event
+		 * taking place. If p~1, this was expected,
+		 * min/max reflected reality. On the other hand
+		 * if p~0, we were either unlucky or more likely
+		 * our lower bound was too high. */
+
+		/* off-by-one because the high bound in MCF
+		 * means "we know the liquidity is below this
+		 * value", which makes some equations take a
+		 * simpler form. */
+		if (!amount_msat_add(&high, *max, AMOUNT_MSAT(1)))
+			high = capacity;
+		/* off-by-one because
+		 * json_askrene_inform_channel already
+		 * substracted 1msat here, meaning we tried x+1
+		 * and it failed. */
+		if (!amount_msat_add(&amount, x, AMOUNT_MSAT(1)))
+			amount = capacity;
+		prob_fail =
+		    1.0 - pickhardt_richter_probability(*min, high, amount);
+		assert(prob_fail >= 0 && prob_fail <= 1.0);
+
+		*max = amount_msat_min(*max, x);
+		if (!amount_msat_scale(min, *min,
+				       1.0 + ASKRENE_FAILURE_RELAX_FRACTION *
+						 (prob_fail - 1.0))) {
+			*min = AMOUNT_MSAT(0);
+		}
+	}
+}
+
+/* When we have been informed of a "succeeded" flow event. */
+static void bounds_by_impression(struct amount_msat x,
+				 struct amount_msat capacity,
+				 struct amount_msat *min,
+				 struct amount_msat *max,
+				 bool reverse)
+{
+	if(reverse){
+		struct amount_msat rev_min, rev_max;
+		reverse_bounds(&rev_min, &rev_max, capacity, *min, *max);
+		bounds_by_impression(x, capacity, &rev_min, &rev_max, false);
+		reverse_bounds(min, max, capacity, rev_min, rev_max);
+		return;
+	}
+	if(!amount_msat_deduct(max, x))
+		*max = AMOUNT_MSAT(0);
+	if(!amount_msat_deduct(min, x))
+		*min = AMOUNT_MSAT(0);
+}
+
+/* Computes min/max bounds based on known constraints. It self-adjusts for
+ * contradictory information giving precedence to more recent constraints.
+ * FIXME: add time decay
+ * FIXME: this approach was completey cooked by hand because it is better than
+ * simply trusting all constraints as we have seen during tests (see CLN #9282).
+ * However it would be nice to have a theoretically sound adjustment, eg.
+ * Maximum Likelyhood, if applicable.
+ * FIXME: unit test it */
+static void get_bounds_adaptively(struct channel_intel *intelarr,
+					      const struct amount_msat capacity,
+					      struct amount_msat *min,
+					      struct amount_msat *max,
+					      int dir)
+{
+	const struct constraint *constraint;
+	const struct impression *impression;
+
+	*min = AMOUNT_MSAT(0);
+	*max = capacity;
+	asort(intelarr, tal_count(intelarr), intel_cmp, NULL);
+	for (size_t i = 0; i < tal_count(intelarr); i++) {
+		if (intelarr[i].constraint) {
+			/* a constraint */
+			assert(!intelarr[i].impression);
+			constraint = intelarr[i].constraint;
+			if (amount_msat_greater_eq(constraint->max,
+						   AMOUNT_MSAT(UINT64_MAX))) {
+				/* this is an "unconstrained" event, a min value
+				 * bound */
+				bounds_by_unconstrained(
+				    constraint->min, capacity, min, max,
+				    dir != constraint->scidd.dir);
+			} else {
+				/* this is a "constrained" event, a max value
+				 * bound */
+				bounds_by_constrained(
+				    constraint->max, capacity, min, max,
+				    dir != constraint->scidd.dir);
+			}
+		} else {
+			/* an impression */
+			assert(intelarr[i].impression);
+			impression = intelarr[i].impression;
+			bounds_by_impression(impression->amount, capacity, min,
+					     max, dir != impression->scidd.dir);
+		}
+	}
+}
+
 void get_constraints(const struct route_query *rq,
 		     const struct gossmap_chan *chan,
 		     int dir,
@@ -24,6 +208,8 @@ void get_constraints(const struct route_query *rq,
 {
 	struct short_channel_id_dir scidd;
 	size_t idx = gossmap_chan_idx(rq->gossmap, chan);
+	struct channel_intel *intelarr;
+	struct amount_msat capacity;
 
 	*min = AMOUNT_MSAT(0);
 
@@ -34,7 +220,8 @@ void get_constraints(const struct route_query *rq,
 	}
 
 	/* Might be here because it's reserved, but capacity is normal. */
-	*max = gossmap_chan_get_capacity(rq->gossmap, chan);
+	*max = capacity = gossmap_chan_get_capacity(rq->gossmap, chan);
+	intelarr = tal_arr(tmpctx, struct channel_intel, 0);
 
 	/* Naive implementation! */
 	scidd.scid = gossmap_chan_scid(rq->gossmap, chan);
@@ -43,8 +230,11 @@ void get_constraints(const struct route_query *rq,
 	/* Look through layers for any constraints (might be dummy
 	 * ones, for created channels!) */
 	for (size_t i = 0; i < tal_count(rq->layers); i++)
-		layer_apply_constraints(rq->layers[i], &scidd, min, max);
+		intelarr = layer_collect_channel_intels(tmpctx, rq->layers[i],
+							&scidd, take(intelarr));
 
+	get_bounds_adaptively(intelarr, capacity, min, max, dir);
+	tal_free(intelarr);
 	/* Finally, if any is in use, subtract that! */
 	reserve_sub(rq->reserved, &scidd, rq->layers, min);
 	reserve_sub(rq->reserved, &scidd, rq->layers, max);

plugins/askrene/layer.c

@@ -34,26 +34,6 @@ struct local_update {
 	const struct amount_msat *htlc_min, *htlc_max;
 };
 
-/* A constraint reflects something we learned about a channel */
-struct constraint {
-	struct short_channel_id_dir scidd;
-	/* Time this constraint was last updated */
-	u64 timestamp;
-	/* Non-zero means set */
-	struct amount_msat min;
-	/* Non-0xFFFFF.... means set */
-	struct amount_msat max;
-};
-
-/* An impression reflects something we did to a channel (successful payments) */
-struct impression {
-	/* This is the direction of the payment, but it affects both ways */
-	struct short_channel_id_dir scidd;
-	/* Time this constraint was last updated */
-	u64 timestamp;
-	struct amount_msat amount;
-};
-
 /* A bias, for special-effects (user-controlled) */
 struct bias {
 	struct short_channel_id_dir scidd;
@@ -69,13 +49,6 @@ struct node_bias {
 	u64 timestamp;
 };
 
-/* A timestamp-ordered list of impresssion and constraint */
-struct channel_intel {
-	/* Only one is set */
-	const struct impression *impression;
-	const struct constraint *constraint;
-};
-
 static struct short_channel_id
 channel_intel_scid(const struct channel_intel *intelarr)
 {
@@ -311,13 +284,6 @@ static struct local_update *add_update_channel(struct layer *layer,
 	return lu;
 }
 
-static u64 channel_intel_timestamp(const struct channel_intel *intel)
-{
-	if (intel->constraint)
-		return intel->constraint->timestamp;
-	return intel->impression->timestamp;
-}
-
 /* Insert this constraint/impression in htable, maintaining timestamp order */
 static void add_channel_intel(struct layer *layer,
 			      const struct constraint *constraint STEALS,
@@ -1147,6 +1113,26 @@ void layer_apply_constraints(const struct layer *layer,
 	}
 }
 
+struct channel_intel *layer_collect_channel_intels(const tal_t *ctx,
+						   const struct layer *layer,
+						   const struct short_channel_id_dir *scidd,
+						   struct channel_intel *in_intelarr TAKES)
+{
+	struct channel_intel *out_intelarr;
+	struct channel_intel *intelarr =
+	    channel_intel_hash_get(layer->channel_intels, scidd->scid);
+
+	if (in_intelarr) {
+		out_intelarr =
+		    tal_dup_talarr(ctx, struct channel_intel, in_intelarr);
+		tal_arr_append(&out_intelarr, intelarr);
+	} else {
+		out_intelarr =
+		    tal_dup_talarr(ctx, struct channel_intel, intelarr);
+	}
+	return out_intelarr;
+}
+
 const struct constraint *layer_add_constraint(struct layer *layer,
 					      const struct short_channel_id_dir *scidd,
 					      u64 timestamp,