From 7a04967ca8aa1a6a7b0113b2cd970c0a5b8cea37 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 31 Jul 2026 10:17:40 -0700 Subject: [PATCH 1/4] feat(repeater): discover and subscribe to the regions of other repeaters Adds the client side of the existing anon-regions request, which until now only had a server side (handleAnonRegionsReq), and uses it two ways: - `discover.regions` asks each heard neighbor in turn which regions it floods, printing results to serial as they arrive. `discover.regions list` reports the last run for remote CLI users. - `regions.subscribe ` nominates a repeater to take regions from. The node fetches its regions on change, 2 minutes after boot, and every 72 hours after that, adding any region it doesn't already have as a flood-allowed child of the wildcard. An unanswered fetch is retried after a minute for the first few attempts, then every 12 hours. Subscribing is additive: a region already on this node keeps its flags and its place in the hierarchy, so a region the operator has denied stays denied, and nothing is ever removed. Private ($) regions are skipped, as their transport keys are not derived from the name. Only one request is in flight at a time. The queried node is matched in searchPeersByHash after the ACL clients, so that querying a node which is also a client cannot swallow that client's own traffic. Also fixes two pre-existing issues this feature depends on: - handleAnonRegionsReq() could build a reply larger than createDatagram() accepts, so a node with a large region map answered nothing at all. - discovery_mod_timestamp was saved but never serialized, so it reset to zero on every reboot. --- docs/cli_commands.md | 62 ++++++ examples/simple_repeater/MyMesh.cpp | 313 +++++++++++++++++++++++++++- examples/simple_repeater/MyMesh.h | 23 ++ src/helpers/CommonCLI.h | 4 + src/helpers/RegionMap.cpp | 47 +++++ src/helpers/RegionMap.h | 10 + 6 files changed, 456 insertions(+), 3 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index b618ae2bfe..a9392bd6bd 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -129,6 +129,26 @@ This document provides an overview of CLI commands that can be sent to MeshCore --- +### Discover the regions of zero hop neighbors + +**Usage:** +- `discover.regions` +- `discover.regions list` + +**Note:** Asks each neighbor in turn which regions it floods, one request at a time. The reply +is `OK - querying neighbors`; results arrive over the following seconds and are printed to +the serial terminal as they land. + +**Note:** `discover.regions list` reports the results of the last run, as `{pubkey-prefix}:{regions}` +per line. A `?` in place of the regions means that neighbor did not answer — it may be out of range, +running older firmware, or simply rate-limiting anonymous requests (repeaters answer at most 4 per +3 minutes). This reply is capped at 134 characters, so on a node with many neighbors it may not +show them all — the serial output always does. + +**Note:** The pubkey prefixes reported here can be passed straight to `regions.subscribe`. + +--- + ## Statistics ### Clear Stats @@ -856,6 +876,48 @@ region save --- +#### Subscribe to the regions of another repeater (Repeater Only) +**Usage:** +- `regions.subscribe` +- `regions.subscribe ` +- `regions.subscribe now` +- `regions.subscribe off` + +**Parameters:** +- `pubkey`: Public key of the repeater to subscribe to. A prefix (4 bytes or more) is accepted if + that node is a current neighbor, such as the prefixes reported by `discover.regions`; otherwise + give the full key. + +**Note:** The node asks the subscribed repeater which regions it floods, and adds any it does not +already have. This is **additive**: a region already on this node is left exactly as it is, keeping +its flags and its place in the hierarchy, so a region you have deliberately denied stays denied. +New regions are added as flood-allowed children of the wildcard `*`, and are saved automatically. + +**Note:** The subscribed node must be within direct radio range — the request and its reply are both +sent zero-hop, so a repeater that is only reachable over several hops will never answer. A node also +answers at most 4 anonymous requests every 3 minutes, so a fetch issued right after `discover.regions` +can go unanswered. Unanswered fetches are retried after 1 minute for the first 5 attempts, and every +12 hours after that until one succeeds; `regions.subscribe` reports the failure count and the next +retry. + +**Note:** The fetch repeats every 72 hours (2 minutes after boot, and immediately when the +subscription is set or `regions.subscribe now` is used), so new regions added to the subscribed node +are picked up. Nothing is ever removed by this command. `region remove` deletes a region locally, but +if the subscribed node still floods it, the next fetch adds it back as flood-allowed — to drop it for +good, either `regions.subscribe off` first, or keep the region and `region denyf` it, which a fetch +will not override. + +**Note:** A reply carries at most ~158 characters of region names. A subscribed node with a larger +map has its list truncated at a name boundary, and the names past that point are not inherited. + +**Note:** Private (`$`) regions are skipped, since their transport keys are not derived from the name. + +**Note:** Bare `regions.subscribe` reports the subscription and the result of the last fetch, for +example `OK - subscribed to a1b2c3d4 (2 added, 5 known, 43 secs ago)`. `regions.subscribe off` +clears it and replies `OK - unsubscribed`. + +--- + #### Remove a region **Usage:** - `region remove ` diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 09f74cbeaf..73492af4a0 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -52,6 +52,10 @@ #define RESP_SERVER_LOGIN_OK 0 // response to ANON_REQ +// createDatagram() rejects a reply longer than this, once the MAC and the cipher block +// padding are added. The exporters below MUST keep within it, or no reply is sent at all. +#define MAX_ANON_REPLY_LEN (MAX_PACKET_PAYLOAD - CIPHER_MAC_SIZE - (CIPHER_BLOCK_SIZE - 1)) + #define ANON_REQ_TYPE_REGIONS 0x01 #define ANON_REQ_TYPE_OWNER 0x02 #define ANON_REQ_TYPE_BASIC 0x03 // just remote clock @@ -60,6 +64,20 @@ #define LAZY_CONTACTS_WRITE_DELAY 5000 +#define REGION_QUERY_NONE 0 +#define REGION_QUERY_DISCOVER 1 // 'discover.regions' pass over the neighbours table +#define REGION_QUERY_SUBSCRIBE 2 // scheduled fetch from the subscribed node + +#define REGION_QUERY_TIMEOUT 20000 // millis to wait for one regions reply +#define REGION_FETCH_START_DELAY 120000 // first fetch after boot +#define REGION_FETCH_INTERVAL (72*3600000) // between successful fetches +#define REGION_FETCH_RETRY 60000 // after an unanswered fetch +#define REGION_FETCH_MAX_TRIES 5 // ..then back off to: +#define REGION_FETCH_BACKOFF (12*3600000) // ..until one succeeds + +// peer index (see searchPeersByHash) for the node we are querying for regions +#define REGION_QUERY_PEER_IDX 1000 + void MyMesh::putNeighbour(const mesh::Identity &id, uint32_t timestamp, float snr) { #if MAX_NEIGHBOURS // check if neighbours enabled // find existing neighbour, else use least recently updated @@ -158,7 +176,9 @@ uint8_t MyMesh::handleAnonRegionsReq(const mesh::Identity& sender, uint32_t send uint32_t now = getRTCClock()->getCurrentTime(); memcpy(&reply_data[4], &now, 4); // include our clock (for easy clock sync, and packet hash uniqueness) - return 8 + region_map.exportNamesTo((char *) &reply_data[8], sizeof(reply_data) - 12, REGION_DENY_FLOOD); // reply length + // NOTE: a region map that doesn't fit is truncated at a name boundary. Asking for more + // than one packet's worth would need a paged request, which this reply has no room for. + return 8 + region_map.exportNamesTo((char *) &reply_data[8], MAX_ANON_REPLY_LEN - 8, REGION_DENY_FLOOD); // reply length } return 0; } @@ -607,17 +627,26 @@ void MyMesh::onAnonDataRecv(mesh::Packet *packet, const uint8_t *secret, const m int MyMesh::searchPeersByHash(const uint8_t *hash) { int n = 0; - for (int i = 0; i < acl.getNumClients(); i++) { + for (int i = 0; i < acl.getNumClients() && n < MAX_CLIENTS; i++) { if (acl.getClientByIdx(i)->id.isHashMatch(hash)) { matching_peer_indexes[n++] = i; // store the INDEXES of matching contacts (for subsequent 'peer' methods) } } + // the node we are querying for regions is not (necessarily) a client, so match it + // separately, to be able to decode its reply. MUST come after the clients: the caller + // stops at the first match that decrypts, and a client that IS the queried node has the + // same shared secret, so matching it here first would swallow its normal traffic. + if (region_query_mode != REGION_QUERY_NONE && region_query_id.isHashMatch(hash) && n < MAX_CLIENTS) { + matching_peer_indexes[n++] = REGION_QUERY_PEER_IDX; + } return n; } void MyMesh::getPeerSharedSecret(uint8_t *dest_secret, int peer_idx) { int i = matching_peer_indexes[peer_idx]; - if (i >= 0 && i < acl.getNumClients()) { + if (i == REGION_QUERY_PEER_IDX) { + self_id.calcSharedSecret(dest_secret, region_query_id); // not a client, so calculate it now + } else if (i >= 0 && i < acl.getNumClients()) { // lookup pre-calculated shared_secret memcpy(dest_secret, acl.getClientByIdx(i)->shared_secret, PUB_KEY_SIZE); } else { @@ -648,12 +677,23 @@ void MyMesh::onAdvertRecv(mesh::Packet *packet, const mesh::Identity &id, uint32 void MyMesh::onPeerDataRecv(mesh::Packet *packet, uint8_t type, int sender_idx, const uint8_t *secret, uint8_t *data, size_t len) { int i = matching_peer_indexes[sender_idx]; + if (i == REGION_QUERY_PEER_IDX) { // a reply to our regions query (not a client packet) + if (type == PAYLOAD_TYPE_RESPONSE) handleRegionsResponse(data, len); + return; + } if (i < 0 || i >= acl.getNumClients()) { // get from our known_clients table (sender SHOULD already be known in this context) MESH_DEBUG_PRINTLN("onPeerDataRecv: invalid peer idx: %d", i); return; } ClientInfo* client = acl.getClientByIdx(i); + // a node we are querying for regions can ALSO be a client, and then resolves to a client + // index (see searchPeersByHash), so its reply lands here instead + if (type == PAYLOAD_TYPE_RESPONSE && region_query_mode != REGION_QUERY_NONE + && client->id.matches(region_query_id) && handleRegionsResponse(data, len)) { + return; + } + if (type == PAYLOAD_TYPE_REQ) { // request (from a Known admin client!) uint32_t timestamp; memcpy(×tamp, data, 4); @@ -843,6 +883,214 @@ void MyMesh::sendNodeDiscoverReq() { } } +// Expand a pubkey prefix to the full key of a heard neighbour, so that operators can +// use the (short) keys that 'discover.regions' reports. +bool MyMesh::resolveNeighbourPubKey(uint8_t *pubkey, int prefix_len) { +#if MAX_NEIGHBOURS + for (int i = 0; i < MAX_NEIGHBOURS; i++) { + auto neighbour = &neighbours[i]; + if (neighbour->heard_timestamp > 0 && neighbour->id.isHashMatch(pubkey, prefix_len)) { + memcpy(pubkey, neighbour->id.pub_key, PUB_KEY_SIZE); + return true; + } + } +#endif + return false; // not found +} + +void MyMesh::formatRegionSubscribeReply(char *reply) { + if (!isRegionSrcSet()) { + strcpy(reply, "OK - not subscribed"); + return; + } + + char hex[10]; + mesh::Utils::toHex(hex, _prefs.region_src_pubkey, 4); + if (region_query_mode == REGION_QUERY_SUBSCRIBE) { + sprintf(reply, "OK - subscribed to %s (querying now)", hex); + } else if (region_fetch_fails > 0) { + long remaining = (long)(next_region_fetch - futureMillis(0)); // can be due already + sprintf(reply, "OK - subscribed to %s (%d failed, retry in %d secs)", hex, + (int) region_fetch_fails, remaining > 0 ? (uint32_t) remaining / 1000 : 0); + } else if (region_fetch_at == 0) { + sprintf(reply, "OK - subscribed to %s (no reply yet)", hex); + } else { + uint32_t secs_ago = getRTCClock()->getCurrentTime() - region_fetch_at; + sprintf(reply, "OK - subscribed to %s (%d added, %d known, %d secs ago)", hex, + (int) region_fetch_added, (int) region_fetch_known, secs_ago); + } +} + +bool MyMesh::isRegionSrcSet() const { + for (int i = 0; i < PUB_KEY_SIZE; i++) { + if (_prefs.region_src_pubkey[i] != 0) return true; + } + return false; // all zeroes means 'not configured' +} + +// Client side of the anon-regions request (handleAnonRegionsReq is the server side). +// Asks 'target' which regions it floods. Only ONE of these is ever in flight. +bool MyMesh::sendRegionsReq(const mesh::Identity &target) { + uint8_t secret[PUB_KEY_SIZE]; + self_id.calcSharedSecret(secret, target); + + region_query_tag = getRTCClock()->getCurrentTimeUnique(); + + uint8_t inner[6]; + memcpy(inner, ®ion_query_tag, 4); // tag, so we can match up the reply + inner[4] = ANON_REQ_TYPE_REGIONS; + inner[5] = 0; // reply-path len, zero == reply direct to us (we are a neighbour) + + auto pkt = createAnonDatagram(PAYLOAD_TYPE_ANON_REQ, self_id, target, secret, inner, sizeof(inner)); + if (pkt == NULL) return false; + + region_query_id = target; + region_query_until = futureMillis(REGION_QUERY_TIMEOUT); + sendDirect(pkt, NULL, 0, 0); + return true; +} + +// The reply payload is: {tag}{their-clock}{comma separated region names} +bool MyMesh::handleRegionsResponse(const uint8_t *data, size_t len) { + if (region_query_mode == REGION_QUERY_NONE || len < 8) return false; + + uint32_t tag; + memcpy(&tag, data, 4); + if (tag != region_query_tag) return false; // not the reply we are waiting for + + if (region_query_mode == REGION_QUERY_SUBSCRIBE) { + mergeSubscribedRegions(&data[8], len - 8); + region_query_mode = REGION_QUERY_NONE; + next_region_fetch = futureMillis(REGION_FETCH_INTERVAL); + } else { + advanceRegionDiscover(&data[8], len - 8); + } + return true; +} + +// Merge the regions of the subscribed node into our region map. The merge itself is +// additive (see RegionMap::importNamesFrom), so the operator's own flags and hierarchy +// always win. +void MyMesh::mergeSubscribedRegions(const uint8_t *names, size_t len) { + int known = 0; + int added = region_map.importNamesFrom((const char *) names, (int) len, &known); + + region_fetch_at = getRTCClock()->getCurrentTime(); + region_fetch_added = added; + region_fetch_known = known; + region_fetch_fails = 0; + + if (added > 0) { // only touch the filesystem when the map actually changed + _prefs.discovery_mod_timestamp = region_fetch_at; + savePrefs(); + saveRegions(); + } + MESH_DEBUG_PRINTLN("mergeSubscribedRegions: %d added, %d already known", added, known); +} + +void MyMesh::startRegionDiscover(char *reply) { + if (region_query_mode != REGION_QUERY_NONE) { + strcpy(reply, "Err - busy, try again shortly"); + return; + } +#if MAX_NEIGHBOURS + region_query_mode = REGION_QUERY_DISCOVER; + region_discover_next = 0; + region_discover_reply[0] = 0; + advanceRegionDiscover(NULL, 0); // query the first neighbour + + if (region_query_mode == REGION_QUERY_NONE) { + strcpy(reply, "Err - no neighbors heard yet"); + } else { + strcpy(reply, "OK - querying neighbors"); + } +#else + strcpy(reply, "Err - neighbors not enabled"); +#endif +} + +// Record the result for the neighbour we just queried ('names' is NULL if it didn't +// answer), then move on to the next one. Queries are sent one at a time. +void MyMesh::advanceRegionDiscover(const uint8_t *names, size_t len) { +#if MAX_NEIGHBOURS + if (region_discover_next > 0) { // ..we have a result to record + char entry[64]; + char hex[10]; + mesh::Utils::toHex(hex, region_query_id.pub_key, 4); + + char *dp = entry + sprintf(entry, "%s:", hex); + if (names == NULL) { + *dp++ = '?'; // no reply + } else { + // stop at the cipher block padding, else it is counted as part of the entry + for (size_t i = 0; i < len && names[i] != 0 && dp - entry < (int)sizeof(entry) - 1; i++) { + *dp++ = names[i]; + } + } + *dp = 0; + Serial.printf("regions: %s\n", entry); // ..for the serial CLI, as they arrive + + int used = strlen(region_discover_reply); + if (used + 1 + (dp - entry) < (int) sizeof(region_discover_reply)) { // else drop, reply is full + if (used > 0) { region_discover_reply[used++] = '\n'; } + strcpy(®ion_discover_reply[used], entry); + } + } + + while (region_discover_next < MAX_NEIGHBOURS) { + auto neighbour = &neighbours[region_discover_next++]; + if (neighbour->heard_timestamp > 0 && sendRegionsReq(neighbour->id)) return; + } +#endif + region_query_mode = REGION_QUERY_NONE; // no more neighbours to query +} + +// Forget the configured source: any in-flight request for it is abandoned, and the +// last result no longer describes what we have. +void MyMesh::cancelRegionSubscribe() { + if (region_query_mode == REGION_QUERY_SUBSCRIBE) { + region_query_mode = REGION_QUERY_NONE; + } + next_region_fetch = 0; + region_fetch_at = 0; + region_fetch_fails = 0; +} + +// A request can go unanswered because it was lost, or because the other node is +// rate-limiting anon requests (it allows only a few per minute). Retry soon for the first +// few attempts, then keep retrying on the backoff (which is shorter than the interval +// between successful fetches, so a node that was down is picked up again sooner). +void MyMesh::scheduleRegionFetchRetry() { + if (region_fetch_fails < 0xFF) region_fetch_fails++; + next_region_fetch = futureMillis( + region_fetch_fails < REGION_FETCH_MAX_TRIES ? REGION_FETCH_RETRY : REGION_FETCH_BACKOFF); + MESH_DEBUG_PRINTLN("regions fetch failed (attempt %d)", (uint32_t) region_fetch_fails); +} + +void MyMesh::startRegionFetch() { + mesh::Identity source(_prefs.region_src_pubkey); + if (sendRegionsReq(source)) { + region_query_mode = REGION_QUERY_SUBSCRIBE; + } else { + scheduleRegionFetchRetry(); + } +} + +void MyMesh::loopRegionQuery() { + if (region_query_mode != REGION_QUERY_NONE) { + if (!millisHasNowPassed(region_query_until)) return; // still waiting for the reply + + if (region_query_mode == REGION_QUERY_SUBSCRIBE) { + region_query_mode = REGION_QUERY_NONE; + scheduleRegionFetchRetry(); + } else { + advanceRegionDiscover(NULL, 0); + } + } else if (next_region_fetch && millisHasNowPassed(next_region_fetch) && isRegionSrcSet()) { + startRegionFetch(); + } +} + MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondClock &ms, mesh::RNG &rng, mesh::RTCClock &rtc, mesh::MeshTables &tables) : mesh::Mesh(radio, ms, rng, rtc, *new StaticPoolPacketManager(32), tables), @@ -866,6 +1114,13 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc _logging = false; region_load_active = false; recv_pkt_region = NULL; + region_query_mode = REGION_QUERY_NONE; + region_query_until = 0; + region_discover_next = 0; + region_discover_reply[0] = 0; + next_region_fetch = 0; + region_fetch_at = 0; + region_fetch_added = region_fetch_known = region_fetch_fails = 0; #if MAX_NEIGHBOURS memset(neighbours, 0, sizeof(neighbours)); @@ -970,6 +1225,10 @@ void MyMesh::begin(FILESYSTEM *fs) { updateAdvertTimer(); updateFloodAdvertTimer(); + if (isRegionSrcSet()) { // give the mesh time to settle before the first fetch + next_region_fetch = futureMillis(REGION_FETCH_START_DELAY); + } + board.setAdcMultiplier(_prefs.adc_multiplier); #if ENV_INCLUDE_GPS == 1 @@ -1256,6 +1515,52 @@ void MyMesh::handleCommand(uint32_t sender_timestamp, char *command, char *reply sendNodeDiscoverReq(); strcpy(reply, "OK - Discover sent"); } + } else if (memcmp(command, "discover.regions", 16) == 0) { // format: discover.regions [list] + const char* sub = command + 16; + while (*sub == ' ') sub++; + if (*sub == 0) { + startRegionDiscover(reply); + } else if (strcmp(sub, "list") == 0) { + strcpy(reply, region_discover_reply[0] ? region_discover_reply : "-none-"); + } else { + strcpy(reply, "Err - unknown option"); + } + } else if (memcmp(command, "regions.subscribe", 17) == 0) { // format: regions.subscribe [{pubkey-hex}|off|now] + char* sub = command + 17; + while (*sub == ' ') sub++; + + if (*sub == 0) { + formatRegionSubscribeReply(reply); + } else if (strcmp(sub, "off") == 0) { + memset(_prefs.region_src_pubkey, 0, PUB_KEY_SIZE); + cancelRegionSubscribe(); + savePrefs(); + strcpy(reply, "OK - unsubscribed"); + } else if (strcmp(sub, "now") == 0) { + if (isRegionSrcSet()) { + next_region_fetch = futureMillis(1); + strcpy(reply, "OK - fetching"); + } else { + strcpy(reply, "Err - not subscribed"); + } + } else { + uint8_t pubkey[PUB_KEY_SIZE]; + int hex_len = strlen(sub); + if (hex_len < 8 || hex_len > PUB_KEY_SIZE*2 || (hex_len & 1) + || !mesh::Utils::fromHex(pubkey, hex_len / 2, sub)) { + strcpy(reply, "Err - bad pubkey"); + } else if (hex_len < PUB_KEY_SIZE*2 && !resolveNeighbourPubKey(pubkey, hex_len / 2)) { + strcpy(reply, "Err - not a known neighbor, need full pubkey"); + } else if (self_id.matches(pubkey)) { + strcpy(reply, "Err - that is this node"); + } else { + cancelRegionSubscribe(); + memcpy(_prefs.region_src_pubkey, pubkey, PUB_KEY_SIZE); + next_region_fetch = futureMillis(1); // fetch from the new source now + savePrefs(); + formatRegionSubscribeReply(reply); + } + } } else{ _cli.handleCommand(sender_timestamp, command, reply); // common CLI commands } @@ -1300,6 +1605,8 @@ void MyMesh::loop() { dirty_contacts_expiry = 0; } + loopRegionQuery(); + // update uptime uint32_t now = millis(); uptime_millis += now - last_millis; diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index aa7d30b062..dd1e533af7 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -114,12 +114,35 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks { uint8_t pending_sf; uint8_t pending_cr; int matching_peer_indexes[MAX_CLIENTS]; + mesh::Identity region_query_id; // node we are currently asking for regions + uint32_t region_query_tag; + unsigned long region_query_until; + uint8_t region_query_mode; // one of SCOPE_QUERY_* + uint8_t region_discover_next; // index into neighbours[], for the 'discover.regions' pass + char region_discover_reply[134]; // results of last pass, for 'discover.regions list' + unsigned long next_region_fetch; // when to re-fetch from the subscribed node (0 = never) + uint32_t region_fetch_at; // when the last fetch completed (0 = not yet) + uint8_t region_fetch_added, region_fetch_known; + uint8_t region_fetch_fails; // consecutive unanswered fetches #if defined(WITH_RS232_BRIDGE) RS232Bridge bridge; #elif defined(WITH_ESPNOW_BRIDGE) ESPNowBridge bridge; #endif + bool isRegionSrcSet() const; + bool resolveNeighbourPubKey(uint8_t* pubkey, int prefix_len); + void formatRegionSubscribeReply(char* reply); + bool sendRegionsReq(const mesh::Identity& target); + bool handleRegionsResponse(const uint8_t* data, size_t len); + void mergeSubscribedRegions(const uint8_t* names, size_t len); + void startRegionDiscover(char* reply); + void advanceRegionDiscover(const uint8_t* names, size_t len); + void cancelRegionSubscribe(); + void scheduleRegionFetchRetry(); + void startRegionFetch(); + void loopRegionQuery(); + void putNeighbour(const mesh::Identity& id, uint32_t timestamp, float snr); uint8_t handleLoginReq(const mesh::Identity& sender, const uint8_t* secret, uint32_t sender_timestamp, const uint8_t* data, bool is_flood); uint8_t handleAnonRegionsReq(const mesh::Identity& sender, uint32_t sender_timestamp, const uint8_t* data); diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 69fe03150c..407eed1ce7 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -68,6 +68,7 @@ class NodePrefs : public ConfigSerializer { uint8_t path_hash_mode = 0; // which path mode to use when sending uint8_t loop_detect = 0; uint8_t cad_enabled = 0; // hardware Channel Activity Detection before TX (boolean) + uint8_t region_src_pubkey[PUB_KEY_SIZE]; // node to subscribe to regions from (all zeroes = none) private: class RadioPrefs : public ConfigSerializer { @@ -146,6 +147,7 @@ class NodePrefs : public ConfigSerializer { def("f_max_uns", _parent->flood_max_unscoped); def("f_max_adv", _parent->flood_max_advert); def("loop", _parent->loop_detect); + def("region_src", _parent->region_src_pubkey, sizeof(_parent->region_src_pubkey)); } public: RepeatPrefs(NodePrefs* parent) : _parent(parent) { } @@ -171,6 +173,7 @@ class NodePrefs : public ConfigSerializer { def("owner", owner_info, sizeof(owner_info)); def("adv_int", advert_interval); def("f_adv_int", flood_advert_interval); + def("disc_mod", discovery_mod_timestamp); // else 'since' filtered DISCOVERs miss us after a reboot def("lat", node_lat); def("lon", node_lon); def("radio", radio); @@ -188,6 +191,7 @@ class NodePrefs : public ConfigSerializer { guest_password[0] = 0; bridge_secret[0] = 0; owner_info[0] = 0; + memset(region_src_pubkey, 0, sizeof(region_src_pubkey)); } }; diff --git a/src/helpers/RegionMap.cpp b/src/helpers/RegionMap.cpp index 4667e0038e..29ee4b184d 100644 --- a/src/helpers/RegionMap.cpp +++ b/src/helpers/RegionMap.cpp @@ -346,3 +346,50 @@ int RegionMap::exportNamesTo(char *dest, int max_len, uint8_t mask, bool invert) *dp = 0; // set null terminator return dp - dest; // return length } + +// Take the next region name from 'src', starting at *cursor. Returns false at the end. +static bool next_name(const char *src, int len, int *cursor, char dest[], int dest_size) { + while (*cursor < len) { + int n = 0; + bool valid = true; + while (*cursor < len && src[*cursor] != ',') { + uint8_t c = src[(*cursor)++]; + if (!RegionMap::is_name_char(c)) { valid = false; continue; } // don't trust the sender + if (n + 1 < dest_size) { dest[n++] = c; } else { valid = false; } // too long for a name + } + (*cursor)++; // skip the separator + + dest[n] = 0; + if (valid && n > 0) return true; + } + return false; // no more names +} + +int RegionMap::importNamesFrom(const char *src, int len, int* num_known) { + char name[sizeof(RegionEntry::name)]; + int cursor = 0, added = 0, known = 0; + + // a decrypted payload is padded out to the cipher block size, so the list can be + // followed by null bytes. Without this, the last name runs into the padding. + for (int i = 0; i < len; i++) { + if (src[i] == 0) { len = i; break; } + } + + while (next_name(src, len, &cursor, name, sizeof(name))) { + // the wildcard is not a Region, and private ('$') Regions have keys we cannot derive + if (name[0] == '*' || name[0] == '$') continue; + + if (findByName(name)) { + known++; // already in the map, leave it exactly as it is + continue; + } + + auto region = putRegion(name, 0); // add as a child of the wildcard + if (region == NULL) break; // full! + region->flags = 0; // allow flood + added++; + } + + if (num_known) { *num_known = known; } + return added; // return number of new Regions +} diff --git a/src/helpers/RegionMap.h b/src/helpers/RegionMap.h index 5eb1442983..038f11d5d6 100644 --- a/src/helpers/RegionMap.h +++ b/src/helpers/RegionMap.h @@ -54,6 +54,16 @@ class RegionMap { const RegionEntry* getByIdx(int i) const { return ®ions[i]; } const RegionEntry* getRoot() const { return &wildcard; } int exportNamesTo(char *dest, int max_len, uint8_t mask, bool invert = false); + + /** + * \brief Add Regions from a comma separated list of names, as written by exportNamesTo(). + * This is additive: a name already in the map is left exactly as it is, keeping its + * flags and its parent. New names are added as flood-allowed children of the wildcard. + * The wildcard and private ('$') Regions in the list are skipped. + * \param num_known OUT - optional, how many names were already in the map + * \returns the number of Regions added + */ + int importNamesFrom(const char *src, int len, int* num_known = NULL); int getTransportKeysFor(const RegionEntry& src, TransportKey dest[], int max_num); void exportTo(Stream& out) const; From b905423f31dfce42c01d048c8db4a53cea6fd9f3 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 31 Jul 2026 11:57:01 -0700 Subject: [PATCH 2/4] fix(repeater): report the entries 'discover.regions list' could not fit The reply is capped at 134 chars, so a pass over several neighbours with long region lists silently showed only the first few. Count what doesn't fit and append '+N more', so a short list isn't mistaken for the whole pass. The serial output is unchanged, and still shows every result. --- docs/cli_commands.md | 5 +++-- examples/simple_repeater/MyMesh.cpp | 14 ++++++++++++-- examples/simple_repeater/MyMesh.h | 1 + 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index a9392bd6bd..46c4661ee4 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -142,8 +142,9 @@ the serial terminal as they land. **Note:** `discover.regions list` reports the results of the last run, as `{pubkey-prefix}:{regions}` per line. A `?` in place of the regions means that neighbor did not answer — it may be out of range, running older firmware, or simply rate-limiting anonymous requests (repeaters answer at most 4 per -3 minutes). This reply is capped at 134 characters, so on a node with many neighbors it may not -show them all — the serial output always does. +3 minutes). This reply is capped at 134 characters, so on a node with many neighbors it may not show +them all; when entries do not fit, the last line is `+N more`. The serial output always shows every +result. **Note:** The pubkey prefixes reported here can be passed straight to `regions.subscribe`. diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 73492af4a0..56a54f9d3e 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -997,6 +997,7 @@ void MyMesh::startRegionDiscover(char *reply) { region_query_mode = REGION_QUERY_DISCOVER; region_discover_next = 0; region_discover_reply[0] = 0; + region_discover_dropped = 0; advanceRegionDiscover(NULL, 0); // query the first neighbour if (region_query_mode == REGION_QUERY_NONE) { @@ -1031,9 +1032,11 @@ void MyMesh::advanceRegionDiscover(const uint8_t *names, size_t len) { Serial.printf("regions: %s\n", entry); // ..for the serial CLI, as they arrive int used = strlen(region_discover_reply); - if (used + 1 + (dp - entry) < (int) sizeof(region_discover_reply)) { // else drop, reply is full + if (used + 1 + (dp - entry) < (int) sizeof(region_discover_reply)) { if (used > 0) { region_discover_reply[used++] = '\n'; } strcpy(®ion_discover_reply[used], entry); + } else if (region_discover_dropped < 0xFF) { + region_discover_dropped++; // 'list' is full, so just report how many are missing } } @@ -1118,6 +1121,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc region_query_until = 0; region_discover_next = 0; region_discover_reply[0] = 0; + region_discover_dropped = 0; next_region_fetch = 0; region_fetch_at = 0; region_fetch_added = region_fetch_known = region_fetch_fails = 0; @@ -1521,7 +1525,13 @@ void MyMesh::handleCommand(uint32_t sender_timestamp, char *command, char *reply if (*sub == 0) { startRegionDiscover(reply); } else if (strcmp(sub, "list") == 0) { - strcpy(reply, region_discover_reply[0] ? region_discover_reply : "-none-"); + if (region_discover_reply[0] == 0) { + strcpy(reply, "-none-"); + } else if (region_discover_dropped > 0) { // ..so a short list isn't mistaken for the whole pass + sprintf(reply, "%s\n+%d more", region_discover_reply, (int) region_discover_dropped); + } else { + strcpy(reply, region_discover_reply); + } } else { strcpy(reply, "Err - unknown option"); } diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index dd1e533af7..2e5e2f3051 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -120,6 +120,7 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks { uint8_t region_query_mode; // one of SCOPE_QUERY_* uint8_t region_discover_next; // index into neighbours[], for the 'discover.regions' pass char region_discover_reply[134]; // results of last pass, for 'discover.regions list' + uint8_t region_discover_dropped; // ..entries from that pass which did not fit unsigned long next_region_fetch; // when to re-fetch from the subscribed node (0 = never) uint32_t region_fetch_at; // when the last fetch completed (0 = not yet) uint8_t region_fetch_added, region_fetch_known; From 5708cfbc16235d987bff7d5cfc7e1030d0f47e77 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 31 Jul 2026 13:57:35 -0700 Subject: [PATCH 3/4] fix(repeater): stop reporting partial region fetches as complete Three ways an incomplete result was presented as a whole one: - 'discover.regions' built each entry in a 64 byte buffer, so a neighbour with more than ~54 chars of region names was cut short with no marker, in the serial log as well as the list. The serial log now prints the whole reply, and an entry that does not fit the list is counted in the '+N more' total rather than stored truncated. - importNamesFrom() stops when the region table is full, and saveRegions() can fail; both were reported as a successful fetch, which cleared the failure count and stood down for 72 hours. A full table is now flagged in the reply, and an unsaved import keeps retrying: the regions are in RAM, where a later fetch counts them as 'known' and would never retry the save on its own. - An automatic fetch could land in the middle of an interactive 'region load', whose temp map then overwrote the imported regions after they had been reported and saved. Fetches now stand aside while a load is active. --- docs/cli_commands.md | 6 ++- examples/simple_repeater/MyMesh.cpp | 75 +++++++++++++++++++---------- examples/simple_repeater/MyMesh.h | 4 +- src/helpers/RegionMap.cpp | 9 +++- src/helpers/RegionMap.h | 3 +- 5 files changed, 67 insertions(+), 30 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 46c4661ee4..5375e4f385 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -915,7 +915,11 @@ map has its list truncated at a name boundary, and the names past that point are **Note:** Bare `regions.subscribe` reports the subscription and the result of the last fetch, for example `OK - subscribed to a1b2c3d4 (2 added, 5 known, 43 secs ago)`. `regions.subscribe off` -clears it and replies `OK - unsubscribed`. +clears it and replies `OK - unsubscribed`. Two suffixes flag an incomplete fetch: +`- region table full` (the node already holds the maximum 32 regions, so the rest of the list was +dropped — remove regions to make room), and `- NOT SAVED` (the regions were added in memory but +could not be written to flash, so they will be lost on reboot; the fetch is retried until a save +succeeds). --- diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 56a54f9d3e..5920fc8ac1 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -906,18 +906,24 @@ void MyMesh::formatRegionSubscribeReply(char *reply) { char hex[10]; mesh::Utils::toHex(hex, _prefs.region_src_pubkey, 4); + long remaining = (long)(next_region_fetch - futureMillis(0)); // can be due already + uint32_t retry_secs = remaining > 0 ? (uint32_t) remaining / 1000 : 0; + if (region_query_mode == REGION_QUERY_SUBSCRIBE) { sprintf(reply, "OK - subscribed to %s (querying now)", hex); + } else if (region_save_pending) { // ..report the save, not the retry counter it bumped + sprintf(reply, "OK - subscribed to %s (%d added but NOT SAVED, retry in %d secs)", hex, + (int) region_fetch_added, retry_secs); } else if (region_fetch_fails > 0) { - long remaining = (long)(next_region_fetch - futureMillis(0)); // can be due already sprintf(reply, "OK - subscribed to %s (%d failed, retry in %d secs)", hex, - (int) region_fetch_fails, remaining > 0 ? (uint32_t) remaining / 1000 : 0); + (int) region_fetch_fails, retry_secs); } else if (region_fetch_at == 0) { sprintf(reply, "OK - subscribed to %s (no reply yet)", hex); } else { uint32_t secs_ago = getRTCClock()->getCurrentTime() - region_fetch_at; - sprintf(reply, "OK - subscribed to %s (%d added, %d known, %d secs ago)", hex, - (int) region_fetch_added, (int) region_fetch_known, secs_ago); + sprintf(reply, "OK - subscribed to %s (%d added, %d known, %d secs ago)%s", hex, + (int) region_fetch_added, (int) region_fetch_known, secs_ago, + region_fetch_full ? " - region table full" : ""); } } @@ -959,9 +965,15 @@ bool MyMesh::handleRegionsResponse(const uint8_t *data, size_t len) { if (tag != region_query_tag) return false; // not the reply we are waiting for if (region_query_mode == REGION_QUERY_SUBSCRIBE) { - mergeSubscribedRegions(&data[8], len - 8); region_query_mode = REGION_QUERY_NONE; - next_region_fetch = futureMillis(REGION_FETCH_INTERVAL); + if (region_load_active) { // 'region load' is about to replace the map, so discard this + scheduleRegionFetchRetry(); + } else if (mergeSubscribedRegions(&data[8], len - 8)) { + region_fetch_fails = 0; + next_region_fetch = futureMillis(REGION_FETCH_INTERVAL); + } else { + scheduleRegionFetchRetry(); // couldn't be saved, so come back and try again + } } else { advanceRegionDiscover(&data[8], len - 8); } @@ -971,21 +983,26 @@ bool MyMesh::handleRegionsResponse(const uint8_t *data, size_t len) { // Merge the regions of the subscribed node into our region map. The merge itself is // additive (see RegionMap::importNamesFrom), so the operator's own flags and hierarchy // always win. -void MyMesh::mergeSubscribedRegions(const uint8_t *names, size_t len) { +bool MyMesh::mergeSubscribedRegions(const uint8_t *names, size_t len) { int known = 0; - int added = region_map.importNamesFrom((const char *) names, (int) len, &known); + bool was_full = false; + int added = region_map.importNamesFrom((const char *) names, (int) len, &known, &was_full); region_fetch_at = getRTCClock()->getCurrentTime(); region_fetch_added = added; region_fetch_known = known; - region_fetch_fails = 0; + region_fetch_full = was_full; - if (added > 0) { // only touch the filesystem when the map actually changed + // a failed save leaves the new regions in RAM only, where the next fetch counts them as + // 'known' and would not try again, so keep asking until one succeeds + if (added > 0 || region_save_pending) { _prefs.discovery_mod_timestamp = region_fetch_at; savePrefs(); - saveRegions(); + region_save_pending = !saveRegions(); } - MESH_DEBUG_PRINTLN("mergeSubscribedRegions: %d added, %d already known", added, known); + MESH_DEBUG_PRINTLN("mergeSubscribedRegions: %d added, %d known, full=%d, unsaved=%d", + added, known, (uint32_t) was_full, (uint32_t) region_save_pending); + return !region_save_pending; } void MyMesh::startRegionDiscover(char *reply) { @@ -1015,28 +1032,34 @@ void MyMesh::startRegionDiscover(char *reply) { void MyMesh::advanceRegionDiscover(const uint8_t *names, size_t len) { #if MAX_NEIGHBOURS if (region_discover_next > 0) { // ..we have a result to record - char entry[64]; char hex[10]; mesh::Utils::toHex(hex, region_query_id.pub_key, 4); - char *dp = entry + sprintf(entry, "%s:", hex); + // the reply is padded out to the cipher block size, so the names stop at the padding + size_t n = 0; + while (names != NULL && n < len && names[n] != 0) n++; + + // print the WHOLE list, however long: the serial log is the complete record if (names == NULL) { - *dp++ = '?'; // no reply + Serial.printf("regions: %s:?\n", hex); } else { - // stop at the cipher block padding, else it is counted as part of the entry - for (size_t i = 0; i < len && names[i] != 0 && dp - entry < (int)sizeof(entry) - 1; i++) { - *dp++ = names[i]; - } + Serial.printf("regions: %s:%.*s\n", hex, (int) n, names); } - *dp = 0; - Serial.printf("regions: %s\n", entry); // ..for the serial CLI, as they arrive + // ..the 'list' reply can only hold so much, so an entry goes in whole, or not at all int used = strlen(region_discover_reply); - if (used + 1 + (dp - entry) < (int) sizeof(region_discover_reply)) { + int need = (used > 0 ? 1 : 0) + 9 + (names == NULL ? 1 : (int) n); // "\n{prefix}:{names}" + if (used + need < (int) sizeof(region_discover_reply)) { if (used > 0) { region_discover_reply[used++] = '\n'; } - strcpy(®ion_discover_reply[used], entry); + used += sprintf(®ion_discover_reply[used], "%s:", hex); + if (names == NULL) { + region_discover_reply[used++] = '?'; + } else { + memcpy(®ion_discover_reply[used], names, n); used += n; + } + region_discover_reply[used] = 0; } else if (region_discover_dropped < 0xFF) { - region_discover_dropped++; // 'list' is full, so just report how many are missing + region_discover_dropped++; // report how many are missing, rather than a partial entry } } @@ -1089,7 +1112,8 @@ void MyMesh::loopRegionQuery() { } else { advanceRegionDiscover(NULL, 0); } - } else if (next_region_fetch && millisHasNowPassed(next_region_fetch) && isRegionSrcSet()) { + } else if (next_region_fetch && millisHasNowPassed(next_region_fetch) && isRegionSrcSet() + && !region_load_active) { // a 'region load' in progress will replace the map startRegionFetch(); } } @@ -1125,6 +1149,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc next_region_fetch = 0; region_fetch_at = 0; region_fetch_added = region_fetch_known = region_fetch_fails = 0; + region_fetch_full = region_save_pending = false; #if MAX_NEIGHBOURS memset(neighbours, 0, sizeof(neighbours)); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 2e5e2f3051..38902143ce 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -125,6 +125,8 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks { uint32_t region_fetch_at; // when the last fetch completed (0 = not yet) uint8_t region_fetch_added, region_fetch_known; uint8_t region_fetch_fails; // consecutive unanswered fetches + bool region_fetch_full; // last fetch hit the region table limit + bool region_save_pending; // imported regions are in RAM but not on disk #if defined(WITH_RS232_BRIDGE) RS232Bridge bridge; #elif defined(WITH_ESPNOW_BRIDGE) @@ -136,7 +138,7 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks { void formatRegionSubscribeReply(char* reply); bool sendRegionsReq(const mesh::Identity& target); bool handleRegionsResponse(const uint8_t* data, size_t len); - void mergeSubscribedRegions(const uint8_t* names, size_t len); + bool mergeSubscribedRegions(const uint8_t* names, size_t len); void startRegionDiscover(char* reply); void advanceRegionDiscover(const uint8_t* names, size_t len); void cancelRegionSubscribe(); diff --git a/src/helpers/RegionMap.cpp b/src/helpers/RegionMap.cpp index 29ee4b184d..ab50d5f7a4 100644 --- a/src/helpers/RegionMap.cpp +++ b/src/helpers/RegionMap.cpp @@ -365,10 +365,12 @@ static bool next_name(const char *src, int len, int *cursor, char dest[], int de return false; // no more names } -int RegionMap::importNamesFrom(const char *src, int len, int* num_known) { +int RegionMap::importNamesFrom(const char *src, int len, int* num_known, bool* was_full) { char name[sizeof(RegionEntry::name)]; int cursor = 0, added = 0, known = 0; + if (was_full) { *was_full = false; } + // a decrypted payload is padded out to the cipher block size, so the list can be // followed by null bytes. Without this, the last name runs into the padding. for (int i = 0; i < len; i++) { @@ -385,7 +387,10 @@ int RegionMap::importNamesFrom(const char *src, int len, int* num_known) { } auto region = putRegion(name, 0); // add as a child of the wildcard - if (region == NULL) break; // full! + if (region == NULL) { // full! (the name is already known to be valid) + if (was_full) { *was_full = true; } + break; // the rest of the list cannot be added + } region->flags = 0; // allow flood added++; } diff --git a/src/helpers/RegionMap.h b/src/helpers/RegionMap.h index 038f11d5d6..55a58ab941 100644 --- a/src/helpers/RegionMap.h +++ b/src/helpers/RegionMap.h @@ -61,9 +61,10 @@ class RegionMap { * flags and its parent. New names are added as flood-allowed children of the wildcard. * The wildcard and private ('$') Regions in the list are skipped. * \param num_known OUT - optional, how many names were already in the map + * \param was_full OUT - optional, true if the map filled up and names had to be dropped * \returns the number of Regions added */ - int importNamesFrom(const char *src, int len, int* num_known = NULL); + int importNamesFrom(const char *src, int len, int* num_known = NULL, bool* was_full = NULL); int getTransportKeysFor(const RegionEntry& src, TransportKey dest[], int max_num); void exportTo(Stream& out) const; From 25508e48e862921973469615cf639e7fbc821bf4 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 31 Jul 2026 16:49:37 -0700 Subject: [PATCH 4/4] style(repeater): trim comments and simplify the region subscribe code Bring the new code in line with the surrounding style, which comments lightly. Drops the function-header blocks, keeping the short notes that record something non-obvious: the peer match ordering, the cipher block padding, and the reply size limit. Also collapses the duplicated no-reply/reply branches in advanceRegionDiscover(), and hoists the repeated prefix out of the 'regions.subscribe' status reply. No behaviour change. Includes the ambiguous-prefix check for 'regions.subscribe {prefix}', which previously took whichever neighbour matched first. --- docs/cli_commands.md | 4 +- examples/simple_repeater/MyMesh.cpp | 153 +++++++++++++--------------- examples/simple_repeater/MyMesh.h | 16 +-- src/helpers/RegionMap.cpp | 20 ++-- src/helpers/RegionMap.h | 11 +- 5 files changed, 92 insertions(+), 112 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 5375e4f385..479903ddf7 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -886,8 +886,8 @@ region save **Parameters:** - `pubkey`: Public key of the repeater to subscribe to. A prefix (4 bytes or more) is accepted if - that node is a current neighbor, such as the prefixes reported by `discover.regions`; otherwise - give the full key. + it uniquely identifies a current neighbor, such as the prefixes reported by `discover.regions`; + an ambiguous prefix is rejected. Otherwise, give the full key. **Note:** The node asks the subscribed repeater which regions it floods, and adds any it does not already have. This is **additive**: a region already on this node is left exactly as it is, keeping diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 5920fc8ac1..42dfcdb676 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -52,8 +52,8 @@ #define RESP_SERVER_LOGIN_OK 0 // response to ANON_REQ -// createDatagram() rejects a reply longer than this, once the MAC and the cipher block -// padding are added. The exporters below MUST keep within it, or no reply is sent at all. +// createDatagram() rejects anything longer, once MAC and block padding are added, so a +// reply over this size is never sent at all #define MAX_ANON_REPLY_LEN (MAX_PACKET_PAYLOAD - CIPHER_MAC_SIZE - (CIPHER_BLOCK_SIZE - 1)) #define ANON_REQ_TYPE_REGIONS 0x01 @@ -64,19 +64,17 @@ #define LAZY_CONTACTS_WRITE_DELAY 5000 -#define REGION_QUERY_NONE 0 -#define REGION_QUERY_DISCOVER 1 // 'discover.regions' pass over the neighbours table -#define REGION_QUERY_SUBSCRIBE 2 // scheduled fetch from the subscribed node +#define REGION_QUERY_NONE 0 +#define REGION_QUERY_DISCOVER 1 // 'discover.regions' pass over the neighbours +#define REGION_QUERY_SUBSCRIBE 2 // scheduled fetch from the subscribed node +#define REGION_QUERY_PEER_IDX 1000 // peer index for the node being queried -#define REGION_QUERY_TIMEOUT 20000 // millis to wait for one regions reply -#define REGION_FETCH_START_DELAY 120000 // first fetch after boot -#define REGION_FETCH_INTERVAL (72*3600000) // between successful fetches -#define REGION_FETCH_RETRY 60000 // after an unanswered fetch -#define REGION_FETCH_MAX_TRIES 5 // ..then back off to: -#define REGION_FETCH_BACKOFF (12*3600000) // ..until one succeeds - -// peer index (see searchPeersByHash) for the node we are querying for regions -#define REGION_QUERY_PEER_IDX 1000 +#define REGION_QUERY_TIMEOUT 20000 // to wait for one regions reply +#define REGION_FETCH_START_DELAY 120000 // first fetch after boot +#define REGION_FETCH_INTERVAL (72*3600000) // between successful fetches +#define REGION_FETCH_RETRY 60000 // after an unanswered fetch +#define REGION_FETCH_MAX_TRIES 5 // ..then back off to: +#define REGION_FETCH_BACKOFF (12*3600000) // ..until one succeeds void MyMesh::putNeighbour(const mesh::Identity &id, uint32_t timestamp, float snr) { #if MAX_NEIGHBOURS // check if neighbours enabled @@ -176,8 +174,7 @@ uint8_t MyMesh::handleAnonRegionsReq(const mesh::Identity& sender, uint32_t send uint32_t now = getRTCClock()->getCurrentTime(); memcpy(&reply_data[4], &now, 4); // include our clock (for easy clock sync, and packet hash uniqueness) - // NOTE: a region map that doesn't fit is truncated at a name boundary. Asking for more - // than one packet's worth would need a paged request, which this reply has no room for. + // NOTE: a map that doesn't fit is truncated at a name boundary (no paged request yet) return 8 + region_map.exportNamesTo((char *) &reply_data[8], MAX_ANON_REPLY_LEN - 8, REGION_DENY_FLOOD); // reply length } return 0; @@ -632,10 +629,9 @@ int MyMesh::searchPeersByHash(const uint8_t *hash) { matching_peer_indexes[n++] = i; // store the INDEXES of matching contacts (for subsequent 'peer' methods) } } - // the node we are querying for regions is not (necessarily) a client, so match it - // separately, to be able to decode its reply. MUST come after the clients: the caller - // stops at the first match that decrypts, and a client that IS the queried node has the - // same shared secret, so matching it here first would swallow its normal traffic. + // the node we are querying for regions needn't be a client, so match it separately. + // MUST stay after the clients: caller stops at the first match that decrypts, and a + // queried node that IS a client shares the same secret, so we'd eat its normal traffic if (region_query_mode != REGION_QUERY_NONE && region_query_id.isHashMatch(hash) && n < MAX_CLIENTS) { matching_peer_indexes[n++] = REGION_QUERY_PEER_IDX; } @@ -883,19 +879,24 @@ void MyMesh::sendNodeDiscoverReq() { } } -// Expand a pubkey prefix to the full key of a heard neighbour, so that operators can -// use the (short) keys that 'discover.regions' reports. -bool MyMesh::resolveNeighbourPubKey(uint8_t *pubkey, int prefix_len) { +// expand a pubkey prefix (as 'discover.regions' reports) to a full key. +// returns the number of matches, capped at 2: none, unique, or ambiguous +int MyMesh::resolveNeighbourPubKey(uint8_t *pubkey, int prefix_len) { #if MAX_NEIGHBOURS + const uint8_t *resolved = NULL; for (int i = 0; i < MAX_NEIGHBOURS; i++) { auto neighbour = &neighbours[i]; if (neighbour->heard_timestamp > 0 && neighbour->id.isHashMatch(pubkey, prefix_len)) { - memcpy(pubkey, neighbour->id.pub_key, PUB_KEY_SIZE); - return true; + if (resolved != NULL) return 2; // ambiguous + resolved = neighbour->id.pub_key; } } + if (resolved != NULL) { + memcpy(pubkey, resolved, PUB_KEY_SIZE); + return 1; + } #endif - return false; // not found + return 0; // not found } void MyMesh::formatRegionSubscribeReply(char *reply) { @@ -906,23 +907,22 @@ void MyMesh::formatRegionSubscribeReply(char *reply) { char hex[10]; mesh::Utils::toHex(hex, _prefs.region_src_pubkey, 4); + char *dp = reply + sprintf(reply, "OK - subscribed to %s ", hex); + long remaining = (long)(next_region_fetch - futureMillis(0)); // can be due already uint32_t retry_secs = remaining > 0 ? (uint32_t) remaining / 1000 : 0; if (region_query_mode == REGION_QUERY_SUBSCRIBE) { - sprintf(reply, "OK - subscribed to %s (querying now)", hex); - } else if (region_save_pending) { // ..report the save, not the retry counter it bumped - sprintf(reply, "OK - subscribed to %s (%d added but NOT SAVED, retry in %d secs)", hex, - (int) region_fetch_added, retry_secs); + strcpy(dp, "(querying now)"); + } else if (region_save_pending) { // report the save, not the retry counter it bumped + sprintf(dp, "(%d added but NOT SAVED, retry in %d secs)", (int) region_fetch_added, retry_secs); } else if (region_fetch_fails > 0) { - sprintf(reply, "OK - subscribed to %s (%d failed, retry in %d secs)", hex, - (int) region_fetch_fails, retry_secs); + sprintf(dp, "(%d failed, retry in %d secs)", (int) region_fetch_fails, retry_secs); } else if (region_fetch_at == 0) { - sprintf(reply, "OK - subscribed to %s (no reply yet)", hex); + strcpy(dp, "(no reply yet)"); } else { - uint32_t secs_ago = getRTCClock()->getCurrentTime() - region_fetch_at; - sprintf(reply, "OK - subscribed to %s (%d added, %d known, %d secs ago)%s", hex, - (int) region_fetch_added, (int) region_fetch_known, secs_ago, + sprintf(dp, "(%d added, %d known, %d secs ago)%s", (int) region_fetch_added, + (int) region_fetch_known, getRTCClock()->getCurrentTime() - region_fetch_at, region_fetch_full ? " - region table full" : ""); } } @@ -934,8 +934,7 @@ bool MyMesh::isRegionSrcSet() const { return false; // all zeroes means 'not configured' } -// Client side of the anon-regions request (handleAnonRegionsReq is the server side). -// Asks 'target' which regions it floods. Only ONE of these is ever in flight. +// ask 'target' which regions it floods (handleAnonRegionsReq is the server side) bool MyMesh::sendRegionsReq(const mesh::Identity &target) { uint8_t secret[PUB_KEY_SIZE]; self_id.calcSharedSecret(secret, target); @@ -980,9 +979,6 @@ bool MyMesh::handleRegionsResponse(const uint8_t *data, size_t len) { return true; } -// Merge the regions of the subscribed node into our region map. The merge itself is -// additive (see RegionMap::importNamesFrom), so the operator's own flags and hierarchy -// always win. bool MyMesh::mergeSubscribedRegions(const uint8_t *names, size_t len) { int known = 0; bool was_full = false; @@ -993,15 +989,14 @@ bool MyMesh::mergeSubscribedRegions(const uint8_t *names, size_t len) { region_fetch_known = known; region_fetch_full = was_full; - // a failed save leaves the new regions in RAM only, where the next fetch counts them as - // 'known' and would not try again, so keep asking until one succeeds + // an unsaved import stays in RAM, where the next fetch counts it as 'known' and would + // not save it again, so keep re-trying the save if (added > 0 || region_save_pending) { _prefs.discovery_mod_timestamp = region_fetch_at; savePrefs(); region_save_pending = !saveRegions(); } - MESH_DEBUG_PRINTLN("mergeSubscribedRegions: %d added, %d known, full=%d, unsaved=%d", - added, known, (uint32_t) was_full, (uint32_t) region_save_pending); + MESH_DEBUG_PRINTLN("regions merged: %d added, %d known", added, known); return !region_save_pending; } @@ -1027,39 +1022,34 @@ void MyMesh::startRegionDiscover(char *reply) { #endif } -// Record the result for the neighbour we just queried ('names' is NULL if it didn't -// answer), then move on to the next one. Queries are sent one at a time. +// record the result for the neighbour just queried ('names' is NULL if it didn't answer), +// then query the next one. Only one request is in flight at a time. void MyMesh::advanceRegionDiscover(const uint8_t *names, size_t len) { #if MAX_NEIGHBOURS if (region_discover_next > 0) { // ..we have a result to record char hex[10]; mesh::Utils::toHex(hex, region_query_id.pub_key, 4); - // the reply is padded out to the cipher block size, so the names stop at the padding - size_t n = 0; - while (names != NULL && n < len && names[n] != 0) n++; - - // print the WHOLE list, however long: the serial log is the complete record - if (names == NULL) { - Serial.printf("regions: %s:?\n", hex); - } else { - Serial.printf("regions: %s:%.*s\n", hex, (int) n, names); + const char* list = "?"; // no reply + size_t list_len = 1; + if (names != NULL) { + list = (const char *) names; + // the reply is padded out to the cipher block size, so stop at the padding + for (list_len = 0; list_len < len && names[list_len] != 0; list_len++) ; } - // ..the 'list' reply can only hold so much, so an entry goes in whole, or not at all + Serial.printf("regions: %s:%.*s\n", hex, (int) list_len, list); // never truncated + + // the 'list' reply is limited, so an entry goes in whole, or not at all int used = strlen(region_discover_reply); - int need = (used > 0 ? 1 : 0) + 9 + (names == NULL ? 1 : (int) n); // "\n{prefix}:{names}" + int need = (used > 0 ? 1 : 0) + (int) strlen(hex) + 1 + (int) list_len; if (used + need < (int) sizeof(region_discover_reply)) { if (used > 0) { region_discover_reply[used++] = '\n'; } used += sprintf(®ion_discover_reply[used], "%s:", hex); - if (names == NULL) { - region_discover_reply[used++] = '?'; - } else { - memcpy(®ion_discover_reply[used], names, n); used += n; - } - region_discover_reply[used] = 0; + memcpy(®ion_discover_reply[used], list, list_len); + region_discover_reply[used + list_len] = 0; } else if (region_discover_dropped < 0xFF) { - region_discover_dropped++; // report how many are missing, rather than a partial entry + region_discover_dropped++; // count it, rather than storing a partial entry } } @@ -1071,8 +1061,8 @@ void MyMesh::advanceRegionDiscover(const uint8_t *names, size_t len) { region_query_mode = REGION_QUERY_NONE; // no more neighbours to query } -// Forget the configured source: any in-flight request for it is abandoned, and the -// last result no longer describes what we have. +// forget the subscription: any in-flight request is abandoned, and the last result no +// longer describes what we have void MyMesh::cancelRegionSubscribe() { if (region_query_mode == REGION_QUERY_SUBSCRIBE) { region_query_mode = REGION_QUERY_NONE; @@ -1082,10 +1072,8 @@ void MyMesh::cancelRegionSubscribe() { region_fetch_fails = 0; } -// A request can go unanswered because it was lost, or because the other node is -// rate-limiting anon requests (it allows only a few per minute). Retry soon for the first -// few attempts, then keep retrying on the backoff (which is shorter than the interval -// between successful fetches, so a node that was down is picked up again sooner). +// a request can go unanswered because it was lost, or because the other node is rate +// limiting anon requests. Retry soon at first, then on the (shorter than normal) backoff void MyMesh::scheduleRegionFetchRetry() { if (region_fetch_fails < 0xFF) region_fetch_fails++; next_region_fetch = futureMillis( @@ -1584,16 +1572,21 @@ void MyMesh::handleCommand(uint32_t sender_timestamp, char *command, char *reply if (hex_len < 8 || hex_len > PUB_KEY_SIZE*2 || (hex_len & 1) || !mesh::Utils::fromHex(pubkey, hex_len / 2, sub)) { strcpy(reply, "Err - bad pubkey"); - } else if (hex_len < PUB_KEY_SIZE*2 && !resolveNeighbourPubKey(pubkey, hex_len / 2)) { - strcpy(reply, "Err - not a known neighbor, need full pubkey"); - } else if (self_id.matches(pubkey)) { - strcpy(reply, "Err - that is this node"); } else { - cancelRegionSubscribe(); - memcpy(_prefs.region_src_pubkey, pubkey, PUB_KEY_SIZE); - next_region_fetch = futureMillis(1); // fetch from the new source now - savePrefs(); - formatRegionSubscribeReply(reply); + int matches = hex_len < PUB_KEY_SIZE*2 ? resolveNeighbourPubKey(pubkey, hex_len / 2) : 1; + if (matches == 0) { + strcpy(reply, "Err - not a known neighbor, need full pubkey"); + } else if (matches > 1) { + strcpy(reply, "Err - ambiguous neighbor pubkey, use a longer prefix"); + } else if (self_id.matches(pubkey)) { + strcpy(reply, "Err - that is this node"); + } else { + cancelRegionSubscribe(); + memcpy(_prefs.region_src_pubkey, pubkey, PUB_KEY_SIZE); + next_region_fetch = futureMillis(1); // fetch from the new source now + savePrefs(); + formatRegionSubscribeReply(reply); + } } } } else{ diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 38902143ce..7d19d80745 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -117,16 +117,16 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks { mesh::Identity region_query_id; // node we are currently asking for regions uint32_t region_query_tag; unsigned long region_query_until; - uint8_t region_query_mode; // one of SCOPE_QUERY_* + uint8_t region_query_mode; // one of REGION_QUERY_* uint8_t region_discover_next; // index into neighbours[], for the 'discover.regions' pass - char region_discover_reply[134]; // results of last pass, for 'discover.regions list' + char region_discover_reply[134]; // last pass results, for 'discover.regions list' uint8_t region_discover_dropped; // ..entries from that pass which did not fit - unsigned long next_region_fetch; // when to re-fetch from the subscribed node (0 = never) - uint32_t region_fetch_at; // when the last fetch completed (0 = not yet) + unsigned long next_region_fetch; // when to re-fetch from the subscribed node (0 = never) + uint32_t region_fetch_at; // when the last fetch completed (0 = not yet) uint8_t region_fetch_added, region_fetch_known; - uint8_t region_fetch_fails; // consecutive unanswered fetches - bool region_fetch_full; // last fetch hit the region table limit - bool region_save_pending; // imported regions are in RAM but not on disk + uint8_t region_fetch_fails; // consecutive unanswered fetches + bool region_fetch_full; // last fetch hit the region table limit + bool region_save_pending; // imported regions are in RAM but not on disk #if defined(WITH_RS232_BRIDGE) RS232Bridge bridge; #elif defined(WITH_ESPNOW_BRIDGE) @@ -134,7 +134,7 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks { #endif bool isRegionSrcSet() const; - bool resolveNeighbourPubKey(uint8_t* pubkey, int prefix_len); + int resolveNeighbourPubKey(uint8_t* pubkey, int prefix_len); void formatRegionSubscribeReply(char* reply); bool sendRegionsReq(const mesh::Identity& target); bool handleRegionsResponse(const uint8_t* data, size_t len); diff --git a/src/helpers/RegionMap.cpp b/src/helpers/RegionMap.cpp index ab50d5f7a4..61aa38c5f1 100644 --- a/src/helpers/RegionMap.cpp +++ b/src/helpers/RegionMap.cpp @@ -347,7 +347,7 @@ int RegionMap::exportNamesTo(char *dest, int max_len, uint8_t mask, bool invert) return dp - dest; // return length } -// Take the next region name from 'src', starting at *cursor. Returns false at the end. +// take the next name from 'src', starting at *cursor. Returns false at the end static bool next_name(const char *src, int len, int *cursor, char dest[], int dest_size) { while (*cursor < len) { int n = 0; @@ -355,7 +355,7 @@ static bool next_name(const char *src, int len, int *cursor, char dest[], int de while (*cursor < len && src[*cursor] != ',') { uint8_t c = src[(*cursor)++]; if (!RegionMap::is_name_char(c)) { valid = false; continue; } // don't trust the sender - if (n + 1 < dest_size) { dest[n++] = c; } else { valid = false; } // too long for a name + if (n + 1 < dest_size) { dest[n++] = c; } else { valid = false; } // too long } (*cursor)++; // skip the separator @@ -371,15 +371,11 @@ int RegionMap::importNamesFrom(const char *src, int len, int* num_known, bool* w if (was_full) { *was_full = false; } - // a decrypted payload is padded out to the cipher block size, so the list can be - // followed by null bytes. Without this, the last name runs into the padding. - for (int i = 0; i < len; i++) { - if (src[i] == 0) { len = i; break; } - } + const char* end = (const char *) memchr(src, 0, len); // stop at any cipher block padding + if (end) { len = end - src; } while (next_name(src, len, &cursor, name, sizeof(name))) { - // the wildcard is not a Region, and private ('$') Regions have keys we cannot derive - if (name[0] == '*' || name[0] == '$') continue; + if (name[0] == '*' || name[0] == '$') continue; // not a Region, and '$' keys aren't derivable if (findByName(name)) { known++; // already in the map, leave it exactly as it is @@ -387,14 +383,14 @@ int RegionMap::importNamesFrom(const char *src, int len, int* num_known, bool* w } auto region = putRegion(name, 0); // add as a child of the wildcard - if (region == NULL) { // full! (the name is already known to be valid) + if (region == NULL) { // full! (name was already validated, so that's the only cause) if (was_full) { *was_full = true; } - break; // the rest of the list cannot be added + break; } region->flags = 0; // allow flood added++; } if (num_known) { *num_known = known; } - return added; // return number of new Regions + return added; } diff --git a/src/helpers/RegionMap.h b/src/helpers/RegionMap.h index 55a58ab941..27a3af6006 100644 --- a/src/helpers/RegionMap.h +++ b/src/helpers/RegionMap.h @@ -54,16 +54,7 @@ class RegionMap { const RegionEntry* getByIdx(int i) const { return ®ions[i]; } const RegionEntry* getRoot() const { return &wildcard; } int exportNamesTo(char *dest, int max_len, uint8_t mask, bool invert = false); - - /** - * \brief Add Regions from a comma separated list of names, as written by exportNamesTo(). - * This is additive: a name already in the map is left exactly as it is, keeping its - * flags and its parent. New names are added as flood-allowed children of the wildcard. - * The wildcard and private ('$') Regions in the list are skipped. - * \param num_known OUT - optional, how many names were already in the map - * \param was_full OUT - optional, true if the map filled up and names had to be dropped - * \returns the number of Regions added - */ + // additive: existing names keep their flags and parent. Returns the number added int importNamesFrom(const char *src, int len, int* num_known = NULL, bool* was_full = NULL); int getTransportKeysFor(const RegionEntry& src, TransportKey dest[], int max_num);