From c10235e4867c2aa6dc55d836d6f36eb81840e63a Mon Sep 17 00:00:00 2001 From: Denis Skripnik <264741654+web3blind@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:03:11 +0000 Subject: [PATCH 1/4] fix(hf16): harden finality, production and operator boundaries --- .github/workflows/tests.yml | 10 +- docs/consensus/hf16.md | 2 +- docs/p2p/messages.md | 6 + docs/plugins/node-admin.md | 4 + docs/plugins/validator.md | 36 ++- docs/storage/block-log-tools.md | 2 +- libraries/chain/database.cpp | 235 +++++++++------- .../chain/include/graphene/chain/database.hpp | 9 + libraries/network/dlt_p2p_node.cpp | 37 +-- .../include/graphene/network/dlt_p2p_node.hpp | 10 +- .../graphene/network/dlt_vote_cache.hpp | 52 ++++ .../graphene/plugins/node_admin/http_only.hpp | 24 ++ plugins/node_admin/node_admin_plugin.cpp | 3 +- plugins/validator/validator.cpp | 52 ++-- programs/util/block-log/block-archive.cjs | 8 +- tests/block_log/archive_merge.test.cjs | 52 ++++ tests/consensus_sim/CMakeLists.txt | 2 + tests/consensus_sim/README.md | 37 +++ .../production_safety_offline.py | 157 +++++++++++ .../scenarios/test_hf16_boundary.cpp | 9 + .../scenarios/test_hf16_production_safety.cpp | 264 ++++++++++++++++++ tests/node_admin/CMakeLists.txt | 11 + tests/node_admin/node_admin_http_test.cpp | 93 ++++++ tests/p2p/test_vote_ingress.py | 260 +++++++++++++++++ .../validator/test_committed_chain_state.cpp | 72 +++++ tests/validator/test_signing_cache.py | 175 ++++++++++++ 26 files changed, 1475 insertions(+), 147 deletions(-) create mode 100644 libraries/network/include/graphene/network/dlt_vote_cache.hpp create mode 100644 plugins/node_admin/include/graphene/plugins/node_admin/http_only.hpp create mode 100644 tests/block_log/archive_merge.test.cjs create mode 100644 tests/consensus_sim/production_safety_offline.py create mode 100644 tests/consensus_sim/scenarios/test_hf16_production_safety.cpp create mode 100644 tests/node_admin/node_admin_http_test.cpp create mode 100644 tests/p2p/test_vote_ingress.py create mode 100644 tests/validator/test_committed_chain_state.cpp create mode 100644 tests/validator/test_signing_cache.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fac4083815..73ebbdca4d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,14 @@ jobs: - name: P2P corruption recovery regression (offline) run: python3 tests/p2p/test_corruption_recovery.py -v + - name: HF16 offline regression checks + run: | + set -euo pipefail + python3 tests/p2p/test_vote_ingress.py + python3 tests/validator/test_signing_cache.py + python3 tests/consensus_sim/production_safety_offline.py + node --test tests/block_log/archive_merge.test.cjs + - name: Install build deps run: | set -euo pipefail @@ -125,7 +133,7 @@ jobs: set -euo pipefail cd build if [ "${{ matrix.validators }}" = "2" ]; then - tests/consensus_sim/consensus_sim_tests --run_test=hf16_boundary_suite --log_level=test_suite + tests/consensus_sim/consensus_sim_tests --run_test=hf16_boundary_suite,hf16_production_safety_suite,validator_committed_state_suite --log_level=test_suite else ctest -R '^consensus_sim$' --output-on-failure fi diff --git a/docs/consensus/hf16.md b/docs/consensus/hf16.md index f88d4dbbff..01de3c6c4a 100644 --- a/docs/consensus/hf16.md +++ b/docs/consensus/hf16.md @@ -10,7 +10,7 @@ After the master image is built, check its revision/version, startup, synchroniz - The runtime slot changes from **3 seconds to 1 second** at HF16. Use the node's current `get_config` value and block timestamps; the compile-time legacy `CHAIN_BLOCK_INTERVAL` is not the post-fork slot. - A QC collects signatures from scheduled validators. The implementation uses `max(floor(2N/3)+1, floor(0.75N))`: **15 of 21**, not the old 14-confirmation post-validation description. -- A producer handover carries a parent QC. Timeout certificates (TCs) allow progress after skipped rounds. LIB uses the **2-chain** rule: certified parent and child on the local main branch, adjacent heights and consecutive rounds. One isolated QC is insufficient. +- A producer handover carries a parent QC. Timeout certificates (TCs) allow progress after skipped rounds. LIB uses the **2-chain** rule: certified parent and child on the local main branch, adjacent heights and consecutive rounds. The signed child header must carry that parent's QC. A child using the late-QC grandparent fallback does not finalize its parent merely because the local vote pool later collects QCs for both. One isolated QC is insufficient. - Votes are limited to one block per validator per round; the highest voted round is persisted before broadcasting. Repeating the same vote is allowed. Conflicting votes can be proved by header evidence and produce virtual operation `validator_equivocation` (ID **106**). - Header extension tags are append-only: **3 QC**, **4 TC**, **5 equivocation proof**. Transaction extensions and existing operation IDs are unchanged. - Block-denominated governance values whose wall-time meaning must survive the slot change are scaled at activation. Do not mechanically replace every 3-second timer: independent retry/shutdown timers remain separate. diff --git a/docs/p2p/messages.md b/docs/p2p/messages.md index 0dad5c4bcd..2cc0c23861 100644 --- a/docs/p2p/messages.md +++ b/docs/p2p/messages.md @@ -6,6 +6,12 @@ DLT P2P uses a binary protocol over raw TCP. Each message is framed with a 4-byt --- +## Vote relay bounds + +Block and timeout vote relay caches retain only chain-accepted hashes. Each cache has at most 8192 entries and a 120-second lifetime. Expiry cleanup examines at most 64 oldest entries per lookup. + +Accepted local block votes have a separate bounded retry cache, so remote relay churn cannot suppress their rebroadcast. Malformed names/heights use the existing strike policy; a generic delegate refusal is not proof of peer misconduct. + ## Message Type Summary | Type ID | Name | Direction | Purpose | diff --git a/docs/plugins/node-admin.md b/docs/plugins/node-admin.md index 847c36840c..044c583ae8 100644 --- a/docs/plugins/node-admin.md +++ b/docs/plugins/node-admin.md @@ -29,6 +29,10 @@ request arrives **from 127.0.0.1**, so checking the source address on the public protects nothing. `node_admin` therefore opens its own HTTP listener, bound to loopback by default, that is never proxied. +This listener is HTTP-only, including MCP. WebSocket upgrade requests are rejected +with HTTP 403 and the connection is closed, even when a bearer token is supplied; +there is no admin WebSocket API on this port. + An endpoint on a non-loopback address is refused unless `admin-allow-remote = true` is set explicitly. A bearer token on an open port is a different threat model, not a convenience. diff --git a/docs/plugins/validator.md b/docs/plugins/validator.md index 28e9b54d7d..0a1bdbda3e 100644 --- a/docs/plugins/validator.md +++ b/docs/plugins/validator.md @@ -157,6 +157,40 @@ Production resumes automatically on the next tick. ## `on_block_applied()` — Signal Handler Connected to `database::applied_block`. Runs for every incoming block. +This deferred callback does not refresh the signing-key snapshot. + +### Committed signing-key snapshot + +Block headers must use the key committed at the current head, not a pending +`validator_update_operation` key. The plugin seeds configured validators' keys at +startup, then replaces the complete snapshot on `database::committed_chain_state`. +That signal runs synchronously under the database write lock after successful +`_push_block()` and memory checking, inside the callback that temporarily removes +pending transactions. It fires before those transactions are restored. Signal +callbacks must not yield or acquire database locks. + +Reading all configured validators at this boundary also captures rotations +reverted by a fork switch, including cases where the adopted blocks contain no +rotation at all. Applying only incoming block operations cannot undo such a +rotation; reading the database in deferred `applied_block` instead sees pending +state. The snapshot stores both head height and ID, and removes validators absent +from the committed index. Visible null keys still disable production; a committed +null key is not replaced by a pending re-enable. + +Offline regression (requires Python 3 and a C++14 compiler): + +```sh +python3 tests/validator/test_signing_cache.py +# Negative control against the pre-fix source; expected to exit 1: +python3 tests/validator/test_signing_cache.py --source-ref d4fd4bcd +``` + +The test extracts and compiles the real plugin helpers with a controlled database +fixture, and guards the source hook placement and connection lifecycle. It does +not replace a native chain integration run. The companion scenario +`tests/validator/test_committed_chain_state.cpp` is included in the testnet +`consensus_sim_tests` target and verifies that real `push_block()` exposes committed keys in +the synchronous signal and restored pending keys in deferred `applied_block`. ### Missed slot detection @@ -205,7 +239,7 @@ Included in P2P FORWARD stagnation logs when the node is stuck with no peer ahea 5. **Slaves must sync before producing in emergency mode** — producing on stale head = minority fork. 6. **Participation < 33% stops production** — network partition guard (overridable). 7. **21 consecutive own-validator blocks → rollback to LIB** — minority fork recovery. -8. **All database reads are fresh** — no state caching; emergency mode can activate/deactivate any block. +8. **Committed signing keys are refreshed at each successful push boundary** — other production state is read fresh; emergency mode can activate/deactivate any block. --- diff --git a/docs/storage/block-log-tools.md b/docs/storage/block-log-tools.md index 523a74aa00..c90fd1ff2d 100644 --- a/docs/storage/block-log-tools.md +++ b/docs/storage/block-log-tools.md @@ -75,7 +75,7 @@ node block-archive.cjs merge /var/lib/vizd/block-archive --from=83700000 --out=/ Order matters: the merged log must reach at least the snapshot block minus one. If it ends earlier, snapshot import sees a gap and resets the log (`plugins/snapshot/plugin.cpp`, "gap detected"). If it ends after the snapshot, the node replays the extra blocks locally before P2P sync. -It refuses gaps in the range (`GAP a-b, merge stopped`) and never overwrites existing files. +It refuses gaps in the range (`GAP a-b, merge stopped`) and never overwrites existing files. Overlapping ranges emit each height once, including nested ranges. Without `--to`, the upper bound is the highest archived height, not the last filename's end. ### Filters diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 3fb6d9492f..6ff6c5c103 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -1753,6 +1753,7 @@ namespace graphene { namespace chain { try { result = _push_block(new_block, skip); check_free_memory(false, new_block.block_num()); + CHAIN_TRY_NOTIFY(committed_chain_state); } catch (const fc::exception &e) { auto msg = std::string(e.what()); // TODO: there is no easy way to catch boost::interprocess::bad_alloc @@ -2857,6 +2858,121 @@ namespace graphene { namespace chain { // while no pending session is active (see the assert below). public_key_type committed_signing_key; + auto assemble_header = [&]() { + pending_block.previous = head_block_id(); + pending_block.timestamp = when; + pending_block.validator = validator_owner; + + const auto &validator = get_validator(validator_owner); + + // The header of THIS block is validated against the validator signing key + // committed at the previous (head) block — a pending validator_update that + // changes our own key takes effect only for the NEXT block. Verify against + // the committed object (visible again after the session reset above), not + // against the session-visible one used at the top of this function. + if (!(skip & skip_validator_signature)) + FC_ASSERT(committed_signing_key == block_signing_private_key.get_public_key(), + "Block signing key does not match the signing key committed at head " + "(a pending validator_update must not change the key used to sign " + "the current block)", + ("validator", validator_owner) + ("committed_key", committed_signing_key) + ("signing_key", block_signing_private_key.get_public_key())); + + if (validator.running_version != CHAIN_VERSION) { + pending_block.extensions.insert(block_header_extensions(CHAIN_VERSION)); + } + + // HF16: hand the turn over only on a certified parent. The validator + // plugin waits for the QC before calling us; reaching here without + // one is a bug, and the block would be rejected anyway. + if (block_needs_parent_qc(validator_owner)) { + fc::optional cert = qc_override; + if (!cert) cert = parent_qc_for_next_block(validator_owner); + FC_ASSERT(cert.valid(), "No QC for head #${n} yet: cannot hand over the turn", + ("n", head_block_num())); + FC_ASSERT(!(qc_override && tc), "The late-QC fallback carries no TC"); + pending_block.extensions.insert(block_header_extensions(*cert)); + if (tc) pending_block.extensions.insert(block_header_extensions(*tc)); + + // Double-vote proofs: newest round per validator that the chain has + // not punished yet; a few per block keeps headers small. + std::lock_guard guard(_block_votes_mutex); + const uint32_t own_round = round_of(when); + std::map pick; + for (auto it = _pending_equivocations.begin(); it != _pending_equivocations.end(); ) { + const auto &p = it->second; + const auto *v = find_validator(p.validator); + if (!v || p.round <= v->last_equivocation_round || uint64_t(p.round) + 3600 / block_interval() < own_round) { + it = _pending_equivocations.erase(it); // punished already or too old + continue; + } + // Votes may arrive up to 64 rounds ahead of head. Keep such + // evidence queued, but it cannot be punished in this block. + if (p.round >= own_round) { + ++it; + continue; + } + // The schedule/key may have changed since these votes arrived. + // Match apply_equivocation_proofs() against committed head state, + // not pending validator_update transactions. + if (!verify_block_vote(p.first.block_num, p.first.block_id, p.round, p.validator, p.first.signature) || + !verify_block_vote(p.second.block_num, p.second.block_id, p.round, p.validator, p.second.signature)) { + it = _pending_equivocations.erase(it); + continue; + } + auto &slot = pick[p.validator]; + if (!slot || slot->round < p.round) slot = &p; + ++it; + } + size_t n = 0; + for (const auto &kv : pick) { + if (n++ >= 5) break; + pending_block.extensions.insert(block_header_extensions(*kv.second)); + } + } + FC_ASSERT(!tc || block_needs_parent_qc(validator_owner), + "A TC goes only next to a parent QC"); + + const auto &hfp = get_hardfork_property_object(); + + // Emergency committee validator should NOT vote for future hardforks + // via block extensions. Its on-chain vote is set to + // current_hardfork_version (status quo) and is synced every + // schedule update. Auto-injection would overwrite this and + // make committee vote for the next hardfork, defeating its + // neutral-voter design. + const auto &dgp_block = get_dynamic_global_properties(); + bool is_emergency_committee = + has_hardfork(CHAIN_HARDFORK_12) && + dgp_block.emergency_consensus_active && + validator_owner == CHAIN_EMERGENCY_VALIDATOR_ACCOUNT; + + if (!is_emergency_committee) { + if (hfp.current_hardfork_version < + CHAIN_HARDFORK_VERSION // Binary is newer hardfork than has been applied + && (validator.hardfork_version_vote != + _hardfork_versions[hfp.last_hardfork + 1] || + validator.hardfork_time_vote != + _hardfork_times[hfp.last_hardfork + + 1])) // Witness vote does not match binary configuration + { + // Make vote match binary configuration + pending_block.extensions.insert(block_header_extensions(hardfork_version_vote(_hardfork_versions[ + hfp.last_hardfork + 1], _hardfork_times[ + hfp.last_hardfork + 1]))); + } else if (hfp.current_hardfork_version == + CHAIN_HARDFORK_VERSION // Binary does not know of a new hardfork + && validator.hardfork_version_vote > + CHAIN_HARDFORK_VERSION) // Voting for hardfork in the future, that we do not know of... + { + // Make vote match binary configuration. This is vote to not apply the new hardfork. + pending_block.extensions.insert(block_header_extensions(hardfork_version_vote(_hardfork_versions[hfp.last_hardfork], _hardfork_times[hfp.last_hardfork]))); + } + } + + }; + with_strong_write_lock([&]() { // // The following code throws away existing pending_tx_session and @@ -2870,6 +2986,15 @@ namespace graphene { namespace chain { // re-apply pending transactions in this method. // _pending_tx_session.reset(); + if (has_hardfork(CHAIN_HARDFORK_16)) { + committed_signing_key = get_validator(validator_owner).signing_key; + assemble_header(); + // QC, TC, proofs and version votes are fixed before packing + // transactions. Reserve four additional bytes for growth of + // the transaction-vector varuint (one byte is already packed). + total_block_size = fc::raw::pack_size(pending_block) + 4; + maximum_block_size = std::min(get_dynamic_global_properties().maximum_block_size, uint32_t(CHAIN_BLOCK_SIZE)); + } _pending_tx_session = start_undo_session(); uint64_t postponed_tx_count = 0; @@ -2930,104 +3055,8 @@ namespace graphene { namespace chain { // However, the push_block() call below will re-create the // _pending_tx_session. - pending_block.previous = head_block_id(); - pending_block.timestamp = when; + if (!has_hardfork(CHAIN_HARDFORK_16)) assemble_header(); pending_block.transaction_merkle_root = pending_block.calculate_merkle_root(); - pending_block.validator = validator_owner; - - const auto &validator = get_validator(validator_owner); - - // The header of THIS block is validated against the validator signing key - // committed at the previous (head) block — a pending validator_update that - // changes our own key takes effect only for the NEXT block. Verify against - // the committed object (visible again after the session reset above), not - // against the session-visible one used at the top of this function. - if (!(skip & skip_validator_signature)) - FC_ASSERT(committed_signing_key == block_signing_private_key.get_public_key(), - "Block signing key does not match the signing key committed at head " - "(a pending validator_update must not change the key used to sign " - "the current block)", - ("validator", validator_owner) - ("committed_key", committed_signing_key) - ("signing_key", block_signing_private_key.get_public_key())); - - if (validator.running_version != CHAIN_VERSION) { - pending_block.extensions.insert(block_header_extensions(CHAIN_VERSION)); - } - - // HF16: hand the turn over only on a certified parent. The validator - // plugin waits for the QC before calling us; reaching here without - // one is a bug, and the block would be rejected anyway. - if (block_needs_parent_qc(validator_owner)) { - fc::optional cert = qc_override; - if (!cert) cert = parent_qc_for_next_block(validator_owner); - FC_ASSERT(cert.valid(), "No QC for head #${n} yet: cannot hand over the turn", - ("n", head_block_num())); - FC_ASSERT(!(qc_override && tc), "The late-QC fallback carries no TC"); - pending_block.extensions.insert(block_header_extensions(*cert)); - if (tc) pending_block.extensions.insert(block_header_extensions(*tc)); - - // Double-vote proofs: newest round per validator that the chain has - // not punished yet; a few per block keeps headers small. - std::lock_guard guard(_block_votes_mutex); - const uint32_t own_round = round_of(when); - std::map pick; - for (auto it = _pending_equivocations.begin(); it != _pending_equivocations.end(); ) { - const auto &p = it->second; - const auto *v = find_validator(p.validator); - if (!v || p.round <= v->last_equivocation_round || uint64_t(p.round) + 3600 / block_interval() < own_round) { - it = _pending_equivocations.erase(it); // punished already or too old - continue; - } - auto &slot = pick[p.validator]; - if (!slot || slot->round < p.round) slot = &p; - ++it; - } - size_t n = 0; - for (const auto &kv : pick) { - if (n++ >= 5) break; - pending_block.extensions.insert(block_header_extensions(*kv.second)); - } - } - FC_ASSERT(!tc || block_needs_parent_qc(validator_owner), - "A TC goes only next to a parent QC"); - - const auto &hfp = get_hardfork_property_object(); - - // Emergency committee validator should NOT vote for future hardforks - // via block extensions. Its on-chain vote is set to - // current_hardfork_version (status quo) and is synced every - // schedule update. Auto-injection would overwrite this and - // make committee vote for the next hardfork, defeating its - // neutral-voter design. - const auto &dgp_block = get_dynamic_global_properties(); - bool is_emergency_committee = - has_hardfork(CHAIN_HARDFORK_12) && - dgp_block.emergency_consensus_active && - validator_owner == CHAIN_EMERGENCY_VALIDATOR_ACCOUNT; - - if (!is_emergency_committee) { - if (hfp.current_hardfork_version < - CHAIN_HARDFORK_VERSION // Binary is newer hardfork than has been applied - && (validator.hardfork_version_vote != - _hardfork_versions[hfp.last_hardfork + 1] || - validator.hardfork_time_vote != - _hardfork_times[hfp.last_hardfork + - 1])) // Witness vote does not match binary configuration - { - // Make vote match binary configuration - pending_block.extensions.insert(block_header_extensions(hardfork_version_vote(_hardfork_versions[ - hfp.last_hardfork + 1], _hardfork_times[ - hfp.last_hardfork + 1]))); - } else if (hfp.current_hardfork_version == - CHAIN_HARDFORK_VERSION // Binary does not know of a new hardfork - && validator.hardfork_version_vote > - CHAIN_HARDFORK_VERSION) // Voting for hardfork in the future, that we do not know of... - { - // Make vote match binary configuration. This is vote to not apply the new hardfork. - pending_block.extensions.insert(block_header_extensions(hardfork_version_vote(_hardfork_versions[hfp.last_hardfork], _hardfork_times[hfp.last_hardfork]))); - } - } if (!(skip & skip_validator_signature)) { pending_block.sign(block_signing_private_key); @@ -7675,6 +7704,18 @@ namespace graphene { namespace chain { // compile-time 3 s step never matched there, so QC-LIB was always 0 // and LIB fell back to post-validation (stand: lag exactly 14 blocks). if (round_of(cb->timestamp) != round_of(pb->timestamp) + 1) continue; + // The late-QC exception permits a child justified by its grandparent. + // A locally collected QC for the parent does not prove that the + // child's quorum learned it before voting. Require that causal link + // in the signed child header, not merely two adjacent cached QCs. + bool carries_parent_qc = false; + for (const auto &ext : cb->extensions) { + if (ext.which() != 3) continue; + const auto &qc = ext.get(); + if (qc.block_num == parent.first && qc.block_id == parent.second && + qc.round == round_of(pb->timestamp)) carries_parent_qc = true; + } + if (!carries_parent_qc) continue; return parent.first; } return 0; diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index cfd8ce483d..de32a7adf3 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -370,6 +370,15 @@ namespace graphene { namespace chain { */ boost::signals2::signal applied_block; + /** + * Emitted synchronously after a successful push_block has selected its + * committed head, before pending transactions are restored. Includes + * fork switches and changes not represented in the incoming block. + * Runs under the database write lock: callbacks must not yield, acquire + * database locks, or retain references into chain state. + */ + fc::signal committed_chain_state; + /** * Emitted when dlt_block_log is reset due to a gap between * dlt_block_log end and fork_db start. The snapshot plugin diff --git a/libraries/network/dlt_p2p_node.cpp b/libraries/network/dlt_p2p_node.cpp index 9dd59e3f02..c825e93efb 100644 --- a/libraries/network/dlt_p2p_node.cpp +++ b/libraries/network/dlt_p2p_node.cpp @@ -2547,8 +2547,13 @@ void dlt_p2p_node::broadcast_block_post_validation( // die in the relay dedup and in the pool's one-vote-per-round check, so it // goes straight to the peers. Peers that already have it drop it there. const auto key = fc::sha256::hash(fc::raw::pack(msg)); - if (_block_votes_seen.count(key)) { - _block_votes_seen[key] = fc::time_point::now(); + const auto now = fc::time_point::now(); + if (_local_block_votes_seen.contains(key, now)) { + gossip_to_capable(message(msg), DLT_CAP_BLOCK_VOTE, INVALID_PEER_ID); + return; + } + if (_block_votes_seen.contains(key, now)) { + _local_block_votes_seen.insert(key, now); gossip_to_capable(message(msg), DLT_CAP_BLOCK_VOTE, INVALID_PEER_ID); return; } @@ -2560,18 +2565,14 @@ void dlt_p2p_node::on_dlt_block_vote(peer_id peer, const dlt_block_vote_message& } void dlt_p2p_node::process_block_vote(const dlt_block_vote_message& msg, peer_id from) { - static constexpr int64_t DLT_BLOCK_VOTE_SEEN_SEC = 120; const auto now = fc::time_point::now(); - - // Prune the relay-dedup map. Votes older than two minutes are far below LIB - // at any block interval we run, so re-accepting one is harmless (the pool - // already counted it or dropped it as stale). - for (auto it = _block_votes_seen.begin(); it != _block_votes_seen.end(); ) { - if (now - it->second > fc::seconds(DLT_BLOCK_VOTE_SEEN_SEC)) it = _block_votes_seen.erase(it); - else ++it; + if (msg.validator.empty() || msg.validator.size() > CHAIN_MAX_ACCOUNT_NAME_LENGTH || + msg.block_num == 0 || graphene::protocol::block_header::num_from_id(msg.block_id) != msg.block_num) { + if (from != INVALID_PEER_ID) record_packet_result(from, false); + return; } const auto key = fc::sha256::hash(fc::raw::pack(msg)); - if (!_block_votes_seen.emplace(key, now).second) return; // already handled + if (_block_votes_seen.contains(key, now)) return; // Verification lives in the chain: scheduled validator + signature over // block_vote_digest. An invalid vote is neither counted nor relayed, so a @@ -2584,6 +2585,10 @@ void dlt_p2p_node::process_block_vote(const dlt_block_vote_message& msg, peer_id return; } + _block_votes_seen.insert(key, now); + // Socket writes may yield to remote traffic. Enrol accepted local retries + // before gossip, while the accepted hash cannot yet have been evicted. + if (from == INVALID_PEER_ID) _local_block_votes_seen.insert(key, now); // Gossip to capable fork peers only: a peer without DLT_CAP_BLOCK_VOTE // would score the message as an unknown packet against us. gossip_to_capable(message(msg), DLT_CAP_BLOCK_VOTE, from); @@ -2619,14 +2624,13 @@ void dlt_p2p_node::on_dlt_timeout_vote(peer_id peer, const dlt_timeout_vote_mess } void dlt_p2p_node::process_timeout_vote(const dlt_timeout_vote_message& msg, peer_id from) { - static constexpr int64_t DLT_TIMEOUT_VOTE_SEEN_SEC = 120; const auto now = fc::time_point::now(); - for (auto it = _timeout_votes_seen.begin(); it != _timeout_votes_seen.end(); ) { - if (now - it->second > fc::seconds(DLT_TIMEOUT_VOTE_SEEN_SEC)) it = _timeout_votes_seen.erase(it); - else ++it; + if (msg.validator.empty() || msg.validator.size() > CHAIN_MAX_ACCOUNT_NAME_LENGTH) { + if (from != INVALID_PEER_ID) record_packet_result(from, false); + return; } const auto key = fc::sha256::hash(fc::raw::pack(msg)); - if (!_timeout_votes_seen.emplace(key, now).second) return; // already handled + if (_timeout_votes_seen.contains(key, now)) return; // Invalid votes are neither counted nor relayed (no amplification). if (!_delegate || !_delegate->accept_timeout_vote(msg)) { @@ -2636,6 +2640,7 @@ void dlt_p2p_node::process_timeout_vote(const dlt_timeout_vote_message& msg, pee } return; } + _timeout_votes_seen.insert(key, now); gossip_to_capable(message(msg), DLT_CAP_TIMEOUT_VOTE, from); } diff --git a/libraries/network/include/graphene/network/dlt_p2p_node.hpp b/libraries/network/include/graphene/network/dlt_p2p_node.hpp index 8930ce01c0..cb52f60e89 100644 --- a/libraries/network/include/graphene/network/dlt_p2p_node.hpp +++ b/libraries/network/include/graphene/network/dlt_p2p_node.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -506,10 +507,11 @@ class dlt_p2p_node { // ── Mempool ────────────────────────────────────────────────── std::map _mempool_by_id; - // Relay dedup for block votes: digest of the vote message -> seen time. - // Bounded by pruning entries older than DLT_BLOCK_VOTE_SEEN_SEC. - std::map _block_votes_seen; - std::map _timeout_votes_seen; + // Accepted vote hashes only, with hard capacity and bounded expiry work. + dlt_vote_cache _block_votes_seen; + dlt_vote_cache _timeout_votes_seen; + // Local accepted retries must not be evicted by remote relay traffic. + dlt_vote_cache _local_block_votes_seen; std::multimap _mempool_by_expiry; uint32_t _mempool_total_bytes = 0; diff --git a/libraries/network/include/graphene/network/dlt_vote_cache.hpp b/libraries/network/include/graphene/network/dlt_vote_cache.hpp new file mode 100644 index 0000000000..3d74681dc1 --- /dev/null +++ b/libraries/network/include/graphene/network/dlt_vote_cache.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace graphene { namespace network { + +// Relay deduplication only; chain validation still decides whether a vote counts. +// Keep insertion order so expiry never scans every retained hash per packet. +class dlt_vote_cache { + using entry = std::pair; + std::list _order; + std::map::iterator> _seen; + static constexpr size_t capacity = 8192; + + void erase_oldest() { + _seen.erase(_order.front().first); + _order.pop_front(); + } + +public: + bool contains(const fc::sha256& key, fc::time_point now) { + // Bound even the first packet after a long idle interval. + for (size_t n = 0; n < 64 && !_order.empty(); ++n) { + if (!(now - _order.front().second > fc::seconds(120))) break; + erase_oldest(); + } + auto it = _seen.find(key); + if (it == _seen.end()) return false; + if (now - it->second->second > fc::seconds(120)) { + _order.erase(it->second); + _seen.erase(it); + return false; + } + return true; + } + + void insert(const fc::sha256& key, fc::time_point now) { + // A local resend must not renew retention indefinitely. + if (_seen.count(key)) return; + if (_seen.size() >= capacity) erase_oldest(); + _order.emplace_back(key, now); + _seen.emplace(key, std::prev(_order.end())); + } + + size_t size() const { return _seen.size(); } +}; + +} } // graphene::network diff --git a/plugins/node_admin/include/graphene/plugins/node_admin/http_only.hpp b/plugins/node_admin/include/graphene/plugins/node_admin/http_only.hpp new file mode 100644 index 0000000000..206595a7cd --- /dev/null +++ b/plugins/node_admin/include/graphene/plugins/node_admin/http_only.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +namespace graphene { namespace plugins { namespace node_admin { namespace util { + +// Shared by the Asio listener and the offline websocketpp transport regression. +template +void configure_http_only(Server &server, HttpHandler handler) { + server.set_http_handler(handler); + // websocketpp accepts valid WebSocket handshakes by default, without calling + // the HTTP handler. This listener has no WebSocket API: reject even when an + // Authorization header is present, then let websocketpp close the transport. + server.set_validate_handler([&server](websocketpp::connection_hdl hdl) { + auto con = server.get_con_from_hdl(hdl); + con->set_status(websocketpp::http::status_code::forbidden); + con->set_body("WebSocket upgrades are not supported on the admin HTTP listener"); + con->replace_header("Connection", "close"); + return false; + }); +} + +} } } } // graphene::plugins::node_admin::util diff --git a/plugins/node_admin/node_admin_plugin.cpp b/plugins/node_admin/node_admin_plugin.cpp index a233b3d135..32076d0bed 100644 --- a/plugins/node_admin/node_admin_plugin.cpp +++ b/plugins/node_admin/node_admin_plugin.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -2467,7 +2468,7 @@ void node_admin_plugin::impl::start() { } connections.insert(hdl); }); - server.set_http_handler([this](connection_hdl hdl) { this->handle_http(hdl); }); + util::configure_http_only(server, [this](connection_hdl hdl) { this->handle_http(hdl); }); server.listen(*endpoint); server.start_accept(); diff --git a/plugins/validator/validator.cpp b/plugins/validator/validator.cpp index eb62f242ba..9727c85a6d 100644 --- a/plugins/validator/validator.cpp +++ b/plugins/validator/validator.cpp @@ -166,11 +166,13 @@ namespace graphene { // (validate_block_header). Production must therefore sign with the // committed key, not with the session-visible one. // - // Seeded at startup and refreshed from applied block CONTENTS: an - // applied_block callback runs after push_block() has re-applied the - // pending pool, so reading the DB there would see uncommitted state. + // Seeded at startup and refreshed synchronously at the committed + // chain boundary, before pending transactions are restored. Reading + // applied block contents misses rotations reverted by a fork switch; + // reading the DB in deferred applied_block sees pending state. std::map _committed_signing_keys; uint64_t _committed_keys_block_num = 0; + block_id_type _committed_keys_block_id; std::mutex _committed_keys_mutex; /** Key the next block header must be signed with (the key committed @@ -180,6 +182,7 @@ namespace graphene { return visible; // blanked / disabled: production is skipped anyway public_key_type committed; uint64_t snapshot_block = 0; + block_id_type snapshot_id; { std::lock_guard lk(_committed_keys_mutex); auto it = _committed_signing_keys.find(name); @@ -187,9 +190,10 @@ namespace graphene { return visible; committed = it->second; snapshot_block = _committed_keys_block_num; + snapshot_id = _committed_keys_block_id; } // DB access outside the lock: a writer must never wait on it. - if (snapshot_block != database().head_block_num()) + if (snapshot_block != database().head_block_num() || snapshot_id != database().head_block_id()) return visible; // snapshot behind head: trust the chain // Snapshot is for head and differs from the session-visible key: // the difference comes from a pending validator_update. Keep @@ -198,20 +202,21 @@ namespace graphene { return committed; } - /** Record the signing keys committed by an applied block. */ - void update_committed_signing_keys(const graphene::chain::signed_block &block) { + /** Called under the DB write lock, with no pending session. */ + void update_committed_signing_keys() { + auto &d = database(); + std::map keys; + for (const auto &name : _validators) { + const auto *validator = d.find_validator(name); + if (validator) + keys[name] = validator->signing_key; + } + const auto head_num = d.head_block_num(); + const auto head_id = d.head_block_id(); std::lock_guard lk(_committed_keys_mutex); - _committed_keys_block_num = block.block_num(); - for (const auto &trx : block.transactions) { - for (const auto &op : trx.operations) { - if (op.which() != operation::tag::value) - continue; - const auto &up = op.get(); - auto it = _committed_signing_keys.find(up.owner); - if (it != _committed_signing_keys.end()) - it->second = up.block_signing_key; - } - } + _committed_signing_keys.swap(keys); + _committed_keys_block_num = head_num; + _committed_keys_block_id = head_id; } fc::time_point last_block_post_validation_time; @@ -408,6 +413,7 @@ namespace graphene { uint64_t _last_applied_block_num = 0; boost::signals2::connection _applied_block_connection; + boost::signals2::scoped_connection _committed_chain_state_connection; // Protects cross-thread diagnostic fields shared between // production_io_thread_ and the P2P thread (on_block_applied / @@ -583,15 +589,20 @@ namespace graphene { pimpl->_committed_signing_keys[w] = d.get_validator(w).signing_key; } catch (const fc::exception& e) { wlog("Validator ${w} is not on chain yet; its committed signing key " - "will be learned from applied blocks", ("w", w)); + "will be learned from committed chain state", ("w", w)); } } pimpl->_committed_keys_block_num = d.head_block_num(); + pimpl->_committed_keys_block_id = d.head_block_id(); if (!pimpl->_validators.empty()) { ilog("Launching block production for ${n} validators.", ("n", pimpl->_validators.size())); pimpl->p2p().set_block_production(true); + pimpl->_committed_chain_state_connection = d.committed_chain_state.connect([this]() { + pimpl->update_committed_signing_keys(); + }); + // Connect to applied_block signal to detect missed slots // that belong to our validators and log diagnostic state. pimpl->_last_applied_block_num = d.head_block_num(); @@ -615,6 +626,7 @@ namespace graphene { graphene::time::shutdown_ntp_time(); if (!pimpl->_validators.empty()) { pimpl->_applied_block_connection.disconnect(); + pimpl->_committed_chain_state_connection.disconnect(); ilog("shutting downing production timer"); // Stop the dedicated io_service so the production thread exits. // io_service::stop() is thread-safe; it causes run() to return @@ -808,10 +820,6 @@ namespace graphene { uint64_t prev_num = _last_applied_block_num; _last_applied_block_num = block_num; - // Keep the committed signing-key snapshot current from block - // contents (a DB read here would also see re-applied pending txs). - update_committed_signing_keys(block); - // === SLOT-HIJACK DETECTION (runs for every block) === // In DLT emergency mode, the emergency master may blank our validator's // signing_key and fill our scheduled slots with committee blocks. This diff --git a/programs/util/block-log/block-archive.cjs b/programs/util/block-log/block-archive.cjs index 260c0b018c..4cd6bdae1c 100644 --- a/programs/util/block-log/block-archive.cjs +++ b/programs/util/block-log/block-archive.cjs @@ -116,17 +116,19 @@ function merge(files, from, to, out) { let next = from; for (const f of sel) { if (f.first > next) { console.error(`GAP ${next}-${f.first - 1}, merge stopped`); process.exit(1); } - next = Math.min(to, f.last) + 1; + next = Math.max(next, Math.min(to, f.last) + 1); } if (!sel.length || next <= to) { console.error(`archive does not cover ${from}-${to}`); process.exit(1); } const log = fs.openSync(logPath, 'wx'), idx = fs.openSync(idxPath, 'wx'); fs.writeSync(idx, u64(from)); let pos = 0, count = 0; + next = from; for (const f of sel) { const src = fs.openSync(f.path, 'r'); const ix = fs.readFileSync(f.path + '.index'); const size = fs.fstatSync(src).size; - const a = Math.max(from, f.first), z = Math.min(to, f.last); + const a = Math.max(next, f.first), z = Math.min(to, f.last); + next = Math.max(next, z + 1); for (let n = a; n <= z; n++) { const i = 8 + (n - f.first) * 8; const start = Number(ix.readBigUInt64LE(i)); @@ -160,7 +162,7 @@ function main() { process.exit(1); } const from = o.from ? Number(o.from) : files[0].first; - const to = o.to ? Number(o.to) : files[files.length - 1].last; + const to = o.to ? Number(o.to) : files.reduce((last, f) => Math.max(last, f.last), files[0].last); if (cmd === 'info') { let prev = null; diff --git a/tests/block_log/archive_merge.test.cjs b/tests/block_log/archive_merge.test.cjs new file mode 100644 index 0000000000..2d42ccf717 --- /dev/null +++ b/tests/block_log/archive_merge.test.cjs @@ -0,0 +1,52 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); +const cli = process.env.ARCHIVE_CLI || path.resolve(__dirname, '../../programs/util/block-log/block-archive.cjs'); +function u64(n) { const b=Buffer.alloc(8); b.writeBigUInt64LE(BigInt(n)); return b; } +function range(dir, first, last) { + const log=[], index=[u64(first)]; + for(let n=first;n<=last;n++) { + const offset=(n-first)*12, row=Buffer.alloc(4); row.writeUInt32LE(n); + log.push(row,u64(offset)); index.push(u64(offset)); + } + const file=path.join(dir,`blocks-${String(first).padStart(10,'0')}-${String(last).padStart(10,'0')}.log`); + fs.writeFileSync(file,Buffer.concat(log)); fs.writeFileSync(file+'.index',Buffer.concat(index)); +} +function fixture(fn) { + const dir=fs.mkdtempSync(path.join(process.env.TMPDIR||os.tmpdir(),'viz-archive-')); + try { fn(dir,path.join(dir,'merged')); } finally { fs.rmSync(dir,{recursive:true,force:true}); } +} +function merge(dir,out,extra=[]) { return spawnSync(process.execPath,[cli,'merge',dir,'--out='+out,...extra],{encoding:'utf8'}); } +function verify(out,first,last) { + const log=fs.readFileSync(out), ix=fs.readFileSync(out+'.index'); + assert.equal(log.length,(last-first+1)*12); assert.equal(ix.length,8+(last-first+1)*8); + assert.equal(Number(ix.readBigUInt64LE(0)),first); + for(let n=first;n<=last;n++) { const p=(n-first)*12; + assert.equal(log.readUInt32LE(p),n); assert.equal(Number(log.readBigUInt64LE(p+4)),p); + assert.equal(Number(ix.readBigUInt64LE(8+(n-first)*8)),p); + } +} +test('nested overlap cannot rewind either cursor',()=>fixture((d,o)=>{ + range(d,1,10); range(d,2,3); range(d,4,12); + const r=merge(d,o,['--from=1','--to=12']); assert.equal(r.status,0,r.stderr); verify(o,1,12); +})); +test('fully covered tail keeps default upper bound',()=>fixture((d,o)=>{ + range(d,1,12); range(d,2,3); + const r=merge(d,o); assert.equal(r.status,0,r.stderr); verify(o,1,12); +})); +test('equal starts and clipped interval are merged once',()=>fixture((d,o)=>{ + range(d,1,4); range(d,1,10); range(d,4,12); + const r=merge(d,o,['--from=3','--to=9']); assert.equal(r.status,0,r.stderr); verify(o,3,9); +})); +test('real gap fails before creating output',()=>fixture((d,o)=>{ + range(d,1,3); range(d,5,7); + const r=merge(d,o,['--to=7']); assert.notEqual(r.status,0); assert.match(r.stderr,/GAP/); assert.equal(fs.existsSync(o),false); +})); +test('existing output is preserved',()=>fixture((d,o)=>{ + range(d,1,3); fs.writeFileSync(o,'keep'); + const r=merge(d,o); assert.notEqual(r.status,0); assert.match(r.stderr,/exists, refusing/); + assert.equal(fs.readFileSync(o,'utf8'),'keep'); +})); diff --git a/tests/consensus_sim/CMakeLists.txt b/tests/consensus_sim/CMakeLists.txt index 8a8f5f3101..5001266e96 100644 --- a/tests/consensus_sim/CMakeLists.txt +++ b/tests/consensus_sim/CMakeLists.txt @@ -74,6 +74,8 @@ if(BUILD_TESTNET) list(APPEND SCENARIO_SOURCES scenarios/test_pm_no_contest_fee.cpp) list(APPEND SCENARIO_SOURCES scenarios/test_agent_access.cpp) list(APPEND SCENARIO_SOURCES scenarios/test_hf16_boundary.cpp) + list(APPEND SCENARIO_SOURCES scenarios/test_hf16_production_safety.cpp) + list(APPEND SCENARIO_SOURCES ../validator/test_committed_chain_state.cpp) else() message(STATUS "PM audit consensus regressions require BUILD_TESTNET=ON") endif() diff --git a/tests/consensus_sim/README.md b/tests/consensus_sim/README.md index 635b383736..b915c32161 100644 --- a/tests/consensus_sim/README.md +++ b/tests/consensus_sim/README.md @@ -30,6 +30,43 @@ The chain code has pre-existing ASan/UBSan findings (a base-pointer delete in with `ASAN_OPTIONS=new_delete_type_mismatch=0:detect_leaks=0 UBSAN_OPTIONS=halt_on_error=0:print_stacktrace=0` to get past them. +## HF16 production safety regressions + +Build with `BUILD_TESTNET=ON` and `BUILD_CONSENSUS_TESTS=ON`, including +`scenarios/test_hf16_production_safety.cpp` in the testnet `SCENARIO_SOURCES`, then run: + +``` +./tests/consensus_sim/consensus_sim_tests --run_test=hf16_production_safety_suite +``` + +These native scenarios exercise `database::generate_block` with real signatures +and FC serialization: future/equal-round evidence is deferred until a strictly +later round; old-key evidence is discarded; header certificates/proofs are +budgeted before transactions, and postponed transactions remain pending. The +key-rotation and near-full-block scenarios use isolated fixture state changes; +none connects to a network or uses a live database. + +HF16 production assembles its header against committed head state before +re-applying pending transactions. The budget includes QC/TC, proofs, version +extensions, the producer name, and transaction-count varuint growth, and is +bounded by both configured and absolute block limits. Consensus validation, +wire formats and pre-HF16 packing policy are unchanged. + +For a fast offline check without the native dependency build: + +``` +python3 tests/consensus_sim/production_safety_offline.py +``` + +This compiles the actual evidence-selection and transaction-packing loops from +`database.cpp` with explicit mock state/size/signature functions, and checks +header/budget ordering. It covers rejection/retention boundaries, replaced +keys, the absolute size cap and count-prefix growth. It **does not** execute the +database, actual signature recovery, FC serialization or consensus validation, +and does not replace the native scenarios. `PRODUCTION_SOURCE=/path/database.cpp` +can select an unmodified baseline for a RED comparison. Temporary compiler +artifacts use `TMPDIR` (or the Hermes scratch directory). + ## Coverage ``` diff --git a/tests/consensus_sim/production_safety_offline.py b/tests/consensus_sim/production_safety_offline.py new file mode 100644 index 0000000000..f52246db3d --- /dev/null +++ b/tests/consensus_sim/production_safety_offline.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Offline extracted-body regression; NOT a consensus/crypto/serialization test. + +Compiles the production evidence-selection loop with explicit fake validator and +signature state. Also checks that the production transaction loop's initial +budget includes the assembled header. Run the native hf16_production_safety_suite tests +for real database, signatures and FC serialization coverage. +""" +import pathlib +import subprocess +import tempfile +import unittest +import os + +SOURCE = pathlib.Path(os.environ.get('PRODUCTION_SOURCE', + str(pathlib.Path(__file__).resolve().parents[2] / 'libraries/chain/database.cpp'))) +TEXT = SOURCE.read_text() +GEN = TEXT[TEXT.index('signed_block database::_generate_block('):TEXT.index('void database::pop_block()')] + +class ProductionSafety(unittest.TestCase): + def test_evidence_selection_actual_body(self): + begin = GEN.index('for (auto it = _pending_equivocations.begin();') + end = GEN.index('size_t n = 0;', begin) + loop = GEN[begin:end] + cpp = r''' +#include +#include +#include +#include +using account_name_type = std::string; +struct Evidence { uint32_t block_num; int block_id; int signature; }; +namespace protocol { struct block_equivocation_proof { + account_name_type validator; uint32_t round; Evidence first, second; +}; } +struct Validator { uint32_t last_equivocation_round = 0; int key = 7; }; +std::map validators; +const Validator* find_validator(const account_name_type& n) { + auto i=validators.find(n); return i==validators.end()?nullptr:&i->second; +} +uint32_t block_interval() { return 1; } +bool verify_block_vote(uint32_t, int, uint32_t, const account_name_type& n, int signature) { + auto v=find_validator(n); return v && signature==v->key; +} +int main() { + int failures=0; + for (int scenario=0; scenario<11; ++scenario) { + validators.clear(); validators["alice"] = Validator{}; + uint32_t own_round=100; + protocol::block_equivocation_proof proof{"alice",99,{10,1,7},{10,2,7}}; + bool expected=true, retained=true; + if(scenario==1) { proof.round=100; expected=false; } + if(scenario==2) { proof.round=101; expected=false; } + if(scenario==3) { validators["alice"].key=8; expected=false; retained=false; } + if(scenario==4) { proof.second.signature=8; expected=false; retained=false; } + if(scenario==5) { validators["alice"].last_equivocation_round=99; expected=false; retained=false; } + if(scenario==6) { validators.clear(); expected=false; retained=false; } + if(scenario==7) { own_round=3700; proof.round=100; } + if(scenario==8) { own_round=3701; proof.round=100; expected=false; retained=false; } + if(scenario==9) { proof.first.signature=8; expected=false; retained=false; } + std::map,protocol::block_equivocation_proof> _pending_equivocations; + _pending_equivocations.emplace(std::make_pair(proof.validator,proof.round),proof); + if(scenario==10) { + auto future=proof; future.round=101; + _pending_equivocations.emplace(std::make_pair(future.validator,future.round),future); + } + std::map pick; +''' + loop + r''' + if(scenario==10 && (!pick.count("alice") || pick["alice"]->round!=99 || _pending_equivocations.size()!=2)) return 2; + if((!pick.empty())!=expected || (!_pending_equivocations.empty())!=retained) { + std::cerr<<"selection case "< 0)', begin) + loop = GEN[begin:end] + if 'total_block_size = fc::raw::pack_size(pending_block)' in GEN: + budget_begin = GEN.index('total_block_size = fc::raw::pack_size(pending_block)') + budget_end = GEN.index('\n }', budget_begin) + budget = GEN[budget_begin:budget_end] + else: + budget = '' # baseline: the real loop starts with its empty-header estimate + cpp = r''' +#include +#include +#include +#include +#include +using std::size_t; +struct signed_transaction { uint64_t expiration=200; size_t bytes=200; }; +struct Block { size_t header=400; std::vector transactions; }; +namespace fc { +struct exception : std::exception { const char* to_detail_string() const {return "mock";} }; +namespace raw { +size_t pack_size(const signed_transaction& t) { return t.bytes; } +size_t pack_size(const Block& b) { + size_t size=b.header+1; // vector count: deliberately exercise the 127/128 boundary + if(b.transactions.size()>=128) ++size; + for(auto t:b.transactions) size+=t.bytes; + return size; +} +}} +struct shared_memory_corruption_exception : fc::exception {}; +struct Session { void squash() {} }; +Session start_undo_session() { return {}; } +void _apply_transaction(const signed_transaction&,uint32_t) {} +#define wlog(...) ((void)0) +constexpr uint32_t CHAIN_BLOCK_SIZE=1000; +constexpr uint32_t CHAIN_BLOCK_GENERATION_POSTPONED_TX_LIMIT=10000; +int main() { + for(int scenario=0; scenario<3; ++scenario) { + Block pending_block; + std::vector _pending_tx(5); + uint32_t maximum_block_size=scenario==1?2000:1000; + struct Props { uint32_t maximum_block_size; }; + auto get_dynamic_global_properties = [&]() { return Props{maximum_block_size}; }; + (void)get_dynamic_global_properties; + if(scenario==2) { pending_block.header=600; _pending_tx.assign(400, signed_transaction{200,1}); } + size_t total_block_size=104; + uint64_t postponed_tx_count=0, when=100; uint32_t skip=0; +''' + budget + loop + r''' + if(fc::raw::pack_size(pending_block)>std::min(maximum_block_size,CHAIN_BLOCK_SIZE)) return 1; + if(scenario<2 && pending_block.transactions.size()!=2) return 2; + if(scenario==2 && pending_block.transactions.size()<128) return 3; + } +} +''' + root = os.environ.get('TMPDIR', str(pathlib.Path.home()/'.hermes/cache/scratch')) + with tempfile.TemporaryDirectory(prefix='production-budget-', dir=root) as tmp: + src = pathlib.Path(tmp)/'budget.cpp' + exe = pathlib.Path(tmp)/'budget' + src.write_text(cpp) + subprocess.run(['g++','-std=c++14','-Wall','-Wextra','-Werror',str(src),'-o',str(exe)],check=True) + subprocess.run([str(exe)],check=True) + + def test_full_header_budget_precedes_transaction_packing(self): + packing = GEN.index('for (const signed_transaction &tx : _pending_tx)') + header = GEN.index('pending_block.extensions.insert(block_header_extensions(*cert))') + self.assertLess(header, packing, 'QC/TC/proofs assembled after transaction packing') + prefix = GEN[:packing] + self.assertIn('fc::raw::pack_size(pending_block)', prefix) + self.assertIn('has_hardfork(CHAIN_HARDFORK_16)', prefix) + self.assertIn('std::min', prefix) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/consensus_sim/scenarios/test_hf16_boundary.cpp b/tests/consensus_sim/scenarios/test_hf16_boundary.cpp index 38016b9c3c..5288f63fca 100644 --- a/tests/consensus_sim/scenarios/test_hf16_boundary.cpp +++ b/tests/consensus_sim/scenarios/test_hf16_boundary.cpp @@ -363,9 +363,18 @@ BOOST_AUTO_TEST_CASE(qc_tc_late_qc_and_equivocation) { const auto qc = db.grandparent_qc_for_next_block(producer, db.round_of(when) + 1); BOOST_REQUIRE(qc.valid()); when += fc::seconds(db.block_interval()); + const uint32_t fallback_parent = db.head_block_num(); db.generate_block(when, producer, key_for(gp, producer), database::skip_nothing, {}, qc); relay(); + // Two consecutive certified blocks are not sufficient when the child used + // the late-QC fallback: its voters need not have learned the parent's QC. + // Tally the child's real quorum separately, as delayed vote delivery can do. + vote_for_head(node, gp); + BOOST_REQUIRE(db.get_block_qc(fallback_parent).valid()); + BOOST_REQUIRE(db.get_block_qc(fallback_parent + 1).valid()); + BOOST_CHECK_LT(db.highest_qc_on_main_branch(db.head_block_num()), fallback_parent); + // A quorum times out a stalled round; its signatures survive wire inclusion. vote_for_head(node, gp); const auto high = db.get_block_qc(db.head_block_num()); diff --git a/tests/consensus_sim/scenarios/test_hf16_production_safety.cpp b/tests/consensus_sim/scenarios/test_hf16_production_safety.cpp new file mode 100644 index 0000000000..65833d34b2 --- /dev/null +++ b/tests/consensus_sim/scenarios/test_hf16_production_safety.cpp @@ -0,0 +1,264 @@ +// HF16 (#6737, Q#3103=A): the block slot shrinks from 3 s to 1 s at the fork block. The switch must be +// seamless at the boundary: +// * block_interval() flips 3 -> 1 exactly when HF16 applies, and the next slot is head + 1 s; +// * rounds are re-anchored at the fork block with no jump: round_of(fork block) is the pre-fork +// round, and every later 1 s block advances the round by exactly one; +// * 1 s blocks after the fork are applied with no missed slots; +// * block-denominated chain props keep their wall-time meaning (Q#3108=A: x3), and the median +// still holds the scaled value after a schedule round recomputes it from the stored votes. +// Only BUILD_TESTNET can reach HF14/HF16 with the single-validator harness (see CMakeLists). +#include +#include "simulated_node.hpp" +#include "genesis_factory.hpp" +#include "virtual_clock.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef BUILD_TESTNET +#include +#else +#include +#endif +#include +#include + +using namespace consensus_sim; +using namespace graphene::chain; +using namespace graphene::protocol; + +namespace { + +account_name_type next_validator(simulated_node& n) { return n.db().get_scheduled_validator(1); } + +fc::ecc::private_key key_for(const genesis_params& gp, const account_name_type& v) { + return (v == gp.genesis_witness_name || v == account_name_type("validator-two")) + ? gp.genesis_witness_key : gp.initiator_key; +} + +// The harness has no p2p: every scheduled validator votes for head directly (see test_pm_no_contest_fee). +void vote_for_head(simulated_node& n, const genesis_params& gp) { + auto& db = n.db(); + if (!db.has_hardfork(CHAIN_HARDFORK_16) || db.head_block_num() == 0 || db.get_block_qc(db.head_block_num())) + return; + const auto head = db.fetch_block_by_number(db.head_block_num()); + const uint32_t round = db.round_of(head->timestamp); + const auto digest = graphene::protocol::block_vote_digest(db.get_chain_id(), head->block_num(), head->id(), round); + const auto& wso = db.get_validator_schedule_object(); + for (int i = 0; i < wso.num_scheduled_validators; i += CHAIN_BLOCK_VALIDATOR_REPEAT) { + const auto v = wso.current_shuffled_validators[i]; + db.push_block_vote(head->block_num(), head->id(), round, v, key_for(gp, v).sign_compact(digest)); + } +} + +void produce(simulated_node& n, const genesis_params& gp, fc::time_point_sec& when) { + vote_for_head(n, gp); + when += fc::seconds(n.db().block_interval()); + const auto v = next_validator(n); + n.produce_block(v, key_for(gp, v), when); +} + +signed_transaction sign_ops(const std::vector& ops, + const fc::ecc::private_key& key, const simulated_node& node) { + signed_transaction tx; + tx.set_reference_block(node.head_block_id()); + tx.set_expiration(node.head_block_time() + fc::seconds(60)); + for (const auto& op : ops) tx.operations.emplace_back(op); + tx.sign(key, node.chain_id()); + return tx; +} + +bool bring_to_hf14(simulated_node& node, const genesis_params& gp, fc::time_point_sec& when) { + try { + produce(node, gp, when); + const share_type bal = node.db().get_account(gp.initiator_name).balance.amount; + transfer_to_vesting_operation tv; + tv.from = gp.initiator_name; tv.to = gp.initiator_name; + tv.amount = asset(share_type(bal.value / 2), TOKEN_SYMBOL); + validator_update_operation vu; + vu.owner = gp.initiator_name; vu.url = "viz"; + vu.block_signing_key = gp.initiator_key.get_public_key(); + account_validator_vote_operation vv; + vv.account = gp.initiator_name; vv.validator = gp.initiator_name; vv.approve = true; + node.push_pending_transaction(sign_ops({tv, vu, vv}, gp.initiator_key, node)); + for (int i = 0; i < 300 && !node.db().has_hardfork(CHAIN_HARDFORK_14); ++i) + produce(node, gp, when); + } catch (const std::exception& e) { + BOOST_TEST_MESSAGE(std::string("could not reach HF14 (harness/genesis limit): ") + e.what()); + } + return node.db().has_hardfork(CHAIN_HARDFORK_14); +} + +// Isolated native production regressions: no p2p, no external chain state. +struct production_fixture { + genesis_params gp = make_genesis_params(0x5AFE16u, 1); + fc::time_point_sec start = fc::time_point_sec(CHAIN_HARDFORK_14_TIME) + fc::seconds(CHAIN_BLOCK_INTERVAL); + virtual_clock clk{start}; + simulated_node node{"production-safety", gp, clk}; + fc::time_point_sec when = start - fc::seconds(CHAIN_BLOCK_INTERVAL); + + production_fixture() { + BOOST_REQUIRE(bring_to_hf14(node, gp, when)); + node.db().set_hardfork(CHAIN_HARDFORK_16, true); + for (int i = 0; i < 3; ++i) produce(node, gp, when); + } + + block_equivocation_proof queue_proof(uint32_t round, const account_name_type& voter) { + auto& db = node.db(); + signed_block first = *db.fetch_block_by_number(db.head_block_num()); + // Keep speculative votes clear of the next few real heights: a one-voter + // fixture otherwise creates a QC for a fake immediate child as a side effect. + if (round > db.round_of(first.timestamp)) + for (int i = 0; i < 32; ++i) first.previous = first.id(); + signed_block second = first; + second.timestamp += fc::seconds(1); + const auto key = key_for(gp, voter); + block_equivocation_proof proof; + proof.validator = voter; proof.round = round; + const auto first_digest = block_vote_digest(db.get_chain_id(), first.block_num(), first.id(), round); + const auto second_digest = block_vote_digest(db.get_chain_id(), second.block_num(), second.id(), round); + proof.first = block_vote_evidence{first.block_num(), first.id(), key.sign_compact(first_digest)}; + proof.second = block_vote_evidence{second.block_num(), second.id(), key.sign_compact(second_digest)}; + BOOST_REQUIRE(db.push_block_vote(proof.first.block_num, proof.first.block_id, round, voter, proof.first.signature)); + BOOST_REQUIRE(!db.push_block_vote(proof.second.block_num, proof.second.block_id, round, voter, proof.second.signature)); + return proof; + } +}; + +bool carries_proof(const signed_block& b) { + for (const auto& e : b.extensions) if (e.which() == 5) return true; + return false; +} + +} // namespace + +BOOST_AUTO_TEST_SUITE(hf16_production_safety_suite) + +BOOST_AUTO_TEST_CASE(production_defers_future_evidence_until_strictly_later_round) { + production_fixture f; + auto& db = f.node.db(); + const auto voter = db.get_validator_schedule_object().current_shuffled_validators[0]; + const uint32_t round = db.round_of(f.when) + 2; + f.queue_proof(round, voter); + for (int i = 0; i < 2; ++i) { + produce(f.node, f.gp, f.when); + BOOST_CHECK(!carries_proof(*db.fetch_block_by_number(db.head_block_num()))); + } + // The equivocator already voted for another block in this round, so its + // local tally correctly refuses another vote for the real head. Supply the + // independently signed parent certificate directly; consensus still verifies + // every signature (no skip flag and no mutation of the vote cache). + const auto head = *db.fetch_block_by_number(db.head_block_num()); + block_qc_certificate cert; + cert.block_num = head.block_num(); cert.block_id = head.id(); cert.round = db.round_of(head.timestamp); + const auto digest = block_vote_digest(db.get_chain_id(), cert.block_num, cert.block_id, cert.round); + const auto& wso = db.get_validator_schedule_object(); + for (int i = 0; i < wso.num_scheduled_validators; i += CHAIN_BLOCK_VALIDATOR_REPEAT) { + const auto v = wso.current_shuffled_validators[i]; + cert.signatures.emplace(v, key_for(f.gp, v).sign_compact(digest)); + } + f.when += fc::seconds(db.block_interval()); + const auto producer = db.get_scheduled_validator(1); + db.generate_block(f.when, producer, key_for(f.gp, producer), database::skip_nothing, {}, cert); + BOOST_CHECK(carries_proof(*db.fetch_block_by_number(db.head_block_num()))); + BOOST_CHECK_EQUAL(db.get_validator(voter).last_equivocation_round, round); +} + +BOOST_AUTO_TEST_CASE(production_discards_evidence_signed_with_replaced_key) { + production_fixture f; + auto& db = f.node.db(); + const auto voter = db.get_validator_schedule_object().current_shuffled_validators[0]; + const uint32_t round = db.round_of(f.when) - 1; + f.queue_proof(round, voter); + const auto punished = db.get_validator(voter).last_equivocation_round; + // Fixture-only committed key rotation. Native signature verification, not a + // skip flag, must make the queued old-key evidence ineligible. + const auto replacement = fc::ecc::private_key::regenerate(fc::sha256::hash("production-rotation")); + db.modify(db.get_validator(voter), [&](validator_object& v) { v.signing_key = replacement.get_public_key(); }); + const auto head = *db.fetch_block_by_number(db.head_block_num()); + const auto digest = block_vote_digest(db.get_chain_id(), head.block_num(), head.id(), db.round_of(head.timestamp)); + const auto& wso = db.get_validator_schedule_object(); + for (int i = 0; i < wso.num_scheduled_validators; i += CHAIN_BLOCK_VALIDATOR_REPEAT) { + const auto v = wso.current_shuffled_validators[i]; + const auto key = v == voter ? replacement : key_for(f.gp, v); + BOOST_REQUIRE(db.push_block_vote(head.block_num(), head.id(), db.round_of(head.timestamp), v, key.sign_compact(digest))); + } + f.when += fc::seconds(db.block_interval()); + const auto producer = db.get_scheduled_validator(1); + const auto block = db.generate_block(f.when, producer, + producer == voter ? replacement : key_for(f.gp, producer), database::skip_nothing); + BOOST_CHECK(!carries_proof(block)); + BOOST_CHECK_EQUAL(db.get_validator(voter).last_equivocation_round, punished); +} + +BOOST_AUTO_TEST_CASE(production_budgets_certificates_and_proofs_before_transactions) { + production_fixture f; + auto& db = f.node.db(); + const auto voter = db.get_validator_schedule_object().current_shuffled_validators[0]; + const auto proof = f.queue_proof(db.round_of(f.when), voter); + vote_for_head(f.node, f.gp); + transfer_operation transfer; + transfer.from = f.gp.initiator_name; transfer.to = f.gp.genesis_witness_name; + transfer.amount = asset(1, TOKEN_SYMBOL); transfer.memo = std::string(512, 'x'); + const auto tx = sign_ops({transfer}, f.gp.initiator_key, f.node); + const auto producer = db.get_scheduled_validator(1); + const auto cert = db.parent_qc_for_next_block(producer); + BOOST_REQUIRE(cert.valid()); + const auto legacy_estimate = fc::raw::pack_size(signed_block_header()) + 4; + const uint32_t limit = std::max(legacy_estimate + fc::raw::pack_size(tx) + 16, + fc::raw::pack_size(tx) + 256); + signed_block projected; + projected.previous = db.head_block_id(); projected.validator = producer; + projected.timestamp = f.when + fc::seconds(db.block_interval()); + projected.extensions.insert(block_header_extensions(*cert)); + projected.extensions.insert(block_header_extensions(proof)); + projected.transactions.push_back(tx); + BOOST_REQUIRE_GT(fc::raw::pack_size(projected), limit); + // Fixture-only near-full block. Set the committed cap BEFORE creating a + // pending undo session; otherwise reset() would undo this fixture change. + // The cap still admits the transaction through the ordinary 256-byte reserve. + db.modify(db.get_dynamic_global_properties(), [&](dynamic_global_property_object& p) { p.maximum_block_size = limit; }); + f.node.push_pending_transaction(tx); + produce(f.node, f.gp, f.when); + const auto block = *db.fetch_block_by_number(db.head_block_num()); + BOOST_CHECK_LE(fc::raw::pack_size(block), limit); + BOOST_CHECK(block.transactions.empty()); + BOOST_CHECK(carries_proof(block)); + // Observe retention, then let the next block consume the same pending pool. + // The one-shot proof is gone, so the transaction fits at the SAME cap. + BOOST_REQUIRE_EQUAL(db._pending_tx.size(), 1u); + BOOST_CHECK(db._pending_tx.front().id() == tx.id()); + produce(f.node, f.gp, f.when); + const auto next = *db.fetch_block_by_number(db.head_block_num()); + BOOST_CHECK_LE(fc::raw::pack_size(next), limit); + BOOST_REQUIRE_EQUAL(next.transactions.size(), 1u); + BOOST_CHECK(next.transactions[0].id() == tx.id()); +} + + +BOOST_AUTO_TEST_CASE(production_preserves_pending_signing_key_rotation) { + production_fixture f; + auto& db = f.node.db(); + vote_for_head(f.node, f.gp); + const auto owner = db.get_scheduled_validator(1); + const auto old_key = key_for(f.gp, owner); + const auto new_key = fc::ecc::private_key::regenerate(fc::sha256::hash("pending-production-rotation")); + validator_update_operation op; + op.owner = owner; + op.url = "https://example.invalid/validator"; + op.block_signing_key = new_key.get_public_key(); + const auto tx = sign_ops({op}, old_key, f.node); + f.node.push_pending_transaction(tx); + BOOST_REQUIRE(db.get_validator(owner).signing_key == new_key.get_public_key()); + f.when += fc::seconds(db.block_interval()); + const auto block = db.generate_block(f.when, owner, old_key, database::skip_nothing); + BOOST_REQUIRE_EQUAL(block.transactions.size(), 1u); + BOOST_CHECK(block.transactions.front().id() == tx.id()); + BOOST_CHECK(db.get_validator(owner).signing_key == new_key.get_public_key()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/node_admin/CMakeLists.txt b/tests/node_admin/CMakeLists.txt index 3487b20185..ca8740c299 100644 --- a/tests/node_admin/CMakeLists.txt +++ b/tests/node_admin/CMakeLists.txt @@ -30,3 +30,14 @@ target_include_directories(node_admin_objects_tests ) add_test(NAME node_admin_objects COMMAND node_admin_objects_tests) + +# Real websocketpp handshake/state machine via its iostream transport: no listener +# sockets, live node, chain fixtures or FC runtime required. +find_package(Threads REQUIRED) +add_executable(node_admin_http_tests node_admin_http_test.cpp) +target_include_directories(node_admin_http_tests PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/../../plugins/node_admin/include" + "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/fc/vendor/websocketpp" +) +target_link_libraries(node_admin_http_tests PRIVATE Boost::boost Threads::Threads) +add_test(NAME node_admin_http COMMAND node_admin_http_tests) diff --git a/tests/node_admin/node_admin_http_test.cpp b/tests/node_admin/node_admin_http_test.cpp new file mode 100644 index 0000000000..8a87bb197b --- /dev/null +++ b/tests/node_admin/node_admin_http_test.cpp @@ -0,0 +1,93 @@ +// Offline regression: real websocketpp parser/handshake/state machine, no sockets. +#include +#include +#include + +#include +#include +#include +#include + +namespace { +using server_type = websocketpp::server; + +void require(bool condition, const char *message) { + if (!condition) throw std::runtime_error(message); +} + +struct result { + std::string response; + int http_calls = 0; + int opens = 0; + int failures = 0; + websocketpp::session::state::value state; +}; + +result request(const std::string &input, bool http_only) { + server_type server; + server.clear_access_channels(websocketpp::log::alevel::all); + server.clear_error_channels(websocketpp::log::elevel::all); + result r; + std::ostringstream output; + server.register_ostream(&output); + auto http = [&](websocketpp::connection_hdl hdl) { + ++r.http_calls; + auto con = server.get_con_from_hdl(hdl); + // Model the existing routing callback: ordinary responses remain its job. + const bool authorized = con->get_request().get_header("Authorization") == "Bearer test"; + con->set_status(authorized ? websocketpp::http::status_code::ok + : websocketpp::http::status_code::unauthorized); + con->set_body(authorized ? "normal HTTP" : "unauthorized"); + con->replace_header("Connection", "close"); + }; + if (http_only) graphene::plugins::node_admin::util::configure_http_only(server, http); + else server.set_http_handler(http); // Negative control: baseline accepts upgrades. + server.set_open_handler([&](websocketpp::connection_hdl) { ++r.opens; }); + server.set_fail_handler([&](websocketpp::connection_hdl) { ++r.failures; }); + auto con = server.get_connection(); + require(bool(con), "get_connection failed"); + con->start(); + require(con->read_all(input.data(), input.size()) == input.size(), "request not consumed"); + r.response = output.str(); + r.state = con->get_state(); + return r; +} + +const std::string upgrade = + "GET /admin HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\n" + "Connection: Upgrade\r\nSec-WebSocket-Version: 13\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"; +} + +int main() { + try { + const auto baseline = request(upgrade + "\r\n", false); + require(baseline.response.find("HTTP/1.1 101 ") == 0, "negative control did not upgrade"); + require(baseline.http_calls == 0 && baseline.opens == 1, "baseline route not confirmed"); + require(baseline.state == websocketpp::session::state::open, "baseline did not remain open"); + std::cout << "Baseline: unauthenticated upgrade opens WebSocket, skips HTTP callback\n"; + for (const std::string &auth : {std::string(), std::string("Authorization: Bearer test\r\n")}) { + const auto rejected = request(upgrade + auth + "\r\n", true); + require(rejected.response.find("HTTP/1.1 403 ") == 0, "HTTP-only listener must reject upgrades with 403"); + require(rejected.http_calls == 0 && rejected.opens == 0 && rejected.failures == 1, + "upgrade rejection must bypass HTTP callback without opening WebSocket"); + require(rejected.state == websocketpp::session::state::closed, "rejected upgrade did not close"); + } + std::cout << "HTTP-only: upgrades rejected and closed, with and without Authorization\n"; + for (const std::string &auth : {std::string(), std::string("Authorization: Bearer test\r\n")}) { + const std::string http = "POST /admin HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n" + auth + "\r\n{}"; + const auto before = request(http, false); + const auto after = request(http, true); + require(after.response == before.response, "ordinary HTTP response changed"); + require(after.http_calls == 1 && after.opens == 0, "ordinary HTTP callback not called"); + require(after.state == websocketpp::session::state::closed, "ordinary HTTP did not close"); + require(after.response.find(auth.empty() ? "HTTP/1.1 401 " : "HTTP/1.1 200 ") == 0, + "ordinary HTTP status changed"); + } + std::cout << "Ordinary HTTP: callback, 200/401 responses and closure preserved\n"; + return 0; + } catch (const std::exception &e) { + std::cerr << "FAIL: " << e.what() << '\n'; + return 1; + } +} diff --git a/tests/p2p/test_vote_ingress.py b/tests/p2p/test_vote_ingress.py new file mode 100644 index 0000000000..6db7e7007f --- /dev/null +++ b/tests/p2p/test_vote_ingress.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Offline real-handler regression; fake time/hash/delegate, never a live node. + +--baseline extracts the committed versions of the production handlers/members. +--probe reports resource measurements without imposing post-fix assertions. +SANITIZE=1 enables ASan/UBSan. No production source is rewritten by this runner. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import tempfile +from test_corruption_recovery import method + +ROOT = Path(__file__).resolve().parents[2] +CPP = 'libraries/network/dlt_p2p_node.cpp' +HPP = 'libraries/network/include/graphene/network/dlt_p2p_node.hpp' + +FIXTURE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define dlog(...) ((void)0) +#define CHAIN_MAX_ACCOUNT_NAME_LENGTH 32 +namespace fc { +static int64_t clock_now = 1000; +static uint64_t age_checks = 0; +struct microseconds { int64_t value; }; +inline microseconds seconds(int64_t n) { return {n}; } +inline bool operator>(microseconds a, microseconds b) { return a.value > b.value; } +struct time_point { + int64_t value = 0; + static time_point now() { return {clock_now}; } +}; +inline microseconds operator-(time_point a, time_point b) { + ++age_checks; return {a.value-b.value}; +} +struct sha256 { + uint64_t value = 0; + static sha256 hash(uint64_t n) { return {n}; } + bool operator<(sha256 b) const { return value < b.value; } +}; +namespace raw { template uint64_t pack(const T& msg) { return msg.key(); } } +} +CACHE_INCLUDE +using peer_id = int; +using block_id_type = uint64_t; +using signature_type = uint64_t; +constexpr int INVALID_PEER_ID = -1; +constexpr int DLT_CAP_BLOCK_VOTE = 1, DLT_CAP_TIMEOUT_VOTE = 2; +namespace graphene { namespace protocol { +struct block_header { static uint32_t num_from_id(block_id_type id) { return uint32_t(id); } }; +} } +struct dlt_block_vote_message { + uint32_t block_num = 1, round = 1; + block_id_type block_id = 1; + std::string validator = "alice"; + signature_type signature = 1; + uint64_t key() const { return signature; } +}; +struct dlt_timeout_vote_message { + uint32_t round = 1, high_qc_round = 0; + std::string validator = "alice"; + signature_type signature = 1; + uint64_t key() const { return signature; } +}; +template int message(const T&) { return 0; } +struct controlled_delegate { + bool accepts = true, throws = false; + size_t calls = 0; + bool accept() { ++calls; if (throws) throw std::runtime_error("controlled failure"); return accepts; } + bool accept_block_vote(const dlt_block_vote_message&) { return accept(); } + bool accept_timeout_vote(const dlt_timeout_vote_message&) { return accept(); } +}; +struct peer_state { uint32_t spam_strikes = 0; fc::time_point last_good_packet_time; }; +struct dlt_p2p_node { +MEMBERS +THRESHOLD + controlled_delegate* _delegate; + std::map _peer_states; + size_t gossips = 0, bans = 0; + std::function during_gossip; + explicit dlt_p2p_node(controlled_delegate* d) : _delegate(d) { _peer_states.emplace(7, peer_state{}); } + void gossip_to_capable(int, int, peer_id) { + ++gossips; + auto callback = std::move(during_gossip); + during_gossip = nullptr; + if (callback) callback(); + } + void soft_ban_peer(peer_id, const std::string&) { ++bans; } + bool record_packet_result(peer_id, bool); + void process_block_vote(const dlt_block_vote_message&, peer_id); + void process_timeout_vote(const dlt_timeout_vote_message&, peer_id); + void on_dlt_block_vote(peer_id, const dlt_block_vote_message&); + void on_dlt_timeout_vote(peer_id, const dlt_timeout_vote_message&); + void broadcast_block_post_validation(const block_id_type&, const std::string&, const signature_type&, uint32_t); + void broadcast_timeout_vote(uint32_t, uint32_t, const std::string&, const signature_type&); +}; +METHODS +void require(bool ok, const char* msg) { if (!ok) throw std::runtime_error(msg); } +int main(int argc, char** argv) { + try { + require(argc >= 3, "scenario and vote type required"); + std::string scenario = argv[1]; bool block = std::string(argv[2]) == "block"; + size_t n = argc > 3 ? std::stoull(argv[3]) : 20000; + bool probe = scenario == "probe"; + controlled_delegate d; dlt_p2p_node node(&d); + dlt_block_vote_message b; dlt_timeout_vote_message t; + auto send = [&](uint64_t key, peer_id peer = 7) { + b.signature = t.signature = key; + if (block) node.on_dlt_block_vote(peer, b); else node.on_dlt_timeout_vote(peer, t); + }; + auto size = [&]() { return block ? node._block_votes_seen.size() : node._timeout_votes_seen.size(); }; + auto started = std::chrono::steady_clock::now(); + if (scenario == "rejected" || probe) { + d.accepts = false; + for (size_t i = 1; i <= n; ++i) send(i); + if (!probe) { require(size() == 0, "rejected votes retained in cache"); require(fc::age_checks <= n*2, "rejected flood scans cache"); } + require(node.gossips == 0, "rejected vote gossiped"); + } else if (scenario == "cap") { + for (size_t i = 1; i <= n; ++i) { send(i); require(size() <= 8192, "cache exceeds hard cap"); } + require(size() == 8192, "cap negative control not reached"); + require(fc::age_checks <= n*2, "accepted flood scans cache"); + send(n); require(d.calls == n, "recent duplicate not suppressed"); + send(1); require(d.calls == n+1, "oldest entry not evicted"); + } else if (scenario == "expiry") { + send(1); require(d.calls == 1 && node.gossips == 1, "accepted vote lost"); + fc::clock_now += 120; send(1); require(d.calls == 1, "TTL boundary changed"); + ++fc::clock_now; send(1); require(d.calls == 2, "expired vote not retried"); + } else if (scenario == "cleanup") { + for (size_t i = 1; i <= 8192; ++i) send(i); + fc::clock_now += 121; fc::age_checks = 0; + send(9000); require(fc::age_checks <= 65, "unbounded expiry cleanup burst"); + require(size() <= 8192, "expiry overflow"); + send(8000); require(d.calls == 8194, "expired tail falsely deduplicated"); + } else if (scenario == "accounting") { + b.validator = t.validator = ""; + for (unsigned i = 0; i < node.SPAM_STRIKE_THRESHOLD; ++i) send(i+1); + require(node.bans == 1 && node._peer_states[7].spam_strikes == node.SPAM_STRIKE_THRESHOLD, + "malformed traffic does not use existing strike policy"); + require(d.calls == 0 && size() == 0, "malformed traffic reached delegate/cache"); + b.validator = t.validator = std::string(33, 'x'); send(99); + require(d.calls == 0, "oversized name reached delegate"); + send(100, INVALID_PEER_ID); require(node._peer_states.size() == 1, "local malformed vote scored peer"); + b.validator = t.validator = "alice"; + if (block) { + b.block_num = 0; send(101); b.block_num = 2; send(102); + require(d.calls == 0, "impossible block vote reached delegate"); b.block_num = 1; + } + d.accepts = false; auto strikes = node._peer_states[7].spam_strikes; + send(103); send(103); require(node._peer_states[7].spam_strikes == strikes, + "ambiguous false (stale/already known/local failure) unfairly strikes peer"); + node._delegate = nullptr; send(104); require(node._peer_states[7].spam_strikes == strikes, "missing delegate strikes peer"); + } else if (scenario == "local_interleaving_retry") { + node.during_gossip = [&]() { + for (size_t i = 2; i <= 8193; ++i) { b.signature = i; node.on_dlt_block_vote(7, b); } + }; + node.broadcast_block_post_validation(1, "alice", 1, 1); + const auto before = node.gossips; + d.accepts = false; + node.broadcast_block_post_validation(1, "alice", 1, 1); + require(node.gossips == before + 1, "initial gossip yield suppresses local retry"); + } else if (scenario == "local_expiry") { + node.broadcast_block_post_validation(1, "alice", 1, 1); + fc::clock_now += 121; + d.accepts = false; + node.broadcast_block_post_validation(1, "alice", 1, 1); + require(node.gossips == 1, "expired local vote bypasses acceptance"); + d.accepts = true; + node.broadcast_block_post_validation(1, "alice", 1, 1); + require(node.gossips == 2, "expired local vote cannot be reaccepted"); + } else if (scenario == "local_eviction_retry") { + node.broadcast_block_post_validation(1, "alice", 1, 1); + require(d.calls == 1 && node.gossips == 1, "initial local vote not accepted"); + for (size_t i = 2; i <= 8193; ++i) { b.signature = i; node.on_dlt_block_vote(7, b); } + const auto before = node.gossips; + d.accepts = false; // chain has already counted the original local vote + node.broadcast_block_post_validation(1, "alice", 1, 1); + require(node.gossips == before + 1, "remote cache churn suppresses local retry"); + } else if (scenario == "retry") { + if (block) { + node.broadcast_block_post_validation(1, "alice", 1, 1); + fc::clock_now += 100; + node.broadcast_block_post_validation(1, "alice", 1, 1); + require(d.calls == 1 && node.gossips == 2, "local retry no longer gossips"); + fc::clock_now += 21; send(1); require(d.calls == 2, "local resend pins expiry"); + } else { + node.broadcast_timeout_vote(1, 0, "alice", 1); + node.broadcast_timeout_vote(1, 0, "alice", 1); + require(d.calls == 1 && node.gossips == 1, "timeout duplicate semantics changed"); + } + } else if (scenario == "failure_retry") { + d.throws = true; + try { send(1); } catch (const std::runtime_error&) {} + require(size() == 0, "exception poisons dedup"); + d.throws = false; send(1); require(node.gossips == 1, "failed vote cannot be retried"); + } else throw std::runtime_error("unknown scenario"); + auto us = std::chrono::duration_cast(std::chrono::steady_clock::now()-started).count(); + std::cout << scenario << ' ' << (block ? "block" : "timeout") << " n=" << n + << " retained=" << size() << " age_checks=" << fc::age_checks + << " delegate_calls=" << d.calls << " elapsed_us=" << us << '\n'; + return 0; + } catch (const std::exception& e) { std::cerr << e.what() << '\n'; return 1; } +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--baseline', action='store_true') + parser.add_argument('--probe', action='store_true') + args = parser.parse_args() + def source(path): + if args.baseline: + return subprocess.check_output(['git', 'show', 'HEAD:' + path], cwd=ROOT, text=True) + return (ROOT / path).read_text() + cpp, hpp = source(CPP), source(HPP) + signatures = ['void dlt_p2p_node::' + name + '(' for name in ( + 'process_block_vote', 'process_timeout_vote', 'on_dlt_block_vote', + 'on_dlt_timeout_vote', 'broadcast_block_post_validation', 'broadcast_timeout_vote')] + signatures.append('bool dlt_p2p_node::record_packet_result(') + members = '\n'.join(re.findall(r'^\s*[^\n]*\s_block_votes_seen;|^\s*[^\n]*\s_timeout_votes_seen;|^\s*[^\n]*\s_local_block_votes_seen;', hpp, re.M)) + threshold = re.search(r' static constexpr uint32_t\s+SPAM_STRIKE_THRESHOLD = \d+;', hpp).group() + include = '' if args.baseline else '#include \nusing graphene::network::dlt_vote_cache;' + code = FIXTURE.replace('CACHE_INCLUDE', include).replace('MEMBERS', members).replace('\nTHRESHOLD\n', '\n' + threshold + '\n').replace('METHODS', '\n'.join(method(cpp, sig) for sig in signatures)) + with tempfile.TemporaryDirectory(prefix='viz-vote-', dir=os.environ.get('TMPDIR')) as tmp: + tmp = Path(tmp) + for stub in ('fc/time.hpp', 'fc/crypto/sha256.hpp'): + p = tmp / stub; p.parent.mkdir(parents=True, exist_ok=True); p.write_text('#pragma once\n') + source_file, binary = tmp / 'votes.cpp', tmp / 'votes' + source_file.write_text(code) + flags = ['-fsanitize=address,undefined', '-fno-omit-frame-pointer', '-g'] if os.environ.get('SANITIZE') else [] + subprocess.run([os.environ.get('CXX', 'c++'), '-std=c++14', '-O2', '-Wall', '-Wextra', '-Werror', *flags, + '-I'+str(tmp), '-I'+str(ROOT / 'libraries/network/include'), str(source_file), '-o', str(binary)], check=True) + failed = 0 + cases = [('probe', str(n)) for n in (1000, 2000, 4000, 8000)] if args.probe else [(name, '20000') for name in ( + 'rejected', 'cap', 'expiry', 'cleanup', 'accounting', 'retry', 'failure_retry', + 'local_eviction_retry', 'local_interleaving_retry', 'local_expiry')] + executed = 0 + for vote in ('block', 'timeout'): + for scenario, n in cases: + if scenario.startswith('local_') and vote == 'timeout': + continue + executed += 1 + result = subprocess.run([str(binary), scenario, vote, n], capture_output=True, text=True) + print(result.stdout or f'FAIL {scenario} {vote}: {result.stderr.strip()}', end='' if result.stdout else '\n') + failed += result.returncode != 0 + if failed: raise SystemExit(f'{failed} scenarios failed') + print(f'PASS: {executed} scenarios') + + +if __name__ == '__main__': + main() diff --git a/tests/validator/test_committed_chain_state.cpp b/tests/validator/test_committed_chain_state.cpp new file mode 100644 index 0000000000..fdd1cba548 --- /dev/null +++ b/tests/validator/test_committed_chain_state.cpp @@ -0,0 +1,72 @@ +// Native integration companion to test_signing_cache.py, built by consensus_sim. +#include +#include "simulated_node.hpp" +#include "genesis_factory.hpp" +#include "virtual_clock.hpp" +#include +#include +#ifdef BUILD_TESTNET +#include +#else +#include +#endif + +using namespace consensus_sim; +using namespace graphene::protocol; + +BOOST_AUTO_TEST_SUITE(validator_committed_state_suite) + +BOOST_AUTO_TEST_CASE(committed_signal_precedes_pending_restoration) { + auto gp = make_genesis_params(0xCA5E16u, 1); + auto t0 = fc::time_point_sec::from_iso_string("2026-01-01T00:00:00"); + virtual_clock clk(t0); + simulated_node producer("committed-producer", gp, clk); + simulated_node receiver("committed-receiver", gp, clk); + auto produce_next = [&](fc::time_point_sec when) { + const auto owner = producer.db().get_scheduled_validator(1); + BOOST_REQUIRE(owner == gp.genesis_witness_name || owner == gp.initiator_name); + return producer.produce_block(owner, owner == gp.genesis_witness_name ? + gp.genesis_witness_key : gp.initiator_key, when); + }; + auto first = produce_next(t0 + CHAIN_BLOCK_INTERVAL); + BOOST_REQUIRE(receiver.receive_block(first) == block_outcome::accepted_extends_head); + auto &db = receiver.db(); + const auto *before = db.find_validator(gp.initiator_name); + const bool existed = before != nullptr; + const auto old_key = before ? before->signing_key : public_key_type(); + const auto rotated_key = fc::ecc::private_key::regenerate(fc::sha256::hash("pending-key")).get_public_key(); + validator_update_operation update; + update.owner = gp.initiator_name; + update.url = "viz"; + update.block_signing_key = rotated_key; + signed_transaction tx; + tx.set_reference_block(receiver.head_block_id()); + tx.set_expiration(receiver.head_block_time() + fc::seconds(60)); + tx.operations.emplace_back(update); + tx.sign(gp.initiator_key, receiver.chain_id()); + receiver.push_pending_transaction(tx); + BOOST_REQUIRE(db.get_validator(gp.initiator_name).signing_key == rotated_key); + + unsigned committed_calls = 0, applied_calls = 0; + boost::signals2::scoped_connection committed = db.committed_chain_state.connect([&]() { + ++committed_calls; + const auto *v = db.find_validator(gp.initiator_name); + BOOST_CHECK_EQUAL(v != nullptr, existed); + if (v) BOOST_CHECK(v->signing_key == old_key); + BOOST_CHECK_EQUAL(db.head_block_num(), 2u); + }); + boost::signals2::scoped_connection applied = db.applied_block.connect([&](const signed_block &) { + ++applied_calls; + BOOST_CHECK_EQUAL(committed_calls, 1u); + BOOST_CHECK(db.get_validator(gp.initiator_name).signing_key == rotated_key); + }); + // Producer never saw the receiver's pending rotation; its next block is empty. + auto second = produce_next(t0 + 2 * CHAIN_BLOCK_INTERVAL); + BOOST_REQUIRE(second.transactions.empty()); + BOOST_REQUIRE(receiver.receive_block(second) == block_outcome::accepted_extends_head); + BOOST_CHECK_EQUAL(committed_calls, 1u); + BOOST_CHECK_EQUAL(applied_calls, 1u); + BOOST_CHECK(db.get_validator(gp.initiator_name).signing_key == rotated_key); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/validator/test_signing_cache.py b/tests/validator/test_signing_cache.py new file mode 100644 index 0000000000..09de662eea --- /dev/null +++ b/tests/validator/test_signing_cache.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Offline regression: compile the actual plugin helpers, not a Python rewrite. + +Run: python3 tests/validator/test_signing_cache.py +RED: python3 tests/validator/test_signing_cache.py --source-ref d4fd4bcd +Only stdlib + a C++14 compiler; temporary output stays under TMPDIR. +The fake DB controls committed/pending state; it does not test chain execution. +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +PLUGIN = "plugins/validator/validator.cpp" + + +def function(text, signature): + start = text.index(signature) + opening = text.index("{", start) + depth = 1 + end = opening + 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-ref") + args = parser.parse_args() + source = (subprocess.check_output(["git", "show", f"{args.source_ref}:{PLUGIN}"], cwd=ROOT, text=True) + if args.source_ref else (ROOT / PLUGIN).read_text()) + effective = function(source, "public_key_type effective_signing_key(") + refresh = function(source, "void update_committed_signing_keys(") + fields_start = source.index("std::map _committed_signing_keys;") + fields_end = source.index("std::mutex _committed_keys_mutex;", fields_start) + fields_end += len("std::mutex _committed_keys_mutex;") + fields = source[fields_start:fields_end] + old = "signed_block &block" in refresh + adapter = "update_committed_signing_keys(b);" if old else "update_committed_signing_keys();" + cpp = r''' +#include +#include +#include +#include +#include +#include +#include +using string = std::string; +using public_key_type = int; +using block_id_type = int; +struct validator_update_operation { string owner; int block_signing_key; }; +struct operation { + template struct tag { enum { value = 1 }; }; + validator_update_operation up; + int which() const { return 1; } + template const T& get() const { return up; } +}; +namespace graphene { namespace chain { +struct signed_block { + struct transaction { std::vector operations; }; + std::vector transactions; + unsigned num = 0; + unsigned block_num() const { return num; } +}; +}} +struct fake_db { + struct validator { int signing_key = 0; }; + std::map visible; + unsigned num = 1; + int id = 11; + unsigned head_block_num() const { return num; } + int head_block_id() const { return id; } + const validator* find_validator(const string& name) const { + auto it = visible.find(name); + return it == visible.end() ? nullptr : &it->second; + } +}; +struct plugin { + fake_db db; + fake_db& database() { return db; } + std::set _validators{"alice", "bob", "not-created"}; +''' + fields + "\n" + effective + "\n" + refresh + r''' + void capture(int key, unsigned num, int id, bool rotation) { + db.num = num; db.id = id; db.visible["alice"].signing_key = key; + graphene::chain::signed_block b; b.num = num; + if (rotation) { + graphene::chain::signed_block::transaction tx; + tx.operations.push_back(operation{{"alice", key}}); + b.transactions.push_back(tx); + } + ''' + adapter + r''' + } + void seed() { + db.visible["alice"].signing_key = 10; + db.visible["bob"].signing_key = 20; + _committed_signing_keys["alice"] = 10; + _committed_signing_keys["bob"] = 20; + _committed_keys_block_num = db.head_block_num(); +''' + ("_committed_keys_block_id = db.head_block_id();" if not old else "") + r''' + } +}; +int failures = 0; +void check(const char* name, bool ok) { + std::cout << (ok ? "PASS " : "FAIL ") << name << '\n'; + if (!ok) ++failures; +} +int main() { + plugin p; p.seed(); + check("startup seed", p.effective_signing_key("alice", 10) == 10); + p.db.visible["alice"].signing_key = 11; + check("pending rotation signs with committed key", p.effective_signing_key("alice", 11) == 10); + p.capture(11, 2, 22, true); + check("committed rotation", p.effective_signing_key("alice", 11) == 11); + p.db.visible["alice"].signing_key = 12; + check("second pending rotation", p.effective_signing_key("alice", 12) == 11); + // Reorg removes the committed 10->11 rotation. The adopted head contains + // no rotation. The orphaned rotation is then restored to the pending pool. + p.capture(10, 3, 33, false); + p.db.visible["alice"].signing_key = 11; + check("reverted rotation restored as pending", p.effective_signing_key("alice", 11) == 10); + check("unrelated validator preserved", p.effective_signing_key("bob", 21) == 20); + check("visible null disables production", p.effective_signing_key("alice", 0) == 0); + p.capture(0, 4, 44, true); + check("committed null remains disabled despite pending enable", p.effective_signing_key("alice", 12) == 0); + p.capture(12, 5, 55, true); + check("committed re-enable", p.effective_signing_key("alice", 13) == 12); + p.db.num = 6; p.db.id = 66; + check("stale height falls back", p.effective_signing_key("alice", 13) == 13); + p.db.num = 5; + check("same-height different head falls back", p.effective_signing_key("alice", 13) == 13); + p.capture(12, 5, 55, false); + p.db.visible.erase("alice"); + p.capture(12, 6, 66, false); + p.db.visible.erase("alice"); +''' + (r''' + p.update_committed_signing_keys(); + check("reverted validator creation removes cache entry", p.effective_signing_key("alice", 14) == 14); +''' if not old else "") + r''' + return failures ? 1 : 0; +} +''' + with tempfile.TemporaryDirectory(prefix="validator-cache-", dir=os.environ.get("TMPDIR")) as tmp: + path = Path(tmp) + (path / "test.cpp").write_text(cpp) + subprocess.run([os.environ.get("CXX", "c++"), "-std=c++14", "-Wall", "-Wextra", "-Werror", "-pthread", + str(path / "test.cpp"), "-o", str(path / "test")], check=True) + result = subprocess.run([str(path / "test")]) + if result.returncode: + return result.returncode + if not args.source_ref: + # Static integration guard: notification must be in the no-pending lambda, + # after both fallible operations, not in a deferred applied_block handler. + database = (ROOT / "libraries/chain/database.cpp").read_text() + push = function(database, "bool database::push_block(") + callback = function(push, "detail::without_pending_transactions(") + assert callback.index("result = _push_block(new_block, skip);") < callback.index( + "check_free_memory(false, new_block.block_num());") < callback.index( + "CHAIN_TRY_NOTIFY(committed_chain_state);") < callback.index("} catch") + assert "with_strong_write_lock([&]()" in push[:push.index("detail::without_pending_transactions(")] + applied = function(source, "void validator_plugin::impl::on_block_applied(") + assert "update_committed_signing_keys" not in applied + assert "scoped_connection _committed_chain_state_connection;" in source + assert "_committed_chain_state_connection.disconnect();" in source + connected = function(source, "pimpl->_committed_chain_state_connection = d.committed_chain_state.connect(") + assert "pimpl->update_committed_signing_keys();" in connected + print("PASS synchronous runtime hook placement / lifecycle (source guard)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 1ec92549e1a376b3e87946694be9837eac0ec8c3 Mon Sep 17 00:00:00 2001 From: Denis Skripnik <264741654+web3blind@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:10:31 +0000 Subject: [PATCH 2/4] fix(snapshot): fail closed on restore transition errors --- .github/workflows/tests.yml | 1 + docs/plugins/snapshot.md | 55 +++++++ plugins/chain/plugin.cpp | 43 ++++- tests/chain/test_snapshot_restore.py | 225 +++++++++++++++++++++++++++ tests/p2p/test_vote_ingress.py | 4 +- 5 files changed, 318 insertions(+), 10 deletions(-) create mode 100644 tests/chain/test_snapshot_restore.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 73ebbdca4d..5ce67169c8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,7 @@ jobs: set -euo pipefail python3 tests/p2p/test_vote_ingress.py python3 tests/validator/test_signing_cache.py + python3 tests/chain/test_snapshot_restore.py python3 tests/consensus_sim/production_safety_offline.py node --test tests/block_log/archive_merge.test.cjs diff --git a/docs/plugins/snapshot.md b/docs/plugins/snapshot.md index 67ab8629d5..7a31b2508e 100644 --- a/docs/plugins/snapshot.md +++ b/docs/plugins/snapshot.md @@ -242,6 +242,61 @@ P2P sync fills the gap to the live chain head. --- +## Admin-Requested In-Process Restore + +`admin.snapshot.restore` schedules a restore on the snapshot worker thread. Its +`status: started` response means the job was accepted, not that import or resume +succeeded. `admin.snapshot.status` exposes whether the job is still running; +completion and failure details are logged by the worker. + +The chain restore method applies these failure boundaries: + +- With a started P2P plugin, a synchronous pause exception aborts before database + close/import and leaves the pre-restore syncing flag unchanged. Missing or + unstarted P2P is skipped, as before. +- After pause, block production is gated through `is_syncing()`. If database + close throws, the method does not import or resume: close may have partially + changed state. It keeps the gate set and requests application shutdown. +- An import exception, zero imported head, or unreadable head returns failure + without resuming P2P or clearing the production gate. Import handling may + already have requested shutdown; the restore method does not assume that it + has, and does not automatically recover from this state. +- An immediate peer-reset/resume exception returns failure and retains the + production gate. P2P flags may already have changed before the exception; this + is not a guarantee that P2P remains paused after a failed resume. +- Only after a nonzero head and no immediate resume error does the method clear + the syncing gate and report success. The existing post-pause P2P catchup gate + still applies. + +These are local error-handling boundaries, **not whole-node quiescence**. The +recovery mutex only excludes cooperating users; pausing P2P and setting the +production gate do not establish that all API calls, transactions or already +in-flight validator/database work have drained. P2P reset and the resume queue +are asynchronous, so later errors in those posted tasks are not observed by +this method. Plugin lifecycle changes during restore are not synchronized here. +After a post-close failure, inspect the logs and use the normal stopped-node +recovery procedure rather than assuming the old database is intact. + +### Offline regression + +Run from the repository root with a C++17 compiler available: + +```bash +python3 tests/chain/test_snapshot_restore.py +``` + +The test extracts the production `restore_from_snapshot()` method and compiles +it against controlled storage, P2P, filesystem and application dependencies. It +covers preflight, absent/unstarted P2P, pause exceptions with both initial syncing +values, partially mutating close failures, failed/empty import, unreadable head, +peer-reset/resume exceptions (including partially changed resume flags), and +successful restore ordering. It starts no node and performs no real DB restore; +it does not prove thread quiescence or test the queued P2P task bodies. +`CXX` selects the compiler; `RESTORE_SOURCE` can select an alternate `plugin.cpp` +for baseline regression checks. + +--- + ## Automatic Runtime Recovery: `--auto-recover-from-snapshot` Enabled by default. When shared memory corruption is detected during block processing or block generation, the node: diff --git a/plugins/chain/plugin.cpp b/plugins/chain/plugin.cpp index 68c136bfad..a613a3b1d4 100644 --- a/plugins/chain/plugin.cpp +++ b/plugins/chain/plugin.cpp @@ -1211,9 +1211,16 @@ namespace chain { wlog("Restore: P2P block processing paused"); } } catch (const fc::exception& e) { - wlog("Restore: failed to pause P2P: ${e}", ("e", e.to_detail_string())); + error = "failed to pause P2P: " + e.to_detail_string(); + } catch (const std::exception& e) { + error = std::string("failed to pause P2P: ") + e.what(); } catch (...) { - wlog("Restore: failed to pause P2P"); + error = "unknown exception while pausing P2P"; + } + if (!error.empty()) { + elog("Restore aborted before database close: ${e}", ("e", error)); + // No DB mutation or syncing change is safe without a successful pause. + return false; } // 2. Gate block production (validator reads is_syncing()). @@ -1223,8 +1230,19 @@ namespace chain { // reopens it and wipes shared memory for a recovery load. try { my->db.close(false); + } catch (const fc::exception& e) { + error = "failed to close database: " + e.to_detail_string(); + } catch (const std::exception& e) { + error = std::string("failed to close database: ") + e.what(); } catch (...) { - wlog("Restore: ignoring error while closing the database"); + error = "unknown exception while closing database"; + } + if (!error.empty()) { + elog("Restore FAILED: ${e}", ("e", error)); + // close() may have partially changed state. Keep P2P paused and + // production gated; neither importing nor resuming is safe now. + appbase::app().quit(); + return false; } my->snapshot_path = snapshot_path; @@ -1255,13 +1273,11 @@ namespace chain { if (error.empty()) error = "snapshot import did not produce state"; elog("Restore FAILED: ${e}", ("e", error)); // P2P stays paused and the syncing flag stays set: the state on disk is not - // safe to consume. do_snapshot_load() has already asked the process to quit; - // if it did not, the node must not resume as if nothing happened. + // safe to consume. do_snapshot_load() may already have requested shutdown; + // even if it did not, the node must not resume as if nothing happened. return false; } - my->currently_syncing.store(false, std::memory_order_release); - // 4. Resume P2P. reset_peers_after_recovery() clears soft bans and stale sync // state, so peers can serve blocks immediately after the import. try { @@ -1271,10 +1287,21 @@ namespace chain { p2p_plug->resume_block_processing(); wlog("Restore: P2P block processing resumed"); } + } catch (const fc::exception& e) { + error = "failed to resume P2P: " + e.to_detail_string(); + } catch (const std::exception& e) { + error = std::string("failed to resume P2P: ") + e.what(); } catch (...) { - wlog("Restore: failed to resume P2P"); + error = "unknown exception while resuming P2P"; + } + if (!error.empty()) { + // Resume may have changed P2P flags before throwing. Retain the + // production gate; do not report this as a resumed node. + elog("Restore imported state but failed to resume: ${e}", ("e", error)); + return false; } + my->currently_syncing.store(false, std::memory_order_release); wlog("=== RESTORE COMPLETE: node resumed at block ${n} ===", ("n", head)); return true; } diff --git a/tests/chain/test_snapshot_restore.py b/tests/chain/test_snapshot_restore.py new file mode 100644 index 0000000000..2ad3a7432f --- /dev/null +++ b/tests/chain/test_snapshot_restore.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Exercise the extracted production restore method; no node, sockets or real DB. + +Only dependencies are controlled. Run with python3 tests/chain/test_snapshot_restore.py. +RESTORE_SOURCE may name an alternate plugin.cpp for baseline RED checks. +""" +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +def method(source, signature): + start = source.index(signature) + masked = re.sub(r'//[^\n]*|/\*.*?\*/|"(?:\\.|[^"\\])*"', + lambda m: ' ' * len(m.group()), source, flags=re.S) + opening = masked.index('{', start) + depth, end = 1, opening + 1 + while depth: + depth += (masked[end] == '{') - (masked[end] == '}') + end += 1 + return source[start:end] + + +FIXTURE = r''' +#include +#include +#include +#include +#include +#include +#include +#define wlog(...) ((void)0) +#define elog(...) ((void)0) +namespace fc { +struct exception : std::runtime_error { + exception() : std::runtime_error("controlled fc failure") {} + std::string to_detail_string() const { return what(); } +}; +} +std::vector events; +void fail(int kind) { + if (kind == 1) throw fc::exception(); + if (kind == 2) throw std::runtime_error("controlled std failure"); + if (kind == 3) throw 42; +} +namespace appbase { struct abstract_plugin { enum state { registered, started }; }; } +namespace graphene { namespace plugins { namespace p2p { +struct p2p_plugin { + int pause_failure = 0, reset_failure = 0, resume_failure = 0; + bool paused = false, resume_partial = false; + appbase::abstract_plugin::state state = appbase::abstract_plugin::started; + auto get_state() const { return state; } + void pause_block_processing_sync() { + events.push_back("pause"); fail(pause_failure); paused = true; + } + void reset_peers_after_recovery() { events.push_back("reset"); fail(reset_failure); } + void resume_block_processing() { + events.push_back("resume"); + if (resume_partial) paused = false; + fail(resume_failure); paused = false; + } +}; +} } } +namespace appbase { +struct application { + graphene::plugins::p2p::p2p_plugin p2p; + bool missing = false; + int quits = 0; + template T* find_plugin() { return missing ? nullptr : &p2p; } + std::string data_dir() { return "offline"; } + void quit() { events.push_back("quit"); ++quits; } +}; +application& app() { static application a; return a; } +} +std::string operator/(const std::string& a, const char* b) { return a + "/" + b; } +namespace bfs { +bool exists(const std::string& p) { return p != "missing"; } +bool is_regular_file(const std::string& p) { return p != "directory"; } +} +struct controlled_db { + int close_failure = 0, head_failure = 0; + uint32_t head = 99; + void close(bool) { events.push_back("close"); head = 0; fail(close_failure); } + uint32_t head_block_num() { events.push_back("head"); fail(head_failure); return head; } +}; +struct implementation { + std::atomic recovery_in_progress{false}, currently_syncing{false}; + std::mutex recovery_read_mutex; + controlled_db db; + std::string snapshot_path = "previous"; +}; +struct plugin { + implementation impl; + implementation* my = &impl; + bool snapshot_load_callback = true, empty_import = false, populated_failure = false; + int import_failure = 0; + void do_snapshot_load(const std::string&, bool recovery) { + if (!recovery) throw std::runtime_error("not a recovery load"); + events.push_back("import"); + if (populated_failure) my->db.head = 123; + fail(import_failure); + my->db.head = empty_import ? 0 : 123; + } + bool restore_from_snapshot(const std::string&, std::string&); +}; +PRODUCTION_METHOD +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} +int main(int argc, char** argv) { + try { + require(argc == 2, "scenario required"); + std::string s = argv[1]; + plugin p; + auto& app = appbase::app(); + auto& net = app.p2p; + auto& db = p.my->db; + bool prior_sync = s.find("syncing") != std::string::npos; + p.my->currently_syncing = prior_sync; + auto starts = [&](const std::string& prefix) { return s.find(prefix) == 0; }; + int kind = s.find("_fc") != std::string::npos ? 1 : + s.find("_std") != std::string::npos ? 2 : 3; + if (starts("pause_")) net.pause_failure = kind; + if (starts("close_")) db.close_failure = kind; + if (starts("import_")) p.import_failure = kind; + if (starts("reset_")) net.reset_failure = kind; + if (starts("resume_")) net.resume_failure = kind; + net.resume_partial = s == "resume_partial_std"; + if (s == "head_throws") db.head_failure = 1; + p.empty_import = s == "zero_head"; + p.populated_failure = s == "import_populated_std"; + app.missing = s == "missing_p2p"; + if (s == "unstarted_p2p") net.state = appbase::abstract_plugin::registered; + std::string path = "snapshot", error; + if (s == "empty_path") path.clear(); + if (s == "missing_file") path = "missing"; + if (s == "directory") path = "directory"; + if (s == "no_callback") p.snapshot_load_callback = false; + bool busy = s == "busy"; + if (busy) p.my->recovery_in_progress = true; + bool ok = p.restore_from_snapshot(path, error); + require(p.my->recovery_in_progress == busy, "recovery ownership/guard changed"); + bool validation = s == "empty_path" || s == "missing_file" || s == "directory" || s == "no_callback" || busy; + bool pause = starts("pause_"); + bool close = starts("close_"); + bool imported_bad = starts("import_") || s == "zero_head" || s == "head_throws"; + bool resume = starts("resume_") || starts("reset_"); + bool failed = validation || pause || close || imported_bad || resume; + std::cerr << "ok=" << ok << " syncing=" << p.my->currently_syncing + << " head=" << db.head << " events="; + for (const auto& event : events) std::cerr << event << ','; + std::cerr << '\n'; + require(ok == !failed, "incorrect restore success/failure result"); + require(error.empty() == !failed, "incorrect error result"); + if (validation || pause) { + require(db.head == 99, "failed preflight/pause mutated database"); + require(p.my->snapshot_path == "previous", "failed pause replaced snapshot path"); + require(p.my->currently_syncing == prior_sync, "failed pause changed pre-restore syncing gate"); + require(events == (pause ? std::vector{"pause"} : std::vector{}), "pre-mutation abort continued restore"); + } else if (close) { + require(events == std::vector({"pause", "close", "quit"}), "failed close imported/resumed or did not request shutdown"); + require(p.my->snapshot_path == "previous", "failed close installed new snapshot path"); + require(net.paused && p.my->currently_syncing, "failed close lost safe gate"); + } else if (imported_bad) { + require(net.paused && p.my->currently_syncing, "failed import lost safe gate"); + for (auto& e : events) require(e != "reset" && e != "resume", "failed import resumed P2P"); + } else if (resume) { + require(p.my->currently_syncing, "failed resume cleared syncing gate"); + require(db.head == 123, "resume failure changed imported state"); + } else { + bool has_p2p = s != "missing_p2p" && s != "unstarted_p2p"; + require(events == (has_p2p ? std::vector{"pause", "close", "import", "head", "reset", "resume"} : std::vector{"close", "import", "head"}), "happy-path ordering changed"); + require(!p.my->currently_syncing && !net.paused && db.head == 123, "successful restore did not resume"); + require(p.my->snapshot_path == path, "successful restore lost path"); + } + require(app.quits == (close ? 1 : 0), "unexpected shutdown policy"); + std::cout << s << " OK\n"; + return 0; + } catch (const std::exception& e) { + std::cerr << e.what() << '\n'; return 1; + } +} +''' + + +class SnapshotRestoreTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + source = Path(os.environ.get('RESTORE_SOURCE', ROOT / 'plugins/chain/plugin.cpp')).read_text() + body = method(source, 'bool plugin::restore_from_snapshot(') + cls.temp = tempfile.TemporaryDirectory(prefix='viz-restore-', dir=os.environ.get('TMPDIR')) + cls.addClassCleanup(cls.temp.cleanup) + cpp = Path(cls.temp.name) / 'restore.cpp' + cpp.write_text(FIXTURE.replace('PRODUCTION_METHOD', body)) + cls.binary = Path(cls.temp.name) / 'restore' + subprocess.run([os.environ.get('CXX', 'c++'), '-std=c++17', '-Wall', '-Wextra', '-Werror', + '-pthread', str(cpp), '-o', str(cls.binary)], check=True) + + +def scenario_test(scenario): + def test(self): + result = subprocess.run([str(self.binary), scenario], text=True, capture_output=True) + self.assertEqual(result.returncode, 0, f'{scenario}: {result.stderr}') + self.assertIn(f'{scenario} OK', result.stdout) + return test + + +SCENARIOS = ['happy', 'happy_syncing', 'missing_p2p', 'unstarted_p2p', + 'empty_path', 'missing_file', 'directory', 'no_callback', 'busy', + 'zero_head', 'head_throws', 'resume_partial_std', 'import_populated_std'] +SCENARIOS += [f'{stage}_{kind}{suffix}' + for stage in ('pause', 'close', 'import', 'reset', 'resume') + for kind in ('fc', 'std', 'unknown') + for suffix in (('', '_syncing') if stage == 'pause' else ('',))] +for scenario in SCENARIOS: + setattr(SnapshotRestoreTests, 'test_' + scenario, scenario_test(scenario)) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/p2p/test_vote_ingress.py b/tests/p2p/test_vote_ingress.py index 6db7e7007f..cc480e6e71 100644 --- a/tests/p2p/test_vote_ingress.py +++ b/tests/p2p/test_vote_ingress.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Offline real-handler regression; fake time/hash/delegate, never a live node. ---baseline extracts the committed versions of the production handlers/members. +--baseline extracts release 4.2.0 (d4fd4bcd), not the moving current HEAD. --probe reports resource measurements without imposing post-fix assertions. SANITIZE=1 enables ASan/UBSan. No production source is rewritten by this runner. """ @@ -219,7 +219,7 @@ def main(): args = parser.parse_args() def source(path): if args.baseline: - return subprocess.check_output(['git', 'show', 'HEAD:' + path], cwd=ROOT, text=True) + return subprocess.check_output(['git', 'show', 'd4fd4bcd:' + path], cwd=ROOT, text=True) return (ROOT / path).read_text() cpp, hpp = source(CPP), source(HPP) signatures = ['void dlt_p2p_node::' + name + '(' for name in ( From eef6c5f6b7aa0a6050b4afc9c5dbac90edcc5e33 Mon Sep 17 00:00:00 2001 From: Denis Skripnik <264741654+web3blind@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:15:08 +0000 Subject: [PATCH 3/4] test(hf16): use portable temporary roots for offline checks --- tests/consensus_sim/README.md | 2 +- tests/consensus_sim/production_safety_offline.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/consensus_sim/README.md b/tests/consensus_sim/README.md index b915c32161..1971803677 100644 --- a/tests/consensus_sim/README.md +++ b/tests/consensus_sim/README.md @@ -65,7 +65,7 @@ keys, the absolute size cap and count-prefix growth. It **does not** execute the database, actual signature recovery, FC serialization or consensus validation, and does not replace the native scenarios. `PRODUCTION_SOURCE=/path/database.cpp` can select an unmodified baseline for a RED comparison. Temporary compiler -artifacts use `TMPDIR` (or the Hermes scratch directory). +artifacts use `TMPDIR`, or the platform's default temporary directory. ## Coverage diff --git a/tests/consensus_sim/production_safety_offline.py b/tests/consensus_sim/production_safety_offline.py index f52246db3d..4b0b91d9e7 100644 --- a/tests/consensus_sim/production_safety_offline.py +++ b/tests/consensus_sim/production_safety_offline.py @@ -73,7 +73,7 @@ def test_evidence_selection_actual_body(self): return failures?1:0; } ''' - root = os.environ.get('TMPDIR', str(pathlib.Path.home()/'.hermes/cache/scratch')) + root = os.environ.get('TMPDIR') with tempfile.TemporaryDirectory(prefix='production-safety-', dir=root) as tmp: src = pathlib.Path(tmp)/'selection.cpp' exe = pathlib.Path(tmp)/'selection' @@ -136,7 +136,7 @@ def test_transaction_budget_actual_body(self): } } ''' - root = os.environ.get('TMPDIR', str(pathlib.Path.home()/'.hermes/cache/scratch')) + root = os.environ.get('TMPDIR') with tempfile.TemporaryDirectory(prefix='production-budget-', dir=root) as tmp: src = pathlib.Path(tmp)/'budget.cpp' exe = pathlib.Path(tmp)/'budget' From 8a3ea96a48cd968580c141ee4638e40ccaeb8264 Mon Sep 17 00:00:00 2001 From: Denis Skripnik <264741654+web3blind@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:19:02 +0000 Subject: [PATCH 4/4] ci: build fork pull requests without registry credentials --- .github/workflows/docker-pr-build.yml | 6 ++++-- docs/development/testing.md | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-pr-build.yml b/.github/workflows/docker-pr-build.yml index 567d2613de..a354c39acc 100644 --- a/.github/workflows/docker-pr-build.yml +++ b/.github/workflows/docker-pr-build.yml @@ -28,7 +28,9 @@ jobs: - uses: docker/setup-buildx-action@v4 id: setup-buildx + # Fork PRs build without registry secrets and never publish images. - uses: docker/login-action@v4 + if: github.event.pull_request.head.repo.full_name == github.repository with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} @@ -84,7 +86,7 @@ jobs: with: context: . file: ./share/vizd/docker/Dockerfile-production - push: true + push: ${{ github.event.pull_request.head.repo.full_name == github.repository }} tags: vizblockchain/vizd:${{ steps.tag.outputs.value }} cache-from: type=gha,scope=pr-${{ github.base_ref }} cache-to: type=gha,scope=pr-${{ github.base_ref }},mode=max @@ -97,7 +99,7 @@ jobs: context: . file: ./share/vizd/docker/Dockerfile-testnet build-args: TESTNET_VALIDATORS=21 - push: true + push: ${{ github.event.pull_request.head.repo.full_name == github.repository }} tags: vizblockchain/vizd:${{ steps.tag.outputs.value }}-testnet21 cache-from: type=gha,scope=pr-testnet21 cache-to: type=gha,scope=pr-testnet21,mode=max diff --git a/docs/development/testing.md b/docs/development/testing.md index 926cc4f0e6..78940b8f46 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -4,6 +4,10 @@ VIZ Ledger uses Boost.Test for unit tests and provides utility programs for inte --- +## Pull-request Docker builds + +Fork pull requests build images without registry login or publication. Internal pull requests retain Docker Hub login and image publication. The separate master/release workflows are unchanged. + ## Unit Tests The unit test binary is built via CMake as part of the `libraries/chain` target.