Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion csnode/src/blockchain.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1852,7 +1852,12 @@ std::optional<csdb::Pool> BlockChain::recordBlock(csdb::Pool& pool, bool isTrust
}

if (!finalizeBlock(pool, isTrusted, lastConfidants)) {
csdebug() << kLogPrefix << "the signatures of the block are insufficient or incorrect";
// an invalid pool has no hash: do not let hash() compute one from its empty binary
const auto finalizeHash = pool.is_valid() ? pool.hash().to_binary() : cs::Bytes{};
const auto finalizePrev = pool.previous_hash().to_binary();
cswarning() << kLogPrefix << "the signatures of the block are insufficient or incorrect: #" << pool.sequence()
<< " hash " << (finalizeHash.empty() ? std::string("-") : cs::Utils::byteStreamToHex(finalizeHash.data(), finalizeHash.size()))
<< " prev " << (finalizePrev.empty() ? std::string("-") : cs::Utils::byteStreamToHex(finalizePrev.data(), finalizePrev.size()));
setBlocksToBeRemoved(1U);
return std::nullopt;
}
Expand Down
20 changes: 19 additions & 1 deletion csnode/src/node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4171,7 +4171,7 @@ void Node::setCurrentRP(const cs::RoundPackage& rp) {
currentRoundPackage_ = rp;
}

void Node::performRoundPackage(cs::RoundPackage& rPackage, const cs::PublicKey& /*sender*/, bool updateRound) {
void Node::performRoundPackage(cs::RoundPackage& rPackage, const cs::PublicKey& sender, bool updateRound) {
csdebug() << __func__;

// got round package in any way, reset default round table flag
Expand All @@ -4182,11 +4182,29 @@ void Node::performRoundPackage(cs::RoundPackage& rPackage, const cs::PublicKey&
if (ptrRT != nullptr) {
const cs::ConfidantsKeys& prevConfidants = ptrRT->confidants;
if (!prevConfidants.empty() && realTrusted.size() == prevConfidants.size()) {
// diagnostics only: which previous confidants this package gray-lists, and whether we are one
std::string grayKeys;
size_t grayCount = 0;
bool ownKey = false;
for (size_t i = 0; i < realTrusted.size(); ++i) {
if (realTrusted[i] == cs::ConfidantConsts::InvalidConfidantIndex) {
solver_->addToGraylist(prevConfidants[i], Consensus::GrayListPunishment);
const cs::PublicKey& grayKey = prevConfidants[i];
grayKeys += (grayCount ? " T" : "T") + std::to_string(i) + ':' + EncodeBase58(grayKey.data(), grayKey.data() + grayKey.size()).substr(0, 8);
++grayCount;
if (grayKey == nodeIdKey_) {
ownKey = true;
grayKeys += "(OWN KEY)";
}
}
}
if (grayCount > 0) {
cslog() << "NODE> round package R-" << rPackage.roundTable().round << " from "
<< EncodeBase58(sender.data(), sender.data() + sender.size()).substr(0, 8)
<< " gray-lists " << grayCount << " of " << prevConfidants.size() << " confidants of R-" << rPackage.roundTable().round - 1
<< " for " << Consensus::GrayListPunishment << " rounds: [" << grayKeys << "]"
<< (ownKey ? " - OWN KEY gray-listed" : "");
}
}
}

Expand Down
1 change: 1 addition & 0 deletions docs/CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ lines under the new release.
| #163 | ci: the testnet script stops at the start when a pruned node is in the run and `TARGET_SEQ` is below 2100: a pruned node prunes first at the checkpoint of block 2000, so a lower target only failed after the whole run with "never pruned its storage" |
| #164 | api: `ContractAllMethodsGet` now returns the annotations the executor reports for each method and argument (`@Getter`, `@UsingContract`, `@ContractAddress`, `@ContractMethod`, in the IDL fields `MethodDescription.annotations` and `MethodArgument.annotations` that were always left empty), so a client can tell a getter; ci: `contracts.py` checks the `@Getter` methods of `samples/contracts/Counter.java` |
| #165 | api: a CS payment to a payable contract was recorded as a contract deployed by the payer, and `SmartContractsListGet` / `SmartContractAddressesListGet` failed on every page holding one ("No more data to read"); only a transaction carrying an invocation now counts as a deploy in that list (API only, `SmartContracts::is_deploy` and consensus unchanged); ci: `contracts.py` pays `samples/contracts/TipJar.java` and checks the lists and the payments in `TransactionsGet` |
| #167 | solver: stage-3 diagnostics at info level, logging only: what this node signed (`stage3 inputs`: sequence, previous hash, timestamp, reward, hashes of the transactions and fees, the three signed hashes), why a peer's stage 3 failed (another block/round hash or a key/message mismatch), the agree/disagree split among the confidants, the keys a round package gray-lists (`OWN KEY` when ours is one), the counts on "not enough active confidants", and the block on "signatures of the block are insufficient" (now a warning); see [consensus-diagnostics.md](consensus-diagnostics.md) |

## community.16 — 2026-11-03

Expand Down
130 changes: 130 additions & 0 deletions docs/consensus-diagnostics.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
# Reading the stage-3 diagnostics

A trusted node sometimes reports, at the end of a round:

```
Stage3 from T[4] - final check ... NOT PASSED! This problem will be resolved automatically.
```

and the debug log adds "Block Signatures are not valid" or "Round Signatures are not valid".
Soon after, the node may wait for a bootstrap round, and its own key may be gray-listed for
`GrayListPunishment` rounds (1000 on mainnet).

Stage 3 carries only signatures: each confidant signs the hash of the block it built, the hash of
the next round table and the hash of the next confidant list, and sends the three signatures
without the hashes. A receiver checks each signature against **its own** three hashes. When a
signature fails, either the key or message is wrong, or the sender built a different block or
round table. The lines below, at info level (no debug log needed), tell these cases apart and
show which input differed. They are written only to the log: nothing is signed, sent or stored
differently, and the consensus result is the same as without them.

## 1. What this node signed: `stage3 inputs`

One line per stage 3 sent by this node:

```
info SolverCore: stage3 inputs: R-37 iter=0 seq=37 prev=7C839893 ts=1793783441147 reward=- tx=1
txIds=40E79CA6 fees=69F07B9F charMask=FC87F4F1/1 newWallets=0 smartSigs=0 conf=5/5 writer=2
block=BC24F874 round=44C9C6F6 trusted=D7B1BD25 mask=[3] [4] [0] [1] [2]
```

(one line in the log; wrapped here.) Hashes are shortened to their first 8 hex characters; `-` means
empty (no transactions, no reward field).

| Field | Meaning | Goes into |
|---|---|---|
| `seq`, `prev` | sequence and previous block hash of the new block | block hash |
| `ts` | block timestamp chosen from the stage-1 timestamps | block hash and round table |
| `reward` | block reward field (hex; `-` before DPoS or with mining off), computed from the confidants' cached wallet data | block hash and round table |
| `tx`, `txIds` | number of transactions and a hash of their signatures in block order | block hash |
| `fees` | hash of the counted fee of every transaction, in order | block hash |
| `charMask` | hash and length of the characteristic (the mask that chose the transactions) | block hash |
| `newWallets` | number of new wallet ids the block assigns | block hash |
| `smartSigs` | number of contract-consensus signatures carried in the block | block hash |
| `conf` | round confirmations in the block: mask size / signature count | block hash |
| `writer`, `mask` | writer index and real-trusted mask of this iteration (`X` = excluded) | compared directly |
| `block`, `round`, `trusted` | the three hashes this node signed | signatures |

For example, in a test network where two nodes were made to build their block with the timestamp
1 ms later, the same round printed on a majority node and on a skewed one:

```
stage3 inputs: R-1 iter=0 seq=1 prev=C0541961 ts=1793784561704 ... newWallets=7 ... block=2E6353D9 round=A5256CC5 trusted=CA0A93D1
stage3 inputs: R-1 iter=0 seq=1 prev=C0541961 ts=1793784561705 ... newWallets=7 ... block=9876C3C4 round=7B1677FB trusted=CA0A93D1
```

`prev` and every other input agree, `ts` differs, and so do `block` and `round`; `trusted` is equal.

**How to use it.** Collect this line for the same round (`R-`) and iteration from several
confidants (or ask the operators). Nodes that agree print the same `block`/`round` hashes. On the
nodes that differ, the first field that differs is the cause:

- `prev` or `seq` differs: the node is on another chain tip (a fork or a missed block);
- `ts` differs: the nodes chose different timestamps from stage 1;
- `reward` differs (with `ts`, `prev` equal): the cached wallet data (stakes, delegations) differs
between the nodes; compare their `STATE DIGEST` lines;
- `txIds`, `fees`, `newWallets` or `charMask` differ: the block content differs;
- `conf` differs: the nodes hold different round confirmations;
- all of these equal but `round` differs: the round table differs (the next confidants, the
packet hashes chosen for the next round, the sub-round or the iteration).

The round table also signs the timestamp, reward, characteristic, sequence and previous hash, so a
difference in one of them makes both `block` and `round` fail (`failed=block+round` below); only
`block` failing points to the block-only inputs (`txIds`, `fees`, `newWallets`, `conf`,
`smartSigs`).

## 2. Why a peer failed: `stage3 check failed`

One line per failing peer:

```
info R-1 SolverCore> stage3 check failed T[4] EJGNJj2S iter=0 msg=ok failed=block+round -> signed another block/round hash (key ok, trusted hash agreed)
```

`T[4]` is the confidant index, followed by the first 8 base58 characters of its public key.
`msg=ok` means the outer signature of the message was valid (it is checked when the message is
received). `failed=` lists the checks that failed: `block`, `round`, `trusted`, `mask`, `writer`,
or `untrusted` (the sender was already excluded in our mask). The verdict:

- **signed another block/round hash (key ok, trusted hash agreed)**: the key is right and the
sender agrees on the next confidant list, but it signed another block and/or round table. One
of the inputs of section 1 differs between this node and the sender.
- **key or message mismatch (trusted signature fails too)**: even the confidant-list signature
fails, so the sender used another key for its index or another confidant list (another round).
- **signatures ok, mask or writer differs**: the signatures match, but the sender excluded other
nodes or chose another writer.

## 3. Who is in the majority: `stage3 split`

After each failure (at most 10 per iteration) the node logs the split it sees. When the node's
own stage 3 arrives after its peers', the early lines list only the peers checked so far: read the
last `stage3 split` line of the iteration.

```
info R-1 SolverCore> stage3 split iter=0: agree(5)=[T0:KasmLeQs T1:7gZ4BWYh T2:B9XbBZUD(own) T3:Br6U154N T6:HhovRb4b] | block+round(2)=[T4:EJGNJj2S T5:GqamRj6f] of 7
```

and on one of the two skewed nodes, the same round seen from the minority:

```
info R-1 SolverCore> stage3 split iter=0: agree(1)=[T4:EJGNJj2S(own)] | block+round(5)=[T0:KasmLeQs T1:7gZ4BWYh T2:B9XbBZUD T3:Br6U154N T6:HhovRb4b] of 7
```

`agree` lists the confidants whose signatures matched our hashes, ourselves marked `(own)`; the
other groups list the failing confidants by failure pattern. The hash a failing sender signed is
not sent, so the node cannot tell whether two failing senders signed the same block; senders that
fail with the same pattern are grouped. When `agree` is the smaller side, **this node** is
probably the one that built a different block: compare its `stage3 inputs` line with a node of
the larger group.

## 4. After the round

- `not enough active confidants to make a decision, Bootstrap required (active 2 of 7, need 4, mask [X] [X] [X] [X] [1] [2] [X] )`
(warning): too many peers were excluded in this iteration, so no writer can be chosen.
- `NODE> round package R-2 from Br6U154N gray-lists 2 of 7 confidants of R-1 for 1000 rounds: [T4:EJGNJj2S T5:GqamRj6f(OWN KEY)] - OWN KEY gray-listed`
(info): the round package from `Br6U154N` excluded these confidants of the previous
round; every node gray-lists them, and they are not chosen as confidants for that many rounds.
`OWN KEY` means this node is among them: the majority did not accept its stage 3.
- `the signatures of the block are insufficient or incorrect: #<seq> hash <hex> prev <hex>`
(warning, was debug): a received block failed the signature check and is dropped; its full hash
and previous hash identify which block it was.
6 changes: 6 additions & 0 deletions docs/differences.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,12 @@ rebuilt from the chain when the checkpoint holds none. See [tokens.md](tokens.md
as `qs/<sequence>/state_digest.bin`, and logs it (`STATE DIGEST #<sequence> ... checkpoint`). A
quick start recomputes it from the loaded caches, logs `... checkpoint loaded`, and moves a
checkpoint whose caches do not match it aside to `qs/<sequence>.bad`.
- **Stage-3 diagnostics.** A trusted node logs at info level what it signed in stage 3 (sequence,
previous hash, timestamp, reward, a hash of the transactions and fees, the three signed hashes),
why a peer's stage 3 failed (another block/round hash, or a key/message mismatch), the
agree/disagree split among the confidants, and which keys a round package gray-lists, with
`OWN KEY` when this node is one of them. Logging only: see
[consensus-diagnostics.md](consensus-diagnostics.md) to tell from these lines which input differed.
- **Stopping on errors.** Some errors that the official node can survive in an inconsistent
state now stop the node, and are logged with `cserror`:
- a block that cannot be stored;
Expand Down
6 changes: 6 additions & 0 deletions solver/include/solver/solvercore.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,9 @@ class SolverCore {
// to use private data while serve for states as SolverCore context:
friend class SolverContext;

// diagnostics only: classify a stage-3 final-check failure and log the agree/disagree split
void logStage3Failure(const cs::StageThree& stageFrom, uint8_t pattern);

enum class Mode {
Default,
Debug,
Expand Down Expand Up @@ -326,6 +329,9 @@ class SolverCore {
std::vector<cs::StageTwo> stageTwoStorage;
std::vector<cs::StageThree> stageThreeStorage;
std::vector<cs::StageThree> trueStageThreeStorage;
// diagnostics only (never sent or stored): stage-3 final-check failures of the current
// iteration as (sender index, failure pattern bits), used to log the agree/disagree split
std::vector<std::pair<uint8_t, uint8_t>> stage3Failures_;
bool realTrustedChanged_;
cs::Bytes tempRealTrusted_;

Expand Down
55 changes: 55 additions & 0 deletions solver/src/solvercore.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,61 @@ void SolverCore::spawn_next_round(const cs::PublicKeys& nodes, const cs::Packets
}

csdebug() << "NODE> StageThree prepared:" << std::endl << cs::StageThree::toString(stage3);

// Diagnostics only (one info line per stage 3): the inputs that went into our block/round hashes.
// Stage-3 carries signatures but no hashes, so a receiver that fails "Block/Round Signatures are
// not valid" cannot see what the sender signed; comparing this line between nodes for the same
// round shows which input differed. Read-only: nothing below is signed, sent or stored.
{
const csdb::Pool& blk = deferredBlock_;
const auto& trxs = blk.transactions();
cs::Bytes txSigs;
cs::Bytes txFees;
txSigs.reserve(trxs.size() * sizeof(cs::Signature));
txFees.reserve(trxs.size() * 2);
for (const auto& t : trxs) {
const cs::Signature& sig = t.signature();
txSigs.insert(txSigs.end(), sig.cbegin(), sig.cend());
auto fee = t.counted_fee();
const uint16_t rawFee = fee.get_raw();
txFees.push_back(static_cast<uint8_t>(rawFee & 0xFF));
txFees.push_back(static_cast<uint8_t>(rawFee >> 8));
}
auto short8 = [](const cs::Byte* data, size_t size) -> std::string {
if (size == 0) {
return "-";
}
return cs::Utils::byteStreamToHex(data, std::min<size_t>(size, 4));
};
auto hash8 = [&short8](const cs::Bytes& bytes) -> std::string {
if (bytes.empty()) {
return "-";
}
const cs::Hash h = cscrypto::calculateHash(bytes.data(), bytes.size());
return short8(h.data(), h.size());
};
const cs::Bytes prevHash = blk.previous_hash().to_binary();
const cs::PoolMetaInfo& meta = justCreatedRoundPackage_.poolMetaInfo();

cslog() << log_prefix << "stage3 inputs: R-" << conveyer.currentRoundNumber()
<< " iter=" << static_cast<int>(stage3.iteration)
<< " seq=" << seq
<< " prev=" << short8(prevHash.data(), prevHash.size())
<< " ts=" << meta.timestamp
<< " reward=" << (meta.reward.empty() ? std::string("-") : cs::Utils::byteStreamToHex(meta.reward.data(), meta.reward.size()))
<< " tx=" << trxs.size()
<< " txIds=" << hash8(txSigs)
<< " fees=" << hash8(txFees)
<< " charMask=" << hash8(block_characteristic->mask) << '/' << block_characteristic->mask.size()
<< " newWallets=" << blk.newWallets().size()
<< " smartSigs=" << blk.smartSignatures().size()
<< " conf=" << static_cast<int>(blk.numberConfirmations()) << '/' << blk.roundConfirmations().size()
<< " writer=" << static_cast<int>(stage3.writer)
<< " block=" << short8(stage3.blockHash.data(), stage3.blockHash.size())
<< " round=" << short8(stage3.roundHash.data(), stage3.roundHash.size())
<< " trusted=" << short8(stage3.trustedHash.data(), stage3.trustedHash.size())
<< " mask=" << cs::TrustedMask::toString(stage3.realTrustedMask);
}
}

void SolverCore::uploadNewStates([[maybe_unused]] std::vector<csdb::Transaction> newStates) {
Expand Down
Loading
Loading