From f646fd57234fb387cc9a057a0af2cfb667bb98a4 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 22:45:26 +0200 Subject: [PATCH 1/3] api: executor version check polls up to the delay; multi-instance nodes keep the others' executors - checkExecutorVersion slept executor_check_version_delay (7 s on mainnet, 60 s in CI) on the node's start path and then asked once: it now asks every second (cs::pollUntil) and goes on as soon as the executor answers, with the same outcome as before when it never does (terminated); a transport error means not listening yet and is retried, any other answer is final; a stop is seen within a second. It stays on the launching thread (two threads opening the shared transport would be the new risk). The log line says how long the answer took. - Executor::stop() killed every contract-executor JVM of the machine (jps): with executor_multi_instance it now terminates only its own, as checkAnotherExecutor already does at start. The default keeps the official behaviour. Nothing here touches consensus. Found in the scale runs: every node start waited the full delay, and every restart test relaunched all the executors of the machine. Co-Authored-By: Claude Opus 5.5 --- api/src/executor.cpp | 41 ++++++++++++++++++++----- docs/CHANGES.md | 1 + lib/include/lib/system/polluntil.hpp | 33 ++++++++++++++++++++ tests/csnode/polluntil_tests.cpp | 45 ++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 lib/include/lib/system/polluntil.hpp create mode 100644 tests/csnode/polluntil_tests.cpp diff --git a/api/src/executor.cpp b/api/src/executor.cpp index 170b38c7a..dbf30a375 100644 --- a/api/src/executor.cpp +++ b/api/src/executor.cpp @@ -13,6 +13,7 @@ #include "serializer.hpp" #include +#include #include #include @@ -208,7 +209,9 @@ void cs::Executor::stop() { std::this_thread::sleep_for(std::chrono::milliseconds(500)); - if (manager_.isExecutorProcessRunning()) { + // every contract-executor JVM of the machine is this node's (official behaviour), unless the + // operator runs several nodes on it (executor_multi_instance): then the others are theirs to keep + if (!cs::ConfigHolder::instance().config()->getApiSettings().executorMultiInstance && manager_.isExecutorProcessRunning()) { manager_.stopExecutorProcess(); } } @@ -759,24 +762,48 @@ void cs::Executor::onExecutorProcessError(const cs::ProcessException& exception) cswarning() << "Executor process error occured " << exception.what() << ", code " << exception.code(); } +// Runs on the thread that launched the executor (the node's start, or the watcher when it relaunches +// one), synchronously: moving it to a thread of its own would open the shared transport from two +// threads. The executor gets executorCheckVersionDelay to answer and is asked every second: a JVM up +// in 2 s lets the node go on after 2 s, dozens of JVMs starting at once on one machine still get the +// whole delay, and the last attempt is made at or after the deadline, so this is never stricter than +// the former single attempt after the delay. A failure ends as before: the executor is terminated. void cs::Executor::checkExecutorVersion() { executor::ExecutorBuildVersionResult _return; bool result = false; - std::this_thread::sleep_for(std::chrono::milliseconds(cs::ConfigHolder::instance().config()->getApiSettings().executorCheckVersionDelay)); + const auto start = std::chrono::steady_clock::now(); + const auto delay = std::chrono::milliseconds(cs::ConfigHolder::instance().config()->getApiSettings().executorCheckVersionDelay); + cs::pollUntil(delay, std::chrono::milliseconds(1000), + [&] { + _return = executor::ExecutorBuildVersionResult{}; + if (!connect()) { + return false; + } + getExecutorBuildVersion(_return); + // an answer, or an error with the connection still open (the executor answered + // with an error, e.g. an unknown method), is final; a transport error closed the + // connection: the executor is not listening yet + return _return.status.code == 0 || isConnected(); + }, + [&] { return requestStop_.load(); }); if (requestStop_) { return; } - connect(); - getExecutorBuildVersion(_return); - + const auto waited = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); + if (_return.status.code == 0 && !isConnected()) { + _return.status.code = 1; + _return.status.message = "no connection to the executor"; + } if (_return.status.code != 0) { - cserror() << "Start contract executor error code " << int(_return.status.code) << ": " << _return.status.message; + cserror() << "Start contract executor error code " << int(_return.status.code) << ": " << _return.status.message + << " (no answer within " << waited << " ms)"; } else { - cslog() << "Start contract executor: " << _return.status.message << ". Executor build number " << _return.commitNumber; + cslog() << "Start contract executor: " << _return.status.message << ". Executor build number " << _return.commitNumber + << ", answered after " << waited << " ms"; } result = _return.commitNumber < commitMin_ || (_return.commitNumber > commitMax_ && commitMax_ != -1); diff --git a/docs/CHANGES.md b/docs/CHANGES.md index b39102762..36d154ea8 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -37,6 +37,7 @@ lines under the new release. | #67 | csdb, ci, docs: conversion follow-ups from the review of #65 (contract-records message, keep a copy of `db/` before converting from berkeleydb, sturdier db-convert scenario) | | #70 | csnode: bootstrap tables of 9 to 25 confidants are accepted (the official node silently drops them, so a network of 9 or more initial confidants never starts); the same choice as the official node up to 8 | | #71 | csnode: a bootstrap round table is taken only for a round next to the node's own, and, unless signed with the blockchain key, only once the node's own round has stalled; the block-removal request it set (never effective) is gone | +| #73 | api: the executor version check asks every second up to `executor_check_version_delay` instead of once after it (the node starts as soon as its executor answers); with `executor_multi_instance` a stopping node no longer kills the other nodes' executors | ## community.9 — 2026-10-22 diff --git a/lib/include/lib/system/polluntil.hpp b/lib/include/lib/system/polluntil.hpp new file mode 100644 index 000000000..f64b46823 --- /dev/null +++ b/lib/include/lib/system/polluntil.hpp @@ -0,0 +1,33 @@ +#ifndef POLLUNTIL_HPP +#define POLLUNTIL_HPP + +#include +#include +#include + +namespace cs { + +// Calls probe() now and then every step until it returns true, the time is up or stop() returns true. +// The last call is made at or after the deadline, so a probe that succeeds only at the very end still +// counts. Returns the result of the last call (false if stop() ended it before any success). +template +bool pollUntil(std::chrono::milliseconds total, std::chrono::milliseconds step, Probe probe, Stop stop) { + const auto deadline = std::chrono::steady_clock::now() + total; + for (;;) { + if (stop()) { + return false; + } + if (probe()) { + return true; + } + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + return false; + } + std::this_thread::sleep_for(std::min(step, std::chrono::duration_cast(deadline - now))); + } +} + +} // namespace cs + +#endif // POLLUNTIL_HPP diff --git a/tests/csnode/polluntil_tests.cpp b/tests/csnode/polluntil_tests.cpp new file mode 100644 index 000000000..76f8af247 --- /dev/null +++ b/tests/csnode/polluntil_tests.cpp @@ -0,0 +1,45 @@ +#include "gtest/gtest.h" + +#include + +#include + +using namespace std::chrono_literals; +using cs::pollUntil; + +namespace { +auto never = [] { return false; }; +} // namespace + +// the executor answers on the third attempt: no need to wait for the whole delay +TEST(PollUntil, StopsAtTheFirstSuccess) { + int calls = 0; + const auto start = std::chrono::steady_clock::now(); + EXPECT_TRUE(pollUntil(5000ms, 20ms, [&] { return ++calls == 3; }, never)); + EXPECT_EQ(calls, 3); + EXPECT_LT(std::chrono::steady_clock::now() - start, 1000ms); +} + +// never answers: tried until the deadline, the last attempt at or after it +TEST(PollUntil, LastAttemptAtTheDeadline) { + int calls = 0; + std::chrono::steady_clock::time_point last; + const auto start = std::chrono::steady_clock::now(); + EXPECT_FALSE(pollUntil(250ms, 100ms, [&] { ++calls; last = std::chrono::steady_clock::now(); return false; }, never)); + EXPECT_GE(calls, 3); + EXPECT_LE(calls, 6); + EXPECT_GE(last - start, 250ms); +} + +TEST(PollUntil, AStopEndsItEarly) { + int calls = 0; + EXPECT_FALSE(pollUntil(5000ms, 10ms, [&] { ++calls; return false; }, [&] { return calls >= 2; })); + EXPECT_EQ(calls, 2); +} + +TEST(PollUntil, NoDelayMeansOneAttempt) { + int calls = 0; + EXPECT_FALSE(pollUntil(0ms, 100ms, [&] { ++calls; return false; }, never)); + EXPECT_EQ(calls, 1); + EXPECT_TRUE(pollUntil(0ms, 100ms, [] { return true; }, never)); +} From 4432d2d25d2df93c7dba721a715a646ee6894aba Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 22:46:00 +0200 Subject: [PATCH 2/3] lib: pollUntil rounds the last wait up, or the last millisecond spins Co-Authored-By: Claude Opus 5.5 --- lib/include/lib/system/polluntil.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/include/lib/system/polluntil.hpp b/lib/include/lib/system/polluntil.hpp index f64b46823..ca7cf17ef 100644 --- a/lib/include/lib/system/polluntil.hpp +++ b/lib/include/lib/system/polluntil.hpp @@ -24,7 +24,8 @@ bool pollUntil(std::chrono::milliseconds total, std::chrono::milliseconds step, if (now >= deadline) { return false; } - std::this_thread::sleep_for(std::min(step, std::chrono::duration_cast(deadline - now))); + // rounded up: rounded down, the last millisecond would spin with zero-length sleeps + std::this_thread::sleep_for(std::min(step, std::chrono::ceil(deadline - now))); } } From 0cbf0de1def263bd13257b233d93aba425f91e8d Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 00:16:52 +0200 Subject: [PATCH 3/3] =?UTF-8?q?api:=20review=20of=20#73=20=E2=80=94=20a=20?= =?UTF-8?q?transport=20error=20is=20told=20apart=20from=20an=20executor's?= =?UTF-8?q?=20answer=20in=20the=20version=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getExecutorBuildVersion closes the connection on every exception, so 'still connected' never told an answer from a transport failure: it now sets ThriftException for a transport error (as execute() does), the poll retries only that, and the log says whether the executor never answered or answered with an error. Co-Authored-By: Claude Opus 5.5 --- api/src/executor.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api/src/executor.cpp b/api/src/executor.cpp index dbf30a375..ff7c78471 100644 --- a/api/src/executor.cpp +++ b/api/src/executor.cpp @@ -170,7 +170,8 @@ void cs::Executor::getExecutorBuildVersion(executor::ExecutorBuildVersionResult& // could still arrive on this connection: replace it now, not on the next call recreateOriginExecutor(); - _return.status.code = 1; + // the executor did not answer: told apart from an answer, as execute() does + _return.status.code = cs::error::ThriftException; _return.status.message = x.what(); notifyError(); @@ -781,10 +782,9 @@ void cs::Executor::checkExecutorVersion() { return false; } getExecutorBuildVersion(_return); - // an answer, or an error with the connection still open (the executor answered - // with an error, e.g. an unknown method), is final; a transport error closed the - // connection: the executor is not listening yet - return _return.status.code == 0 || isConnected(); + // any answer is final, an error one too (e.g. an unknown method); a transport + // error means the executor is not listening yet + return _return.status.code != cs::error::ThriftException; }, [&] { return requestStop_.load(); }); @@ -794,12 +794,12 @@ void cs::Executor::checkExecutorVersion() { const auto waited = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); if (_return.status.code == 0 && !isConnected()) { - _return.status.code = 1; + _return.status.code = cs::error::ThriftException; _return.status.message = "no connection to the executor"; } if (_return.status.code != 0) { cserror() << "Start contract executor error code " << int(_return.status.code) << ": " << _return.status.message - << " (no answer within " << waited << " ms)"; + << (_return.status.code == cs::error::ThriftException ? " (no answer within " : " (after ") << waited << " ms)"; } else { cslog() << "Start contract executor: " << _return.status.message << ". Executor build number " << _return.commitNumber