From 9d802a2c19da72c802c2afd3646ecb667148c082 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Thu, 22 Oct 2026 22:37:24 +0200 Subject: [PATCH 1/4] ci(testnet): the contract load window starts after funding, writers spread over their period, block rates from the first block From the slowdown analysis of the scale runs: - load(): the window was set before funding N writer accounts (a block each), leaving ~100 s of 120 at 9 nodes and ~30 s at 51: the achieved rates fell with N for that reason only - the writers started together with the same period and stayed in lockstep: every write landed in one block with the others (51 calls executed one by one), which made write latency look like it grew from 3.6 s to 4.8 s at 51 nodes; they now start spread over the period, with jitter - sampler: per-interval block rates from the first block on (the start, with the executors' version check, is not a slowdown) Co-Authored-By: Claude Opus 5.5 --- ci/testnet/contracts.py | 10 ++++++++-- ci/testnet/sampler.py | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index 8f69e421c..06978c0c8 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -31,6 +31,7 @@ import hashlib import json import os +import random import statistics import struct import sys @@ -307,7 +308,6 @@ def load(args, client_for, master, counter, nodes): # others in a round; a rejected call is an answer, not a fault, and is counted on its own rejected = [0] lock = threading.Lock() - stop_at = time.time() + args.load_seconds writers = list(range(1, nodes + 1)) accounts = {i: Account(hashlib.sha256(master.pk + b"contract writer %d" % i).digest()) for i in writers} funder = Client(9090, 300) @@ -319,10 +319,16 @@ def load(args, client_for, master, counter, nodes): deadline = time.time() + 300 while time.time() < deadline and any(balance(funder, a.pk) < 1000 for a in accounts.values()): time.sleep(5) + # the load window starts once the writers are funded (funding N accounts takes a block each) + stop_at = time.time() + args.load_seconds def writer(i): account = accounts[i] period = len(writers) / args.write_rate if args.write_rate > 0 else None + if period: + # spread over the period, or every writer sends in the same block (one batch of N calls, + # executed one by one, instead of single calls) + time.sleep(period * (i - 1) / len(writers)) while period and time.time() < stop_at: t0 = time.time() ok = False @@ -347,7 +353,7 @@ def writer(i): errors["write"] += 1 if errors["write"] <= 5: print(f"load write via n{i} FAILED: {msg!r}", flush=True) - time.sleep(max(0.0, period - (time.time() - t0))) + time.sleep(max(0.0, period - (time.time() - t0)) * random.uniform(0.8, 1.2)) def reader(k): period = 1.0 / args.read_rate * 4 if args.read_rate > 0 else None diff --git a/ci/testnet/sampler.py b/ci/testnet/sampler.py index 4fff14c4e..f2b247324 100644 --- a/ci/testnet/sampler.py +++ b/ci/testnet/sampler.py @@ -175,7 +175,8 @@ def key(k): out.append(f" peak RSS of all nodes and executors together: {max(per_ts.values())} MB") prog = [(num(r["t"]), num(r["seq_min"]), num(r["seq_max"])) for r in host] - prog = [p for p in prog if None not in p] + # from the first block on: before it the nodes start (and wait for their executors) + prog = [p for p in prog if None not in p and p[2] > 0] if len(prog) > 1: overall = (prog[-1][2] - prog[0][2]) / max(prog[-1][0] - prog[0][0], 1) rates = [(b[2] - a[2]) / (b[0] - a[0]) for a, b in zip(prog, prog[1:]) if b[0] > a[0]] From 1dca8af862a97217df8b16647f07028a9390d1bf Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 02:06:03 +0200 Subject: [PATCH 2/4] docs: CHANGES.md, #75 Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index e6730bd8c..a460502e3 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -42,6 +42,7 @@ lines under the new release. | #72 | ci: contract reads on every node and a contract load phase; networks of up to 61 nodes with 5 to 25 initial confidants, stake-based election and nodes below the minimum stake; a scale workflow | | #73 | api: the executor version check asks every second up to `executor_check_version_delay` instead of once after it (the node starts as soon as its executor answers); with `executor_multi_instance` a stopping node no longer kills the other nodes' executors | | #74 | api: contract getters and the deployer's contract list take the deploy the smart contracts keep (a pruned node no longer has its block, and answered getters with an empty success); a getter with nothing to execute, or no executor, now fails | +| #75 | ci(testnet): the contract load window starts after funding its writers, the writers spread over their period, block rates from the first block (found in the slowdown analysis of the scale runs) | ## community.9 — 2026-10-22 From 1eb1e63bed22b8a54c3273993d2be4a1a6e3a191 Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 02:57:17 +0200 Subject: [PATCH 3/4] ci(testnet): writers start together again (contract calls in one block run in one cycle) Spreading the writers made every call land in its own block and queue behind the previous call's new state: write p50 12.6 s at 9 nodes and 37 s at 51 (51 refused), against 3.0 and 4.8 s in step. One contract runs one execution cycle at a time (about 6 rounds); calls of the same block share it. The load window after funding and the block rates from the first block stay. Co-Authored-By: Claude Opus 5.5 --- ci/testnet/contracts.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index c47b0d4d2..79ab65ddf 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -32,7 +32,6 @@ import hashlib import json import os -import random import statistics import struct import sys @@ -329,10 +328,10 @@ def load(args, client_for, master, counter, nodes, read_nodes): def writer(i): account = accounts[i] period = len(writers) / args.write_rate if args.write_rate > 0 else None - if period: - # spread over the period, or every writer sends in the same block (one batch of N calls, - # executed one by one, instead of single calls) - time.sleep(period * (i - 1) / len(writers)) + # The writers start together and stay in step: their calls land in one block and run in one + # execution cycle of the contract. Spread over the period they land in separate blocks, and + # every call waits for the previous one's new state (about 6 rounds): with 2 calls a second + # the queue grows (scale run 39876040185: write p50 12.6 s at 9 nodes, 37 s at 51) while period and time.time() < stop_at: t0 = time.time() ok = False @@ -357,7 +356,7 @@ def writer(i): errors["write"] += 1 if errors["write"] <= 5: print(f"load write via n{i} FAILED: {msg!r}", flush=True) - time.sleep(max(0.0, period - (time.time() - t0)) * random.uniform(0.8, 1.2)) + time.sleep(max(0.0, period - (time.time() - t0))) def reader(k): period = 1.0 / args.read_rate * 4 if args.read_rate > 0 else None From d4f59039e989965f113a7b009a0c7c5c6f45ba6e Mon Sep 17 00:00:00 2001 From: EnzinoBB Date: Fri, 23 Oct 2026 02:57:27 +0200 Subject: [PATCH 4/4] docs: CHANGES.md, #75 without the writer spread Co-Authored-By: Claude Opus 5.5 --- docs/CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index a460502e3..e0203040a 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -42,7 +42,7 @@ lines under the new release. | #72 | ci: contract reads on every node and a contract load phase; networks of up to 61 nodes with 5 to 25 initial confidants, stake-based election and nodes below the minimum stake; a scale workflow | | #73 | api: the executor version check asks every second up to `executor_check_version_delay` instead of once after it (the node starts as soon as its executor answers); with `executor_multi_instance` a stopping node no longer kills the other nodes' executors | | #74 | api: contract getters and the deployer's contract list take the deploy the smart contracts keep (a pruned node no longer has its block, and answered getters with an empty success); a getter with nothing to execute, or no executor, now fails | -| #75 | ci(testnet): the contract load window starts after funding its writers, the writers spread over their period, block rates from the first block (found in the slowdown analysis of the scale runs) | +| #75 | ci(testnet): the contract load window starts after funding its writers, block rates from the first block (found in the slowdown analysis of the scale runs) | ## community.9 — 2026-10-22