Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions .github/actions/contract-executor/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
name: Contract executor
description: >
The contract executor shipped with the official mainnet release 4.2.532.0 (the one operators
run), with the JDK it runs on (ojdkbuild 11.0.4, which ci/testnet/run.sh takes from the release
folder). Sets EXECUTOR_JAR. Needs a checkout of this repository first.
outputs:
jar:
description: Path of the contract-executor jar
value: ${{ steps.locate.outputs.jar }}
runs:
using: composite
steps:
# its sources no longer build as published: unreleased sc-api/thrift-api-client versions and
# Eclipse version ranges that now resolve to Java 17 jars
- name: Restore contract executor
id: cache-executor
uses: actions/cache@v4
with:
path: executor
key: contract-executor-mainnet-4.2.532.0-v2

- name: Fetch contract executor
if: steps.cache-executor.outputs.cache-hit != 'true'
shell: bash
run: |
curl -sSfL -o release.tar.gz \
https://github.com/CREDITSCOM/node/releases/download/mainnet-4.2.532.0/Ubuntu20_Mainnet-4.2.532.0.tar.gz
mkdir -p executor
tar -xzf release.tar.gz -C executor
rm -rf executor/csnode/javafx-sdk-13
rm release.tar.gz
ls -la executor/csnode | grep -v '\.log$' | head -40
unzip -p executor/csnode/contract-executor.jar META-INF/MANIFEST.MF | head -20

# a fallback only: run.sh uses the release's ojdkbuild JDK when it finds it next to the jar
- name: Set up Java 11
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: '11'

- name: Locate contract executor
id: locate
shell: bash
run: |
jar=$(find "$PWD/executor" -type f -name '*contract-executor*.jar' | head -1)
test -n "$jar"
echo "EXECUTOR_JAR=$jar" >> "$GITHUB_ENV"
echo "jar=$jar" >> "$GITHUB_OUTPUT"
echo "executor jar: $jar"
32 changes: 31 additions & 1 deletion .github/workflows/robustness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ name: Robustness
# - a pruned node killed while it prunes;
# - every block database, full and pruned, in one network, also with crashes and a power cut;
# - packet loss, corruption, delay, reordering and a partition on some nodes, slow nodes, random
# datagrams, and all of them at once.
# datagrams, and all of them at once;
# - smart contracts across crashes, a power cut, packet loss, a pruned node killed and a node joining,
# and a database conversion.
# All scenarios run funded transfers and the usual restart test.
on:
# a robustness-* tag runs it on the tagged commit (a manual run needs the workflow on the default branch)
Expand Down Expand Up @@ -272,6 +274,30 @@ jobs:
env: DB_BACKEND=lmdb PRUNE_NODE=6 STORAGE_SWITCH_AT=2300 TARGET_SEQ=3300
- name: db-convert
env: DB_BACKEND=berkeleydb CONVERT_AT=1500 TARGET_SEQ=2500
# smart contracts (each node with its executor, contracts.py through n1) across the faults:
# six SIGKILLs of n6 and its executor while contracts are deployed and called
- name: contracts-crash
env: DB_BACKEND=lmdb CRASH_NODE=6 CONTRACTS=1 TARGET_SEQ=1600
contracts: true
# every node and executor killed after the first calls, calls again after the restart
- name: contracts-power-cut
env: DB_BACKEND=mixed CRASH_ALL_AT=600 CONTRACTS=1 CONTRACTS_AFTER_AT=700 TARGET_SEQ=1300
contracts: true
- name: contracts-net-loss
env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 CONTRACTS=1
contracts: true
# a pruned node quick-starting from checkpoints that hold the contract states
- name: contracts-pruned-term-kill
env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=3000
contracts: true
# a node joining from block 0 past the deploys, then following live calls
- name: contracts-pruned-join
env: DB_BACKEND=lmdb PRUNE_NODE=6 JOIN_PRUNED_AT=500 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=2300
contracts: true
# block databases converted with contract records in them
- name: contracts-db-convert
env: DB_BACKEND=berkeleydb CONVERT_AT=1500 CONTRACTS=1 CONTRACTS_AFTER_AT=1700 TARGET_SEQ=2500
contracts: true
name: ${{ matrix.name }}
# a manual run can pick scenarios; the others stop at once
env:
Expand Down Expand Up @@ -301,6 +327,10 @@ jobs:
python3 -m pip install --user --quiet --break-system-packages thriftpy2
sudo sysctl -w kernel.core_pattern="${{ github.workspace }}/testnet-run/core.%e.%p"

- name: Contract executor
if: env.SELECTED == 'true' && matrix.contracts
uses: ./.github/actions/contract-executor

- name: Download node
if: env.SELECTED == 'true'
uses: actions/download-artifact@v4
Expand Down
40 changes: 3 additions & 37 deletions .github/workflows/testnet.yml
Original file line number Diff line number Diff line change
Expand Up @@ -179,44 +179,10 @@ jobs:
echo "BIN_ALT=$PWD/build-alt/client/node" >> "$GITHUB_ENV"
echo "node under test: $(git -C node-alt log -1 --format='%h %s')"

# the contract executor shipped with the official mainnet release (the one operators run); its
# sources no longer build as published: unreleased sc-api/thrift-api-client versions and
# Eclipse version ranges that now resolve to Java 17 jars
- name: Restore contract executor
# the contract executor of the official mainnet release (the one operators run)
- name: Contract executor
if: inputs.contracts == '1'
id: cache-executor
uses: actions/cache@v4
with:
path: executor
key: contract-executor-mainnet-4.2.532.0-v2

- name: Fetch contract executor
if: inputs.contracts == '1' && steps.cache-executor.outputs.cache-hit != 'true'
run: |
curl -sSfL -o release.tar.gz \
https://github.com/CREDITSCOM/node/releases/download/mainnet-4.2.532.0/Ubuntu20_Mainnet-4.2.532.0.tar.gz
mkdir -p executor
tar -xzf release.tar.gz -C executor
rm -rf executor/csnode/javafx-sdk-13
rm release.tar.gz
ls -la executor/csnode | grep -v '\.log$' | head -40
grep -ihE 'executor|jdk|java' executor/csnode/config.ini executor/csnode/*.sh executor/csnode/settings.properties 2>/dev/null | head -20
unzip -p executor/csnode/contract-executor.jar META-INF/MANIFEST.MF | head -20

- name: Set up Java 11
if: inputs.contracts == '1'
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: '11'

- name: Locate contract executor
if: inputs.contracts == '1'
run: |
jar=$(find "$PWD/executor" -type f -name '*contract-executor*.jar' | head -1)
test -n "$jar"
echo "EXECUTOR_JAR=$jar" >> "$GITHUB_ENV"
echo "executor jar: $jar"
uses: ./.github/actions/contract-executor

# mixed executors: the even nodes run an executor built from EnzinoBB/contract-executor, so a
# result that differs from the release executor's shows up as a failed contract consensus
Expand Down
33 changes: 27 additions & 6 deletions ci/testnet/contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

Usage: contracts.py <work_dir> <nodes> <master_seed_b58> [--calls-before 3] [--calls-after 5]
[--ready-file PATH] [--wait-file PATH] [--wait-timeout 1800]
[--check-nodes N] [--final-file PATH]

The master key deploys Counter, whose next() adds one to its state and returns it, and calls it
--calls-before times. It then creates --ready-file and, once --wait-file exists (run.sh creates
Expand All @@ -16,8 +17,14 @@
(Violations::SubsequentCall), and executor build 1518 fails an undeclared invokeExternalContract
with an AccessControlException (its configuration is created lazily inside the contract sandbox).

Transactions go through node 1's API (port 9090); the final check reads every node's API
(node i > 1 listens on 9100 + i).
Then, once --final-file exists (run.sh creates it when every node reached its target sequence),
it compares the contract states on nodes 1 to --check-nodes (default <nodes>; run.sh adds a node
that joined later).

Transactions go through node 1's API (port 9090), so a test scenario must never kill, partition or
slow node 1: a call during an outage ends the script. Only a transaction node 1 refuses because it
is not synchronized yet (after a power cut) is sent again, for up to 120 s: nothing was accepted.
The final check reads every node's API (node i > 1 listens on 9100 + i).
"""
import argparse
import hashlib
Expand Down Expand Up @@ -218,9 +225,16 @@ def deploy(client, master, name, source):


def call(client, master, contract, method, label, expected=None):
next_inner_id(client, master)
sci = invocation(method=method)
result = client.TransactionFlow(smart_transaction(master, contract, sci, TT_EXECUTE))
deadline = time.time() + 120
while True:
next_inner_id(client, master)
sci = invocation(method=method)
result = client.TransactionFlow(smart_transaction(master, contract, sci, TT_EXECUTE))
# refused before it was accepted (api/src/apihandler.cpp "Node is not syncronized"): send it again
if result.status.code == 0 or "not syncronized" not in (result.status.message or "") or time.time() >= deadline:
break
print(f"{label}: node not synchronized yet, sending again", flush=True)
time.sleep(5)
value = result.smart_contract_result.v_int if result.smart_contract_result else None
ok = result.status.code == 0 and value == expected
outcome = ("ok" if ok else "FAILED") if expected is not None else "reported"
Expand All @@ -238,6 +252,8 @@ def main():
parser.add_argument("--ready-file", default="")
parser.add_argument("--wait-file", default="")
parser.add_argument("--wait-timeout", type=int, default=1800)
parser.add_argument("--check-nodes", type=int, default=0)
parser.add_argument("--final-file", default="")
args = parser.parse_args()

def client_for(i):
Expand Down Expand Up @@ -282,10 +298,15 @@ def call_callers(when, via):
call_callers("after restart", 1)
call_callers("after restart", 2)

if args.final_file:
deadline = time.time() + args.wait_timeout
while not os.path.exists(args.final_file) and time.time() < deadline:
time.sleep(5)
print(f"final file {'found' if os.path.exists(args.final_file) else 'MISSING'}", flush=True)
time.sleep(60) # let the last state reach every node
for name, contract in (("Counter", counter), ("Ledger", ledger)) + callers:
states = {}
for i in range(1, args.nodes + 1):
for i in range(1, (args.check_nodes or args.nodes) + 1):
try:
got = client_for(i).SmartContractGet(contract)
states[i] = hashlib.sha256(got.smartContract.objectState).hexdigest()[:16] if got.status.code == 0 else "error"
Expand Down
56 changes: 50 additions & 6 deletions ci/testnet/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@
# limit (DELEGATION_SECONDS, default 240), and some delegations are withdrawn later
# CONTRACTS (default 0, needs FUND=1 and EXECUTOR_JAR, the contract-executor jar): every node
# runs its own executor; contracts.py deploys a counter contract and calls it before and after
# the restart test, which waits for the first calls; all nodes must end with the same state
# the restart test, which waits for the first calls; all nodes must end with the same state.
# A power cut (CRASH_ALL_AT) also waits for the first calls. The calls after go once the
# restarted nodes, and the network after a power cut, are 30 blocks past it, and the lowest
# node reached CONTRACTS_AFTER_AT (default 0); the states are compared once every node,
# a joining one too, reached TARGET_SEQ. contracts.py sends everything through n1: a
# contract scenario never kills, partitions or slows n1
# EXECUTOR_JAR_ALT (optional, with CONTRACTS): the even nodes run this executor jar instead, on the
# same JDK, so any result that differs from the release executor's fails the contract consensus
# BIN_ALT (optional): the even nodes run this node binary instead, so a block or contract result
Expand Down Expand Up @@ -136,6 +141,9 @@ MIN_STAKE=${MIN_STAKE:-0}
SYNCRO_ROUND=${SYNCRO_ROUND:-0}
JOIN_FULL=${JOIN_FULL:-0}
CONTRACTS=${CONTRACTS:-0}
CONTRACTS_AFTER_AT=${CONTRACTS_AFTER_AT:-0}
[ "$CONTRACTS_AFTER_AT" -eq 0 ] || [ "$CONTRACTS_AFTER_AT" -le $(( ${TARGET_SEQ:-1100} - 100 )) ] \
|| { echo "CONTRACTS_AFTER_AT must be at most TARGET_SEQ - 100"; exit 1; }
PRUNE_NODE=${PRUNE_NODE:-0}
NODE_STORAGE=${NODE_STORAGE:-}
DB_BACKEND=${DB_BACKEND:-}
Expand Down Expand Up @@ -451,6 +459,19 @@ digest_offline() {
}

# the lowest sequence among the nodes of the restart test
# a SIGKILLed node leaves its executor JVM running (it has no death signal), still bound to its port
kill_executor() {
pkill -KILL -f "cs.testnet.node=n$1 " 2>/dev/null || true
}

# the calls after the restart (contracts.py --wait-file) go once the nodes stopped by the restart
# test and by a power cut are back and 30 blocks past it, and the lowest node is at CONTRACTS_AFTER_AT
contracts_go_ready() {
{ [ "$RESTART_AT" -eq 0 ] || { [ "$restarted" -eq 1 ] && [ "$(restarted_seq)" -ge $((stopped_at + 30)) ]; }; } \
&& { [ "$CRASH_ALL_AT" -eq 0 ] || { [ "$crashed_all" -eq 1 ] && [ "$min" -ge $((CRASH_ALL_AT + 30)) ]; }; } \
&& [ "$min" -ge "$CONTRACTS_AFTER_AT" ]
}

restarted_seq() {
local low="" s
for i in $RESTART_NODES; do
Expand Down Expand Up @@ -550,6 +571,7 @@ while :; do
echo "crash test: SIGKILL n$CRASH_NODE (kill $crashes of $CRASH_TIMES) at sequence $(seq_of "$CRASH_NODE"), newest checkpoint ${newest:-none}"
kill -KILL "${PIDS[$CRASH_NODE]}" 2>/dev/null || true
wait "${PIDS[$CRASH_NODE]}" 2>/dev/null || true
kill_executor "$CRASH_NODE"
sleep 5
start_node "$CRASH_NODE"
crash_wait_qs=""
Expand All @@ -558,14 +580,17 @@ while :; do
fi

# the whole network killed at once, as after a power cut
if [ "$CRASH_ALL_AT" -gt 0 ] && [ "$crashed_all" -eq 0 ] && [ "$min" -ge "$CRASH_ALL_AT" ]; then
# with contracts, only once the first calls are done: a cut during the deploys would end contracts.py
if [ "$CRASH_ALL_AT" -gt 0 ] && [ "$crashed_all" -eq 0 ] && [ "$min" -ge "$CRASH_ALL_AT" ] \
&& { [ "$CONTRACTS" != "1" ] || [ -e "$WORK/contracts-ready" ]; }; then
echo "crash test: SIGKILL every node at sequence $min"
for i in $(seq 1 "$ALL_NODES"); do
[ -n "${PIDS[$i]:-}" ] && kill -KILL "${PIDS[$i]}" 2>/dev/null || true
done
for i in $(seq 1 "$ALL_NODES"); do
[ -n "${PIDS[$i]:-}" ] && wait "${PIDS[$i]}" 2>/dev/null || true
done
pkill -KILL -f "cs.testnet.node=" 2>/dev/null || true
sleep 10
for i in $(seq 1 "$NODES"); do start_node "$i"; done
crashed_all=1
Expand Down Expand Up @@ -884,26 +909,45 @@ while :; do
if [ "$CONTRACTS" = "1" ] && [ -z "$CONTRACTS_PID" ] && grep -q "master transactions done" "$WORK/fund.log" 2>/dev/null; then
echo "contracts: deploying and calling them through node 1"
(cd "$HERE" && python3 contracts.py "$WORK" "$NODES" "$MASTER_SEED" \
--ready-file "$WORK/contracts-ready" --wait-file "$WORK/restart-done") > "$WORK/contracts.log" 2>&1 &
--ready-file "$WORK/contracts-ready" --wait-file "$WORK/restart-done" --wait-timeout $((TIMEOUT_MIN * 60)) \
--check-nodes "$ALL_NODES" --final-file "$WORK/contracts-final") > "$WORK/contracts.log" 2>&1 &
CONTRACTS_PID=$!
fi
# the calls after the restart must run on the restarted node too, so let it catch up first
if [ "$restarted" -eq 1 ] && [ ! -e "$WORK/restart-done" ] && [ "$(restarted_seq)" -ge $((stopped_at + 30)) ] 2>/dev/null; then
# the calls after the restart must run on the restarted nodes too, so let them catch up first
if [ "$CONTRACTS" = "1" ] && [ ! -e "$WORK/restart-done" ] && contracts_go_ready 2>/dev/null; then
touch "$WORK/restart-done"
fi

# with contracts, not before the calls after the restart were let go
if [ "$min" -ge "$TARGET_SEQ" ] && { [ "$JOIN_NODE" -eq 0 ] || [ "$join_seq" -ge "$TARGET_SEQ" ]; } \
&& { [ "$CONTRACTS" != "1" ] || ! kill -0 "${CONTRACTS_PID:-0}" 2>/dev/null; }; then break; fi
&& { [ "$CONTRACTS" != "1" ] || [ -e "$WORK/restart-done" ]; }; then break; fi
if [ "$(date +%s)" -ge "$deadline" ]; then
echo "FAIL: not every node reached sequence $TARGET_SEQ within $TIMEOUT_MIN min"
exit 1
fi
done

if [ -n "$CONTRACTS_PID" ]; then
# every node, a joining one too, is at TARGET_SEQ: contracts.py may compare the states now
touch "$WORK/contracts-final"
# its own deadline: TIMEOUT_MIN is for reaching the target, and contracts.py still waits 60 s
# and reads every node after it
final_deadline=$(( $(date +%s) + 600 ))
while kill -0 "$CONTRACTS_PID" 2>/dev/null && [ "$(date +%s)" -lt "$final_deadline" ]; do sleep 5; done
if kill -0 "$CONTRACTS_PID" 2>/dev/null; then
kill "$CONTRACTS_PID" 2>/dev/null || true
cat "$WORK/contracts.log"
echo "FAIL: contracts.py did not finish its final checks within 10 min of the target"
exit 1
fi
contracts_status=0
wait "$CONTRACTS_PID" || contracts_status=$?
cat "$WORK/contracts.log"
# an executor left running by a killed node keeps the port its successor needs
if grep -l 'Address already in use' "$WORK"/n*/executor.log 2>/dev/null; then
echo "FAIL: an executor could not bind its port (an orphan executor of a killed node)"
exit 1
fi
for i in $(seq 1 "$NODES"); do
echo "n$i executor: $(cat "$WORK/n$i"/log/*.txt 2>/dev/null | grep -ac . || true) log lines, $(grep -aci "SmartContractGet\|exception" "$WORK/n$i/node.log" 2>/dev/null || true) node log lines on SmartContractGet/exceptions"
done
Expand Down