diff --git a/.github/actions/contract-executor/action.yml b/.github/actions/contract-executor/action.yml new file mode 100644 index 000000000..f2bb9c0d0 --- /dev/null +++ b/.github/actions/contract-executor/action.yml @@ -0,0 +1,50 @@ +name: Contract executor +description: > + The contract executor shipped with the official mainnet release 4.2.532.0 (the one operators + run), with the JDK it runs on (ojdkbuild 11.0.4, which ci/testnet/run.sh takes from the release + folder). Sets EXECUTOR_JAR. Needs a checkout of this repository first. +outputs: + jar: + description: Path of the contract-executor jar + value: ${{ steps.locate.outputs.jar }} +runs: + using: composite + steps: + # its sources no longer build as published: unreleased sc-api/thrift-api-client versions and + # Eclipse version ranges that now resolve to Java 17 jars + - name: Restore contract executor + id: cache-executor + uses: actions/cache@v4 + with: + path: executor + key: contract-executor-mainnet-4.2.532.0-v2 + + - name: Fetch contract executor + if: steps.cache-executor.outputs.cache-hit != 'true' + shell: bash + run: | + curl -sSfL -o release.tar.gz \ + https://github.com/CREDITSCOM/node/releases/download/mainnet-4.2.532.0/Ubuntu20_Mainnet-4.2.532.0.tar.gz + mkdir -p executor + tar -xzf release.tar.gz -C executor + rm -rf executor/csnode/javafx-sdk-13 + rm release.tar.gz + ls -la executor/csnode | grep -v '\.log$' | head -40 + unzip -p executor/csnode/contract-executor.jar META-INF/MANIFEST.MF | head -20 + + # a fallback only: run.sh uses the release's ojdkbuild JDK when it finds it next to the jar + - name: Set up Java 11 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '11' + + - name: Locate contract executor + id: locate + shell: bash + run: | + jar=$(find "$PWD/executor" -type f -name '*contract-executor*.jar' | head -1) + test -n "$jar" + echo "EXECUTOR_JAR=$jar" >> "$GITHUB_ENV" + echo "jar=$jar" >> "$GITHUB_OUTPUT" + echo "executor jar: $jar" diff --git a/.github/workflows/robustness.yml b/.github/workflows/robustness.yml index bdea659f6..782fff53f 100644 --- a/.github/workflows/robustness.yml +++ b/.github/workflows/robustness.yml @@ -11,7 +11,9 @@ name: Robustness # - a pruned node killed while it prunes; # - every block database, full and pruned, in one network, also with crashes and a power cut; # - packet loss, corruption, delay, reordering and a partition on some nodes, slow nodes, random -# datagrams, and all of them at once. +# datagrams, and all of them at once; +# - smart contracts across crashes, a power cut, packet loss, a pruned node killed and a node joining, +# and a database conversion. # All scenarios run funded transfers and the usual restart test. on: # a robustness-* tag runs it on the tagged commit (a manual run needs the workflow on the default branch) @@ -272,6 +274,30 @@ jobs: env: DB_BACKEND=lmdb PRUNE_NODE=6 STORAGE_SWITCH_AT=2300 TARGET_SEQ=3300 - name: db-convert env: DB_BACKEND=berkeleydb CONVERT_AT=1500 TARGET_SEQ=2500 + # smart contracts (each node with its executor, contracts.py through n1) across the faults: + # six SIGKILLs of n6 and its executor while contracts are deployed and called + - name: contracts-crash + env: DB_BACKEND=lmdb CRASH_NODE=6 CONTRACTS=1 TARGET_SEQ=1600 + contracts: true + # every node and executor killed after the first calls, calls again after the restart + - name: contracts-power-cut + env: DB_BACKEND=mixed CRASH_ALL_AT=600 CONTRACTS=1 CONTRACTS_AFTER_AT=700 TARGET_SEQ=1300 + contracts: true + - name: contracts-net-loss + env: NET_FAULT=loss NET_FAULT_PCT=15 NET_FAULT_NODES=3,5 CONTRACTS=1 + contracts: true + # a pruned node quick-starting from checkpoints that hold the contract states + - name: contracts-pruned-term-kill + env: DB_BACKEND=lmdb PRUNE_NODE=6 TERM_KILL_NODE=6 TERM_KILL_FROM=2000 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=3000 + contracts: true + # a node joining from block 0 past the deploys, then following live calls + - name: contracts-pruned-join + env: DB_BACKEND=lmdb PRUNE_NODE=6 JOIN_PRUNED_AT=500 CONTRACTS=1 CONTRACTS_AFTER_AT=2100 TARGET_SEQ=2300 + contracts: true + # block databases converted with contract records in them + - name: contracts-db-convert + env: DB_BACKEND=berkeleydb CONVERT_AT=1500 CONTRACTS=1 CONTRACTS_AFTER_AT=1700 TARGET_SEQ=2500 + contracts: true name: ${{ matrix.name }} # a manual run can pick scenarios; the others stop at once env: @@ -301,6 +327,10 @@ jobs: python3 -m pip install --user --quiet --break-system-packages thriftpy2 sudo sysctl -w kernel.core_pattern="${{ github.workspace }}/testnet-run/core.%e.%p" + - name: Contract executor + if: env.SELECTED == 'true' && matrix.contracts + uses: ./.github/actions/contract-executor + - name: Download node if: env.SELECTED == 'true' uses: actions/download-artifact@v4 diff --git a/.github/workflows/testnet.yml b/.github/workflows/testnet.yml index 0b99cdbf6..efeb6724c 100644 --- a/.github/workflows/testnet.yml +++ b/.github/workflows/testnet.yml @@ -179,44 +179,10 @@ jobs: echo "BIN_ALT=$PWD/build-alt/client/node" >> "$GITHUB_ENV" echo "node under test: $(git -C node-alt log -1 --format='%h %s')" - # the contract executor shipped with the official mainnet release (the one operators run); its - # sources no longer build as published: unreleased sc-api/thrift-api-client versions and - # Eclipse version ranges that now resolve to Java 17 jars - - name: Restore contract executor + # the contract executor of the official mainnet release (the one operators run) + - name: Contract executor if: inputs.contracts == '1' - id: cache-executor - uses: actions/cache@v4 - with: - path: executor - key: contract-executor-mainnet-4.2.532.0-v2 - - - name: Fetch contract executor - if: inputs.contracts == '1' && steps.cache-executor.outputs.cache-hit != 'true' - run: | - curl -sSfL -o release.tar.gz \ - https://github.com/CREDITSCOM/node/releases/download/mainnet-4.2.532.0/Ubuntu20_Mainnet-4.2.532.0.tar.gz - mkdir -p executor - tar -xzf release.tar.gz -C executor - rm -rf executor/csnode/javafx-sdk-13 - rm release.tar.gz - ls -la executor/csnode | grep -v '\.log$' | head -40 - grep -ihE 'executor|jdk|java' executor/csnode/config.ini executor/csnode/*.sh executor/csnode/settings.properties 2>/dev/null | head -20 - unzip -p executor/csnode/contract-executor.jar META-INF/MANIFEST.MF | head -20 - - - name: Set up Java 11 - if: inputs.contracts == '1' - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: '11' - - - name: Locate contract executor - if: inputs.contracts == '1' - run: | - jar=$(find "$PWD/executor" -type f -name '*contract-executor*.jar' | head -1) - test -n "$jar" - echo "EXECUTOR_JAR=$jar" >> "$GITHUB_ENV" - echo "executor jar: $jar" + uses: ./.github/actions/contract-executor # mixed executors: the even nodes run an executor built from EnzinoBB/contract-executor, so a # result that differs from the release executor's shows up as a failed contract consensus diff --git a/ci/testnet/contracts.py b/ci/testnet/contracts.py index c564ad788..fa732d246 100644 --- a/ci/testnet/contracts.py +++ b/ci/testnet/contracts.py @@ -3,6 +3,7 @@ Usage: contracts.py [--calls-before 3] [--calls-after 5] [--ready-file PATH] [--wait-file PATH] [--wait-timeout 1800] + [--check-nodes N] [--final-file PATH] The master key deploys Counter, whose next() adds one to its state and returns it, and calls it --calls-before times. It then creates --ready-file and, once --wait-file exists (run.sh creates @@ -16,8 +17,14 @@ (Violations::SubsequentCall), and executor build 1518 fails an undeclared invokeExternalContract with an AccessControlException (its configuration is created lazily inside the contract sandbox). -Transactions go through node 1's API (port 9090); the final check reads every node's API -(node i > 1 listens on 9100 + i). +Then, once --final-file exists (run.sh creates it when every node reached its target sequence), +it compares the contract states on nodes 1 to --check-nodes (default ; run.sh adds a node +that joined later). + +Transactions go through node 1's API (port 9090), so a test scenario must never kill, partition or +slow node 1: a call during an outage ends the script. Only a transaction node 1 refuses because it +is not synchronized yet (after a power cut) is sent again, for up to 120 s: nothing was accepted. +The final check reads every node's API (node i > 1 listens on 9100 + i). """ import argparse import hashlib @@ -218,9 +225,16 @@ def deploy(client, master, name, source): def call(client, master, contract, method, label, expected=None): - next_inner_id(client, master) - sci = invocation(method=method) - result = client.TransactionFlow(smart_transaction(master, contract, sci, TT_EXECUTE)) + deadline = time.time() + 120 + while True: + next_inner_id(client, master) + sci = invocation(method=method) + result = client.TransactionFlow(smart_transaction(master, contract, sci, TT_EXECUTE)) + # refused before it was accepted (api/src/apihandler.cpp "Node is not syncronized"): send it again + if result.status.code == 0 or "not syncronized" not in (result.status.message or "") or time.time() >= deadline: + break + print(f"{label}: node not synchronized yet, sending again", flush=True) + time.sleep(5) value = result.smart_contract_result.v_int if result.smart_contract_result else None ok = result.status.code == 0 and value == expected outcome = ("ok" if ok else "FAILED") if expected is not None else "reported" @@ -238,6 +252,8 @@ def main(): parser.add_argument("--ready-file", default="") parser.add_argument("--wait-file", default="") parser.add_argument("--wait-timeout", type=int, default=1800) + parser.add_argument("--check-nodes", type=int, default=0) + parser.add_argument("--final-file", default="") args = parser.parse_args() def client_for(i): @@ -282,10 +298,15 @@ def call_callers(when, via): call_callers("after restart", 1) call_callers("after restart", 2) + if args.final_file: + deadline = time.time() + args.wait_timeout + while not os.path.exists(args.final_file) and time.time() < deadline: + time.sleep(5) + print(f"final file {'found' if os.path.exists(args.final_file) else 'MISSING'}", flush=True) time.sleep(60) # let the last state reach every node for name, contract in (("Counter", counter), ("Ledger", ledger)) + callers: states = {} - for i in range(1, args.nodes + 1): + for i in range(1, (args.check_nodes or args.nodes) + 1): try: got = client_for(i).SmartContractGet(contract) states[i] = hashlib.sha256(got.smartContract.objectState).hexdigest()[:16] if got.status.code == 0 else "error" diff --git a/ci/testnet/run.sh b/ci/testnet/run.sh index 76b4bcaf1..69b7c028d 100755 --- a/ci/testnet/run.sh +++ b/ci/testnet/run.sh @@ -22,7 +22,12 @@ # limit (DELEGATION_SECONDS, default 240), and some delegations are withdrawn later # CONTRACTS (default 0, needs FUND=1 and EXECUTOR_JAR, the contract-executor jar): every node # runs its own executor; contracts.py deploys a counter contract and calls it before and after -# the restart test, which waits for the first calls; all nodes must end with the same state +# the restart test, which waits for the first calls; all nodes must end with the same state. +# A power cut (CRASH_ALL_AT) also waits for the first calls. The calls after go once the +# restarted nodes, and the network after a power cut, are 30 blocks past it, and the lowest +# node reached CONTRACTS_AFTER_AT (default 0); the states are compared once every node, +# a joining one too, reached TARGET_SEQ. contracts.py sends everything through n1: a +# contract scenario never kills, partitions or slows n1 # EXECUTOR_JAR_ALT (optional, with CONTRACTS): the even nodes run this executor jar instead, on the # same JDK, so any result that differs from the release executor's fails the contract consensus # BIN_ALT (optional): the even nodes run this node binary instead, so a block or contract result @@ -136,6 +141,9 @@ MIN_STAKE=${MIN_STAKE:-0} SYNCRO_ROUND=${SYNCRO_ROUND:-0} JOIN_FULL=${JOIN_FULL:-0} CONTRACTS=${CONTRACTS:-0} +CONTRACTS_AFTER_AT=${CONTRACTS_AFTER_AT:-0} +[ "$CONTRACTS_AFTER_AT" -eq 0 ] || [ "$CONTRACTS_AFTER_AT" -le $(( ${TARGET_SEQ:-1100} - 100 )) ] \ + || { echo "CONTRACTS_AFTER_AT must be at most TARGET_SEQ - 100"; exit 1; } PRUNE_NODE=${PRUNE_NODE:-0} NODE_STORAGE=${NODE_STORAGE:-} DB_BACKEND=${DB_BACKEND:-} @@ -451,6 +459,19 @@ digest_offline() { } # the lowest sequence among the nodes of the restart test +# a SIGKILLed node leaves its executor JVM running (it has no death signal), still bound to its port +kill_executor() { + pkill -KILL -f "cs.testnet.node=n$1 " 2>/dev/null || true +} + +# the calls after the restart (contracts.py --wait-file) go once the nodes stopped by the restart +# test and by a power cut are back and 30 blocks past it, and the lowest node is at CONTRACTS_AFTER_AT +contracts_go_ready() { + { [ "$RESTART_AT" -eq 0 ] || { [ "$restarted" -eq 1 ] && [ "$(restarted_seq)" -ge $((stopped_at + 30)) ]; }; } \ + && { [ "$CRASH_ALL_AT" -eq 0 ] || { [ "$crashed_all" -eq 1 ] && [ "$min" -ge $((CRASH_ALL_AT + 30)) ]; }; } \ + && [ "$min" -ge "$CONTRACTS_AFTER_AT" ] +} + restarted_seq() { local low="" s for i in $RESTART_NODES; do @@ -550,6 +571,7 @@ while :; do echo "crash test: SIGKILL n$CRASH_NODE (kill $crashes of $CRASH_TIMES) at sequence $(seq_of "$CRASH_NODE"), newest checkpoint ${newest:-none}" kill -KILL "${PIDS[$CRASH_NODE]}" 2>/dev/null || true wait "${PIDS[$CRASH_NODE]}" 2>/dev/null || true + kill_executor "$CRASH_NODE" sleep 5 start_node "$CRASH_NODE" crash_wait_qs="" @@ -558,7 +580,9 @@ while :; do fi # the whole network killed at once, as after a power cut - if [ "$CRASH_ALL_AT" -gt 0 ] && [ "$crashed_all" -eq 0 ] && [ "$min" -ge "$CRASH_ALL_AT" ]; then + # with contracts, only once the first calls are done: a cut during the deploys would end contracts.py + if [ "$CRASH_ALL_AT" -gt 0 ] && [ "$crashed_all" -eq 0 ] && [ "$min" -ge "$CRASH_ALL_AT" ] \ + && { [ "$CONTRACTS" != "1" ] || [ -e "$WORK/contracts-ready" ]; }; then echo "crash test: SIGKILL every node at sequence $min" for i in $(seq 1 "$ALL_NODES"); do [ -n "${PIDS[$i]:-}" ] && kill -KILL "${PIDS[$i]}" 2>/dev/null || true @@ -566,6 +590,7 @@ while :; do for i in $(seq 1 "$ALL_NODES"); do [ -n "${PIDS[$i]:-}" ] && wait "${PIDS[$i]}" 2>/dev/null || true done + pkill -KILL -f "cs.testnet.node=" 2>/dev/null || true sleep 10 for i in $(seq 1 "$NODES"); do start_node "$i"; done crashed_all=1 @@ -884,16 +909,18 @@ while :; do if [ "$CONTRACTS" = "1" ] && [ -z "$CONTRACTS_PID" ] && grep -q "master transactions done" "$WORK/fund.log" 2>/dev/null; then echo "contracts: deploying and calling them through node 1" (cd "$HERE" && python3 contracts.py "$WORK" "$NODES" "$MASTER_SEED" \ - --ready-file "$WORK/contracts-ready" --wait-file "$WORK/restart-done") > "$WORK/contracts.log" 2>&1 & + --ready-file "$WORK/contracts-ready" --wait-file "$WORK/restart-done" --wait-timeout $((TIMEOUT_MIN * 60)) \ + --check-nodes "$ALL_NODES" --final-file "$WORK/contracts-final") > "$WORK/contracts.log" 2>&1 & CONTRACTS_PID=$! fi - # the calls after the restart must run on the restarted node too, so let it catch up first - if [ "$restarted" -eq 1 ] && [ ! -e "$WORK/restart-done" ] && [ "$(restarted_seq)" -ge $((stopped_at + 30)) ] 2>/dev/null; then + # the calls after the restart must run on the restarted nodes too, so let them catch up first + if [ "$CONTRACTS" = "1" ] && [ ! -e "$WORK/restart-done" ] && contracts_go_ready 2>/dev/null; then touch "$WORK/restart-done" fi + # with contracts, not before the calls after the restart were let go if [ "$min" -ge "$TARGET_SEQ" ] && { [ "$JOIN_NODE" -eq 0 ] || [ "$join_seq" -ge "$TARGET_SEQ" ]; } \ - && { [ "$CONTRACTS" != "1" ] || ! kill -0 "${CONTRACTS_PID:-0}" 2>/dev/null; }; then break; fi + && { [ "$CONTRACTS" != "1" ] || [ -e "$WORK/restart-done" ]; }; then break; fi if [ "$(date +%s)" -ge "$deadline" ]; then echo "FAIL: not every node reached sequence $TARGET_SEQ within $TIMEOUT_MIN min" exit 1 @@ -901,9 +928,26 @@ while :; do done if [ -n "$CONTRACTS_PID" ]; then + # every node, a joining one too, is at TARGET_SEQ: contracts.py may compare the states now + touch "$WORK/contracts-final" + # its own deadline: TIMEOUT_MIN is for reaching the target, and contracts.py still waits 60 s + # and reads every node after it + final_deadline=$(( $(date +%s) + 600 )) + while kill -0 "$CONTRACTS_PID" 2>/dev/null && [ "$(date +%s)" -lt "$final_deadline" ]; do sleep 5; done + if kill -0 "$CONTRACTS_PID" 2>/dev/null; then + kill "$CONTRACTS_PID" 2>/dev/null || true + cat "$WORK/contracts.log" + echo "FAIL: contracts.py did not finish its final checks within 10 min of the target" + exit 1 + fi contracts_status=0 wait "$CONTRACTS_PID" || contracts_status=$? cat "$WORK/contracts.log" + # an executor left running by a killed node keeps the port its successor needs + if grep -l 'Address already in use' "$WORK"/n*/executor.log 2>/dev/null; then + echo "FAIL: an executor could not bind its port (an orphan executor of a killed node)" + exit 1 + fi for i in $(seq 1 "$NODES"); do echo "n$i executor: $(cat "$WORK/n$i"/log/*.txt 2>/dev/null | grep -ac . || true) log lines, $(grep -aci "SmartContractGet\|exception" "$WORK/n$i/node.log" 2>/dev/null || true) node log lines on SmartContractGet/exceptions" done