diff --git a/.github/workflows/setup-test.yml b/.github/workflows/setup-test.yml index debd1c26e..bb9cff4b4 100644 --- a/.github/workflows/setup-test.yml +++ b/.github/workflows/setup-test.yml @@ -190,6 +190,27 @@ jobs: grep -a "contract-executor.jar and sc-api-support.jar replaced: build [0-9]* (community)" setup5.txt test -f csnode-exec/contract-executor.jar && test -f csnode-exec/sc-api-support.jar + - name: Full rocksdb set-up from a snapshot, with and without the backfill + run: | + cp -a csnode-clean csnode-full && rm -f csnode-full/config.ini.bak-* + # storage, database, first start (network), backfill (Enter: yes), share (no), executor, start, write + bash setup-run.sh csnode-full 'full\nrocksdb\nnetwork\n\nno\n\nno\n\n' 2>&1 | tee setup6.txt | tail -14 + grep -a "fetches the older blocks from" setup6.txt + grep -a "backfill: yes" setup6.txt + python check-config.py csnode-full/config.ini snapshot "serve=off fetch=auto" storage "db_backend=rocksdb backfill=on" + # a tuning key set by hand is kept; backfill no + printf 'backfill_rate=500\n' >> csnode-full/config.ini + bash setup-run.sh csnode-full 'full\nrocksdb\nnetwork\nno\nno\n\nno\n\n' 2>&1 | tee setup7.txt | tail -8 + python check-config.py csnode-full/config.ini snapshot "serve=off fetch=auto" storage "db_backend=rocksdb backfill=off backfill_rate=500" + + - name: A pruned folder switched to full keeps its blocks and backfills + run: | + # csnode holds the pruned lmdb database of the first set-up: storage (full), database (Enter: + # lmdb), backfill (Enter: yes), share, executor, start (no), write + bash setup-run.sh csnode 'full\n\n\n\n\nno\n\n' 2>&1 | tee setup8.txt | tail -14 + grep -a "keeps its blocks and fetches the ones" setup8.txt + python check-config.py csnode/config.ini snapshot "serve=on fetch=off" storage "db_backend=lmdb backfill=on" + - name: Collect if: always() run: | diff --git a/docs/CHANGES.md b/docs/CHANGES.md index f956ccd67..3ba65e803 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -45,6 +45,7 @@ lines under the new release. | #151 | csnode: every backfill progress line (each 100,000 blocks) is followed by the average cost of a block per phase (verify, store, confirmations), the transactions per block and the share of the processor thread the backfill took | | #152 | csnode: the backfill checks the signatures of the fetched blocks, and the round confirmations of the stored ones, on verifier threads (`backfill_verify_threads`, default min(4, cores − 1), `0` = as before); the thread that runs consensus only links and stores the blocks, about 0.5 ms a block instead of 2.5 ms on mainnet | | #153 | docs: the README links the guided set-up (`packaging/setup/`, `csnode/setup.sh` / `setup.bat` in every package) | +| #154 | packaging: the guided set-up knows the backfill: a full node on lmdb or rocksdb starts from a snapshot by default and is asked whether to fetch the older blocks in the background (`backfill=on/off`, kept when not asked, `backfill_*` keys set by hand kept); a pruned folder can switch to full and fetches the blocks it removed (the set-up used to say a full node from a snapshot holds only the snapshot's blocks and defaulted full nodes to a sync from block 0) | ## community.15 — 2026-11-01 diff --git a/packaging/setup/setup.ps1 b/packaging/setup/setup.ps1 index 3e6a15e97..dc0f55d67 100644 --- a/packaging/setup/setup.ps1 +++ b/packaging/setup/setup.ps1 @@ -155,9 +155,11 @@ Write-Host '1. Storage' Write-Host ' pruned: keeps only recent blocks (a few GB), starts in minutes from a snapshot. Enough to run a node.' Write-Host ' full: keeps every block since the start of the chain (tens of GB), for history and explorers.' $storage = Ask ' Pruned or full node? (pruned/full)' $storageDefault @('pruned', 'full') +$prunedToFull = $false if ($storage -eq 'full' -and $have -and $curPrune -and $curPrune -ne '0') { - Write-Host ' This folder holds a pruned database: its removed blocks are gone. To get a full node,' - Write-Host ' move db, caches and qs away and run the set-up again on the empty folder.' + Write-Host ' This folder holds a pruned database: as a full node it keeps its blocks and fetches the ones' + Write-Host ' it removed from other nodes in the background (backfill, question 3).' + $prunedToFull = $true } Write-Host '' @@ -201,14 +203,31 @@ if (-not $have) { Write-Host ' sync: sync from block 0 by itself (about 35 hours on mainnet).' $firstDefault = 'github' if ($storage -eq 'full') { - Write-Host ' A full node started from a snapshot holds the blocks from the snapshot on only.' - $firstDefault = 'sync' + Write-Host ' A full node started from a snapshot runs in minutes and fetches the older blocks from' + Write-Host ' other nodes in the background (backfill, next question).' } $first = Ask ' How should the node get the chain? (github/network/sync)' $firstDefault @('github', 'network', 'sync') } Write-Host '' } +# 3b. backfill: a full node that starts above block 0 (from a snapshot, or a pruned folder) fetches +# the older blocks from other nodes in the background +$backfill = '' +if ($storage -eq 'full' -and ($first -eq 'github' -or $first -eq 'network' -or $prunedToFull)) { + $backfillDefault = if ((Current 'storage' 'backfill') -eq 'off') { 'no' } else { 'yes' } + Write-Host '3b. History (backfill)' + Write-Host ' yes: fetch every older block down to block 0 in the background, each checked against the chain;' + Write-Host ' the node runs and takes part in the network meanwhile. On mainnet about 186 million blocks:' + Write-Host ' one to five days, and the disk of a full node (about 130 GB or more with rocksdb, 400 GB' + Write-Host ' with lmdb or berkeleydb).' + Write-Host ' no: keep only the blocks from the snapshot on (a few GB); the history before is not served.' + $backfill = Ask ' Fetch the older blocks? (yes/no)' $backfillDefault @('yes', 'no') + Write-Host '' +} elseif ($storage -eq 'full' -and (Current 'storage' 'backfill') -eq 'off') { + $backfill = 'no' # not asked: a choice made before is kept +} + # 4. sharing snapshots $serveDefault = if ($curServe -eq 'off') { 'no' } else { 'yes' } Write-Host '4. Sharing' @@ -269,6 +288,7 @@ switch ($first) { 'github' { Write-Host ' first start: snapshot downloaded from GitHub now' } 'sync' { Write-Host ' first start: sync from block 0' } } +if ($backfill) { Write-Host " backfill: $backfill" } Write-Host " share: $serve" if ($executorChoice -ne 'none') { switch ($executorAction) { @@ -312,6 +332,15 @@ if ($storage -eq 'pruned') { $out.Add('checkpoint_keep=2') $out.Add('prune_keep_blocks=1000') } +if ($backfill -eq 'yes') { $out.Add('backfill=on') } elseif ($backfill -eq 'no') { $out.Add('backfill=off') } +# the backfill's tuning keys set by hand (backfill_rate, backfill_batch...) are kept +if ($storage -eq 'full') { + $inStorage = $false + foreach ($l in $lines) { + if ($l -match '^\s*\[') { $inStorage = ($l -match '^\s*\[storage\]'); continue } + if ($inStorage -and $l -match '^\s*backfill_[a-z_]+\s*=') { $out.Add($l.Trim()) } + } +} # the executor: put the chosen jars in place, then make config.ini accept their build $executorNotes = New-Object System.Collections.Generic.List[string] diff --git a/packaging/setup/setup.sh b/packaging/setup/setup.sh index dcb6cf2d5..f71ad7860 100755 --- a/packaging/setup/setup.sh +++ b/packaging/setup/setup.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Guided set-up of the node in this folder: storage (pruned or full), database, how the first start -# gets the chain, whether to share snapshots, which contract executor to run. Writes the [snapshot] +# gets the chain, whether a full node fetches the older blocks in the background (backfill), whether +# to share snapshots, which contract executor to run. Writes the [snapshot] # and [storage] sections of config.ini and the executor builds it accepts (the previous file is # kept as config.ini.bak-), can replace the executor jars, and can start the node. # @@ -171,9 +172,11 @@ echo " pruned: keeps only recent blocks (a few GB), starts in minutes from a s echo " full: keeps every block since the start of the chain (tens of GB), for history and explorers." ask " Pruned or full node? (pruned/full)" "$storage_default" pruned full storage=$answer +pruned_to_full=no if [ "$storage" = full ] && [ -n "$have" ] && [ -n "$cur_prune" ] && [ "$cur_prune" != 0 ]; then - echo " This folder holds a pruned database: its removed blocks are gone. To get a full node," - echo " move db, caches and qs away and run the set-up again on the empty folder." + echo " This folder holds a pruned database: as a full node it keeps its blocks and fetches the ones" + echo " it removed from other nodes in the background (backfill, question 3)." + pruned_to_full=yes fi echo @@ -228,8 +231,8 @@ if [ -z "$have" ]; then echo " sync: sync from block 0 by itself (about 35 hours on mainnet)." first_default=github if [ "$storage" = full ]; then - echo " A full node started from a snapshot holds the blocks from the snapshot on only." - first_default=sync + echo " A full node started from a snapshot runs in minutes and fetches the older blocks from" + echo " other nodes in the background (backfill, next question)." fi ask " How should the node get the chain? (github/network/sync)" "$first_default" github network sync first=$answer @@ -237,6 +240,25 @@ if [ -z "$have" ]; then echo fi +# 3b. backfill: a full node that starts above block 0 (from a snapshot, or a pruned folder) fetches +# the older blocks from other nodes in the background +backfill= +if [ "$storage" = full ] && { [ "$first" = github ] || [ "$first" = network ] || [ "$pruned_to_full" = yes ]; }; then + backfill_default=yes + [ "$(current storage backfill)" != off ] || backfill_default=no + echo "3b. History (backfill)" + echo " yes: fetch every older block down to block 0 in the background, each checked against the chain;" + echo " the node runs and takes part in the network meanwhile. On mainnet about 186 million blocks:" + echo " one to five days, and the disk of a full node (about 130 GB or more with rocksdb, 400 GB" + echo " with lmdb or berkeleydb)." + echo " no: keep only the blocks from the snapshot on (a few GB); the history before is not served." + ask " Fetch the older blocks? (yes/no)" "$backfill_default" yes no + backfill=$answer + echo +elif [ "$storage" = full ] && [ "$(current storage backfill)" = off ]; then + backfill=no # not asked: a choice made before is kept +fi + # 4. sharing snapshots serve_default=yes [ "$cur_serve" != off ] || serve_default=no @@ -303,6 +325,7 @@ case "$first" in github) echo " first start: snapshot downloaded from GitHub now" ;; sync) echo " first start: sync from block 0" ;; esac +[ -z "$backfill" ] || echo " backfill: $backfill" echo " share: $serve" case "$executor_choice/$executor_action" in none/*) ;; @@ -343,6 +366,15 @@ awk ' echo "checkpoint_keep=2" echo "prune_keep_blocks=1000" fi + case "$backfill" in + yes) echo "backfill=on" ;; + no) echo "backfill=off" ;; + esac + # the backfill's tuning keys set by hand (backfill_rate, backfill_batch...) are kept + if [ "$storage" = full ]; then + awk '/^[[:space:]]*\[/ { in_s = ($0 ~ /^[[:space:]]*\[storage\]/); next } + in_s && /^[[:space:]]*backfill_[a-z_]+[[:space:]]*=/ { sub(/^[[:space:]]+/, ""); print }' "$backup" | tr -d '\r' + fi } >> "$tmp" mv "$tmp" "$config" echo "config.ini written; the previous one is $backup"