Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Continuous Integration Checks - CLN
name: CI Checks - CLN Integration Tests

on: [push, pull_request]

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/kotlin.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Continuous Integration Checks - Kotlin
name: CI Checks - Kotlin Tests

on: [push, pull_request]

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/python.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Continuous Integration Checks - Python
name: CI Checks - Python Tests

on: [push, pull_request]

Expand Down
19 changes: 13 additions & 6 deletions .github/workflows/build.yml → .github/workflows/rust.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Continuous Integration Checks
name: CI Checks - Rust Tests

on: [push, pull_request]

Expand Down Expand Up @@ -46,13 +46,18 @@ jobs:
cargo update -p proptest --precise "1.2.0" --verbose # proptest 1.3.0 requires rustc 1.64.0
cargo update -p reqwest --precise "0.11.20" --verbose # reqwest 0.11.21 broke 1.63.0 MSRV
cargo update -p regex --precise "1.9.6" --verbose # regex 1.10.0 requires rustc 1.65.0
cargo update -p jobserver --precise "0.1.26" --verbose # jobserver 0.1.27 requires rustc 1.66.0
cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5" --verbose # zstd-sys 2.0.9+zstd.1.5.5 requires rustc 1.64.0
cargo update -p petgraph --precise "0.6.3" --verbose # petgraph v0.6.4, requires rustc 1.64 or newer
cargo update -p home --precise "0.5.5" --verbose # home v0.5.9, requires rustc 1.70 or newer
- name: Set RUSTFLAGS to deny warnings
if: "matrix.toolchain == 'stable'"
run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV"
- name: Download bitcoind/electrs and set environment variables
if: "matrix.platform != 'windows-latest'"
run: |
source ./scripts/download_bitcoind_electrs.sh
cp "$BITCOIND_EXE" "$HOME"/bitcoind
cp "$ELECTRS_EXE" "$HOME"/electrs
echo "BITCOIND_EXE=$HOME/bitcoind" >> "$GITHUB_ENV"
echo "ELECTRS_EXE=$HOME/electrs" >> "$GITHUB_ENV"
- name: Build on Rust ${{ matrix.toolchain }}
run: cargo build --verbose --color always
- name: Build with UniFFI support on Rust ${{ matrix.toolchain }}
Expand All @@ -69,10 +74,12 @@ jobs:
run: cargo check --release --features uniffi --verbose --color always
- name: Test on Rust ${{ matrix.toolchain }}
if: "matrix.platform != 'windows-latest'"
run: cargo test
run: |
RUSTFLAGS="--cfg no_download" cargo test
- name: Test with UniFFI support on Rust ${{ matrix.toolchain }}
if: "matrix.platform != 'windows-latest' && matrix.build-uniffi"
run: cargo test --features uniffi
run: |
RUSTFLAGS="--cfg no_download" cargo test --features uniffi
- name: Check formatting on Rust ${{ matrix.toolchain }}
if: matrix.check-fmt
run: rustup component add rustfmt && cargo fmt --all -- --check
6 changes: 3 additions & 3 deletions .github/workflows/vss-integration.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: VSS Integration Test
name: CI Checks - VSS Integration Tests

on: [push, pull_request]

Expand Down Expand Up @@ -74,8 +74,8 @@ jobs:
run: |
cd ldk-node
export TEST_VSS_BASE_URL="http://localhost:8080/vss"
RUSTFLAGS="--cfg=vss_test --cfg=vss" cargo build --verbose --color always
RUSTFLAGS="--cfg=vss_test --cfg=vss" cargo test -- --nocapture
RUSTFLAGS="--cfg vss_test --cfg vss" cargo build --verbose --color always
RUSTFLAGS="--cfg vss_test --cfg vss" cargo test --test integration_tests_vss

- name: Cleanup
run: |
Expand Down
11 changes: 8 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -76,14 +76,19 @@ winapi = { version = "0.3", features = ["winbase"] }
[dev-dependencies]
lightning = { version = "0.0.119", features = ["std", "_test_utils"] }
#lightning = { git = "https://github.com/lightningdevkit/rust-lightning", branch="main", features = ["std", "_test_utils"] }
electrsd = { version = "0.26.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }
electrum-client = { version = "0.18.0", default-features = true }
electrum-client = { version = "0.15.1", default-features = true }
bitcoincore-rpc = { version = "0.17.0", default-features = false }
proptest = "1.0.0"
regex = "1.5.6"

[target.'cfg(not(no_download))'.dev-dependencies]
electrsd = { version = "0.26.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }

[target.'cfg(no_download)'.dev-dependencies]
electrsd = { version = "0.26.0", features = ["legacy"] }

[target.'cfg(cln_test)'.dev-dependencies]
clightningrpc = { version = "0.3.0-beta.8", default-features = false }
bitcoincore-rpc = { version = "0.17.0", default-features = false }

[build-dependencies]
uniffi = { version = "0.25.1", features = ["build"], optional = true }
Expand Down
2 changes: 1 addition & 1 deletion rustfmt.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ hard_tabs = true # use tab characters for indentation, spaces for alignment
use_field_init_shorthand = true
max_width = 100
use_small_heuristics = "Max"
fn_args_layout = "Compressed"
fn_params_layout = "Compressed"
43 changes: 43 additions & 0 deletions scripts/download_bitcoind_electrs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Our Esplora-based tests require `electrs` and `bitcoind`

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't we do this as part of setup_bitcoind_and_electrsd() in test code?

so that our CI and local tests work consistently and in same manner.

This will avoid another flag.
Moreover, the auto-download feature of bitcoind doesn't work on my mac.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't we do this as part of setup_bitcoind_and_electrsd() in test code?

Electrsd and bitcoind already do this in Rust but pull in quite a bit of annoying unnecessary dev dependencies to do so. Rather than constantly pinning new dependencies (and even risking to mask some actual dependency bump requirements), we can just use CI's curl and be done with it.

so that our CI and local tests work consistently and in same manner.

We leave the auto-download feature as the default behavior, so nothing changes for local testing.

Moreover, the auto-download feature of bitcoind doesn't work on my mac.

I assume this is due to some local anti-virus software or similar, which we probably can't make work in the build scripts. If manual downloads work for you, you can use the newly added download script, or just point BITCOIND_EXE and ELECTRS_EXE env. variables to some preinstalled binaries before running with the no_download cfg flag.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I am fine either ways, feel free to continue with current approach.
I was hoping that if we do it ourselves we will just need zip and reqwest and we can use fixed lower versions of those crates.

# binaries. Here, we download the binaries, validate them, and export their
# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the
# `electrsd`/`bitcoind` crates in our tests.

HOST_PLATFORM="$(rustc --version --verbose | grep "host:" | awk '{ print $2 }')"
ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases"
ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254"
BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/"
BITCOIND_VERSION="25.1"
if [[ "$HOST_PLATFORM" == *linux* ]]; then
ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip
ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1"
BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz
BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b"
elif [[ "$HOST_PLATFORM" == *darwin* ]]; then
ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip
ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd"
BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz
BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1"
else
echo "\n\nUnsupported platform: $HOST_PLATFORM Exiting.."
exit 1
fi

DL_TMP_DIR=$(mktemp -d)
trap 'rm -rf -- "$DL_TMP_DIR"' EXIT

pushd "$DL_TMP_DIR"
ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME"
curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL"
echo "$ELECTRS_DL_HASH $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c
unzip "$ELECTRS_DL_FILE_NAME"
export ELECTRS_EXE="$DL_TMP_DIR"/electrs
chmod +x "$ELECTRS_EXE"

BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME"
curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL"
echo "$BITCOIND_DL_HASH $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c
tar xzf "$BITCOIND_DL_FILE_NAME"
export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind
chmod +x "$BITCOIND_EXE"
popd
3 changes: 1 addition & 2 deletions src/event.rs
Original file line number Diff line number Diff line change
Expand Up @@ -795,8 +795,7 @@ where
#[cfg(test)]
mod tests {
use super::*;
use crate::test::utils::TestLogger;
use lightning::util::test_utils::TestStore;
use lightning::util::test_utils::{TestLogger, TestStore};

#[test]
fn event_queue_persistence() {
Expand Down
3 changes: 1 addition & 2 deletions src/io/sqlite_store/migrations.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,7 @@ pub(super) fn migrate_schema(
#[cfg(test)]
mod tests {
use crate::io::sqlite_store::SqliteStore;
use crate::io::test_utils::do_read_write_remove_list_persist;
use crate::test::utils::random_storage_path;
use crate::io::test_utils::{do_read_write_remove_list_persist, random_storage_path};

use lightning::util::persist::KVStore;

Expand Down
5 changes: 3 additions & 2 deletions src/io/sqlite_store/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -284,8 +284,9 @@ impl KVStore for SqliteStore {
#[cfg(test)]
mod tests {
use super::*;
use crate::io::test_utils::{do_read_write_remove_list_persist, do_test_store};
use crate::test::utils::random_storage_path;
use crate::io::test_utils::{
do_read_write_remove_list_persist, do_test_store, random_storage_path,
};

impl Drop for SqliteStore {
fn drop(&mut self) {
Expand Down
162 changes: 12 additions & 150 deletions src/io/test_utils.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
use crate::io::sqlite_store::SqliteStore;
use lightning_persister::fs_store::FilesystemStore;

use lightning::ln::functional_test_utils::{
connect_block, create_announced_chan_between_nodes, create_chanmon_cfgs, create_dummy_block,
create_network, create_node_cfgs, create_node_chanmgrs, send_payment,
Expand All @@ -9,12 +6,22 @@ use lightning::util::persist::{read_channel_monitors, KVStore, KVSTORE_NAMESPACE

use lightning::chain::channelmonitor::CLOSED_CHANNEL_UPDATE_ID;
use lightning::events::ClosureReason;
use lightning::util::test_utils::{self, TestStore};
use lightning::util::test_utils;
use lightning::{check_added_monitors, check_closed_broadcast, check_closed_event};

use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};

use std::panic::RefUnwindSafe;
use std::path::PathBuf;
use std::sync::RwLock;

pub(crate) fn random_storage_path() -> PathBuf {
let mut temp_path = std::env::temp_dir();
let mut rng = thread_rng();
let rand_dir: String = (0..7).map(|_| rng.sample(Alphanumeric) as char).collect();
temp_path.push(rand_dir);
temp_path
}

pub(crate) fn do_read_write_remove_list_persist<K: KVStore + RefUnwindSafe>(kv_store: &K) {
let data = [42u8; 32];
Expand Down Expand Up @@ -173,148 +180,3 @@ pub(crate) fn do_test_store<K: KVStore>(store_0: &K, store_1: &K) {
// Make sure everything is persisted as expected after close.
check_persisted_data!(CLOSED_CHANNEL_UPDATE_ID);
}

// A `KVStore` impl for testing purposes that wraps all our `KVStore`s and asserts their synchronicity.
pub(crate) struct TestSyncStore {
serializer: RwLock<()>,
test_store: TestStore,
fs_store: FilesystemStore,
sqlite_store: SqliteStore,
}

impl TestSyncStore {
pub(crate) fn new(dest_dir: PathBuf) -> Self {
let serializer = RwLock::new(());
let mut fs_dir = dest_dir.clone();
fs_dir.push("fs_store");
let fs_store = FilesystemStore::new(fs_dir);
let mut sql_dir = dest_dir.clone();
sql_dir.push("sqlite_store");
let sqlite_store = SqliteStore::new(
sql_dir,
Some("test_sync_db".to_string()),
Some("test_sync_table".to_string()),
)
.unwrap();
let test_store = TestStore::new(false);
Self { serializer, fs_store, sqlite_store, test_store }
}

fn do_list(
&self, primary_namespace: &str, secondary_namespace: &str,
) -> std::io::Result<Vec<String>> {
let fs_res = self.fs_store.list(primary_namespace, secondary_namespace);
let sqlite_res = self.sqlite_store.list(primary_namespace, secondary_namespace);
let test_res = self.test_store.list(primary_namespace, secondary_namespace);

match fs_res {
Ok(mut list) => {
list.sort();

let mut sqlite_list = sqlite_res.unwrap();
sqlite_list.sort();
assert_eq!(list, sqlite_list);

let mut test_list = test_res.unwrap();
test_list.sort();
assert_eq!(list, test_list);

Ok(list)
}
Err(e) => {
assert!(sqlite_res.is_err());
assert!(test_res.is_err());
Err(e)
}
}
}
}

impl KVStore for TestSyncStore {
fn read(
&self, primary_namespace: &str, secondary_namespace: &str, key: &str,
) -> std::io::Result<Vec<u8>> {
let _guard = self.serializer.read().unwrap();

let fs_res = self.fs_store.read(primary_namespace, secondary_namespace, key);
let sqlite_res = self.sqlite_store.read(primary_namespace, secondary_namespace, key);
let test_res = self.test_store.read(primary_namespace, secondary_namespace, key);

match fs_res {
Ok(read) => {
assert_eq!(read, sqlite_res.unwrap());
assert_eq!(read, test_res.unwrap());
Ok(read)
}
Err(e) => {
assert!(sqlite_res.is_err());
assert_eq!(e.kind(), unsafe { sqlite_res.unwrap_err_unchecked().kind() });
assert!(test_res.is_err());
assert_eq!(e.kind(), unsafe { test_res.unwrap_err_unchecked().kind() });
Err(e)
}
}
}

fn write(
&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8],
) -> std::io::Result<()> {
let _guard = self.serializer.write().unwrap();
let fs_res = self.fs_store.write(primary_namespace, secondary_namespace, key, buf);
let sqlite_res = self.sqlite_store.write(primary_namespace, secondary_namespace, key, buf);
let test_res = self.test_store.write(primary_namespace, secondary_namespace, key, buf);

assert!(self
.do_list(primary_namespace, secondary_namespace)
.unwrap()
.contains(&key.to_string()));

match fs_res {
Ok(()) => {
assert!(sqlite_res.is_ok());
assert!(test_res.is_ok());
Ok(())
}
Err(e) => {
assert!(sqlite_res.is_err());
assert!(test_res.is_err());
Err(e)
}
}
}

fn remove(
&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool,
) -> std::io::Result<()> {
let _guard = self.serializer.write().unwrap();
let fs_res = self.fs_store.remove(primary_namespace, secondary_namespace, key, lazy);
let sqlite_res =
self.sqlite_store.remove(primary_namespace, secondary_namespace, key, lazy);
let test_res = self.test_store.remove(primary_namespace, secondary_namespace, key, lazy);

assert!(!self
.do_list(primary_namespace, secondary_namespace)
.unwrap()
.contains(&key.to_string()));

match fs_res {
Ok(()) => {
assert!(sqlite_res.is_ok());
assert!(test_res.is_ok());
Ok(())
}
Err(e) => {
assert!(sqlite_res.is_err());
assert!(test_res.is_err());
Err(e)
}
}
}

fn list(
&self, primary_namespace: &str, secondary_namespace: &str,
) -> std::io::Result<Vec<String>> {
let _guard = self.serializer.read().unwrap();
self.do_list(primary_namespace, secondary_namespace)
}
}
2 changes: 0 additions & 2 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,6 @@ mod logger;
mod payment_store;
mod peer_store;
mod sweep;
#[cfg(test)]
mod test;
mod tx_broadcaster;
mod types;
#[cfg(feature = "uniffi")]
Expand Down
3 changes: 1 addition & 2 deletions src/payment_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,7 @@ where
#[cfg(test)]
mod tests {
use super::*;
use crate::test::utils::TestLogger;
use lightning::util::test_utils::TestStore;
use lightning::util::test_utils::{TestLogger, TestStore};
use std::sync::Arc;

#[test]
Expand Down
4 changes: 1 addition & 3 deletions src/peer_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,7 @@ impl_writeable_tlv_based!(PeerInfo, {
#[cfg(test)]
mod tests {
use super::*;
use crate::test::utils::TestLogger;

use lightning::util::test_utils::TestStore;
use lightning::util::test_utils::{TestLogger, TestStore};

use std::str::FromStr;
use std::sync::Arc;
Expand Down
Loading