From 3dacbad89d4d7cd131d71ea42a454d9726f9f5c9 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Thu, 27 Aug 2020 15:52:05 +0900 Subject: [PATCH 01/29] start generate-data command --- codechain/codechain.yml | 2 ++ codechain/subcommand/generate_data_command.rs | 7 +++++++ codechain/subcommand/mod.rs | 3 +++ 3 files changed, 12 insertions(+) create mode 100644 codechain/subcommand/generate_data_command.rs diff --git a/codechain/codechain.yml b/codechain/codechain.yml index 43dba554be..d3abdc6fdc 100644 --- a/codechain/codechain.yml +++ b/codechain/codechain.yml @@ -379,3 +379,5 @@ subcommands: - accountId - commit-hash: about: Print the commit hash at the build time. + - generate-data: + about: Generate large trie for test diff --git a/codechain/subcommand/generate_data_command.rs b/codechain/subcommand/generate_data_command.rs new file mode 100644 index 0000000000..725b440ee2 --- /dev/null +++ b/codechain/subcommand/generate_data_command.rs @@ -0,0 +1,7 @@ +use clap::ArgMatches; + +/// Generate large trie to test update speed +pub fn run_generate_data_command(_matches: &ArgMatches) -> Result<(), String> { + println!("HI generate data"); + Ok(()) +} diff --git a/codechain/subcommand/mod.rs b/codechain/subcommand/mod.rs index 51843fbb1b..207d9e6d18 100644 --- a/codechain/subcommand/mod.rs +++ b/codechain/subcommand/mod.rs @@ -16,17 +16,20 @@ mod account_command; mod convert_command; +mod generate_data_command; use clap::ArgMatches; use self::account_command::run_account_command; use self::convert_command::run_convert_command; +use self::generate_data_command::run_generate_data_command; pub fn run_subcommand(matches: &ArgMatches) -> Result<(), String> { let subcommand = matches.subcommand.as_ref().unwrap(); match subcommand.name.as_str() { "account" => run_account_command(&subcommand.matches), "convert" => run_convert_command(&subcommand.matches), + "generate-data" => run_generate_data_command(&subcommand.matches), "commit-hash" => { println!("{}", env!("VERGEN_SHA")); Ok(()) From f22d767e4fe67c4d8668d5042e4c76df6da6167f Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Thu, 27 Aug 2020 19:01:24 +0900 Subject: [PATCH 02/29] Create command that generates DB for test --- Cargo.lock | 1 + Cargo.toml | 1 + codechain/codechain.yml | 6 ++ codechain/main.rs | 2 + codechain/subcommand/generate_data_command.rs | 63 ++++++++++++++++++- 5 files changed, 71 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5826e8306..5cf63fb5c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,6 +234,7 @@ dependencies = [ "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "finally-block 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", + "journaldb 0.1.0", "kvdb 0.1.0", "kvdb-rocksdb 0.1.0", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 0a5ba08425..8ed13a89c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ codechain-crypto = { git = "https://github.com/CodeChain-io/rust-codechain-crypt codechain-discovery = { path = "discovery" } codechain-logger = { path = "util/logger" } codechain-key = { path = "key" } +journaldb = { path = "./util/journaldb" } codechain-keystore = { path = "keystore" } codechain-merkle = { path = "util/merkle" } codechain-network = { path = "network" } diff --git a/codechain/codechain.yml b/codechain/codechain.yml index d3abdc6fdc..4ee8d858bb 100644 --- a/codechain/codechain.yml +++ b/codechain/codechain.yml @@ -381,3 +381,9 @@ subcommands: about: Print the commit hash at the build time. - generate-data: about: Generate large trie for test + args: + - number: + short: n + long: number + help: Set the number of db values + takes_value: true diff --git a/codechain/main.rs b/codechain/main.rs index 98a5f34ee9..d6d8d1f8bd 100644 --- a/codechain/main.rs +++ b/codechain/main.rs @@ -35,6 +35,7 @@ extern crate codechain_keystore as ckeystore; #[macro_use] extern crate codechain_logger as clogger; extern crate cidr; +extern crate codechain_crypto as ccrypto; extern crate codechain_network as cnetwork; extern crate codechain_rpc as crpc; extern crate codechain_state as cstate; @@ -45,6 +46,7 @@ extern crate ctrlc; extern crate env_logger; extern crate fdlimit; extern crate finally_block; +extern crate journaldb; extern crate kvdb; extern crate kvdb_rocksdb; extern crate never_type; diff --git a/codechain/subcommand/generate_data_command.rs b/codechain/subcommand/generate_data_command.rs index 725b440ee2..1752e9937c 100644 --- a/codechain/subcommand/generate_data_command.rs +++ b/codechain/subcommand/generate_data_command.rs @@ -1,7 +1,66 @@ +use ccore::ClientConfig; +use ccrypto::BLAKE_NULL_RLP; +use ckey::Address; use clap::ArgMatches; +use clogger::{self, LoggerConfig}; +use cstate::{StateDB, StateWithCache, TopLevelState, TopState}; +use journaldb; +use kvdb::{DBTransaction, KeyValueDB}; +use kvdb_rocksdb::{Database, DatabaseConfig}; +use std::{path::Path, sync::Arc}; + +pub const COL_STATE: Option = Some(0); /// Generate large trie to test update speed -pub fn run_generate_data_command(_matches: &ArgMatches) -> Result<(), String> { - println!("HI generate data"); +pub fn run_generate_data_command(matches: &ArgMatches) -> Result<(), String> { + clogger::init(&LoggerConfig::new(777), None).expect("Logger must be successfully initialized"); + let db = open_db()?; + + let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); + let state_db = StateDB::new(journal_db); + let root = BLAKE_NULL_RLP; + let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); + + let num = matches.value_of("number").unwrap(); + let num: u32 = num.parse().unwrap(); + + //for i in 0..1_000_000_000 { + for i in 0..10_u64.pow(num) { + let address = Address::random(); + toplevel_state.add_balance(&address, i).unwrap(); + } + toplevel_state.commit().unwrap(); + + let mut batch = DBTransaction::new(); + let updated = toplevel_state.journal_under(&mut batch, 0).unwrap(); + //let updated = state_db.journal_under(&mut batch, 0, root).map_err(|err| err.to_string())?; + db.write(batch).map_err(|err| err.to_string())?; + + println!("updated {}", updated); + Ok(()) } + +pub const DEFAULT_DB_PATH: &str = "db_test"; +pub const NUM_COLUMNS: Option = Some(6); + +pub fn open_db() -> Result, String> { + let base_path = ".".to_owned(); + let db_path = base_path + "/" + DEFAULT_DB_PATH; + let client_path = Path::new(&db_path); + let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS); + + let client_config = ClientConfig::default(); + + db_config.memory_budget = client_config.db_cache_size; + db_config.compaction = client_config.db_compaction.compaction_profile(client_path); + db_config.wal = client_config.db_wal; + + let db = Arc::new( + Database::open(&db_config, &client_path.to_str().expect("DB path could not be converted to string.")) + .map_err(|_e| "Low level database error. Some issue with disk?".to_string())?, + ); + + Ok(db) +} + From 729c51cbfd04d21a587b77ee2fe613d8d73bb492 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Thu, 3 Sep 2020 15:15:13 +0900 Subject: [PATCH 03/29] adding perf script --- codechain/subcommand/mod.rs | 1 + codechain/subcommand/perf_write_command.rs | 44 ++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 codechain/subcommand/perf_write_command.rs diff --git a/codechain/subcommand/mod.rs b/codechain/subcommand/mod.rs index 207d9e6d18..74340c4d55 100644 --- a/codechain/subcommand/mod.rs +++ b/codechain/subcommand/mod.rs @@ -17,6 +17,7 @@ mod account_command; mod convert_command; mod generate_data_command; +mod perf_write_command; use clap::ArgMatches; diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs new file mode 100644 index 0000000000..0cd80852b9 --- /dev/null +++ b/codechain/subcommand/perf_write_command.rs @@ -0,0 +1,44 @@ +use ccore::ClientConfig; +use clogger::{self, LoggerConfig}; +use clap::ArgMatches; +use kvdb::KeyValueDB; +use kvdb_rocksdb::{Database, DatabaseConfig}; +use std::{path::Path, sync::Arc}; + +pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { + clogger::init(&LoggerConfig::new(777), None).expect("Logger must be successfully initialized"); + let db = open_db()?; + + for i in 0..10 { + let address = Address::random(); + toplevel_state.add_balance(&address, i).unwrap(); + } + toplevel_state.commit().unwrap(); + + + unimplemented!(); +} + +pub const DEFAULT_DB_PATH: &str = "db_test"; +pub const NUM_COLUMNS: Option = Some(6); + + +pub fn open_db() -> Result, String> { + let base_path = ".".to_owned(); + let db_path = base_path + "/" + DEFAULT_DB_PATH; + let client_path = Path::new(&db_path); + let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS); + + let client_config = ClientConfig::default(); + + db_config.memory_budget = client_config.db_cache_size; + db_config.compaction = client_config.db_compaction.compaction_profile(client_path); + db_config.wal = client_config.db_wal; + + let db = Arc::new( + Database::open(&db_config, &client_path.to_str().expect("DB path could not be converted to string.")) + .map_err(|_e| "Low level database error. Some issue with disk?".to_string())?, + ); + + Ok(db) +} From fb2ead9c0b72f499c2acc5f13444124c0b954b9d Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Thu, 10 Sep 2020 16:34:56 +0900 Subject: [PATCH 04/29] Create perf checker --- codechain/subcommand/generate_data_command.rs | 9 ++++-- codechain/subcommand/mod.rs | 2 ++ codechain/subcommand/perf_write_command.rs | 28 ++++++++++++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/codechain/subcommand/generate_data_command.rs b/codechain/subcommand/generate_data_command.rs index 1752e9937c..6d6ef491f7 100644 --- a/codechain/subcommand/generate_data_command.rs +++ b/codechain/subcommand/generate_data_command.rs @@ -9,6 +9,7 @@ use kvdb::{DBTransaction, KeyValueDB}; use kvdb_rocksdb::{Database, DatabaseConfig}; use std::{path::Path, sync::Arc}; +pub const COL_EXTRA: Option = Some(3); pub const COL_STATE: Option = Some(0); /// Generate large trie to test update speed @@ -29,7 +30,12 @@ pub fn run_generate_data_command(matches: &ArgMatches) -> Result<(), String> { let address = Address::random(); toplevel_state.add_balance(&address, i).unwrap(); } - toplevel_state.commit().unwrap(); + let new_root = toplevel_state.commit().unwrap(); + { + let mut batch = DBTransaction::new(); + batch.put(COL_EXTRA, b"perf_data_root", new_root.as_ref()); + db.write(batch).unwrap(); + } let mut batch = DBTransaction::new(); let updated = toplevel_state.journal_under(&mut batch, 0).unwrap(); @@ -63,4 +69,3 @@ pub fn open_db() -> Result, String> { Ok(db) } - diff --git a/codechain/subcommand/mod.rs b/codechain/subcommand/mod.rs index 74340c4d55..6d7b034a95 100644 --- a/codechain/subcommand/mod.rs +++ b/codechain/subcommand/mod.rs @@ -24,6 +24,7 @@ use clap::ArgMatches; use self::account_command::run_account_command; use self::convert_command::run_convert_command; use self::generate_data_command::run_generate_data_command; +use self::perf_write_command::run_perf_write_command; pub fn run_subcommand(matches: &ArgMatches) -> Result<(), String> { let subcommand = matches.subcommand.as_ref().unwrap(); @@ -31,6 +32,7 @@ pub fn run_subcommand(matches: &ArgMatches) -> Result<(), String> { "account" => run_account_command(&subcommand.matches), "convert" => run_convert_command(&subcommand.matches), "generate-data" => run_generate_data_command(&subcommand.matches), + "perf-write" => run_perf_write_command(&subcommand.matches), "commit-hash" => { println!("{}", env!("VERGEN_SHA")); Ok(()) diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs index 0cd80852b9..9b3e1d6247 100644 --- a/codechain/subcommand/perf_write_command.rs +++ b/codechain/subcommand/perf_write_command.rs @@ -1,22 +1,42 @@ use ccore::ClientConfig; -use clogger::{self, LoggerConfig}; use clap::ArgMatches; +use clogger::{self, LoggerConfig}; use kvdb::KeyValueDB; use kvdb_rocksdb::{Database, DatabaseConfig}; -use std::{path::Path, sync::Arc}; +use std::{path::Path, sync::Arc, time::Instant}; +use ckey::Address; +use cstate::{TopState, TopLevelState, StateDB, StateWithCache}; +use primitives::H256; + +pub const COL_STATE: Option = Some(0); +pub const COL_EXTRA: Option = Some(3); -pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { +pub fn run_perf_write_command(_matches: &ArgMatches) -> Result<(), String> { clogger::init(&LoggerConfig::new(777), None).expect("Logger must be successfully initialized"); let db = open_db()?; + let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); + let state_db = StateDB::new(journal_db); + + let root = { + let bytes = db.get(COL_EXTRA, b"perf_data_root").unwrap().unwrap(); + H256::from(bytes.as_ref()) + }; + + let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); for i in 0..10 { + let now = Instant::now(); let address = Address::random(); toplevel_state.add_balance(&address, i).unwrap(); + if now.elapsed().as_secs() >= 1 { + println!("{}", now.elapsed().as_secs()); + } } toplevel_state.commit().unwrap(); + println!("Finished"); - unimplemented!(); + Ok(()) } pub const DEFAULT_DB_PATH: &str = "db_test"; From 932e27f869bec8837cfdfe6ebed76ae3e42be676 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Thu, 10 Sep 2020 19:16:18 +0900 Subject: [PATCH 05/29] Flush data frequently The previous version flush data at once. It requires too much memory. --- codechain/subcommand/generate_data_command.rs | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/codechain/subcommand/generate_data_command.rs b/codechain/subcommand/generate_data_command.rs index 6d6ef491f7..0167ef212d 100644 --- a/codechain/subcommand/generate_data_command.rs +++ b/codechain/subcommand/generate_data_command.rs @@ -19,31 +19,35 @@ pub fn run_generate_data_command(matches: &ArgMatches) -> Result<(), String> { let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); let state_db = StateDB::new(journal_db); - let root = BLAKE_NULL_RLP; - let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); + let mut root = BLAKE_NULL_RLP; let num = matches.value_of("number").unwrap(); let num: u32 = num.parse().unwrap(); //for i in 0..1_000_000_000 { for i in 0..10_u64.pow(num) { - let address = Address::random(); - toplevel_state.add_balance(&address, i).unwrap(); - } - let new_root = toplevel_state.commit().unwrap(); - { + let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); + for j in 0..1000 { + let address = Address::random(); + toplevel_state.add_balance(&address, i * 1000 + j).unwrap(); + } + root = toplevel_state.commit().unwrap(); + println!("write root {:?}", root); + { + let mut batch = DBTransaction::new(); + batch.put(COL_EXTRA, b"perf_data_root", root.as_ref()); + db.write(batch).unwrap(); + } let mut batch = DBTransaction::new(); - batch.put(COL_EXTRA, b"perf_data_root", new_root.as_ref()); - db.write(batch).unwrap(); + let updated = toplevel_state.journal_under(&mut batch, 0).unwrap(); + println!("write to db"); + //let updated = state_db.journal_under(&mut batch, 0, root).map_err(|err| err.to_string())?; + db.write(batch).map_err(|err| err.to_string())?; + println!("flush the db"); + db.flush().unwrap(); + println!("updated {}", updated); } - let mut batch = DBTransaction::new(); - let updated = toplevel_state.journal_under(&mut batch, 0).unwrap(); - //let updated = state_db.journal_under(&mut batch, 0, root).map_err(|err| err.to_string())?; - db.write(batch).map_err(|err| err.to_string())?; - - println!("updated {}", updated); - Ok(()) } From a4528019e93da173cd63f7add53a0497f562c1f7 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Thu, 10 Sep 2020 19:16:50 +0900 Subject: [PATCH 06/29] Make the number of write customizable --- codechain/codechain.yml | 8 ++++++++ codechain/subcommand/perf_write_command.rs | 22 +++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/codechain/codechain.yml b/codechain/codechain.yml index 4ee8d858bb..cbcd5cec4d 100644 --- a/codechain/codechain.yml +++ b/codechain/codechain.yml @@ -379,6 +379,14 @@ subcommands: - accountId - commit-hash: about: Print the commit hash at the build time. + - perf-write: + about: Check the write performance + args: + - number: + short: n + long: number + help: Set the number of db values + takes_value: true - generate-data: about: Generate large trie for test args: diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs index 9b3e1d6247..50c9a88def 100644 --- a/codechain/subcommand/perf_write_command.rs +++ b/codechain/subcommand/perf_write_command.rs @@ -1,21 +1,31 @@ use ccore::ClientConfig; +use ckey::Address; use clap::ArgMatches; use clogger::{self, LoggerConfig}; +use cstate::{StateDB, StateWithCache, TopLevelState, TopState}; use kvdb::KeyValueDB; use kvdb_rocksdb::{Database, DatabaseConfig}; -use std::{path::Path, sync::Arc, time::Instant}; -use ckey::Address; -use cstate::{TopState, TopLevelState, StateDB, StateWithCache}; use primitives::H256; +use std::{path::Path, sync::Arc, time::Instant}; pub const COL_STATE: Option = Some(0); pub const COL_EXTRA: Option = Some(3); -pub fn run_perf_write_command(_matches: &ArgMatches) -> Result<(), String> { +pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { clogger::init(&LoggerConfig::new(777), None).expect("Logger must be successfully initialized"); + + let num = matches.value_of("number").unwrap(); + let num: u32 = num.parse().unwrap(); + + println!("start"); let db = open_db()?; + println!("open db"); let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); + println!("open journal db"); let state_db = StateDB::new(journal_db); + println!("open state db"); + + println!("before getting root"); let root = { let bytes = db.get(COL_EXTRA, b"perf_data_root").unwrap().unwrap(); @@ -24,7 +34,9 @@ pub fn run_perf_write_command(_matches: &ArgMatches) -> Result<(), String> { let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); - for i in 0..10 { + println!("before loop"); + for i in 0..10_u64.pow(num) { + // println!("loop"); let now = Instant::now(); let address = Address::random(); toplevel_state.add_balance(&address, i).unwrap(); From 4fb77e796f3d11ed5e3f728021a0bee9e9e027a8 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Mon, 14 Sep 2020 15:24:03 +0900 Subject: [PATCH 07/29] Use different db directories for performance check --- codechain/codechain.yml | 5 ++ codechain/subcommand/generate_data_command.rs | 14 ++-- codechain/subcommand/perf_write_command.rs | 65 ++++++++++++++++--- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/codechain/codechain.yml b/codechain/codechain.yml index cbcd5cec4d..630d65e273 100644 --- a/codechain/codechain.yml +++ b/codechain/codechain.yml @@ -382,6 +382,11 @@ subcommands: - perf-write: about: Check the write performance args: + - db-dir: + short: d + long: db-dir + help: The db directory + takes_value: true - number: short: n long: number diff --git a/codechain/subcommand/generate_data_command.rs b/codechain/subcommand/generate_data_command.rs index 0167ef212d..f956e8295c 100644 --- a/codechain/subcommand/generate_data_command.rs +++ b/codechain/subcommand/generate_data_command.rs @@ -15,15 +15,16 @@ pub const COL_STATE: Option = Some(0); /// Generate large trie to test update speed pub fn run_generate_data_command(matches: &ArgMatches) -> Result<(), String> { clogger::init(&LoggerConfig::new(777), None).expect("Logger must be successfully initialized"); - let db = open_db()?; + + let num = matches.value_of("number").unwrap(); + let num: u32 = num.parse().unwrap(); + + let db = open_db(num + 3)?; let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); let state_db = StateDB::new(journal_db); let mut root = BLAKE_NULL_RLP; - let num = matches.value_of("number").unwrap(); - let num: u32 = num.parse().unwrap(); - //for i in 0..1_000_000_000 { for i in 0..10_u64.pow(num) { let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); @@ -46,6 +47,7 @@ pub fn run_generate_data_command(matches: &ArgMatches) -> Result<(), String> { println!("flush the db"); db.flush().unwrap(); println!("updated {}", updated); + std::thread::sleep(std::time::Duration::from_millis(50)); } Ok(()) @@ -54,9 +56,9 @@ pub fn run_generate_data_command(matches: &ArgMatches) -> Result<(), String> { pub const DEFAULT_DB_PATH: &str = "db_test"; pub const NUM_COLUMNS: Option = Some(6); -pub fn open_db() -> Result, String> { +pub fn open_db(num: u32) -> Result, String> { let base_path = ".".to_owned(); - let db_path = base_path + "/" + DEFAULT_DB_PATH; + let db_path = base_path + "/" + DEFAULT_DB_PATH + &format!("_{}", num); let client_path = Path::new(&db_path); let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS); diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs index 50c9a88def..41aecfb06b 100644 --- a/codechain/subcommand/perf_write_command.rs +++ b/codechain/subcommand/perf_write_command.rs @@ -3,7 +3,7 @@ use ckey::Address; use clap::ArgMatches; use clogger::{self, LoggerConfig}; use cstate::{StateDB, StateWithCache, TopLevelState, TopState}; -use kvdb::KeyValueDB; +use kvdb::{DBTransaction, KeyValueDB}; use kvdb_rocksdb::{Database, DatabaseConfig}; use primitives::H256; use std::{path::Path, sync::Arc, time::Instant}; @@ -17,8 +17,10 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { let num = matches.value_of("number").unwrap(); let num: u32 = num.parse().unwrap(); + let db_dir = matches.value_of("db-dir").unwrap(); + println!("start"); - let db = open_db()?; + let db = open_db(db_dir)?; println!("open db"); let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); println!("open journal db"); @@ -27,23 +29,67 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { println!("before getting root"); - let root = { + let mut root = { let bytes = db.get(COL_EXTRA, b"perf_data_root").unwrap().unwrap(); H256::from(bytes.as_ref()) }; + let mut addresses = Vec::new(); + let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); println!("before loop"); + + let mut total_elapsed_micros = 0_u128; + let mut max_elapsed_micros = 0_u128; + for i in 0..10_u64.pow(num) { + let address = Address::random(); + addresses.push(address); + let now = Instant::now(); + toplevel_state.add_balance(&address, i).unwrap(); + let elapsed = now.elapsed().as_micros(); + if elapsed >= 1 * 1000 * 1000 { + println!("{}", elapsed); + } + total_elapsed_micros += elapsed; + if elapsed > max_elapsed_micros { + max_elapsed_micros = elapsed; + } + } + println!("Average {}us from DB", total_elapsed_micros / 10_u128.pow(num)); + println!("Max {}us from DB", max_elapsed_micros); + + root = toplevel_state.commit().unwrap(); + { + let mut batch = DBTransaction::new(); + let updated = toplevel_state.journal_under(&mut batch, 0).unwrap(); + println!("write to db {}", updated); + db.write(batch).map_err(|err| err.to_string())?; + println!("flush the db"); + db.flush().unwrap(); + } + + toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); + + let mut total_elapsed_micros = 0_u128; + let mut max_elapsed_micros = 0_u128; for i in 0..10_u64.pow(num) { // println!("loop"); let now = Instant::now(); - let address = Address::random(); + let address = addresses[i as usize]; toplevel_state.add_balance(&address, i).unwrap(); - if now.elapsed().as_secs() >= 1 { - println!("{}", now.elapsed().as_secs()); + let elapsed = now.elapsed().as_micros(); + if elapsed >= 1 * 1000 * 1000 { + println!("{}", elapsed); + } + total_elapsed_micros += now.elapsed().as_micros(); + if elapsed > max_elapsed_micros { + max_elapsed_micros = elapsed; } } + println!("Average {}us from cache", total_elapsed_micros / 10_u128.pow(num)); + println!("Max {}us from cache", max_elapsed_micros); + toplevel_state.commit().unwrap(); println!("Finished"); @@ -51,13 +97,12 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { Ok(()) } -pub const DEFAULT_DB_PATH: &str = "db_test"; +// pub const DEFAULT_DB_PATH: &str = "db_test"; pub const NUM_COLUMNS: Option = Some(6); - -pub fn open_db() -> Result, String> { +pub fn open_db(db_dir: &str) -> Result, String> { let base_path = ".".to_owned(); - let db_path = base_path + "/" + DEFAULT_DB_PATH; + let db_path = base_path + "/" + db_dir; let client_path = Path::new(&db_path); let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS); From bdf1371add2a30edc94814d4ac47660f36d2eb7f Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Mon, 14 Sep 2020 17:24:45 +0900 Subject: [PATCH 08/29] Write log of test --- codechain/subcommand/generate_data_command.rs | 2 +- codechain/subcommand/perf_write_command.rs | 95 +++++++++++-------- 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/codechain/subcommand/generate_data_command.rs b/codechain/subcommand/generate_data_command.rs index f956e8295c..b6af02173e 100644 --- a/codechain/subcommand/generate_data_command.rs +++ b/codechain/subcommand/generate_data_command.rs @@ -30,7 +30,7 @@ pub fn run_generate_data_command(matches: &ArgMatches) -> Result<(), String> { let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); for j in 0..1000 { let address = Address::random(); - toplevel_state.add_balance(&address, i * 1000 + j).unwrap(); + toplevel_state.add_balance(&address, i * 1000 + j + 1).unwrap(); } root = toplevel_state.commit().unwrap(); println!("write root {:?}", root); diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs index 41aecfb06b..c6557b63d3 100644 --- a/codechain/subcommand/perf_write_command.rs +++ b/codechain/subcommand/perf_write_command.rs @@ -6,7 +6,7 @@ use cstate::{StateDB, StateWithCache, TopLevelState, TopState}; use kvdb::{DBTransaction, KeyValueDB}; use kvdb_rocksdb::{Database, DatabaseConfig}; use primitives::H256; -use std::{path::Path, sync::Arc, time::Instant}; +use std::{path::Path, sync::Arc, time::Instant, u128, u32, u64}; pub const COL_STATE: Option = Some(0); pub const COL_EXTRA: Option = Some(3); @@ -41,56 +41,71 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { println!("before loop"); let mut total_elapsed_micros = 0_u128; - let mut max_elapsed_micros = 0_u128; + /// (index, elapsed, read count) + let mut max_elapsed = (0_u64, 0_u128, 0_u32); + /// (index, elapsed, read count) + let mut max_height = (0_u64, 0_u128, 0_u32); + /// (index, elapsed, read count) + let mut min_elapsed = (u64::MAX, u128::MAX, u32::MAX); for i in 0..10_u64.pow(num) { - let address = Address::random(); + // let address = Address::random(); + let address = Address::from(i as u64); addresses.push(address); let now = Instant::now(); - toplevel_state.add_balance(&address, i).unwrap(); + let read_count = toplevel_state.add_balance_debug(&address, i + 1).unwrap(); let elapsed = now.elapsed().as_micros(); - if elapsed >= 1 * 1000 * 1000 { - println!("{}", elapsed); - } total_elapsed_micros += elapsed; + let (_, max_elapsed_micros, _) = max_elapsed; if elapsed > max_elapsed_micros { - max_elapsed_micros = elapsed; + max_elapsed = (i, elapsed, read_count); } - } - println!("Average {}us from DB", total_elapsed_micros / 10_u128.pow(num)); - println!("Max {}us from DB", max_elapsed_micros); - - root = toplevel_state.commit().unwrap(); - { - let mut batch = DBTransaction::new(); - let updated = toplevel_state.journal_under(&mut batch, 0).unwrap(); - println!("write to db {}", updated); - db.write(batch).map_err(|err| err.to_string())?; - println!("flush the db"); - db.flush().unwrap(); - } - - toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); - - let mut total_elapsed_micros = 0_u128; - let mut max_elapsed_micros = 0_u128; - for i in 0..10_u64.pow(num) { - // println!("loop"); - let now = Instant::now(); - let address = addresses[i as usize]; - toplevel_state.add_balance(&address, i).unwrap(); - let elapsed = now.elapsed().as_micros(); - if elapsed >= 1 * 1000 * 1000 { - println!("{}", elapsed); + let (_, min_elapsed_micros, _) = min_elapsed; + if elapsed < min_elapsed_micros { + min_elapsed = (i, elapsed, read_count); } - total_elapsed_micros += now.elapsed().as_micros(); - if elapsed > max_elapsed_micros { - max_elapsed_micros = elapsed; + let (_, _, max_height_) = max_height; + if read_count > max_height_ { + max_height = (i, elapsed, read_count); } + println!("read count {}", read_count); } - println!("Average {}us from cache", total_elapsed_micros / 10_u128.pow(num)); - println!("Max {}us from cache", max_elapsed_micros); + println!("Average {}us from DB", total_elapsed_micros / 10_u128.pow(num)); + println!("Max {}us from DB address: {} read_count {}", max_elapsed.1, max_elapsed.0, max_elapsed.2); + println!("Min {}us from DB address: {} read_count {}", min_elapsed.1, min_elapsed.0, min_elapsed.2); + println!("Max height {}us from DB address: {} read_count {}", max_height.1, max_height.0, max_height.2); - toplevel_state.commit().unwrap(); + root = toplevel_state.commit().unwrap(); + // { + // let mut batch = DBTransaction::new(); + // let updated = toplevel_state.journal_under(&mut batch, 0).unwrap(); + // println!("write to db {}", updated); + // db.write(batch).map_err(|err| err.to_string())?; + // println!("flush the db"); + // db.flush().unwrap(); + // } + + // toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); + + // let mut total_elapsed_micros = 0_u128; + // let mut max_elapsed_micros = 0_u128; + // for i in 0..10_u64.pow(num) { + // // println!("loop"); + // let now = Instant::now(); + // let address = addresses[i as usize]; + // toplevel_state.add_balance(&address, i).unwrap(); + // let elapsed = now.elapsed().as_micros(); + // if elapsed >= 1 * 1000 * 1000 { + // println!("{}", elapsed); + // } + // total_elapsed_micros += now.elapsed().as_micros(); + // if elapsed > max_elapsed_micros { + // max_elapsed_micros = elapsed; + // } + // } + // println!("Average {}us from cache", total_elapsed_micros / 10_u128.pow(num)); + // println!("Max {}us from cache", max_elapsed_micros); + + // toplevel_state.commit().unwrap(); println!("Finished"); From ef3eb78c5dfc55657f804722ef3c1c29aa2b2d52 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Mon, 14 Sep 2020 17:25:05 +0900 Subject: [PATCH 09/29] Print the db read count --- state/src/cache/top_cache.rs | 4 ++++ state/src/cache/write_back.rs | 32 +++++++++++++++++++++++++ state/src/impls/top_level.rs | 19 +++++++++++++++ state/src/traits.rs | 2 ++ util/merkle/src/lib.rs | 6 +++++ util/merkle/src/triedb.rs | 45 +++++++++++++++++++++++++++++++++++ 6 files changed, 108 insertions(+) diff --git a/state/src/cache/top_cache.rs b/state/src/cache/top_cache.rs index 91c442078b..39065a34e2 100644 --- a/state/src/cache/top_cache.rs +++ b/state/src/cache/top_cache.rs @@ -98,6 +98,10 @@ impl TopCache { self.account.get_mut(a, db) } + pub fn account_mut_debug(&self, a: &Address, db: &TrieDB) -> TrieResult<(RefMut, u32)> { + self.account.get_mut_debug(a, db) + } + pub fn remove_account(&self, address: &Address) { self.account.remove(address) } diff --git a/state/src/cache/write_back.rs b/state/src/cache/write_back.rs index f2323ec4d1..95fa8e3b75 100644 --- a/state/src/cache/write_back.rs +++ b/state/src/cache/write_back.rs @@ -234,6 +234,38 @@ where })) } + /// Pull item `a` in our cache from the trie DB. + /// If it doesn't exist, make item equal the evaluation of `default`. + pub fn get_mut_debug(&self, a: &Item::Address, db: &TrieDB) -> cmerkle::Result<(RefMut, u32)> { + let contains_key = self.cache.borrow().contains_key(a); + let mut read_count = 0_u32; + if !contains_key { + let (maybe_item, read_count_) = db.get_with_debug(a.as_ref(), ::rlp::decode::)?; + self.insert(a, Entry::::new_clean(maybe_item)); + read_count = read_count_; + } + self.note(a); + + // at this point the entry is guaranteed to be in the cache. + Ok(( + RefMut::map(self.cache.borrow_mut(), |c| { + let entry = c.get_mut(a).expect("entry known to exist in the cache; qed"); + + match &mut entry.item { + Some(_) => {} + slot @ None => *slot = Some(Item::default()), + } + + // set the dirty flag after changing data. + entry.is_dirty = true; + entry.touched = touched_count(); + entry.item.as_mut().expect("Required item must always exist; qed") + }), + read_count, + )) + } + + pub fn create Item>(&self, a: &Item::Address, f: F) -> cmerkle::Result { if let Some(cached) = self.cache.borrow().get(a) { assert!(cached.item.is_none()); diff --git a/state/src/impls/top_level.rs b/state/src/impls/top_level.rs index b871efaa58..c261364dd3 100644 --- a/state/src/impls/top_level.rs +++ b/state/src/impls/top_level.rs @@ -639,6 +639,15 @@ impl TopLevelState { self.top_cache.account_mut(&a, &trie) } + fn get_account_mut_debug(&self, a: &Address) -> TrieResult<(RefMut, u32)> { + debug_assert_eq!(Ok(false), self.regular_account_exists_and_not_null_by_address(a)); + + let db = self.db.borrow(); + let trie = TrieFactory::readonly(db.as_hashdb(), &self.root)?; + self.top_cache.account_mut_debug(&a, &trie) + } + + fn get_regular_account_mut(&self, public: &Public) -> TrieResult> { let regular_account_address = RegularAccountAddress::new(public); let db = self.db.borrow(); @@ -813,6 +822,16 @@ impl TopState for TopLevelState { Ok(()) } + fn add_balance_debug(&mut self, a: &Address, incr: u64) -> TrieResult { + ctrace!(STATE, "add_balance({}, {}): {}", a, incr, self.balance(a)?); + if incr != 0 { + let (mut account, read_count) = self.get_account_mut_debug(a)?; + account.add_balance(incr); + return Ok(read_count); + } + Ok(0) + } + fn sub_balance(&mut self, a: &Address, decr: u64) -> StateResult<()> { ctrace!(STATE, "sub_balance({}, {}): {}", a, decr, self.balance(a)?); if decr == 0 { diff --git a/state/src/traits.rs b/state/src/traits.rs index 40d94688f7..9e974252d4 100644 --- a/state/src/traits.rs +++ b/state/src/traits.rs @@ -155,6 +155,8 @@ pub trait TopState { /// Add `incr` to the balance of account `a`. fn add_balance(&mut self, a: &Address, incr: u64) -> TrieResult<()>; + + fn add_balance_debug(&mut self, a: &Address, incr: u64) -> TrieResult; /// Subtract `decr` from the balance of account `a`. fn sub_balance(&mut self, a: &Address, decr: u64) -> StateResult<()>; /// Subtracts `by` from the balance of `from` and adds it to that of `to`. diff --git a/util/merkle/src/lib.rs b/util/merkle/src/lib.rs index 63f7b708f7..ff3f88a367 100644 --- a/util/merkle/src/lib.rs +++ b/util/merkle/src/lib.rs @@ -109,6 +109,12 @@ pub trait Trie { /// Search for the key with the given query parameter. See the docs of the `Query` /// trait for more details. fn get_with(&self, key: &[u8], query: Q) -> Result>; + + /// Search for the key with the given query parameter. See the docs of the `Query` + /// trait for more details. + fn get_with_debug(&self, key: &[u8], query: Q) -> Result<(Option, u32)> { + Ok((self.get_with(key, query)?, 999)) + } } /// A key-value datastore implemented as a database-backed modified Merkle tree. diff --git a/util/merkle/src/triedb.rs b/util/merkle/src/triedb.rs index c65ce58c6a..01f6fa47f9 100644 --- a/util/merkle/src/triedb.rs +++ b/util/merkle/src/triedb.rs @@ -105,6 +105,44 @@ impl<'db> TrieDB<'db> { None => Ok(None), } } + + /// Get auxiliary + fn get_aux_debug( + &self, + path: &NibbleSlice, + cur_node_hash: Option, + query: Q, + ) -> crate::Result<(Option, u32)> { + match cur_node_hash { + Some(hash) => { + let node_rlp = self.db.get(&hash).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; + + match RlpNode::decoded(&node_rlp) { + Some(RlpNode::Leaf(partial, value)) => { + if &partial == path { + Ok((Some(query.decode(value)), 0)) + } else { + Ok((None, 0)) + } + } + Some(RlpNode::Branch(partial, children)) => { + if path.starts_with(&partial) { + let (result, count) = self.get_aux_debug( + &path.mid(partial.len() + 1), + children[path.mid(partial.len()).at(0) as usize], + query, + )?; + Ok((result, count + 1)) + } else { + Ok((None, 0)) + } + } + None => Ok((None, 0)), + } + } + None => Ok((None, 0)), + } + } } impl<'db> Trie for TrieDB<'db> { @@ -118,6 +156,13 @@ impl<'db> Trie for TrieDB<'db> { self.get_aux(&NibbleSlice::new(&path), Some(root), query) } + + fn get_with_debug(&self, key: &[u8], query: Q) -> crate::Result<(Option, u32)> { + let path = blake256(key); + let root = *self.root; + + self.get_aux_debug(&NibbleSlice::new(&path), Some(root), query) + } } #[cfg(test)] From c03cb481ffd1dc90aa383bd3d183201b6ce24bd6 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Mon, 14 Sep 2020 17:42:37 +0900 Subject: [PATCH 10/29] Use debug info --- Cargo.lock | 1 + codechain/subcommand/perf_write_command.rs | 25 +++++++++++----------- state/src/cache/top_cache.rs | 3 ++- state/src/cache/write_back.rs | 11 +++++----- state/src/impls/top_level.rs | 12 +++++------ state/src/traits.rs | 4 ++-- types/src/lib.rs | 17 +++++++++++++++ util/merkle/Cargo.toml | 1 + util/merkle/src/lib.rs | 6 ++++-- util/merkle/src/triedb.rs | 20 +++++++++-------- 10 files changed, 63 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5cf63fb5c5..809fafea99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -413,6 +413,7 @@ name = "codechain-merkle" version = "0.1.0" dependencies = [ "codechain-crypto 0.1.0 (git+https://github.com/CodeChain-io/rust-codechain-crypto.git)", + "codechain-types 0.1.0", "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "hashdb 0.1.1", "memorydb 0.1.1", diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs index c6557b63d3..516a553ef9 100644 --- a/codechain/subcommand/perf_write_command.rs +++ b/codechain/subcommand/perf_write_command.rs @@ -3,6 +3,7 @@ use ckey::Address; use clap::ArgMatches; use clogger::{self, LoggerConfig}; use cstate::{StateDB, StateWithCache, TopLevelState, TopState}; +use ctypes::DebugInfo; use kvdb::{DBTransaction, KeyValueDB}; use kvdb_rocksdb::{Database, DatabaseConfig}; use primitives::H256; @@ -42,37 +43,37 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { let mut total_elapsed_micros = 0_u128; /// (index, elapsed, read count) - let mut max_elapsed = (0_u64, 0_u128, 0_u32); + let mut max_elapsed = (0_u64, 0_u128, DebugInfo::empty()); /// (index, elapsed, read count) - let mut max_height = (0_u64, 0_u128, 0_u32); + let mut max_height = (0_u64, 0_u128, DebugInfo::empty()); /// (index, elapsed, read count) - let mut min_elapsed = (u64::MAX, u128::MAX, u32::MAX); + let mut min_elapsed = (u64::MAX, u128::MAX, DebugInfo::empty()); for i in 0..10_u64.pow(num) { // let address = Address::random(); let address = Address::from(i as u64); addresses.push(address); let now = Instant::now(); - let read_count = toplevel_state.add_balance_debug(&address, i + 1).unwrap(); + let debug_info = toplevel_state.add_balance_debug(&address, i + 1).unwrap(); let elapsed = now.elapsed().as_micros(); total_elapsed_micros += elapsed; let (_, max_elapsed_micros, _) = max_elapsed; if elapsed > max_elapsed_micros { - max_elapsed = (i, elapsed, read_count); + max_elapsed = (i, elapsed, debug_info); } let (_, min_elapsed_micros, _) = min_elapsed; if elapsed < min_elapsed_micros { - min_elapsed = (i, elapsed, read_count); + min_elapsed = (i, elapsed, debug_info); } let (_, _, max_height_) = max_height; - if read_count > max_height_ { - max_height = (i, elapsed, read_count); + if debug_info.read_count > max_height_.read_count { + max_height = (i, elapsed, debug_info); } - println!("read count {}", read_count); + println!("debug info {:?}", debug_info); } println!("Average {}us from DB", total_elapsed_micros / 10_u128.pow(num)); - println!("Max {}us from DB address: {} read_count {}", max_elapsed.1, max_elapsed.0, max_elapsed.2); - println!("Min {}us from DB address: {} read_count {}", min_elapsed.1, min_elapsed.0, min_elapsed.2); - println!("Max height {}us from DB address: {} read_count {}", max_height.1, max_height.0, max_height.2); + println!("Max {}us from DB address: {} debug_info {:?}", max_elapsed.1, max_elapsed.0, max_elapsed.2); + println!("Min {}us from DB address: {} debug_info {:?}", min_elapsed.1, min_elapsed.0, min_elapsed.2); + println!("Max height {}us from DB address: {} debug_info {:?}", max_height.1, max_height.0, max_height.2); root = toplevel_state.commit().unwrap(); // { diff --git a/state/src/cache/top_cache.rs b/state/src/cache/top_cache.rs index 39065a34e2..210ca123f6 100644 --- a/state/src/cache/top_cache.rs +++ b/state/src/cache/top_cache.rs @@ -24,6 +24,7 @@ use super::WriteBack; use crate::{ Account, ActionData, Metadata, MetadataAddress, RegularAccount, RegularAccountAddress, Shard, ShardAddress, Text, }; +use ctypes::DebugInfo; pub struct TopCache { account: WriteBack, @@ -98,7 +99,7 @@ impl TopCache { self.account.get_mut(a, db) } - pub fn account_mut_debug(&self, a: &Address, db: &TrieDB) -> TrieResult<(RefMut, u32)> { + pub fn account_mut_debug(&self, a: &Address, db: &TrieDB) -> TrieResult<(RefMut, DebugInfo)> { self.account.get_mut_debug(a, db) } diff --git a/state/src/cache/write_back.rs b/state/src/cache/write_back.rs index 95fa8e3b75..692b6a1bba 100644 --- a/state/src/cache/write_back.rs +++ b/state/src/cache/write_back.rs @@ -25,6 +25,7 @@ use std::vec::Vec; use cmerkle::{self, Result as TrieResult, Trie, TrieDB, TrieMut}; use super::CacheableItem; +use ctypes::DebugInfo; static TOUCHED_COUNT: AtomicUsize = AtomicUsize::new(0); fn touched_count() -> usize { @@ -236,13 +237,13 @@ where /// Pull item `a` in our cache from the trie DB. /// If it doesn't exist, make item equal the evaluation of `default`. - pub fn get_mut_debug(&self, a: &Item::Address, db: &TrieDB) -> cmerkle::Result<(RefMut, u32)> { + pub fn get_mut_debug(&self, a: &Item::Address, db: &TrieDB) -> cmerkle::Result<(RefMut, DebugInfo)> { let contains_key = self.cache.borrow().contains_key(a); - let mut read_count = 0_u32; + let mut debug_info = DebugInfo::empty(); if !contains_key { - let (maybe_item, read_count_) = db.get_with_debug(a.as_ref(), ::rlp::decode::)?; + let (maybe_item, debug_info_) = db.get_with_debug(a.as_ref(), ::rlp::decode::)?; self.insert(a, Entry::::new_clean(maybe_item)); - read_count = read_count_; + debug_info = debug_info_; } self.note(a); @@ -261,7 +262,7 @@ where entry.touched = touched_count(); entry.item.as_mut().expect("Required item must always exist; qed") }), - read_count, + debug_info, )) } diff --git a/state/src/impls/top_level.rs b/state/src/impls/top_level.rs index c261364dd3..de031870c6 100644 --- a/state/src/impls/top_level.rs +++ b/state/src/impls/top_level.rs @@ -48,7 +48,7 @@ use ctypes::transaction::{ use ctypes::util::unexpected::Mismatch; #[cfg(test)] use ctypes::Tracker; -use ctypes::{BlockNumber, CommonParams, ShardId, TxHash}; +use ctypes::{BlockNumber, CommonParams, ShardId, TxHash, DebugInfo}; use cvm::ChainTimeInfo; use hashdb::AsHashDB; use kvdb::DBTransaction; @@ -639,7 +639,7 @@ impl TopLevelState { self.top_cache.account_mut(&a, &trie) } - fn get_account_mut_debug(&self, a: &Address) -> TrieResult<(RefMut, u32)> { + fn get_account_mut_debug(&self, a: &Address) -> TrieResult<(RefMut, DebugInfo)> { debug_assert_eq!(Ok(false), self.regular_account_exists_and_not_null_by_address(a)); let db = self.db.borrow(); @@ -822,14 +822,14 @@ impl TopState for TopLevelState { Ok(()) } - fn add_balance_debug(&mut self, a: &Address, incr: u64) -> TrieResult { + fn add_balance_debug(&mut self, a: &Address, incr: u64) -> TrieResult { ctrace!(STATE, "add_balance({}, {}): {}", a, incr, self.balance(a)?); if incr != 0 { - let (mut account, read_count) = self.get_account_mut_debug(a)?; + let (mut account, debug_info) = self.get_account_mut_debug(a)?; account.add_balance(incr); - return Ok(read_count); + return Ok(debug_info); } - Ok(0) + Ok(DebugInfo::empty()) } fn sub_balance(&mut self, a: &Address, decr: u64) -> StateResult<()> { diff --git a/state/src/traits.rs b/state/src/traits.rs index 9e974252d4..f6f1802a0b 100644 --- a/state/src/traits.rs +++ b/state/src/traits.rs @@ -17,7 +17,7 @@ use ckey::{public_to_address, Address, Public, Signature}; use cmerkle::Result as TrieResult; use ctypes::transaction::ShardTransaction; -use ctypes::{BlockNumber, CommonParams, ShardId, Tracker, TxHash}; +use ctypes::{BlockNumber, CommonParams, ShardId, Tracker, TxHash, DebugInfo}; use cvm::ChainTimeInfo; use primitives::{Bytes, H160, H256}; @@ -156,7 +156,7 @@ pub trait TopState { /// Add `incr` to the balance of account `a`. fn add_balance(&mut self, a: &Address, incr: u64) -> TrieResult<()>; - fn add_balance_debug(&mut self, a: &Address, incr: u64) -> TrieResult; + fn add_balance_debug(&mut self, a: &Address, incr: u64) -> TrieResult; /// Subtract `decr` from the balance of account `a`. fn sub_balance(&mut self, a: &Address, decr: u64) -> StateResult<()>; /// Subtracts `by` from the balance of `from` and adds it to that of `to`. diff --git a/types/src/lib.rs b/types/src/lib.rs index 05e9fbf1c4..a560e9b2d9 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -45,3 +45,20 @@ pub use common_params::CommonParams; pub use header::Header; pub use tracker::Tracker; pub use tx_hash::TxHash; + +#[derive(Debug, Copy, Clone)] +pub struct DebugInfo { + pub read_count: u32, +} + +impl DebugInfo { + pub fn empty() -> Self { + Self { + read_count: 0, + } + } + + pub fn inc_read_count(&mut self) { + self.read_count += 1; + } +} diff --git a/util/merkle/Cargo.toml b/util/merkle/Cargo.toml index 4cf53e157a..239e02ba2d 100644 --- a/util/merkle/Cargo.toml +++ b/util/merkle/Cargo.toml @@ -10,6 +10,7 @@ hashdb = {path = "../hashdb" } codechain-crypto = { git = "https://github.com/CodeChain-io/rust-codechain-crypto.git", version = "0.1" } primitives = { git = "https://github.com/CodeChain-io/rust-codechain-primitives.git", version = "0.4" } rlp = {path = "../rlp" } +codechain-types= {path = "../../types"} [dev-dependencies] trie-standardmap = { path = "../trie-standardmap" } diff --git a/util/merkle/src/lib.rs b/util/merkle/src/lib.rs index ff3f88a367..428cf8dffb 100644 --- a/util/merkle/src/lib.rs +++ b/util/merkle/src/lib.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +extern crate codechain_types as ctypes; extern crate codechain_crypto as ccrypto; extern crate elastic_array; extern crate hashdb; @@ -25,6 +26,7 @@ extern crate rlp; #[cfg(test)] extern crate trie_standardmap as standardmap; +use ctypes::DebugInfo; use std::fmt; use ccrypto::BLAKE_NULL_RLP; @@ -112,8 +114,8 @@ pub trait Trie { /// Search for the key with the given query parameter. See the docs of the `Query` /// trait for more details. - fn get_with_debug(&self, key: &[u8], query: Q) -> Result<(Option, u32)> { - Ok((self.get_with(key, query)?, 999)) + fn get_with_debug(&self, key: &[u8], query: Q) -> Result<(Option, DebugInfo)> { + Ok((self.get_with(key, query)?, DebugInfo::empty())) } } diff --git a/util/merkle/src/triedb.rs b/util/merkle/src/triedb.rs index 01f6fa47f9..79b203af24 100644 --- a/util/merkle/src/triedb.rs +++ b/util/merkle/src/triedb.rs @@ -18,6 +18,7 @@ use ccrypto::blake256; use hashdb::HashDB; use primitives::H256; +use ctypes::DebugInfo; use crate::nibbleslice::NibbleSlice; use crate::node::Node as RlpNode; use crate::{Query, Trie, TrieError}; @@ -112,7 +113,7 @@ impl<'db> TrieDB<'db> { path: &NibbleSlice, cur_node_hash: Option, query: Q, - ) -> crate::Result<(Option, u32)> { + ) -> crate::Result<(Option, DebugInfo)> { match cur_node_hash { Some(hash) => { let node_rlp = self.db.get(&hash).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; @@ -120,27 +121,28 @@ impl<'db> TrieDB<'db> { match RlpNode::decoded(&node_rlp) { Some(RlpNode::Leaf(partial, value)) => { if &partial == path { - Ok((Some(query.decode(value)), 0)) + Ok((Some(query.decode(value)), DebugInfo::empty())) } else { - Ok((None, 0)) + Ok((None, DebugInfo::empty())) } } Some(RlpNode::Branch(partial, children)) => { if path.starts_with(&partial) { - let (result, count) = self.get_aux_debug( + let (result, mut debug_info) = self.get_aux_debug( &path.mid(partial.len() + 1), children[path.mid(partial.len()).at(0) as usize], query, )?; - Ok((result, count + 1)) + debug_info.inc_read_count(); + Ok((result, debug_info)) } else { - Ok((None, 0)) + Ok((None, DebugInfo::empty())) } } - None => Ok((None, 0)), + None => Ok((None, DebugInfo::empty())), } } - None => Ok((None, 0)), + None => Ok((None, DebugInfo::empty())), } } } @@ -157,7 +159,7 @@ impl<'db> Trie for TrieDB<'db> { self.get_aux(&NibbleSlice::new(&path), Some(root), query) } - fn get_with_debug(&self, key: &[u8], query: Q) -> crate::Result<(Option, u32)> { + fn get_with_debug(&self, key: &[u8], query: Q) -> crate::Result<(Option, DebugInfo)> { let path = blake256(key); let root = *self.root; From c14e44d83e1eaf4581da1159b0020fecc09a4125 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Mon, 14 Sep 2020 19:12:06 +0900 Subject: [PATCH 11/29] adding db read script --- Cargo.lock | 2 + Cargo.toml | 1 + codechain/main.rs | 1 + codechain/subcommand/mod.rs | 3 ++ codechain/subcommand/perf_write_command.rs | 20 ++++----- codechain/subcommand/read_db_command.rs | 51 ++++++++++++++++++++++ state/src/cache/write_back.rs | 7 ++- types/Cargo.toml | 1 + types/src/lib.rs | 21 ++++++++- util/merkle/src/triedb.rs | 18 +++++--- 10 files changed, 106 insertions(+), 19 deletions(-) create mode 100644 codechain/subcommand/read_db_command.rs diff --git a/Cargo.lock b/Cargo.lock index 809fafea99..60f0117e30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -243,6 +243,7 @@ dependencies = [ "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "primitives 0.4.0 (git+https://github.com/CodeChain-io/rust-codechain-primitives.git)", "rpassword 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", @@ -578,6 +579,7 @@ dependencies = [ "primitives 0.4.0 (git+https://github.com/CodeChain-io/rust-codechain-primitives.git)", "rlp 0.2.1", "rlp_derive 0.1.0", + "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 8ed13a89c4..6250b32c72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ ctrlc = { git = "https://github.com/paritytech/rust-ctrlc.git" } fdlimit = "0.1" finally-block = "0.1" futures = "0.1" +rustc-hex = "1.0" kvdb = { path = "util/kvdb" } kvdb-rocksdb = { path = "util/kvdb-rocksdb" } log = "0.4.6" diff --git a/codechain/main.rs b/codechain/main.rs index d6d8d1f8bd..554a668840 100644 --- a/codechain/main.rs +++ b/codechain/main.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +extern crate rustc_hex; #[macro_use] extern crate clap; extern crate futures; diff --git a/codechain/subcommand/mod.rs b/codechain/subcommand/mod.rs index 6d7b034a95..72450b9335 100644 --- a/codechain/subcommand/mod.rs +++ b/codechain/subcommand/mod.rs @@ -18,6 +18,7 @@ mod account_command; mod convert_command; mod generate_data_command; mod perf_write_command; +mod read_db_command; use clap::ArgMatches; @@ -25,6 +26,7 @@ use self::account_command::run_account_command; use self::convert_command::run_convert_command; use self::generate_data_command::run_generate_data_command; use self::perf_write_command::run_perf_write_command; +use self::read_db_command::run_read_db_command; pub fn run_subcommand(matches: &ArgMatches) -> Result<(), String> { let subcommand = matches.subcommand.as_ref().unwrap(); @@ -33,6 +35,7 @@ pub fn run_subcommand(matches: &ArgMatches) -> Result<(), String> { "convert" => run_convert_command(&subcommand.matches), "generate-data" => run_generate_data_command(&subcommand.matches), "perf-write" => run_perf_write_command(&subcommand.matches), + "read-db" => run_read_db_command(&subcommand.matches), "commit-hash" => { println!("{}", env!("VERGEN_SHA")); Ok(()) diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs index 516a553ef9..8b7445906e 100644 --- a/codechain/subcommand/perf_write_command.rs +++ b/codechain/subcommand/perf_write_command.rs @@ -4,7 +4,7 @@ use clap::ArgMatches; use clogger::{self, LoggerConfig}; use cstate::{StateDB, StateWithCache, TopLevelState, TopState}; use ctypes::DebugInfo; -use kvdb::{DBTransaction, KeyValueDB}; +use kvdb::KeyValueDB; use kvdb_rocksdb::{Database, DatabaseConfig}; use primitives::H256; use std::{path::Path, sync::Arc, time::Instant, u128, u32, u64}; @@ -30,7 +30,7 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { println!("before getting root"); - let mut root = { + let root = { let bytes = db.get(COL_EXTRA, b"perf_data_root").unwrap().unwrap(); H256::from(bytes.as_ref()) }; @@ -42,11 +42,11 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { println!("before loop"); let mut total_elapsed_micros = 0_u128; - /// (index, elapsed, read count) + // (index, elapsed, read count) let mut max_elapsed = (0_u64, 0_u128, DebugInfo::empty()); - /// (index, elapsed, read count) + // (index, elapsed, read count) let mut max_height = (0_u64, 0_u128, DebugInfo::empty()); - /// (index, elapsed, read count) + // (index, elapsed, read count) let mut min_elapsed = (u64::MAX, u128::MAX, DebugInfo::empty()); for i in 0..10_u64.pow(num) { // let address = Address::random(); @@ -58,15 +58,15 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { total_elapsed_micros += elapsed; let (_, max_elapsed_micros, _) = max_elapsed; if elapsed > max_elapsed_micros { - max_elapsed = (i, elapsed, debug_info); + max_elapsed = (i, elapsed, debug_info.clone()); } let (_, min_elapsed_micros, _) = min_elapsed; if elapsed < min_elapsed_micros { - min_elapsed = (i, elapsed, debug_info); + min_elapsed = (i, elapsed, debug_info.clone()); } - let (_, _, max_height_) = max_height; + let (_, _, max_height_) = &max_height; if debug_info.read_count > max_height_.read_count { - max_height = (i, elapsed, debug_info); + max_height = (i, elapsed, debug_info.clone()); } println!("debug info {:?}", debug_info); } @@ -75,7 +75,7 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { println!("Min {}us from DB address: {} debug_info {:?}", min_elapsed.1, min_elapsed.0, min_elapsed.2); println!("Max height {}us from DB address: {} debug_info {:?}", max_height.1, max_height.0, max_height.2); - root = toplevel_state.commit().unwrap(); + toplevel_state.commit().unwrap(); // { // let mut batch = DBTransaction::new(); // let updated = toplevel_state.journal_under(&mut batch, 0).unwrap(); diff --git a/codechain/subcommand/read_db_command.rs b/codechain/subcommand/read_db_command.rs new file mode 100644 index 0000000000..27f4e3b868 --- /dev/null +++ b/codechain/subcommand/read_db_command.rs @@ -0,0 +1,51 @@ +use ccore::ClientConfig; +use clap::ArgMatches; +use clogger::{self, LoggerConfig}; +// use cstate::StateDB; +use kvdb::KeyValueDB; +use kvdb_rocksdb::{Database, DatabaseConfig}; +use rustc_hex::{FromHex, ToHex}; +use std::{path::Path, sync::Arc, time::Instant, u32}; + +// pub const COL_EXTRA: Option = Some(3); +pub const COL_STATE: Option = Some(0); +pub const NUM_COLUMNS: Option = Some(6); + +pub fn run_read_db_command(matches: &ArgMatches) -> Result<(), String> { + clogger::init(&LoggerConfig::new(777), None).expect("Logger must be successfully initialized"); + + let db_dir = matches.value_of("db-dir").unwrap(); + let key = matches.value_of("key").unwrap().to_string().from_hex().unwrap(); + + let db = open_db(db_dir)?; + // let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); + // let state_db = StateDB::new(journal_db); + + let now = Instant::now(); + let value = db.get(COL_STATE, &key).unwrap().unwrap(); + let elapsed = now.elapsed().as_micros(); + + println!("elapsed {}us value: {}", elapsed, value.to_hex()); + + Ok(()) +} + +pub fn open_db(db_dir: &str) -> Result, String> { + let base_path = ".".to_owned(); + let db_path = base_path + "/" + db_dir; + let client_path = Path::new(&db_path); + let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS); + + let client_config = ClientConfig::default(); + + db_config.memory_budget = client_config.db_cache_size; + db_config.compaction = client_config.db_compaction.compaction_profile(client_path); + db_config.wal = client_config.db_wal; + + let db = Arc::new( + Database::open(&db_config, &client_path.to_str().expect("DB path could not be converted to string.")) + .map_err(|_e| "Low level database error. Some issue with disk?".to_string())?, + ); + + Ok(db) +} diff --git a/state/src/cache/write_back.rs b/state/src/cache/write_back.rs index 692b6a1bba..26be9cdbc3 100644 --- a/state/src/cache/write_back.rs +++ b/state/src/cache/write_back.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::convert::AsRef; use std::fmt; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::vec::Vec; +use std::{time::Instant, vec::Vec}; use cmerkle::{self, Result as TrieResult, Trie, TrieDB, TrieMut}; @@ -241,7 +241,10 @@ where let contains_key = self.cache.borrow().contains_key(a); let mut debug_info = DebugInfo::empty(); if !contains_key { - let (maybe_item, debug_info_) = db.get_with_debug(a.as_ref(), ::rlp::decode::)?; + let now = Instant::now(); + let (maybe_item, mut debug_info_) = db.get_with_debug(a.as_ref(), ::rlp::decode::)?; + let elapsed = now.elapsed().as_micros(); + debug_info_.db_read_time_us = elapsed; self.insert(a, Entry::::new_clean(maybe_item)); debug_info = debug_info_; } diff --git a/types/Cargo.toml b/types/Cargo.toml index 379558ca39..96fd06f8d1 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -12,6 +12,7 @@ rlp = { path = "../util/rlp" } rlp_derive = { path = "../util/rlp_derive" } serde = "1.0" serde_derive = "1.0" +rustc-hex = "1.0" [dev-dependencies] serde_json = "1.0" diff --git a/types/src/lib.rs b/types/src/lib.rs index a560e9b2d9..dee2410b64 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -26,6 +26,7 @@ extern crate serde; extern crate serde_derive; #[cfg(test)] extern crate serde_json; +extern crate rustc_hex; mod block_hash; mod common_params; @@ -43,22 +44,40 @@ pub type ShardId = u16; pub use block_hash::BlockHash; pub use common_params::CommonParams; pub use header::Header; +use rustc_hex::ToHex; pub use tracker::Tracker; pub use tx_hash::TxHash; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Clone)] +pub struct Read { + time_us: u128, + key: String, +} + +#[derive(Debug, Clone)] pub struct DebugInfo { pub read_count: u32, + pub db_read_time_us: u128, + pub reads: Vec, } impl DebugInfo { pub fn empty() -> Self { Self { read_count: 0, + db_read_time_us: 0, + reads: Vec::new(), } } pub fn inc_read_count(&mut self) { self.read_count += 1; } + + pub fn add_read(&mut self, time_us: u128, key: &[u8]) { + self.reads.push(Read { + time_us, + key: key.to_hex(), + }) + } } diff --git a/util/merkle/src/triedb.rs b/util/merkle/src/triedb.rs index 79b203af24..b9fdf109ae 100644 --- a/util/merkle/src/triedb.rs +++ b/util/merkle/src/triedb.rs @@ -18,10 +18,11 @@ use ccrypto::blake256; use hashdb::HashDB; use primitives::H256; -use ctypes::DebugInfo; use crate::nibbleslice::NibbleSlice; use crate::node::Node as RlpNode; use crate::{Query, Trie, TrieError}; +use ctypes::DebugInfo; +use std::time::Instant; /// A `Trie` implementation using a generic `HashDB` backing database. /// /// Use it as a `Trie` trait object. You can use `db()` to get the backing database object. @@ -116,14 +117,18 @@ impl<'db> TrieDB<'db> { ) -> crate::Result<(Option, DebugInfo)> { match cur_node_hash { Some(hash) => { + let read_start = Instant::now(); let node_rlp = self.db.get(&hash).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; + let elapsed = read_start.elapsed().as_micros(); + let mut empty = DebugInfo::empty(); + empty.add_read(elapsed, hash.as_ref()); match RlpNode::decoded(&node_rlp) { Some(RlpNode::Leaf(partial, value)) => { if &partial == path { - Ok((Some(query.decode(value)), DebugInfo::empty())) + Ok((Some(query.decode(value)), empty)) } else { - Ok((None, DebugInfo::empty())) + Ok((None, empty)) } } Some(RlpNode::Branch(partial, children)) => { @@ -134,12 +139,13 @@ impl<'db> TrieDB<'db> { query, )?; debug_info.inc_read_count(); + debug_info.add_read(elapsed, hash.as_ref()); Ok((result, debug_info)) } else { - Ok((None, DebugInfo::empty())) + Ok((None, empty)) } } - None => Ok((None, DebugInfo::empty())), + None => Ok((None, empty)), } } None => Ok((None, DebugInfo::empty())), @@ -159,7 +165,7 @@ impl<'db> Trie for TrieDB<'db> { self.get_aux(&NibbleSlice::new(&path), Some(root), query) } - fn get_with_debug(&self, key: &[u8], query: Q) -> crate::Result<(Option, DebugInfo)> { + fn get_with_debug(&self, key: &[u8], query: Q) -> crate::Result<(Option, DebugInfo)> { let path = blake256(key); let root = *self.root; From 4adc309a851e9775c5cd5afa628fea16cb94c0f6 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Tue, 15 Sep 2020 14:53:35 +0900 Subject: [PATCH 12/29] Use local rust rocksdb Use https://github.com/majecty/rust-rocksdb --- Cargo.lock | 14 +++++--------- Cargo.toml | 4 ++++ rust-rocksdb | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) create mode 160000 rust-rocksdb diff --git a/Cargo.lock b/Cargo.lock index 60f0117e30..738437204c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,7 +1345,7 @@ dependencies = [ "primitives 0.4.0 (git+https://github.com/CodeChain-io/rust-codechain-primitives.git)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", - "rocksdb 0.4.5 (git+https://github.com/paritytech/rust-rocksdb?rev=ecf06adf3148ab10f6f7686b724498382ff4f36e)", + "rocksdb 0.4.5", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2234,22 +2234,20 @@ dependencies = [ [[package]] name = "rocksdb" version = "0.4.5" -source = "git+https://github.com/paritytech/rust-rocksdb?rev=ecf06adf3148ab10f6f7686b724498382ff4f36e#ecf06adf3148ab10f6f7686b724498382ff4f36e" dependencies = [ "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "local-encoding 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rocksdb-sys 0.3.0 (git+https://github.com/paritytech/rust-rocksdb?rev=ecf06adf3148ab10f6f7686b724498382ff4f36e)", + "rocksdb-sys 0.3.0", ] [[package]] name = "rocksdb-sys" version = "0.3.0" -source = "git+https://github.com/paritytech/rust-rocksdb?rev=ecf06adf3148ab10f6f7686b724498382ff4f36e#ecf06adf3148ab10f6f7686b724498382ff4f36e" dependencies = [ "cc 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "local-encoding 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "snappy-sys 0.1.0 (git+https://github.com/paritytech/rust-snappy)", + "snappy-sys 0.1.0 (git+https://github.com/paritytech/rust-snappy?rev=40ac9a0d9fd613e7f38df800a11a589b7296da73)", ] [[package]] @@ -2506,7 +2504,7 @@ dependencies = [ [[package]] name = "snappy-sys" version = "0.1.0" -source = "git+https://github.com/paritytech/rust-snappy#40ac9a0d9fd613e7f38df800a11a589b7296da73" +source = "git+https://github.com/paritytech/rust-snappy?rev=40ac9a0d9fd613e7f38df800a11a589b7296da73#40ac9a0d9fd613e7f38df800a11a589b7296da73" dependencies = [ "cc 1.0.29 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3406,8 +3404,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum reqwest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ab52e462d1e15891441aeefadff68bdea005174328ce3da0a314f2ad313ec837" "checksum ring 0.14.6 (registry+https://github.com/rust-lang/crates.io-index)" = "426bc186e3e95cac1e4a4be125a4aca7e84c2d616ffc02244eef36e2a60a093c" -"checksum rocksdb 0.4.5 (git+https://github.com/paritytech/rust-rocksdb?rev=ecf06adf3148ab10f6f7686b724498382ff4f36e)" = "" -"checksum rocksdb-sys 0.3.0 (git+https://github.com/paritytech/rust-rocksdb?rev=ecf06adf3148ab10f6f7686b724498382ff4f36e)" = "" "checksum rotor 0.6.3 (git+https://github.com/tailhook/rotor)" = "" "checksum rpassword 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d127299b02abda51634f14025aec43ae87a7aa7a95202b6a868ec852607d1451" "checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" @@ -3440,7 +3436,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum smallvec 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f90c5e5fe535e48807ab94fc611d323935f39d4660c52b26b96446a7b33aef10" "checksum smallvec 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "211a489e65e94b103926d2054ae515a1cdb5d515ea0ef414fee23b7e043ce748" "checksum snap 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "174451758f7045084ae92070f18e5d8e5c53a716f4172a9c6b17ce03e7b82573" -"checksum snappy-sys 0.1.0 (git+https://github.com/paritytech/rust-snappy)" = "" +"checksum snappy-sys 0.1.0 (git+https://github.com/paritytech/rust-snappy?rev=40ac9a0d9fd613e7f38df800a11a589b7296da73)" = "" "checksum socket2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "06dc9f86ee48652b7c80f3d254e3b9accb67a928c562c64d10d7b016d3d98dab" "checksum spin 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44363f6f51401c34e7be73db0db371c04705d35efbe9f7d6082e03a921a32c55" "checksum spmc 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cd1f11d1fb5fd41834e55ce0b85a186efbf2f2afd9fdb09e2c8d72f9bff1ad1a" diff --git a/Cargo.toml b/Cargo.toml index 6250b32c72..7dca268ceb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,9 @@ cidr = "0.0.4" [build-dependencies] vergen = "2" +[patch."https://github.com/paritytech/rust-rocksdb"] +rocksdb = { path = "./rust-rocksdb/" } + [[bin]] path = "codechain/main.rs" name = "codechain" @@ -76,4 +79,5 @@ members = [ "sync", "types", "vm", + "rust-rocksdb", ] diff --git a/rust-rocksdb b/rust-rocksdb new file mode 160000 index 0000000000..beb64b7d98 --- /dev/null +++ b/rust-rocksdb @@ -0,0 +1 @@ +Subproject commit beb64b7d981d1e8f29f3b0d3a93e99b99fb6a600 From 5d2901737f2e3ecda1a82a677c783c6d396cd880 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Tue, 15 Sep 2020 16:44:04 +0900 Subject: [PATCH 13/29] Print RocksdbStatstics --- Cargo.lock | 1 + Cargo.toml | 1 + codechain/main.rs | 1 + codechain/subcommand/perf_write_command.rs | 10 +++++++--- util/kvdb-rocksdb/src/lib.rs | 4 ++++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 738437204c..3430e91cc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -242,6 +242,7 @@ dependencies = [ "panic_hook 0.1.0", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "primitives 0.4.0 (git+https://github.com/CodeChain-io/rust-codechain-primitives.git)", + "rocksdb 0.4.5", "rpassword 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 7dca268ceb..4810eee846 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ serde_json = "1.0" tokio-core = "0.1.17" toml = "0.4" cidr = "0.0.4" +rocksdb = { git = "https://github.com/paritytech/rust-rocksdb", rev = "ecf06adf3148ab10f6f7686b724498382ff4f36e" } [build-dependencies] vergen = "2" diff --git a/codechain/main.rs b/codechain/main.rs index 554a668840..2f00a897b3 100644 --- a/codechain/main.rs +++ b/codechain/main.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +extern crate rocksdb; extern crate rustc_hex; #[macro_use] extern crate clap; diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs index 8b7445906e..7c70da6b06 100644 --- a/codechain/subcommand/perf_write_command.rs +++ b/codechain/subcommand/perf_write_command.rs @@ -6,7 +6,9 @@ use cstate::{StateDB, StateWithCache, TopLevelState, TopState}; use ctypes::DebugInfo; use kvdb::KeyValueDB; use kvdb_rocksdb::{Database, DatabaseConfig}; +use parking_lot::Mutex; use primitives::H256; +use rocksdb::Options; use std::{path::Path, sync::Arc, time::Instant, u128, u32, u64}; pub const COL_STATE: Option = Some(0); @@ -21,7 +23,7 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { let db_dir = matches.value_of("db-dir").unwrap(); println!("start"); - let db = open_db(db_dir)?; + let (db, opts) = open_db(db_dir)?; println!("open db"); let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); println!("open journal db"); @@ -108,6 +110,7 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { // toplevel_state.commit().unwrap(); + println!("{}", opts.lock().print_statistics()); println!("Finished"); Ok(()) @@ -116,7 +119,7 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { // pub const DEFAULT_DB_PATH: &str = "db_test"; pub const NUM_COLUMNS: Option = Some(6); -pub fn open_db(db_dir: &str) -> Result, String> { +pub fn open_db(db_dir: &str) -> Result<(Arc, Arc>), String> { let base_path = ".".to_owned(); let db_path = base_path + "/" + db_dir; let client_path = Path::new(&db_path); @@ -133,5 +136,6 @@ pub fn open_db(db_dir: &str) -> Result, String> { .map_err(|_e| "Low level database error. Some issue with disk?".to_string())?, ); - Ok(db) + let opts = db.opts.clone(); + Ok((db, opts)) } diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index 7b1ba8822c..262f025422 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -27,6 +27,7 @@ extern crate rocksdb; extern crate kvdb; extern crate rlp; +use std::sync::Arc; use std::cmp; use std::collections::HashMap; use std::marker::PhantomData; @@ -244,6 +245,7 @@ fn col_config(config: &DatabaseConfig, block_opts: &BlockBasedOptions) -> Result /// Key-Value database. pub struct Database { db: RwLock>, + pub opts: Arc>, config: DatabaseConfig, write_opts: WriteOptions, read_opts: ReadOptions, @@ -296,6 +298,7 @@ impl Database { opts.set_parsed_options("bytes_per_sync=1048576")?; opts.set_db_write_buffer_size(config.memory_budget_per_col() / 2); opts.increase_parallelism(cmp::max(1, ::num_cpus::get() as i32 / 2)); + opts.enable_statistics(); let mut block_opts = BlockBasedOptions::new(); @@ -386,6 +389,7 @@ impl Database { db, cfs, })), + opts: Arc::new(Mutex::new(opts)), config: *config, write_opts, overlay: RwLock::new((0..=num_cols).map(|_| HashMap::new()).collect()), From d1c609e8f4933d163c3e9e9963ecb92223c8a9c3 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Wed, 23 Sep 2020 11:34:57 +0900 Subject: [PATCH 14/29] Add read-db command option --- codechain/codechain.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/codechain/codechain.yml b/codechain/codechain.yml index 630d65e273..4bb7f80c00 100644 --- a/codechain/codechain.yml +++ b/codechain/codechain.yml @@ -400,3 +400,16 @@ subcommands: long: number help: Set the number of db values takes_value: true + - read-db: + about: Read a value from DB + args: + - db-dir: + short: d + long: db-dir + help: The db directory + takes_value: true + - key: + short: k + long: key + help: The key to find a value in DB + takes_value: true From b710badcfd1f96dd72916f051c964b1e1b01cbac Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Wed, 23 Sep 2020 15:25:07 +0900 Subject: [PATCH 15/29] Log used time to read a db value deeply --- Cargo.lock | 4 ++ types/src/lib.rs | 41 ++++++++++++++------ util/hashdb/Cargo.toml | 1 + util/hashdb/src/lib.rs | 8 ++++ util/journaldb/Cargo.toml | 1 + util/journaldb/src/archivedb.rs | 18 ++++++++- util/journaldb/src/lib.rs | 2 + util/kvdb-rocksdb/Cargo.toml | 1 + util/kvdb-rocksdb/src/lib.rs | 68 ++++++++++++++++++++++++++++++++- util/kvdb/Cargo.toml | 1 + util/kvdb/src/lib.rs | 6 +++ util/merkle/src/triedb.rs | 11 ++++-- 12 files changed, 144 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3430e91cc8..f46e9cdc0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1066,6 +1066,7 @@ dependencies = [ name = "hashdb" version = "0.1.1" dependencies = [ + "codechain-types 0.1.0", "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "primitives 0.4.0 (git+https://github.com/CodeChain-io/rust-codechain-primitives.git)", ] @@ -1207,6 +1208,7 @@ name = "journaldb" version = "0.1.0" dependencies = [ "codechain-crypto 0.1.0 (git+https://github.com/CodeChain-io/rust-codechain-crypto.git)", + "codechain-types 0.1.0", "hashdb 0.1.1", "kvdb 0.1.0", "kvdb-memorydb 0.1.0", @@ -1319,6 +1321,7 @@ dependencies = [ name = "kvdb" version = "0.1.0" dependencies = [ + "codechain-types 0.1.0", "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "primitives 0.4.0 (git+https://github.com/CodeChain-io/rust-codechain-primitives.git)", @@ -1337,6 +1340,7 @@ dependencies = [ name = "kvdb-rocksdb" version = "0.1.0" dependencies = [ + "codechain-types 0.1.0", "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "kvdb 0.1.0", diff --git a/types/src/lib.rs b/types/src/lib.rs index dee2410b64..1d7a3e813c 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -24,9 +24,9 @@ extern crate rlp_derive; extern crate serde; #[macro_use] extern crate serde_derive; +extern crate rustc_hex; #[cfg(test)] extern crate serde_json; -extern crate rustc_hex; mod block_hash; mod common_params; @@ -49,16 +49,35 @@ pub use tracker::Tracker; pub use tx_hash::TxHash; #[derive(Debug, Clone)] -pub struct Read { - time_us: u128, - key: String, +pub struct DebugRead { + pub time_us: u128, + pub key: String, + pub archive_db_overlay_us: u128, + pub kvrocks_overlay_us: u128, + pub kvrocks_flushing_us: u128, + pub kvrocks_get_opt_us: u128, + pub kvrocks_get_cf_opt_us: u128, +} + +impl DebugRead { + pub fn empty() -> Self { + Self { + time_us: 0, + key: "".to_string(), + archive_db_overlay_us: 0, + kvrocks_overlay_us: 0, + kvrocks_flushing_us: 0, + kvrocks_get_opt_us: 0, + kvrocks_get_cf_opt_us: 0, + } + } } #[derive(Debug, Clone)] pub struct DebugInfo { pub read_count: u32, pub db_read_time_us: u128, - pub reads: Vec, + pub reads: Vec, } impl DebugInfo { @@ -74,10 +93,10 @@ impl DebugInfo { self.read_count += 1; } - pub fn add_read(&mut self, time_us: u128, key: &[u8]) { - self.reads.push(Read { - time_us, - key: key.to_hex(), - }) - } + // pub fn add_read(&mut self, time_us: u128, key: &[u8]) { + // self.reads.push(DebugRead { + // time_us, + // key: key.to_hex(), + // }) + // } } diff --git a/util/hashdb/Cargo.toml b/util/hashdb/Cargo.toml index 0694acbc2e..3f638fed43 100644 --- a/util/hashdb/Cargo.toml +++ b/util/hashdb/Cargo.toml @@ -8,3 +8,4 @@ license = "GPL-3.0" [dependencies] elastic-array = "0.10" primitives = { git = "https://github.com/CodeChain-io/rust-codechain-primitives.git", version = "0.4" } +codechain-types = { path = "../../types" } diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 3550d0619b..87b6cec0ac 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -17,7 +17,9 @@ //! Database of byte-slices keyed to their Keccak hash. extern crate elastic_array; extern crate primitives; +extern crate codechain_types as ctypes; +use ctypes::DebugRead; use elastic_array::ElasticArray128; use primitives::H256; use std::collections::HashMap; @@ -34,6 +36,12 @@ pub trait HashDB: AsHashDB + Send + Sync { /// hash is not known. fn get(&self, key: &H256) -> Option; + /// Look up a given hash into the bytes that hash to it, returning None if the + /// hash is not known. + fn get_debug(&self, key: &H256, debug_info: &mut DebugRead) -> Option { + self.get(key) + } + /// Check for the existance of a hash-key. fn contains(&self, key: &H256) -> bool; diff --git a/util/journaldb/Cargo.toml b/util/journaldb/Cargo.toml index 1b4ac31d4f..2947f02578 100644 --- a/util/journaldb/Cargo.toml +++ b/util/journaldb/Cargo.toml @@ -12,6 +12,7 @@ kvdb = { path = "../kvdb" } memorydb = { path = "../memorydb" } rlp = { path = "../rlp" } util-error = { path = "../error" } +codechain-types = { path = "../../types" } [dev-dependencies] codechain-crypto = { git = "https://github.com/CodeChain-io/rust-codechain-crypto.git", version = "0.1" } diff --git a/util/journaldb/src/archivedb.rs b/util/journaldb/src/archivedb.rs index 62e5729780..f0c4ab867c 100644 --- a/util/journaldb/src/archivedb.rs +++ b/util/journaldb/src/archivedb.rs @@ -25,8 +25,9 @@ use primitives::{Bytes, H256}; use rlp::{decode, encode}; use std::collections::hash_map::Entry; use std::collections::HashMap; -use std::sync::Arc; +use std::{sync::Arc, time::Instant}; use traits::JournalDB; +use ctypes::DebugRead; /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay /// and latent-removal semantics. @@ -58,6 +59,10 @@ impl ArchiveDB { fn payload(&self, key: &H256) -> Option { self.backing.get(self.column, key).expect("Low-level database error. Some issue with your hard disk?") } + + fn payload_debug(&self, key: &H256, debug_read: &mut DebugRead) -> Option { + self.backing.get_debug(self.column, key, debug_read).expect("Low-level database error. Some issue with your hard disk?") + } } impl HashDB for ArchiveDB { @@ -87,6 +92,17 @@ impl HashDB for ArchiveDB { self.payload(key) } + fn get_debug(&self, key: &H256, debug_read: &mut DebugRead) -> Option { + let now = Instant::now(); + if let Some((d, rc)) = self.overlay.raw(key) { + debug_read.archive_db_overlay_us = now.elapsed().as_micros(); + if rc > 0 { + return Some(d) + } + } + self.payload_debug(key, debug_read) + } + fn contains(&self, key: &H256) -> bool { self.get(key).is_some() } diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index 4614077fa0..3c68620a14 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -22,12 +22,14 @@ extern crate memorydb; extern crate primitives; extern crate rlp; extern crate util_error as error; +extern crate codechain_types as ctypes; #[cfg(test)] extern crate codechain_crypto as crypto; #[cfg(test)] extern crate kvdb_memorydb; +use ctypes::DebugInfo; use std::sync::Arc; use std::{fmt, str}; diff --git a/util/kvdb-rocksdb/Cargo.toml b/util/kvdb-rocksdb/Cargo.toml index 2d443da235..b5f48f8066 100644 --- a/util/kvdb-rocksdb/Cargo.toml +++ b/util/kvdb-rocksdb/Cargo.toml @@ -13,6 +13,7 @@ regex = "0.2" rlp = { path = "../rlp" } rocksdb = { git = "https://github.com/paritytech/rust-rocksdb", rev = "ecf06adf3148ab10f6f7686b724498382ff4f36e" } interleaved-ordered = "0.1.0" +codechain-types = { path = "../../types" } [dev-dependencies] primitives = { git = "https://github.com/CodeChain-io/rust-codechain-primitives.git", version = "0.4" } diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index 262f025422..448886a68b 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -24,14 +24,16 @@ extern crate parking_lot; extern crate regex; extern crate rocksdb; +extern crate codechain_types as ctypes; extern crate kvdb; extern crate rlp; -use std::sync::Arc; +use ctypes::DebugRead; use std::cmp; use std::collections::HashMap; use std::marker::PhantomData; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::{fs, io, mem, result}; use interleaved_ordered::{interleave_ordered, InterleaveOrdered}; @@ -50,7 +52,7 @@ use regex::Regex; #[cfg(target_os = "linux")] use std::fs::File; #[cfg(target_os = "linux")] -use std::process::Command; +use std::{process::Command, time::Instant}; const DB_DEFAULT_MEMORY_BUDGET_MB: usize = 128; @@ -584,6 +586,64 @@ impl Database { } } + pub fn get_debug(&self, col: Option, key: &[u8], debug_read: &mut DebugRead) -> Result> { + match *self.db.read() { + Some(DBAndColumns { + ref db, + ref cfs, + }) => { + let overlay = &self.overlay.read()[col_to_overlay_column(col)]; + let overlay_get_result = { + let now = Instant::now(); + let ret = overlay.get(key); + debug_read.kvrocks_overlay_us = now.elapsed().as_micros(); + + ret + }; + match overlay_get_result { + Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => { + Ok(Some(value.clone())) + } + Some(&KeyState::Delete) => Ok(None), + None => { + let flushing = &self.flushing.read()[col_to_overlay_column(col)]; + let flushing_get = { + let now = Instant::now(); + let ret = flushing.get(key); + debug_read.kvrocks_flushing_us = now.elapsed().as_micros(); + ret + }; + match flushing_get { + Some(&KeyState::Insert(ref value)) | Some(&KeyState::InsertCompressed(ref value)) => { + Ok(Some(value.clone())) + } + Some(&KeyState::Delete) => Ok(None), + None => match col { + Some(c) => { + let now = Instant::now(); + let ret = db + .get_cf_opt(cfs[c as usize], key, &self.read_opts) + .map(|r| r.map(|v| DBValue::from_slice(&v))); + debug_read.kvrocks_get_cf_opt_us = now.elapsed().as_micros(); + ret + } + None => { + let now = Instant::now(); + let ret = + db.get_opt(key, &self.read_opts).map(|r| r.map(|v| DBValue::from_slice(&v))); + + debug_read.kvrocks_get_opt_us = now.elapsed().as_micros(); + ret + } + } + .map_err(Into::into), + } + } + } + } + None => Ok(None), + } + } /// Get value by partial key. Prefix size should match configured prefix size. Only searches flushed values. // TODO: support prefix seek for unflushed data pub fn get_by_prefix(&self, col: Option, prefix: &[u8]) -> Option> { @@ -776,6 +836,10 @@ impl KeyValueDB for Database { Database::get(self, col, key) } + fn get_debug(&self, col: Option, key: &[u8], debug_read: &mut DebugRead) -> Result> { + Database::get_debug(self, col, key, debug_read) + } + fn get_by_prefix(&self, col: Option, prefix: &[u8]) -> Option> { Database::get_by_prefix(self, col, prefix) } diff --git a/util/kvdb/Cargo.toml b/util/kvdb/Cargo.toml index 7b627e8371..945d6920b8 100644 --- a/util/kvdb/Cargo.toml +++ b/util/kvdb/Cargo.toml @@ -7,3 +7,4 @@ authors = ["Parity Technologies "] elastic-array = "0.10" error-chain = { version = "0.12", default-features = false } primitives = { git = "https://github.com/CodeChain-io/rust-codechain-primitives.git", version = "0.4" } +codechain-types = { path = "../../types" } diff --git a/util/kvdb/src/lib.rs b/util/kvdb/src/lib.rs index cdee56678f..ef4e8ebf36 100644 --- a/util/kvdb/src/lib.rs +++ b/util/kvdb/src/lib.rs @@ -20,7 +20,9 @@ extern crate error_chain; extern crate elastic_array; extern crate primitives; +extern crate codechain_types as ctypes; +use ctypes::DebugRead; use elastic_array::{ElasticArray128, ElasticArray32}; use primitives::Bytes; @@ -158,6 +160,10 @@ pub trait KeyValueDB: Sync + Send { /// Get a value by key. fn get(&self, col: Option, key: &[u8]) -> Result>; + fn get_debug(&self, col: Option, key: &[u8], debug_read: &mut DebugRead) -> Result> { + self.get(col, key) + } + /// Get a value by partial key. Only works for flushed data. fn get_by_prefix(&self, col: Option, prefix: &[u8]) -> Option>; diff --git a/util/merkle/src/triedb.rs b/util/merkle/src/triedb.rs index b9fdf109ae..9ed50c2988 100644 --- a/util/merkle/src/triedb.rs +++ b/util/merkle/src/triedb.rs @@ -21,7 +21,7 @@ use primitives::H256; use crate::nibbleslice::NibbleSlice; use crate::node::Node as RlpNode; use crate::{Query, Trie, TrieError}; -use ctypes::DebugInfo; +use ctypes::{DebugInfo, DebugRead}; use std::time::Instant; /// A `Trie` implementation using a generic `HashDB` backing database. /// @@ -118,10 +118,13 @@ impl<'db> TrieDB<'db> { match cur_node_hash { Some(hash) => { let read_start = Instant::now(); - let node_rlp = self.db.get(&hash).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; + let mut read = DebugRead::empty(); + let node_rlp = self.db.get_debug(&hash, &mut read).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; let elapsed = read_start.elapsed().as_micros(); let mut empty = DebugInfo::empty(); - empty.add_read(elapsed, hash.as_ref()); + read.time_us = elapsed; + read.key = hash.to_string(); + empty.reads.push(read.clone()); match RlpNode::decoded(&node_rlp) { Some(RlpNode::Leaf(partial, value)) => { @@ -139,7 +142,7 @@ impl<'db> TrieDB<'db> { query, )?; debug_info.inc_read_count(); - debug_info.add_read(elapsed, hash.as_ref()); + debug_info.reads.push(read); Ok((result, debug_info)) } else { Ok((None, empty)) From 59d9ae3a3b35636649d109450b9a0e710a64c4a9 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Wed, 23 Sep 2020 15:37:06 +0900 Subject: [PATCH 16/29] Do not print rocksdb statistics in perf-write command --- codechain/subcommand/perf_write_command.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs index 7c70da6b06..782a617f30 100644 --- a/codechain/subcommand/perf_write_command.rs +++ b/codechain/subcommand/perf_write_command.rs @@ -23,7 +23,7 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { let db_dir = matches.value_of("db-dir").unwrap(); println!("start"); - let (db, opts) = open_db(db_dir)?; + let (db, _) = open_db(db_dir)?; println!("open db"); let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); println!("open journal db"); @@ -110,7 +110,6 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { // toplevel_state.commit().unwrap(); - println!("{}", opts.lock().print_statistics()); println!("Finished"); Ok(()) From d5051844c3907816a376eaed9f1380bf12397798 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Wed, 23 Sep 2020 15:37:24 +0900 Subject: [PATCH 17/29] Print rocks db statistics in read-db command --- codechain/subcommand/read_db_command.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/codechain/subcommand/read_db_command.rs b/codechain/subcommand/read_db_command.rs index 27f4e3b868..6c71df257d 100644 --- a/codechain/subcommand/read_db_command.rs +++ b/codechain/subcommand/read_db_command.rs @@ -6,6 +6,8 @@ use kvdb::KeyValueDB; use kvdb_rocksdb::{Database, DatabaseConfig}; use rustc_hex::{FromHex, ToHex}; use std::{path::Path, sync::Arc, time::Instant, u32}; +use rocksdb::Options; +use parking_lot::Mutex; // pub const COL_EXTRA: Option = Some(3); pub const COL_STATE: Option = Some(0); @@ -17,7 +19,7 @@ pub fn run_read_db_command(matches: &ArgMatches) -> Result<(), String> { let db_dir = matches.value_of("db-dir").unwrap(); let key = matches.value_of("key").unwrap().to_string().from_hex().unwrap(); - let db = open_db(db_dir)?; + let (db, opts) = open_db(db_dir)?; // let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); // let state_db = StateDB::new(journal_db); @@ -25,12 +27,13 @@ pub fn run_read_db_command(matches: &ArgMatches) -> Result<(), String> { let value = db.get(COL_STATE, &key).unwrap().unwrap(); let elapsed = now.elapsed().as_micros(); + println!("{}", opts.lock().print_statistics()); println!("elapsed {}us value: {}", elapsed, value.to_hex()); Ok(()) } -pub fn open_db(db_dir: &str) -> Result, String> { +pub fn open_db(db_dir: &str) -> Result<(Arc, Arc>), String> { let base_path = ".".to_owned(); let db_path = base_path + "/" + db_dir; let client_path = Path::new(&db_path); @@ -47,5 +50,6 @@ pub fn open_db(db_dir: &str) -> Result, String> { .map_err(|_e| "Low level database error. Some issue with disk?".to_string())?, ); - Ok(db) + let opts = db.opts.clone(); + Ok((db, opts)) } From 8846825637f614da0170603ef2354056a4ff19a5 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Wed, 23 Sep 2020 16:10:48 +0900 Subject: [PATCH 18/29] Print full key --- Cargo.lock | 1 + util/merkle/Cargo.toml | 1 + util/merkle/src/lib.rs | 1 + util/merkle/src/triedb.rs | 3 ++- 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index f46e9cdc0c..e2877c3506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -422,6 +422,7 @@ dependencies = [ "primitives 0.4.0 (git+https://github.com/CodeChain-io/rust-codechain-primitives.git)", "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.2.1", + "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap 0.1.0", ] diff --git a/util/merkle/Cargo.toml b/util/merkle/Cargo.toml index 239e02ba2d..f3e75a7015 100644 --- a/util/merkle/Cargo.toml +++ b/util/merkle/Cargo.toml @@ -6,6 +6,7 @@ authors = ["CodeChain Team "] [dependencies] elastic-array = "0.10" rand = "0.6.1" +rustc-hex = "1.0" hashdb = {path = "../hashdb" } codechain-crypto = { git = "https://github.com/CodeChain-io/rust-codechain-crypto.git", version = "0.1" } primitives = { git = "https://github.com/CodeChain-io/rust-codechain-primitives.git", version = "0.4" } diff --git a/util/merkle/src/lib.rs b/util/merkle/src/lib.rs index 428cf8dffb..56be61e36e 100644 --- a/util/merkle/src/lib.rs +++ b/util/merkle/src/lib.rs @@ -15,6 +15,7 @@ // along with this program. If not, see . extern crate codechain_types as ctypes; +extern crate rustc_hex; extern crate codechain_crypto as ccrypto; extern crate elastic_array; extern crate hashdb; diff --git a/util/merkle/src/triedb.rs b/util/merkle/src/triedb.rs index 9ed50c2988..f22f35c55a 100644 --- a/util/merkle/src/triedb.rs +++ b/util/merkle/src/triedb.rs @@ -17,6 +17,7 @@ use ccrypto::blake256; use hashdb::HashDB; use primitives::H256; +use rustc_hex::ToHex; use crate::nibbleslice::NibbleSlice; use crate::node::Node as RlpNode; @@ -123,7 +124,7 @@ impl<'db> TrieDB<'db> { let elapsed = read_start.elapsed().as_micros(); let mut empty = DebugInfo::empty(); read.time_us = elapsed; - read.key = hash.to_string(); + read.key = (hash.as_ref() as &[u8]).to_hex(); empty.reads.push(read.clone()); match RlpNode::decoded(&node_rlp) { From 496e0a629d0f5b0decccc4e74e2c114987ebbd9a Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Wed, 23 Sep 2020 16:48:08 +0900 Subject: [PATCH 19/29] Log get_cf_opt time 10 times if it is slow --- types/src/lib.rs | 2 ++ util/kvdb-rocksdb/src/lib.rs | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/types/src/lib.rs b/types/src/lib.rs index 1d7a3e813c..11ed2299a1 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -57,6 +57,7 @@ pub struct DebugRead { pub kvrocks_flushing_us: u128, pub kvrocks_get_opt_us: u128, pub kvrocks_get_cf_opt_us: u128, + pub kvrocks_get_cf_opt_us_vec: Vec, } impl DebugRead { @@ -69,6 +70,7 @@ impl DebugRead { kvrocks_flushing_us: 0, kvrocks_get_opt_us: 0, kvrocks_get_cf_opt_us: 0, + kvrocks_get_cf_opt_us_vec: Vec::new(), } } } diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index 448886a68b..17b0821401 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -625,6 +625,15 @@ impl Database { .get_cf_opt(cfs[c as usize], key, &self.read_opts) .map(|r| r.map(|v| DBValue::from_slice(&v))); debug_read.kvrocks_get_cf_opt_us = now.elapsed().as_micros(); + if (now.elapsed().as_micros() > 1000) { + for i in 0..10 { + db.get_cf_opt(cfs[c as usize], key, &self.read_opts) + .map(|r| r.map(|v| DBValue::from_slice(&v))); + debug_read.kvrocks_get_cf_opt_us_vec.push(now.elapsed().as_micros()); + } + } + + ret } None => { From e951892e997f32206282ee312b12c9e694f346de Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Wed, 23 Sep 2020 17:12:03 +0900 Subject: [PATCH 20/29] Print debug information in read-db --- codechain/subcommand/read_db_command.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/codechain/subcommand/read_db_command.rs b/codechain/subcommand/read_db_command.rs index 6c71df257d..5e33e82c1f 100644 --- a/codechain/subcommand/read_db_command.rs +++ b/codechain/subcommand/read_db_command.rs @@ -8,6 +8,7 @@ use rustc_hex::{FromHex, ToHex}; use std::{path::Path, sync::Arc, time::Instant, u32}; use rocksdb::Options; use parking_lot::Mutex; +use ctypes::DebugRead; // pub const COL_EXTRA: Option = Some(3); pub const COL_STATE: Option = Some(0); @@ -24,11 +25,13 @@ pub fn run_read_db_command(matches: &ArgMatches) -> Result<(), String> { // let state_db = StateDB::new(journal_db); let now = Instant::now(); - let value = db.get(COL_STATE, &key).unwrap().unwrap(); + let mut debug_read = DebugRead::empty(); + let value = db.get_debug(COL_STATE, &key, &mut debug_read).unwrap().unwrap(); let elapsed = now.elapsed().as_micros(); println!("{}", opts.lock().print_statistics()); println!("elapsed {}us value: {}", elapsed, value.to_hex()); + println!("{:?}", debug_read); Ok(()) } From 10c50f93951207f52221423fd7a0d443d89a4491 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Thu, 24 Sep 2020 10:57:33 +0900 Subject: [PATCH 21/29] Add read-toplevel subcommand --- codechain/codechain.yml | 17 ++++ codechain/subcommand/mod.rs | 3 + codechain/subcommand/read_toplevel_command.rs | 79 +++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 codechain/subcommand/read_toplevel_command.rs diff --git a/codechain/codechain.yml b/codechain/codechain.yml index 4bb7f80c00..6592b623ba 100644 --- a/codechain/codechain.yml +++ b/codechain/codechain.yml @@ -413,3 +413,20 @@ subcommands: long: key help: The key to find a value in DB takes_value: true + - read-toplevel: + about: Read a value from Toplevel state + args: + - prepare: + short: p + long: prepare + help: if key is n, read 0..n keys before reading the key + - db-dir: + short: d + long: db-dir + help: The db directory + takes_value: true + - key: + short: k + long: key + help: The key to find a value in DB + takes_value: true diff --git a/codechain/subcommand/mod.rs b/codechain/subcommand/mod.rs index 72450b9335..2bc39943b2 100644 --- a/codechain/subcommand/mod.rs +++ b/codechain/subcommand/mod.rs @@ -19,6 +19,7 @@ mod convert_command; mod generate_data_command; mod perf_write_command; mod read_db_command; +mod read_toplevel_command; use clap::ArgMatches; @@ -27,6 +28,7 @@ use self::convert_command::run_convert_command; use self::generate_data_command::run_generate_data_command; use self::perf_write_command::run_perf_write_command; use self::read_db_command::run_read_db_command; +use self::read_toplevel_command::run_read_toplevel_command; pub fn run_subcommand(matches: &ArgMatches) -> Result<(), String> { let subcommand = matches.subcommand.as_ref().unwrap(); @@ -36,6 +38,7 @@ pub fn run_subcommand(matches: &ArgMatches) -> Result<(), String> { "generate-data" => run_generate_data_command(&subcommand.matches), "perf-write" => run_perf_write_command(&subcommand.matches), "read-db" => run_read_db_command(&subcommand.matches), + "read-toplevel" => run_read_toplevel_command(&subcommand.matches), "commit-hash" => { println!("{}", env!("VERGEN_SHA")); Ok(()) diff --git a/codechain/subcommand/read_toplevel_command.rs b/codechain/subcommand/read_toplevel_command.rs new file mode 100644 index 0000000000..a0c85a6400 --- /dev/null +++ b/codechain/subcommand/read_toplevel_command.rs @@ -0,0 +1,79 @@ +use ccore::ClientConfig; +use clap::ArgMatches; +use clogger::{self, LoggerConfig}; +// use cstate::StateDB; +use ckey::Address; +use cstate::{StateDB, TopLevelState, TopState}; +use ctypes::DebugRead; +use kvdb::KeyValueDB; +use kvdb_rocksdb::{Database, DatabaseConfig}; +use parking_lot::Mutex; +use primitives::H256; +use rocksdb::Options; +use rustc_hex::{FromHex, ToHex}; +use std::{path::Path, sync::Arc, time::Instant, u32}; + +pub const COL_EXTRA: Option = Some(3); +pub const COL_STATE: Option = Some(0); +pub const NUM_COLUMNS: Option = Some(6); + +pub fn run_read_toplevel_command(matches: &ArgMatches) -> Result<(), String> { + clogger::init(&LoggerConfig::new(777), None).expect("Logger must be successfully initialized"); + + let db_dir = matches.value_of("db-dir").unwrap(); + let key: u64 = matches.value_of("key").unwrap().to_string().parse().unwrap(); + let prepare = matches.is_present("prepare"); + + let (db, opts) = open_db(db_dir)?; + let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); + let state_db = StateDB::new(journal_db); + let root = { + let bytes = db.get(COL_EXTRA, b"perf_data_root").unwrap().unwrap(); + H256::from(bytes.as_ref()) + }; + + let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); + + if prepare { + for key_ in (0..key) { + let address = Address::random(); + let now = Instant::now(); + toplevel_state.add_balance_debug(&address, 7).unwrap(); + let elapsed = now.elapsed().as_micros(); + if (elapsed > 1000) { + println!("elapsed {}us key: {}", elapsed, key_); + } + } + } + + let address = Address::from(key); + let now = Instant::now(); + let debug_info = toplevel_state.add_balance_debug(&address, 7).unwrap(); + let elapsed = now.elapsed().as_micros(); + + println!("{}", opts.lock().print_statistics()); + println!("elapsed {}us debug info: {:?}", elapsed, debug_info); + + Ok(()) +} + +pub fn open_db(db_dir: &str) -> Result<(Arc, Arc>), String> { + let base_path = ".".to_owned(); + let db_path = base_path + "/" + db_dir; + let client_path = Path::new(&db_path); + let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS); + + let client_config = ClientConfig::default(); + + db_config.memory_budget = client_config.db_cache_size; + db_config.compaction = client_config.db_compaction.compaction_profile(client_path); + db_config.wal = client_config.db_wal; + + let db = Arc::new( + Database::open(&db_config, &client_path.to_str().expect("DB path could not be converted to string.")) + .map_err(|_e| "Low level database error. Some issue with disk?".to_string())?, + ); + + let opts = db.opts.clone(); + Ok((db, opts)) +} From d1cf5f7448cfc322c0ff61ee64a2434756562113 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Thu, 24 Sep 2020 12:28:35 +0900 Subject: [PATCH 22/29] Set cache from environment variable --- util/kvdb-rocksdb/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index 17b0821401..8560cf0407 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -306,7 +306,13 @@ impl Database { { block_opts.set_block_size(config.compaction.block_size); - let cache_size = cmp::max(8, config.memory_budget() / 3); + let mut cache_size = cmp::max(8, config.memory_budget() / 3); + println!("Cache size is {}", cache_size); + cache_size = match std::env::var("JH_CACHE") { + Ok(val) => val.parse().unwrap(), + Err(_) => cache_size, + }; + println!("Cache size is {}", cache_size); let cache = Cache::new(cache_size); block_opts.set_cache(cache); } From 518e3963ef9990aec18fdeb6d072e22ba7a42bc5 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Thu, 24 Sep 2020 13:36:20 +0900 Subject: [PATCH 23/29] Do not get multiple time from DB --- util/kvdb-rocksdb/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/util/kvdb-rocksdb/src/lib.rs b/util/kvdb-rocksdb/src/lib.rs index 8560cf0407..fef0c80765 100644 --- a/util/kvdb-rocksdb/src/lib.rs +++ b/util/kvdb-rocksdb/src/lib.rs @@ -632,11 +632,11 @@ impl Database { .map(|r| r.map(|v| DBValue::from_slice(&v))); debug_read.kvrocks_get_cf_opt_us = now.elapsed().as_micros(); if (now.elapsed().as_micros() > 1000) { - for i in 0..10 { - db.get_cf_opt(cfs[c as usize], key, &self.read_opts) - .map(|r| r.map(|v| DBValue::from_slice(&v))); - debug_read.kvrocks_get_cf_opt_us_vec.push(now.elapsed().as_micros()); - } + // for i in 0..10 { + // db.get_cf_opt(cfs[c as usize], key, &self.read_opts) + // .map(|r| r.map(|v| DBValue::from_slice(&v))); + // debug_read.kvrocks_get_cf_opt_us_vec.push(now.elapsed().as_micros()); + // } } From e3cca762cfa5996d9a6ae48b3d0d79470697a656 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Mon, 5 Oct 2020 11:41:32 +0900 Subject: [PATCH 24/29] Make perf-write and read-toplevel commands receive root from env variable --- codechain/subcommand/perf_write_command.rs | 9 ++++++--- codechain/subcommand/read_toplevel_command.rs | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/codechain/subcommand/perf_write_command.rs b/codechain/subcommand/perf_write_command.rs index 782a617f30..5de7b90a72 100644 --- a/codechain/subcommand/perf_write_command.rs +++ b/codechain/subcommand/perf_write_command.rs @@ -32,9 +32,12 @@ pub fn run_perf_write_command(matches: &ArgMatches) -> Result<(), String> { println!("before getting root"); - let root = { - let bytes = db.get(COL_EXTRA, b"perf_data_root").unwrap().unwrap(); - H256::from(bytes.as_ref()) + let root = match std::env::var("ROOT") { + Ok(val) => val.parse().unwrap(), + Err(_) => { + let bytes = db.get(COL_EXTRA, b"perf_data_root").unwrap().unwrap(); + H256::from(bytes.as_ref()) + } }; let mut addresses = Vec::new(); diff --git a/codechain/subcommand/read_toplevel_command.rs b/codechain/subcommand/read_toplevel_command.rs index a0c85a6400..48fbd85cc2 100644 --- a/codechain/subcommand/read_toplevel_command.rs +++ b/codechain/subcommand/read_toplevel_command.rs @@ -27,9 +27,13 @@ pub fn run_read_toplevel_command(matches: &ArgMatches) -> Result<(), String> { let (db, opts) = open_db(db_dir)?; let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); let state_db = StateDB::new(journal_db); - let root = { - let bytes = db.get(COL_EXTRA, b"perf_data_root").unwrap().unwrap(); - H256::from(bytes.as_ref()) + + let root = match std::env::var("ROOT") { + Ok(val) => val.parse().unwrap(), + Err(_) => { + let bytes = db.get(COL_EXTRA, b"perf_data_root").unwrap().unwrap(); + H256::from(bytes.as_ref()) + } }; let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); From 0f19484e5bbd5efdf02ec9440db240ab099126a9 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Mon, 5 Oct 2020 12:17:50 +0900 Subject: [PATCH 25/29] Make read_toplevel receive string key and get prepare number --- codechain/codechain.yml | 5 +++++ codechain/subcommand/read_toplevel_command.rs | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/codechain/codechain.yml b/codechain/codechain.yml index 6592b623ba..ef8f706a40 100644 --- a/codechain/codechain.yml +++ b/codechain/codechain.yml @@ -420,6 +420,7 @@ subcommands: short: p long: prepare help: if key is n, read 0..n keys before reading the key + takes_value: true - db-dir: short: d long: db-dir @@ -430,3 +431,7 @@ subcommands: long: key help: The key to find a value in DB takes_value: true + - key-str: + long: key-str + help: The key to find a value in DB + takes_value: true diff --git a/codechain/subcommand/read_toplevel_command.rs b/codechain/subcommand/read_toplevel_command.rs index 48fbd85cc2..ff40bc8b04 100644 --- a/codechain/subcommand/read_toplevel_command.rs +++ b/codechain/subcommand/read_toplevel_command.rs @@ -21,8 +21,15 @@ pub fn run_read_toplevel_command(matches: &ArgMatches) -> Result<(), String> { clogger::init(&LoggerConfig::new(777), None).expect("Logger must be successfully initialized"); let db_dir = matches.value_of("db-dir").unwrap(); - let key: u64 = matches.value_of("key").unwrap().to_string().parse().unwrap(); - let prepare = matches.is_present("prepare"); + let num_address = matches.value_of("key").map(|s| { + let num: u64 = s.parse().unwrap(); + Address::from(num) + }); + let address = num_address.unwrap_or_else(|| { + let strKey = matches.value_of("key-str").unwrap(); + strKey.parse().unwrap() + }); + let prepare: Option = matches.value_of("prepare").map(|s| s.parse().unwrap()); let (db, opts) = open_db(db_dir)?; let journal_db = journaldb::new(Arc::clone(&db), journaldb::Algorithm::Archive, COL_STATE); @@ -38,9 +45,10 @@ pub fn run_read_toplevel_command(matches: &ArgMatches) -> Result<(), String> { let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); - if prepare { - for key_ in (0..key) { - let address = Address::random(); + if let Some(prepare_max) = prepare { + for key_ in (0..prepare_max) { + // let address = Address::random(); + let address = Address::from(key_); let now = Instant::now(); toplevel_state.add_balance_debug(&address, 7).unwrap(); let elapsed = now.elapsed().as_micros(); @@ -50,7 +58,6 @@ pub fn run_read_toplevel_command(matches: &ArgMatches) -> Result<(), String> { } } - let address = Address::from(key); let now = Instant::now(); let debug_info = toplevel_state.add_balance_debug(&address, 7).unwrap(); let elapsed = now.elapsed().as_micros(); From d87f93def44be7d1fc33984dd0c6941662245707 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Mon, 5 Oct 2020 12:27:07 +0900 Subject: [PATCH 26/29] Run formatter --- codechain/subcommand/read_db_command.rs | 6 +++--- codechain/subcommand/read_toplevel_command.rs | 10 ++++------ state/src/impls/top_level.rs | 4 ++-- state/src/traits.rs | 2 +- util/hashdb/src/lib.rs | 2 +- util/journaldb/src/archivedb.rs | 6 ++++-- util/journaldb/src/lib.rs | 2 +- util/kvdb/src/lib.rs | 2 +- util/merkle/src/lib.rs | 4 ++-- util/merkle/src/triedb.rs | 3 ++- 10 files changed, 21 insertions(+), 20 deletions(-) diff --git a/codechain/subcommand/read_db_command.rs b/codechain/subcommand/read_db_command.rs index 5e33e82c1f..8fbffb45fc 100644 --- a/codechain/subcommand/read_db_command.rs +++ b/codechain/subcommand/read_db_command.rs @@ -2,13 +2,13 @@ use ccore::ClientConfig; use clap::ArgMatches; use clogger::{self, LoggerConfig}; // use cstate::StateDB; +use ctypes::DebugRead; use kvdb::KeyValueDB; use kvdb_rocksdb::{Database, DatabaseConfig}; +use parking_lot::Mutex; +use rocksdb::Options; use rustc_hex::{FromHex, ToHex}; use std::{path::Path, sync::Arc, time::Instant, u32}; -use rocksdb::Options; -use parking_lot::Mutex; -use ctypes::DebugRead; // pub const COL_EXTRA: Option = Some(3); pub const COL_STATE: Option = Some(0); diff --git a/codechain/subcommand/read_toplevel_command.rs b/codechain/subcommand/read_toplevel_command.rs index ff40bc8b04..879a2c9e8e 100644 --- a/codechain/subcommand/read_toplevel_command.rs +++ b/codechain/subcommand/read_toplevel_command.rs @@ -4,13 +4,11 @@ use clogger::{self, LoggerConfig}; // use cstate::StateDB; use ckey::Address; use cstate::{StateDB, TopLevelState, TopState}; -use ctypes::DebugRead; use kvdb::KeyValueDB; use kvdb_rocksdb::{Database, DatabaseConfig}; use parking_lot::Mutex; use primitives::H256; use rocksdb::Options; -use rustc_hex::{FromHex, ToHex}; use std::{path::Path, sync::Arc, time::Instant, u32}; pub const COL_EXTRA: Option = Some(3); @@ -26,8 +24,8 @@ pub fn run_read_toplevel_command(matches: &ArgMatches) -> Result<(), String> { Address::from(num) }); let address = num_address.unwrap_or_else(|| { - let strKey = matches.value_of("key-str").unwrap(); - strKey.parse().unwrap() + let str_key = matches.value_of("key-str").unwrap(); + str_key.parse().unwrap() }); let prepare: Option = matches.value_of("prepare").map(|s| s.parse().unwrap()); @@ -46,13 +44,13 @@ pub fn run_read_toplevel_command(matches: &ArgMatches) -> Result<(), String> { let mut toplevel_state = TopLevelState::from_existing(state_db.clone(&root), root).unwrap(); if let Some(prepare_max) = prepare { - for key_ in (0..prepare_max) { + for key_ in 0..prepare_max { // let address = Address::random(); let address = Address::from(key_); let now = Instant::now(); toplevel_state.add_balance_debug(&address, 7).unwrap(); let elapsed = now.elapsed().as_micros(); - if (elapsed > 1000) { + if elapsed > 1000 { println!("elapsed {}us key: {}", elapsed, key_); } } diff --git a/state/src/impls/top_level.rs b/state/src/impls/top_level.rs index de031870c6..d61bb705e1 100644 --- a/state/src/impls/top_level.rs +++ b/state/src/impls/top_level.rs @@ -48,7 +48,7 @@ use ctypes::transaction::{ use ctypes::util::unexpected::Mismatch; #[cfg(test)] use ctypes::Tracker; -use ctypes::{BlockNumber, CommonParams, ShardId, TxHash, DebugInfo}; +use ctypes::{BlockNumber, CommonParams, DebugInfo, ShardId, TxHash}; use cvm::ChainTimeInfo; use hashdb::AsHashDB; use kvdb::DBTransaction; @@ -827,7 +827,7 @@ impl TopState for TopLevelState { if incr != 0 { let (mut account, debug_info) = self.get_account_mut_debug(a)?; account.add_balance(incr); - return Ok(debug_info); + return Ok(debug_info) } Ok(DebugInfo::empty()) } diff --git a/state/src/traits.rs b/state/src/traits.rs index f6f1802a0b..9b4cd195d6 100644 --- a/state/src/traits.rs +++ b/state/src/traits.rs @@ -17,7 +17,7 @@ use ckey::{public_to_address, Address, Public, Signature}; use cmerkle::Result as TrieResult; use ctypes::transaction::ShardTransaction; -use ctypes::{BlockNumber, CommonParams, ShardId, Tracker, TxHash, DebugInfo}; +use ctypes::{BlockNumber, CommonParams, DebugInfo, ShardId, Tracker, TxHash}; use cvm::ChainTimeInfo; use primitives::{Bytes, H160, H256}; diff --git a/util/hashdb/src/lib.rs b/util/hashdb/src/lib.rs index 87b6cec0ac..c0552249c1 100644 --- a/util/hashdb/src/lib.rs +++ b/util/hashdb/src/lib.rs @@ -15,9 +15,9 @@ // along with Parity. If not, see . //! Database of byte-slices keyed to their Keccak hash. +extern crate codechain_types as ctypes; extern crate elastic_array; extern crate primitives; -extern crate codechain_types as ctypes; use ctypes::DebugRead; use elastic_array::ElasticArray128; diff --git a/util/journaldb/src/archivedb.rs b/util/journaldb/src/archivedb.rs index f0c4ab867c..2d80402bc9 100644 --- a/util/journaldb/src/archivedb.rs +++ b/util/journaldb/src/archivedb.rs @@ -18,6 +18,7 @@ use super::memorydb::*; use super::{DB_PREFIX_LEN, LATEST_ERA_KEY}; +use ctypes::DebugRead; use error::{BaseDataError, UtilError}; use hashdb::*; use kvdb::{DBTransaction, KeyValueDB}; @@ -27,7 +28,6 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use std::{sync::Arc, time::Instant}; use traits::JournalDB; -use ctypes::DebugRead; /// Implementation of the `HashDB` trait for a disk-backed database with a memory overlay /// and latent-removal semantics. @@ -61,7 +61,9 @@ impl ArchiveDB { } fn payload_debug(&self, key: &H256, debug_read: &mut DebugRead) -> Option { - self.backing.get_debug(self.column, key, debug_read).expect("Low-level database error. Some issue with your hard disk?") + self.backing + .get_debug(self.column, key, debug_read) + .expect("Low-level database error. Some issue with your hard disk?") } } diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index 3c68620a14..d90ef021b6 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -16,13 +16,13 @@ //! `JournalDB` interface and implementation. +extern crate codechain_types as ctypes; extern crate hashdb; extern crate kvdb; extern crate memorydb; extern crate primitives; extern crate rlp; extern crate util_error as error; -extern crate codechain_types as ctypes; #[cfg(test)] extern crate codechain_crypto as crypto; diff --git a/util/kvdb/src/lib.rs b/util/kvdb/src/lib.rs index ef4e8ebf36..9b2c6da850 100644 --- a/util/kvdb/src/lib.rs +++ b/util/kvdb/src/lib.rs @@ -18,9 +18,9 @@ #[macro_use] extern crate error_chain; +extern crate codechain_types as ctypes; extern crate elastic_array; extern crate primitives; -extern crate codechain_types as ctypes; use ctypes::DebugRead; use elastic_array::{ElasticArray128, ElasticArray32}; diff --git a/util/merkle/src/lib.rs b/util/merkle/src/lib.rs index 56be61e36e..15d36f870e 100644 --- a/util/merkle/src/lib.rs +++ b/util/merkle/src/lib.rs @@ -14,15 +14,15 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -extern crate codechain_types as ctypes; -extern crate rustc_hex; extern crate codechain_crypto as ccrypto; +extern crate codechain_types as ctypes; extern crate elastic_array; extern crate hashdb; #[cfg(test)] extern crate memorydb; extern crate primitives; extern crate rlp; +extern crate rustc_hex; #[cfg(test)] extern crate trie_standardmap as standardmap; diff --git a/util/merkle/src/triedb.rs b/util/merkle/src/triedb.rs index f22f35c55a..46d5521139 100644 --- a/util/merkle/src/triedb.rs +++ b/util/merkle/src/triedb.rs @@ -120,7 +120,8 @@ impl<'db> TrieDB<'db> { Some(hash) => { let read_start = Instant::now(); let mut read = DebugRead::empty(); - let node_rlp = self.db.get_debug(&hash, &mut read).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; + let node_rlp = + self.db.get_debug(&hash, &mut read).ok_or_else(|| TrieError::IncompleteDatabase(hash))?; let elapsed = read_start.elapsed().as_micros(); let mut empty = DebugInfo::empty(); read.time_us = elapsed; From 29e9f2a9b9e9f679859a8b57478d80b16f91a68b Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Mon, 5 Oct 2020 18:06:36 +0900 Subject: [PATCH 27/29] Add execute-block command --- codechain/codechain.yml | 13 ++ codechain/subcommand/execute_block_command.rs | 158 ++++++++++++++++++ codechain/subcommand/mod.rs | 3 + core/src/block.rs | 10 ++ core/src/client/client.rs | 4 + core/src/client/importer.rs | 53 +++++- core/src/client/mod.rs | 3 + 7 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 codechain/subcommand/execute_block_command.rs diff --git a/codechain/codechain.yml b/codechain/codechain.yml index ef8f706a40..4c1f96aef9 100644 --- a/codechain/codechain.yml +++ b/codechain/codechain.yml @@ -435,3 +435,16 @@ subcommands: long: key-str help: The key to find a value in DB takes_value: true + - execute-block: + about: Execute a block + args: + - db-dir: + short: d + long: db-dir + help: The db directory + takes_value: true + - block-num: + short: n + long: block-num + help: The number of a block + takes_value: true diff --git a/codechain/subcommand/execute_block_command.rs b/codechain/subcommand/execute_block_command.rs new file mode 100644 index 0000000000..2ca4d5742b --- /dev/null +++ b/codechain/subcommand/execute_block_command.rs @@ -0,0 +1,158 @@ +use ccore::{ + AccountProvider, BlockChainTrait, BlockId, ClientConfig, ClientService, ImportBlock, Miner, Scheme, TimeGapParams, +}; +use ckey::NetworkId; +use ckeystore::{accounts_dir::RootDiskDirectory, KeyStore}; +use clap::ArgMatches; +use clogger::{self, LoggerConfig}; +use cnetwork::{Filters, NetworkConfig, NetworkService, RoutingTable, SocketAddr}; +use config::ChainType; +use ctimer::TimerLoop; +use ctypes::DebugRead; +use kvdb::KeyValueDB; +use kvdb_rocksdb::{Database, DatabaseConfig}; +use parking_lot::Mutex; +use rocksdb::Options; +use std::{ + path::Path, + str::FromStr, + sync::Arc, + time::{Duration, Instant}, + u32, +}; + +pub const NUM_COLUMNS: Option = Some(6); + +pub fn run_execute_block_command(matches: &ArgMatches) -> Result<(), String> { + clogger::init(&LoggerConfig::new(777), None).expect("Logger must be successfully initialized"); + + let db_dir = matches.value_of("db-dir").unwrap(); + let block_num: u64 = matches.value_of("block-num").unwrap().parse().unwrap(); + // let key = matches.value_of("key").unwrap().to_string().from_hex().unwrap(); + + let timer_loop = TimerLoop::new(2); + + let time_gap_params = TimeGapParams { + allowed_past_gap: Duration::from_millis(3000), + allowed_future_gap: Duration::from_millis(3000), + }; + let scheme = ChainType::Corgi.scheme()?; + scheme.engine.register_time_gap_config_to_worker(time_gap_params); + let client_config: ClientConfig = Default::default(); + let (db, opts) = open_db(db_dir)?; + // let db = open_db(&config.operating, &client_config)?; + + let keys_path = "./keys".to_string(); + let ap = prepare_account_provider(&keys_path)?; + let miner = new_miner(&scheme, ap.clone(), Arc::clone(&db))?; + + println!("start client"); + + let client = client_start(&client_config, &timer_loop, db, &scheme, miner.clone())?; + + { + let routing_table = RoutingTable::new(); + let network_config = NetworkConfig { + address: "127.0.0.1".to_string(), + port: 3485, + bootstrap_addresses: Vec::new(), + min_peers: 1, + max_peers: 1, + whitelist: Vec::new(), + blacklist: Vec::new(), + }; + println!("network start"); + let service = + network_start(NetworkId::from_str("wc").unwrap(), timer_loop, &network_config, Arc::clone(&routing_table))?; + println!("register network extension to service"); + scheme.engine.register_network_extension_to_service(&service); + } + + println!("read block "); + let block = client.client().block(&BlockId::Number(block_num)).expect("Read block from DB"); + + println!("import block "); + let now = Instant::now(); + let debug_read = DebugRead::empty(); + + client.client().import_block_debug(block.into_inner()); + let elapsed = now.elapsed().as_micros(); + + println!("{}", opts.lock().print_statistics()); + println!("elapsed {}us", elapsed); + println!("{:?}", debug_read); + + Ok(()) +} + +pub fn open_db(db_dir: &str) -> Result<(Arc, Arc>), String> { + let base_path = ".".to_owned(); + let db_path = base_path + "/" + db_dir; + let client_path = Path::new(&db_path); + let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS); + + let client_config = ClientConfig::default(); + + db_config.memory_budget = client_config.db_cache_size; + db_config.compaction = client_config.db_compaction.compaction_profile(client_path); + db_config.wal = client_config.db_wal; + + let db = Arc::new( + Database::open(&db_config, &client_path.to_str().expect("DB path could not be converted to string.")) + .map_err(|_e| "Low level database error. Some issue with disk?".to_string())?, + ); + + let opts = db.opts.clone(); + Ok((db, opts)) +} + +fn new_miner(scheme: &Scheme, ap: Arc, db: Arc) -> Result, String> { + let miner = Miner::new(Default::default(), scheme, Some(ap), db); + Ok(miner) +} + +fn prepare_account_provider(keys_path: &str) -> Result, String> { + let keystore_dir = RootDiskDirectory::create(keys_path).map_err(|_| "Cannot read key path directory")?; + let keystore = KeyStore::open(Box::new(keystore_dir)).map_err(|_| "Cannot open key store")?; + Ok(AccountProvider::new(keystore)) +} + +fn client_start( + client_config: &ClientConfig, + timer_loop: &TimerLoop, + db: Arc, + scheme: &Scheme, + miner: Arc, +) -> Result { + cinfo!(CLIENT, "Starting client"); + let reseal_timer = timer_loop.new_timer_with_name("Client reseal timer"); + let service = ClientService::start(client_config, &scheme, db, miner, reseal_timer.clone()) + .map_err(|e| format!("Client service error: {}", e))?; + reseal_timer.set_handler(Arc::downgrade(&service.client())); + + Ok(service) +} + +fn network_start( + network_id: NetworkId, + timer_loop: TimerLoop, + cfg: &NetworkConfig, + routing_table: Arc, +) -> Result, String> { + let addr = cfg.address.parse().map_err(|_| format!("Invalid NETWORK listen host given: {}", cfg.address))?; + let sockaddress = SocketAddr::new(addr, cfg.port); + let filters = Filters::new(cfg.whitelist.clone(), cfg.blacklist.clone()); + let service = NetworkService::start( + network_id, + timer_loop, + sockaddress, + cfg.bootstrap_addresses.clone(), + cfg.min_peers, + cfg.max_peers, + filters, + routing_table, + ) + .map_err(|e| format!("Network service error: {:?}", e))?; + + Ok(service) +} diff --git a/codechain/subcommand/mod.rs b/codechain/subcommand/mod.rs index 2bc39943b2..0577d047ea 100644 --- a/codechain/subcommand/mod.rs +++ b/codechain/subcommand/mod.rs @@ -16,6 +16,7 @@ mod account_command; mod convert_command; +mod execute_block_command; mod generate_data_command; mod perf_write_command; mod read_db_command; @@ -25,6 +26,7 @@ use clap::ArgMatches; use self::account_command::run_account_command; use self::convert_command::run_convert_command; +use self::execute_block_command::run_execute_block_command; use self::generate_data_command::run_generate_data_command; use self::perf_write_command::run_perf_write_command; use self::read_db_command::run_read_db_command; @@ -39,6 +41,7 @@ pub fn run_subcommand(matches: &ArgMatches) -> Result<(), String> { "perf-write" => run_perf_write_command(&subcommand.matches), "read-db" => run_read_db_command(&subcommand.matches), "read-toplevel" => run_read_toplevel_command(&subcommand.matches), + "execute-block" => run_execute_block_command(&subcommand.matches), "commit-hash" => { println!("{}", env!("VERGEN_SHA")); Ok(()) diff --git a/core/src/block.rs b/core/src/block.rs index 9498ef9337..45134fe9c8 100644 --- a/core/src/block.rs +++ b/core/src/block.rs @@ -133,7 +133,9 @@ impl<'x> OpenBlock<'x> { author: Address, extra_data: Bytes, ) -> Result { + println!("create state from existing"); let state = TopLevelState::from_existing(db, *parent.state_root()).map_err(StateError::from)?; + println!("create open block"); let mut r = OpenBlock { block: ExecutedBlock::new(state, parent), engine, @@ -143,7 +145,10 @@ impl<'x> OpenBlock<'x> { r.block.header.set_extra_data(extra_data); r.block.header.note_dirty(); + println!("engine.machine.populate_from_parent"); engine.machine().populate_from_parent(&mut r.block.header, parent); + println!("engine.populate_from_parent"); + // FIXME: comment the line below not to use tendermint in perf engine.populate_from_parent(&mut r.block.header, parent); Ok(r) @@ -498,11 +503,15 @@ pub fn enact( db: StateDB, parent: &Header, ) -> Result { + println!("open block"); let mut b = OpenBlock::try_new(engine, db, parent, Address::default(), vec![])?; + println!("populate block"); b.populate_from(header); + println!("push transactions"); b.push_transactions(transactions, client, parent.number(), parent.timestamp())?; + println!("common params"); let parent_common_params = client.common_params((*header.parent_hash()).into()).unwrap(); let term_common_params = { let block_number = client @@ -514,6 +523,7 @@ pub fn enact( Some(client.common_params((block_number).into()).expect("Common params should exist")) } }; + println!("close and lock"); b.close_and_lock(parent, &parent_common_params, term_common_params.as_ref()) } diff --git a/core/src/client/client.rs b/core/src/client/client.rs index add839f9fb..4331d3bc3f 100644 --- a/core/src/client/client.rs +++ b/core/src/client/client.rs @@ -654,6 +654,10 @@ impl ImportBlock for Client { Ok(self.importer.block_queue.import(unverified)?) } + fn import_block_debug(&self, bytes: Bytes) { + self.importer.import_block_debug(self, bytes) + } + fn import_header(&self, bytes: Bytes) -> Result { let unverified = ::encoded::Header::new(bytes).decode(); { diff --git a/core/src/client/importer.rs b/core/src/client/importer.rs index b9bf9479a5..a57d458681 100644 --- a/core/src/client/importer.rs +++ b/core/src/client/importer.rs @@ -36,8 +36,12 @@ use crate::service::ClientIoMessage; use crate::types::BlockId; use crate::verification::queue::{BlockQueue, HeaderQueue}; use crate::verification::{self, PreverifiedBlock, Verifier}; -use crate::views::{BlockView, HeaderView}; +use crate::{ + views::{BlockView, HeaderView}, + SignedTransaction, +}; use client::EngineInfo; +use primitives::Bytes; pub struct Importer { /// Lock used during block import @@ -86,6 +90,53 @@ impl Importer { }) } + pub fn import_block_debug(&self, client: &Client, bytes: Bytes) { + println!("import_block_debug"); + let mut transactions = Vec::new(); + { + let v = BlockView::new(&bytes); + for t in v.transactions() { + let signed = SignedTransaction::try_new(t).unwrap(); + transactions.push(signed); + } + } + + let header = BlockView::new(&bytes).header(); + let block = PreverifiedBlock { + header: header.clone(), + transactions, + bytes, + }; + + let chain = client.block_chain(); + + println!("read parent header"); + let parent = chain + .block_header(header.parent_hash()) + .ok_or_else(|| { + cwarn!( + CLIENT, + "Block import failed for #{} ({}): Parent not found ({}) ", + header.number(), + header.hash(), + header.parent_hash() + ); + }) + .unwrap(); + + println!("Read state root"); + + let db = client.state_db().read().clone(&parent.state_root()); + + let engine = &*self.engine; + println!("enact"); + let enact_result = enact(&block.header, &block.transactions, engine, client, db, &parent); + match enact_result { + Ok(_) => cwarn!(CLIENT, "block {} is executed", header.hash()), + Err(err) => cwarn!(CLIENT, "executing block {} has error {}", header.hash(), err), + } + } + /// This is triggered by a message coming from a block queue when the block is ready for insertion pub fn import_verified_blocks(&self, client: &Client) -> usize { let (imported_blocks, import_results, invalid_blocks, imported, duration, is_empty) = { diff --git a/core/src/client/mod.rs b/core/src/client/mod.rs index 6b565e903a..586c0e6f6b 100644 --- a/core/src/client/mod.rs +++ b/core/src/client/mod.rs @@ -196,6 +196,9 @@ pub trait Shard { pub trait ImportBlock { /// Import a block into the blockchain. fn import_block(&self, bytes: Bytes) -> Result; + fn import_block_debug(&self, bytes: Bytes) { + self.import_block(bytes).unwrap(); + } /// Import a header into the blockchain fn import_header(&self, bytes: Bytes) -> Result; From d7f334c17ef6435c0578f1cd45609a7aa085bb40 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Tue, 6 Oct 2020 19:13:22 +0900 Subject: [PATCH 28/29] Print db usage in term_close --- core/src/codechain_machine.rs | 7 +++++++ core/src/consensus/tendermint/engine.rs | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/codechain_machine.rs b/core/src/codechain_machine.rs index 9d10dcf0aa..445bfb60a6 100644 --- a/core/src/codechain_machine.rs +++ b/core/src/codechain_machine.rs @@ -228,6 +228,13 @@ impl CodeChainMachine { Ok(()) } + pub fn add_balance_debug(&self, live: &mut ExecutedBlock, address: &Address, amount: u64) -> Result<(), Error> { + let debug_info = live.state_mut().add_balance_debug(address, amount).map_err(StateError::from)?; + println!("add_balance {:?}", debug_info); + Ok(()) + } + + pub fn increase_term_id(&self, live: &mut ExecutedBlock, last_term_finished_block_num: u64) -> Result<(), Error> { live.state_mut().increase_term_id(last_term_finished_block_num)?; Ok(()) diff --git a/core/src/consensus/tendermint/engine.rs b/core/src/consensus/tendermint/engine.rs index 3261e087a5..f86119c6a9 100644 --- a/core/src/consensus/tendermint/engine.rs +++ b/core/src/consensus/tendermint/engine.rs @@ -155,14 +155,14 @@ impl ConsensusEngine for Tendermint { let mut distributor = stake::fee_distribute(total_min_fee, &stakes); for (address, share) in &mut distributor { - self.machine.add_balance(block, &address, share)? + self.machine.add_balance_debug(block, &address, share)? } let block_author_reward = total_reward - total_min_fee + distributor.remaining_fee(); let metadata = block.state().metadata()?.expect("Metadata must exist"); if metadata.current_term_id() == 0 { - self.machine.add_balance(block, &author, block_author_reward)?; + self.machine.add_balance_debug(block, &author, block_author_reward)?; if let Some(block_number) = block_number_if_term_changed(block.header(), parent_header, parent_common_params) @@ -231,7 +231,7 @@ impl ConsensusEngine for Tendermint { )?; for (address, reward) in pending_rewards { - self.machine.add_balance(block, &address, reward)?; + self.machine.add_balance_debug(block, &address, reward)?; } let validators = stake::Validators::load_from_state(block.state())? From 3a9af39ea5d1884c61175c20b16a6acf6e00d869 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Tue, 6 Oct 2020 19:13:35 +0900 Subject: [PATCH 29/29] Add read_from_cache information in debug_info --- state/src/cache/write_back.rs | 2 ++ types/src/lib.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/state/src/cache/write_back.rs b/state/src/cache/write_back.rs index 26be9cdbc3..5c5b74519c 100644 --- a/state/src/cache/write_back.rs +++ b/state/src/cache/write_back.rs @@ -247,6 +247,8 @@ where debug_info_.db_read_time_us = elapsed; self.insert(a, Entry::::new_clean(maybe_item)); debug_info = debug_info_; + } else { + debug_info.read_from_cache = true; } self.note(a); diff --git a/types/src/lib.rs b/types/src/lib.rs index 11ed2299a1..e87354c44a 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -80,6 +80,7 @@ pub struct DebugInfo { pub read_count: u32, pub db_read_time_us: u128, pub reads: Vec, + pub read_from_cache: bool, } impl DebugInfo { @@ -88,6 +89,7 @@ impl DebugInfo { read_count: 0, db_read_time_us: 0, reads: Vec::new(), + read_from_cache: false, } }