From 0dfa4cc1fec948966f1fc9daeddf0ecafb5ce467 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 8 May 2020 09:10:44 -0500 Subject: [PATCH] add test for block size limits, add cfg(test) only blocks_processed monitor, remove need for gap sleeps in neon_integration tests --- testnet/stacks-node/src/neon_node.rs | 46 ++-- testnet/stacks-node/src/run_loop/neon.rs | 43 +++- .../stacks-node/src/tests/bitcoin_regtest.rs | 1 - testnet/stacks-node/src/tests/mod.rs | 11 + .../src/tests/neon_integrations.rs | 221 +++++++++++++++--- 5 files changed, 272 insertions(+), 50 deletions(-) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index 25dbf9604..9e45bd706 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -72,7 +72,7 @@ pub struct InitializedNeonNode { last_burn_block: Option, active_keys: Vec, sleep_before_tenure: u64, - is_miner: bool + is_miner: bool, } pub struct NeonGenesisNode { @@ -81,6 +81,21 @@ pub struct NeonGenesisNode { event_dispatcher: EventDispatcher, } +#[cfg(test)] +type BlocksProcessedCounter = std::sync::Arc; + +#[cfg(not(test))] +type BlocksProcessedCounter = (); + +#[cfg(test)] +fn bump_processed_counter(blocks_processed: &BlocksProcessedCounter) { + blocks_processed.fetch_add(1, std::sync::atomic::Ordering::SeqCst); +} + +#[cfg(not(test))] +fn bump_processed_counter(_blocks_processed: &BlocksProcessedCounter) { +} + /// Process artifacts from the tenure. /// At this point, we're modifying the chainstate, and merging the artifacts from the previous tenure. fn inner_process_tenure( @@ -255,7 +270,8 @@ fn spawn_miner_relayer(mut relayer: Relayer, local_peer: LocalPeer, config: Config, mut keychain: Keychain, burn_db_path: String, stacks_chainstate_path: String, relay_channel: Receiver, - mut event_dispatcher: EventDispatcher) -> Result<(), NetError> { + mut event_dispatcher: EventDispatcher, + blocks_processed: BlocksProcessedCounter) -> Result<(), NetError> { // Note: the relayer is *the* block processor, it is responsible for writes to the chainstate -- // no other codepaths should be writing once this is spawned. // @@ -369,9 +385,11 @@ fn spawn_miner_relayer(mut relayer: Relayer, local_peer: LocalPeer, last_mined_block = InitializedNeonNode::relayer_run_tenure( registered_key, &mut chainstate, &burndb, last_burn_block, &mut keychain, &mut mem_pool, burn_fee_cap, &mut bitcoin_controller); + bump_processed_counter(&blocks_processed); }, RelayerDirective::RegisterKey(ref last_burn_block) => { - rotate_vrf_and_register(&mut keychain, last_burn_block, &mut bitcoin_controller) + rotate_vrf_and_register(&mut keychain, last_burn_block, &mut bitcoin_controller); + bump_processed_counter(&blocks_processed); }, } } @@ -401,8 +419,8 @@ fn dispatcher_announce(blocks_path: &str, event_dispatcher: &mut EventDispatcher impl InitializedNeonNode { fn new(config: Config, keychain: Keychain, event_dispatcher: EventDispatcher, - last_burn_block: Option, registered_key: Option, - miner: bool) -> InitializedNeonNode { + last_burn_block: Option, + miner: bool, blocks_processed: BlocksProcessedCounter) -> InitializedNeonNode { // we can call _open_ here rather than _connect_, since connect is first called in // make_genesis_block let burndb = BurnDB::open(&config.get_burn_db_file_path(), false) @@ -478,7 +496,8 @@ impl InitializedNeonNode { config.clone(), keychain, config.get_burn_db_file_path(), config.get_chainstate_path(), - relay_recv, event_dispatcher) + relay_recv, event_dispatcher, + blocks_processed.clone()) .expect("Failed to initialize mine/relay thread"); spawn_peer(p2p_net, &p2p_sock, &rpc_sock, @@ -493,10 +512,7 @@ impl InitializedNeonNode { let is_miner = miner; - let mut active_keys = vec![]; - if let Some(key) = registered_key { - active_keys.push(key); - } + let active_keys = vec![]; InitializedNeonNode { relay_channel: relay_send, @@ -504,7 +520,7 @@ impl InitializedNeonNode { burnchain_signer, is_miner, sleep_before_tenure, - active_keys + active_keys, } } @@ -785,21 +801,21 @@ impl NeonGenesisNode { } } - pub fn into_initialized_leader_node(self, burnchain_tip: BurnchainTip) -> InitializedNeonNode { + pub fn into_initialized_leader_node(self, burnchain_tip: BurnchainTip, blocks_processed: BlocksProcessedCounter) -> InitializedNeonNode { let config = self.config; let keychain = self.keychain; let event_dispatcher = self.event_dispatcher; InitializedNeonNode::new(config, keychain, event_dispatcher, Some(burnchain_tip), - None, true) + true, blocks_processed) } - pub fn into_initialized_node(self, burnchain_tip: BurnchainTip) -> InitializedNeonNode { + pub fn into_initialized_node(self, burnchain_tip: BurnchainTip, blocks_processed: BlocksProcessedCounter) -> InitializedNeonNode { let config = self.config; let keychain = self.keychain; let event_dispatcher = self.event_dispatcher; InitializedNeonNode::new(config, keychain, event_dispatcher, Some(burnchain_tip), - None, false) + false, blocks_processed) } } diff --git a/testnet/stacks-node/src/run_loop/neon.rs b/testnet/stacks-node/src/run_loop/neon.rs index 9eb1db59b..f0254db0b 100644 --- a/testnet/stacks-node/src/run_loop/neon.rs +++ b/testnet/stacks-node/src/run_loop/neon.rs @@ -10,6 +10,14 @@ use stacks::burnchains::bitcoin::{BitcoinNetworkType, use super::RunLoopCallbacks; /// Coordinating a node running in neon mode. +#[cfg(test)] +pub struct RunLoop { + config: Config, + pub callbacks: RunLoopCallbacks, + blocks_processed: std::sync::Arc, +} + +#[cfg(not(test))] pub struct RunLoop { config: Config, pub callbacks: RunLoopCallbacks, @@ -18,13 +26,41 @@ pub struct RunLoop { impl RunLoop { /// Sets up a runloop and node, given a config. + #[cfg(not(test))] pub fn new(config: Config) -> Self { Self { config, - callbacks: RunLoopCallbacks::new() + callbacks: RunLoopCallbacks::new(), } } + #[cfg(test)] + pub fn new(config: Config) -> Self { + Self { + config, + callbacks: RunLoopCallbacks::new(), + blocks_processed: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), + } + } + + #[cfg(test)] + pub fn get_blocks_processed_arc(&self) -> std::sync::Arc { + self.blocks_processed.clone() + } + + #[cfg(not(test))] + fn get_blocks_processed_arc(&self) { + } + + #[cfg(test)] + fn bump_blocks_processed(&self) { + self.blocks_processed.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + + #[cfg(not(test))] + fn bump_blocks_processed(&self) { + } + /// Starts the testnet runloop. /// /// This function will block by looping infinitely. @@ -67,9 +103,9 @@ impl RunLoop { // setup genesis let node = NeonGenesisNode::new(self.config.clone(), |_| {}); let mut node = if is_miner { - node.into_initialized_leader_node(burnchain_tip.clone()) + node.into_initialized_leader_node(burnchain_tip.clone(), self.get_blocks_processed_arc()) } else { - node.into_initialized_node(burnchain_tip.clone()) + node.into_initialized_node(burnchain_tip.clone(), self.get_blocks_processed_arc()) }; // TODO (hack) instantiate the burndb in the burnchain @@ -77,6 +113,7 @@ impl RunLoop { // Start the runloop info!("Begin run loop"); + self.bump_blocks_processed(); loop { burnchain_tip = burnchain.sync(); diff --git a/testnet/stacks-node/src/tests/bitcoin_regtest.rs b/testnet/stacks-node/src/tests/bitcoin_regtest.rs index 794ffed23..3d64d8639 100644 --- a/testnet/stacks-node/src/tests/bitcoin_regtest.rs +++ b/testnet/stacks-node/src/tests/bitcoin_regtest.rs @@ -36,7 +36,6 @@ impl BitcoinCoreController { let mut command = Command::new("bitcoind"); command .stdout(Stdio::piped()) - .arg("-conf=/dev/null") // todo(ludo): nix only .arg("-regtest") .arg("-nodebug") .arg("-nodebuglogfile") diff --git a/testnet/stacks-node/src/tests/mod.rs b/testnet/stacks-node/src/tests/mod.rs index da106a7b3..95a88158c 100644 --- a/testnet/stacks-node/src/tests/mod.rs +++ b/testnet/stacks-node/src/tests/mod.rs @@ -86,6 +86,17 @@ pub fn make_contract_publish(sender: &StacksPrivateKey, nonce: u64, fee_rate: u6 serialize_sign_standard_single_sig_tx(payload.into(), sender, nonce, fee_rate) } +pub fn make_contract_publish_microblock_only(sender: &StacksPrivateKey, nonce: u64, fee_rate: u64, + contract_name: &str, contract_content: &str) -> Vec { + let name = ContractName::from(contract_name); + let code_body = StacksString::from_string(&contract_content.to_string()).unwrap(); + + let payload = TransactionSmartContract { name, code_body }; + + serialize_sign_standard_single_sig_tx_anchor_mode(payload.into(), sender, nonce, fee_rate, + TransactionAnchorMode::OffChainOnly) +} + pub fn new_test_conf() -> Config { // secretKey: "b1cf9cee5083f421c84d7cb53be5edf2801c3c78d63d53917aee0bdc8bd160ee01", diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 1bc049b81..81d776560 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -1,4 +1,5 @@ -use super::{make_stacks_transfer_mblock_only, SK_1, ADDR_4, to_addr}; +use super::{make_stacks_transfer_mblock_only, SK_1, ADDR_4, to_addr, + make_contract_publish, make_contract_publish_microblock_only}; use stacks::burnchains::Address; use stacks::chainstate::stacks::{ StacksTransaction, StacksPrivateKey, StacksAddress }; @@ -11,6 +12,9 @@ use crate::{ use stacks::net::AccountEntryResponse; use super::bitcoin_regtest::BitcoinCoreController; use std::{thread, env}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + fn neon_integration_test_conf() -> (Config, StacksAddress) { let mut conf = super::new_test_conf(); @@ -31,6 +35,30 @@ fn neon_integration_test_conf() -> (Config, StacksAddress) { (conf, miner_account) } +const PANIC_TIMEOUT_SECS: u64 = 60; +fn next_block_and_wait(btc_controller: &mut BitcoinRegtestController, blocks_processed: &Arc) { + let current = blocks_processed.load(Ordering::SeqCst); + eprintln!("Issuing block, waiting for bump"); + btc_controller.build_next_block(1); + let start = std::time::Instant::now(); + while blocks_processed.load(Ordering::SeqCst) <= current { + if start.elapsed() > std::time::Duration::from_secs(PANIC_TIMEOUT_SECS) { + panic!("Timed out waiting for block to process"); + } + thread::sleep(std::time::Duration::from_millis(100)); + } +} + +fn wait_for_runloop(blocks_processed: &Arc) { + let start = std::time::Instant::now(); + while blocks_processed.load(Ordering::SeqCst) == 0 { + if start.elapsed() > std::time::Duration::from_secs(PANIC_TIMEOUT_SECS) { + panic!("Timed out waiting for run loop to start"); + } + thread::sleep(std::time::Duration::from_millis(100)); + } +} + #[test] #[ignore] fn bitcoind_integration_test() { @@ -56,6 +84,7 @@ fn bitcoind_integration_test() { eprintln!("Chain bootstrapped..."); let mut run_loop = neon::RunLoop::new(conf); + let blocks_processed = run_loop.get_blocks_processed_arc(); let client = reqwest::blocking::Client::new(); thread::spawn(move || { @@ -63,24 +92,16 @@ fn bitcoind_integration_test() { }); // give the run loop some time to start up! - thread::sleep(std::time::Duration::from_millis(gap_ms)); + wait_for_runloop(&blocks_processed); // first block wakes up the run loop - btc_regtest_controller.build_next_block(1); - eprintln!("== REGTEST BLOCK MINED == "); - // give the run loop some time to figure things out! - thread::sleep(std::time::Duration::from_millis(gap_ms)); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); // first block will hold our VRF registration - btc_regtest_controller.build_next_block(1); - eprintln!("== REGTEST BLOCK MINED == "); - // give the run loop some time to figure things out! - thread::sleep(std::time::Duration::from_millis(gap_ms)); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); // second block will be the first mined Stacks block - btc_regtest_controller.build_next_block(1); - eprintln!("== REGTEST BLOCK MINED == "); - thread::sleep(std::time::Duration::from_millis(gap_ms)); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); // let's query the miner's account nonce: @@ -130,6 +151,7 @@ fn microblock_integration_test() { eprintln!("Chain bootstrapped..."); let mut run_loop = neon::RunLoop::new(conf); + let blocks_processed = run_loop.get_blocks_processed_arc(); let client = reqwest::blocking::Client::new(); thread::spawn(move || { @@ -137,24 +159,16 @@ fn microblock_integration_test() { }); // give the run loop some time to start up! - thread::sleep(std::time::Duration::from_millis(gap_ms)); + wait_for_runloop(&blocks_processed); // first block wakes up the run loop - btc_regtest_controller.build_next_block(1); - eprintln!("== REGTEST BLOCK MINED == "); - // give the run loop some time to figure things out! - thread::sleep(std::time::Duration::from_millis(gap_ms)); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); // first block will hold our VRF registration - btc_regtest_controller.build_next_block(1); - eprintln!("== REGTEST BLOCK MINED == "); - // give the run loop some time to figure things out! - thread::sleep(std::time::Duration::from_millis(gap_ms)); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); // second block will be the first mined Stacks block - btc_regtest_controller.build_next_block(1); - eprintln!("== REGTEST BLOCK MINED == "); - thread::sleep(std::time::Duration::from_millis(gap_ms)); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); // let's query the miner's account nonce: @@ -192,21 +206,166 @@ fn microblock_integration_test() { // now let's mine a couple blocks, and then check the sender's nonce. // this one wakes up our node, so that it'll mine a microblock _and_ an anchor block. - btc_regtest_controller.build_next_block(1); - eprintln!("== REGTEST BLOCK MINED == "); - thread::sleep(std::time::Duration::from_millis(gap_ms)); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); // this one will contain the sortition from above anchor block, // which *should* have also confirmed the microblock. - btc_regtest_controller.build_next_block(1); - eprintln!("== REGTEST BLOCK MINED == "); - thread::sleep(std::time::Duration::from_millis(gap_ms)); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); let path = format!("{}/v2/accounts/{}?proof=0", &http_origin, &spender_addr); let res = client.get(&path).send().unwrap().json::().unwrap(); eprintln!("{:#?}", res); + assert_eq!(res.nonce, 1); assert_eq!(u128::from_str_radix(&res.balance[2..], 16).unwrap(), 98300); +} + +#[test] +#[ignore] +fn size_check_integration_test() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return + } + + // used to specify how long to wait in between blocks. + // we could _probably_ add a hook to the neon node that + // would remove some of the need for this + let mut giant_contract = "(define-public (f) (ok 1))".to_string(); + for _i in 0..(1024*1024 + 500) { + giant_contract.push_str(" "); + } + + let spender_sks: Vec<_> = (0..10).into_iter().map(|_| StacksPrivateKey::new()).collect(); + let spender_addrs: Vec = + spender_sks.iter().map(|x| to_addr(x).into()).collect(); + // make a bunch of txs that will only fit one per block. + let txs: Vec<_> = spender_sks.iter().enumerate().map( + |(ix, spender_sk)| { + if ix % 2 == 0 { + make_contract_publish(spender_sk, 0, 1049230, "large-0", + &giant_contract) + } else { + make_contract_publish_microblock_only( + spender_sk, 0, 1049230, "large-0", + &giant_contract) + } + }).collect(); + + let (mut conf, miner_account) = neon_integration_test_conf(); + + for spender_addr in spender_addrs.iter() { + conf.initial_balances.push(InitialBalance { + address: spender_addr.clone(), + amount: 1049230 + }); + } + + conf.node.mine_microblocks = true; + + let mut btcd_controller = BitcoinCoreController::new(conf.clone()); + btcd_controller.start_bitcoind().map_err(|_e| ()).expect("Failed starting bitcoind"); + + let mut btc_regtest_controller = BitcoinRegtestController::new(conf.clone()); + let http_origin = format!("http://{}", &conf.node.rpc_bind); + + btc_regtest_controller.bootstrap_chain(201); + + eprintln!("Chain bootstrapped..."); + + let mut run_loop = neon::RunLoop::new(conf); + let blocks_processed = run_loop.get_blocks_processed_arc(); + let client = reqwest::blocking::Client::new(); + + thread::spawn(move || { + run_loop.start(0) + }); + + // give the run loop some time to start up! + wait_for_runloop(&blocks_processed); + + // first block wakes up the run loop + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // first block will hold our VRF registration + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // second block will be the first mined Stacks block + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // let's query the miner's account nonce: + + eprintln!("Miner account: {}", miner_account); + + let path = format!("{}/v2/accounts/{}?proof=0", + &http_origin, &miner_account); + eprintln!("Test: GET {}", path); + let res = client.get(&path).send().unwrap().json::().unwrap(); + assert_eq!(u128::from_str_radix(&res.balance[2..], 16).unwrap(), 0); assert_eq!(res.nonce, 1); + // and our potential spenders: + + for spender_addr in spender_addrs.iter() { + let path = format!("{}/v2/accounts/{}?proof=0", + &http_origin, spender_addr); + let res = client.get(&path).send().unwrap().json::().unwrap(); + assert_eq!(u128::from_str_radix(&res.balance[2..], 16).unwrap(), 1049230); + assert_eq!(res.nonce, 0); + } + + for tx in txs.iter() { + // okay, let's push a bunch of transactions that can only fit one per block! + let path = format!("{}/v2/transactions", &http_origin); + let res = client.post(&path) + .header("Content-Type", "application/octet-stream") + .body(tx.clone()) + .send() + .unwrap(); + eprintln!("{:#?}", res); + if res.status().is_success() { + let res: String = res + .json() + .unwrap(); + assert_eq!(res, StacksTransaction::consensus_deserialize(&mut &tx[..]).unwrap().txid().to_string()); + } else { + eprintln!("{}", res.text().unwrap()); + panic!(""); + } + } + + // now let's mine a couple blocks, and then check the sender's nonce. + // at the end of mining three blocks, there should be _one_ transaction from the microblock + // only set that got mined (since the block before this one was empty, a microblock can + // be added), + // and _two_ transactions from the two anchor blocks that got mined (and processed) + // + // this one wakes up our node, so that it'll mine a microblock _and_ an anchor block. + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + // this one will contain the sortition from above anchor block, + // which *should* have also confirmed the microblock. + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // let's figure out how many micro-only and anchor-only txs got accepted + // by examining our account nonces: + let mut micro_block_txs = 0; + let mut anchor_block_txs = 0; + for (ix, spender_addr) in spender_addrs.iter().enumerate() { + let path = format!("{}/v2/accounts/{}?proof=0", + &http_origin, spender_addr); + let res = client.get(&path).send().unwrap().json::().unwrap(); + if res.nonce == 1 { + if ix % 2 == 0 { + anchor_block_txs += 1; + } else { + micro_block_txs += 1; + } + } else if res.nonce != 0 { + panic!("Spender address nonce incremented past 1"); + } + } + + assert_eq!(anchor_block_txs, 2); + assert_eq!(micro_block_txs, 1); }