From 45d0f49b6cb27da3cf75ad43aeadb5c21c6c3d0d Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 10 Jun 2022 16:08:58 -0500 Subject: [PATCH] document new RPC method, clean up some of the changes from demo-day, add testing to rpc and event changes in l1_observer_test --- LOCAL_TESTING.md | 2 +- clarity/src/vm/contexts.rs | 8 +- clarity/src/vm/events.rs | 10 +- contrib/conf/hyperchain-l2.toml | 1 + contrib/conf/stacks-l1-mocknet-local.toml | 2 +- .../core-node/get-stx-withdrawal.example.json | 5 + .../core-node/get-stx-withdrawal.schema.json | 19 +++ docs/rpc/openapi.yaml | 12 ++ src/chainstate/stacks/db/blocks.rs | 2 +- src/chainstate/stacks/miner.rs | 2 +- src/clarity_vm/withdrawal.rs | 114 +++++++++++----- src/net/http.rs | 2 +- src/net/rpc.rs | 26 +++- testnet/stacks-node/src/neon_node.rs | 67 +++++++-- .../stacks-node/src/tests/l1_observer_test.rs | 129 +++++++++++++++--- .../src/tests/neon_integrations.rs | 60 +++++++- 16 files changed, 381 insertions(+), 80 deletions(-) create mode 100644 docs/rpc/api/core-node/get-stx-withdrawal.example.json create mode 100644 docs/rpc/api/core-node/get-stx-withdrawal.schema.json diff --git a/LOCAL_TESTING.md b/LOCAL_TESTING.md index bed2ec2f2..d7d74d362 100644 --- a/LOCAL_TESTING.md +++ b/LOCAL_TESTING.md @@ -136,7 +136,7 @@ curl -s localhost:19443/v2/withdrawal/stx/14/ST18F1AHKW194BWQ3CEFDPWVRARA79RBGFE Perform the withdrawal on layer-1 ```js -let json_merkle_entry = await fetch("http://localhost:19443/v2/withdrawal/stx/14/ST18F1AHKW194BWQ3CEFDPWVRARA79RBGFEWSDQR8/0/50000").then(x => x.json()) +let json_merkle_entry = await fetch("http://localhost:19443/v2/withdrawal/stx/45/ST18F1AHKW194BWQ3CEFDPWVRARA79RBGFEWSDQR8/0/50000").then(x => x.json()) let cv_merkle_entry = { withdrawal_leaf_hash: transactions.deserializeCV(json_merkle_entry.withdrawal_leaf_hash), withdrawal_root: transactions.deserializeCV(json_merkle_entry.withdrawal_root), diff --git a/clarity/src/vm/contexts.rs b/clarity/src/vm/contexts.rs index 0bb760c0f..568d5477a 100644 --- a/clarity/src/vm/contexts.rs +++ b/clarity/src/vm/contexts.rs @@ -1233,7 +1233,11 @@ impl<'a, 'b> Environment<'a, 'b> { sender: PrincipalData, amount: u128, ) -> Result<()> { - let event_data = STXWithdrawEventData { sender, amount }; + let event_data = STXWithdrawEventData { + sender, + amount, + withdrawal_id: None, + }; if let Some(batch) = self.global_context.event_batches.last_mut() { batch.events.push(StacksTransactionEvent::STXEvent( @@ -1315,6 +1319,7 @@ impl<'a, 'b> Environment<'a, 'b> { sender, asset_identifier, value, + withdrawal_id: None, }; if let Some(batch) = self.global_context.event_batches.last_mut() { @@ -1401,6 +1406,7 @@ impl<'a, 'b> Environment<'a, 'b> { sender, asset_identifier, amount, + withdrawal_id: None, }; if let Some(batch) = self.global_context.event_batches.last_mut() { diff --git a/clarity/src/vm/events.rs b/clarity/src/vm/events.rs index e973a9b7f..53f5411e0 100644 --- a/clarity/src/vm/events.rs +++ b/clarity/src/vm/events.rs @@ -234,13 +234,15 @@ impl STXBurnEventData { pub struct STXWithdrawEventData { pub sender: PrincipalData, pub amount: u128, + pub withdrawal_id: Option, } impl STXWithdrawEventData { pub fn json_serialize(&self) -> serde_json::Value { json!({ - "sender": format!("{}", self.sender), - "amount": format!("{}", self.amount), + "sender": self.sender.to_string(), + "amount": self.amount.to_string(), + "withdrawal_id": self.withdrawal_id.unwrap_or(0), }) } } @@ -324,6 +326,7 @@ pub struct NFTWithdrawEventData { pub asset_identifier: AssetIdentifier, pub sender: PrincipalData, pub value: Value, + pub withdrawal_id: Option, } impl NFTWithdrawEventData { @@ -339,6 +342,7 @@ impl NFTWithdrawEventData { "sender": format!("{}",self.sender), "value": self.value, "raw_value": format!("0x{}", raw_value.join("")), + "withdrawal_id": self.withdrawal_id.unwrap_or(0), }) } } @@ -401,6 +405,7 @@ pub struct FTWithdrawEventData { pub asset_identifier: AssetIdentifier, pub sender: PrincipalData, pub amount: u128, + pub withdrawal_id: Option, } impl FTWithdrawEventData { @@ -409,6 +414,7 @@ impl FTWithdrawEventData { "asset_identifier": format!("{}", self.asset_identifier), "sender": format!("{}",self.sender), "amount": format!("{}", self.amount), + "withdrawal_id": self.withdrawal_id.unwrap_or(0), }) } } diff --git a/contrib/conf/hyperchain-l2.toml b/contrib/conf/hyperchain-l2.toml index e8227c043..bf49c7e9d 100644 --- a/contrib/conf/hyperchain-l2.toml +++ b/contrib/conf/hyperchain-l2.toml @@ -19,3 +19,4 @@ rpc_port = 20443 peer_host = "127.0.0.1" first_burn_header_height = 1 contract_identifier = "ST2GE6HSXT81X9X3ATQ14WPT49X915R8X7FVERMBP.hyperchain" +observer_port = 49303 diff --git a/contrib/conf/stacks-l1-mocknet-local.toml b/contrib/conf/stacks-l1-mocknet-local.toml index 749b58187..d14ad7dae 100644 --- a/contrib/conf/stacks-l1-mocknet-local.toml +++ b/contrib/conf/stacks-l1-mocknet-local.toml @@ -33,6 +33,6 @@ address = "STSTW15D618BSZQB85R058DS46THH86YQQY6XCB7" amount = 100000000000000 [[events_observer]] -endpoint = "localhost:50303" +endpoint = "localhost:49303" retry_count = 255 events_keys = ["*"] diff --git a/docs/rpc/api/core-node/get-stx-withdrawal.example.json b/docs/rpc/api/core-node/get-stx-withdrawal.example.json new file mode 100644 index 000000000..c30c9278a --- /dev/null +++ b/docs/rpc/api/core-node/get-stx-withdrawal.example.json @@ -0,0 +1,5 @@ +{ + "withdrawal_root": "0x0200000020898a1d67146f768bea82df555bebad41d2919518c843bdce83057f970efb3889", + "withdrawal_leaf_hash": "0x0200000020a6b03891a27f3cbea3b64c24fed1740740785c8da960bb11cacb55333e8191bc", + "sibling_hashes": "0x0b000000010c0000000204686173680200000020a6b03891a27f3cbea3b64c24fed1740740785c8da960bb11cacb55333e8191bc0c69732d6c6566742d7369646504" +} diff --git a/docs/rpc/api/core-node/get-stx-withdrawal.schema.json b/docs/rpc/api/core-node/get-stx-withdrawal.schema.json new file mode 100644 index 000000000..46efdb858 --- /dev/null +++ b/docs/rpc/api/core-node/get-stx-withdrawal.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "GET request for STX withdrawal data", + "title": "WithdrawalStxResponse", + "type": "object", + "additionalProperties": false, + "required": ["withdrawal_root", "withdrawal_leaf_hash", "sibling_hashes"], + "properties": { + "withdrawal_root": { + "type": "string" + }, + "withdrawal_leaf_hash": { + "type": "string" + }, + "sibling_hashes": { + "type": "string" + } + } +} diff --git a/docs/rpc/openapi.yaml b/docs/rpc/openapi.yaml index 0c3d7c09e..82e349a64 100644 --- a/docs/rpc/openapi.yaml +++ b/docs/rpc/openapi.yaml @@ -387,6 +387,18 @@ paths: example: $ref: ./api/core-node/get-fee-transfer.example.json + /v2/withdrawal/stx/{block_height}/{sender}/{withdrawal_id}/{amount}: + get: + summary: Get merkle tree data associated with a processed withdrawal. + responses: + 200: + description: The merkle leaf hash, root hash, and merkle proof path for the requested withdrawal entry. These are used as parameters to the `stx-withdraw` method of a layer-1 hyperchains contract, and returned as hex-encoded Clarity serialized values. + content: + application/json: + schema: + $ref: ./api/core-node/get-stx-withdrawal.schema.json + example: + $ref: ./api/core-node/get-stx-withdrawal.example.json /v2/info: get: summary: Get Core API info diff --git a/src/chainstate/stacks/db/blocks.rs b/src/chainstate/stacks/db/blocks.rs index d9364ed89..b87ba85a9 100644 --- a/src/chainstate/stacks/db/blocks.rs +++ b/src/chainstate/stacks/db/blocks.rs @@ -5528,7 +5528,7 @@ impl StacksChainState { // Check withdrawal state merkle root // Process withdrawal events let withdrawal_tree = - create_withdrawal_merkle_tree(&tx_receipts, block.header.total_work.work); + create_withdrawal_merkle_tree(&mut tx_receipts, block.header.total_work.work); let withdrawal_root_hash = withdrawal_tree.root(); if withdrawal_root_hash != block.header.withdrawal_merkle_root { diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index aaa9c5f99..a50e8828e 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -1479,7 +1479,7 @@ impl StacksBlockBuilder { self.header.state_index_root = state_root_hash; let withdrawal_tree = - create_withdrawal_merkle_tree(&self.tx_receipts, self.header.total_work.work); + create_withdrawal_merkle_tree(&mut self.tx_receipts, self.header.total_work.work); let withdrawal_merkle_root = withdrawal_tree.root(); self.header.withdrawal_merkle_root = withdrawal_merkle_root; diff --git a/src/clarity_vm/withdrawal.rs b/src/clarity_vm/withdrawal.rs index dcbcaa629..d4c40b6ba 100644 --- a/src/clarity_vm/withdrawal.rs +++ b/src/clarity_vm/withdrawal.rs @@ -6,7 +6,7 @@ use clarity::vm::events::{ FTEventType, FTWithdrawEventData, NFTEventType, NFTWithdrawEventData, STXEventType, STXWithdrawEventData, StacksTransactionEvent, }; -use clarity::vm::types::PrincipalData; +use clarity::vm::types::{AssetIdentifier, PrincipalData}; use clarity::vm::Value; use regex::internal::Input; @@ -27,21 +27,42 @@ pub fn buffer_from_hash(hash: Sha512Trunc256Sum) -> Value { Value::buff_from(hash.0.to_vec()).expect("Failed to construct buffer from hash") } -pub fn make_key_for_ft_withdrawal(data: &FTWithdrawEventData, withdrawal_id: u32) -> String { - let str_data = format!("ft::{}::{}", data.asset_identifier, data.amount); - make_key_for_withdrawal(str_data, &data.sender, withdrawal_id) +pub fn make_key_for_ft_withdrawal_event(data: &FTWithdrawEventData, block_height: u64) -> String { + let withdrawal_id = data + .withdrawal_id + .expect("Tried to serialize a withdraw event before setting withdrawal ID"); + info!("Parsed L2 withdrawal event"; + "type" => "ft", + "block_height" => block_height, + "sender" => %data.sender, + "withdrawal_id" => withdrawal_id, + "amount" => %data.amount, + "asset_id" => %data.asset_identifier); + make_key_for_ft_withdrawal( + &data.sender, + withdrawal_id, + &data.asset_identifier, + data.amount, + ) } -pub fn make_key_for_nft_withdrawal(data: &NFTWithdrawEventData, withdrawal_id: u32) -> String { - let str_data = format!("nft::{}", data.asset_identifier); - make_key_for_withdrawal(str_data, &data.sender, withdrawal_id) +pub fn make_key_for_nft_withdrawal_event(data: &NFTWithdrawEventData, block_height: u64) -> String { + let withdrawal_id = data + .withdrawal_id + .expect("Tried to serialize a withdraw event before setting withdrawal ID"); + info!("Parsed L2 withdrawal event"; + "type" => "nft", + "block_height" => block_height, + "sender" => %data.sender, + "withdrawal_id" => withdrawal_id, + "asset_id" => %data.asset_identifier); + make_key_for_nft_withdrawal(&data.sender, withdrawal_id, &data.asset_identifier) } -pub fn make_key_for_stx_withdrawal_event( - data: &STXWithdrawEventData, - withdrawal_id: u32, - block_height: u64, -) -> String { +pub fn make_key_for_stx_withdrawal_event(data: &STXWithdrawEventData, block_height: u64) -> String { + let withdrawal_id = data + .withdrawal_id + .expect("Tried to serialize a withdraw event before setting withdrawal ID"); info!("Parsed L2 withdrawal event"; "type" => "stx", "block_height" => block_height, @@ -51,6 +72,25 @@ pub fn make_key_for_stx_withdrawal_event( make_key_for_stx_withdrawal(&data.sender, withdrawal_id, data.amount) } +pub fn make_key_for_nft_withdrawal( + sender: &PrincipalData, + withdrawal_id: u32, + asset_identifier: &AssetIdentifier, +) -> String { + let str_data = format!("nft::{}", asset_identifier); + make_key_for_withdrawal(str_data, sender, withdrawal_id) +} + +pub fn make_key_for_ft_withdrawal( + sender: &PrincipalData, + withdrawal_id: u32, + asset_identifier: &AssetIdentifier, + amount: u128, +) -> String { + let str_data = format!("ft::{}::{}", asset_identifier, amount); + make_key_for_withdrawal(str_data, sender, withdrawal_id) +} + pub fn make_key_for_stx_withdrawal( sender: &PrincipalData, withdrawal_id: u32, @@ -60,21 +100,26 @@ pub fn make_key_for_stx_withdrawal( make_key_for_withdrawal(str_data, sender, withdrawal_id) } +/// The supplied withdrawal ID is inserted into the supplied withdraw event +/// (this is why the event are supplied as a mutable argument). pub fn generate_key_from_event( - event: &StacksTransactionEvent, + event: &mut StacksTransactionEvent, withdrawal_id: u32, block_height: u64, ) -> Option { match event { StacksTransactionEvent::NFTEvent(NFTEventType::NFTWithdrawEvent(data)) => { - Some(make_key_for_nft_withdrawal(data, withdrawal_id)) + data.withdrawal_id = Some(withdrawal_id); + Some(make_key_for_nft_withdrawal_event(data, block_height)) } StacksTransactionEvent::FTEvent(FTEventType::FTWithdrawEvent(data)) => { - Some(make_key_for_ft_withdrawal(data, withdrawal_id)) + data.withdrawal_id = Some(withdrawal_id); + Some(make_key_for_ft_withdrawal_event(data, block_height)) + } + StacksTransactionEvent::STXEvent(STXEventType::STXWithdrawEvent(data)) => { + data.withdrawal_id = Some(withdrawal_id); + Some(make_key_for_stx_withdrawal_event(data, block_height)) } - StacksTransactionEvent::STXEvent(STXEventType::STXWithdrawEvent(data)) => Some( - make_key_for_stx_withdrawal_event(data, withdrawal_id, block_height), - ), _ => None, } } @@ -87,13 +132,13 @@ pub fn convert_withdrawal_key_to_bytes(key: &str) -> Vec { /// that correspond to each event. These IDs are used to generate the withdrawal key that is /// ultimately inserted in the withdrawal Merkle tree. pub fn generate_withdrawal_keys( - tx_receipts: &Vec, + tx_receipts: &mut [StacksTransactionReceipt], block_height: u64, ) -> Vec> { let mut items = Vec::new(); let mut withdrawal_id = 0; - for receipt in tx_receipts { - for event in &receipt.events { + for receipt in tx_receipts.iter_mut() { + for event in receipt.events.iter_mut() { if let Some(key) = generate_key_from_event(event, withdrawal_id, block_height) { withdrawal_id += 1; items.push(key); @@ -107,10 +152,12 @@ pub fn generate_withdrawal_keys( .collect() } -/// Put all withdrawal keys and values into a single Merkle tree -/// The order of the transaction receipts will affect the final tree +/// Put all withdrawal keys and values into a single Merkle tree. +/// The order of the transaction receipts will affect the final tree. +/// The generated withdrawal IDs are inserted into the supplied withdraw events +/// (this is why the receipts are supplied as a mutable argument). pub fn create_withdrawal_merkle_tree( - tx_receipts: &Vec, + tx_receipts: &mut [StacksTransactionReceipt], block_height: u64, ) -> MerkleTree { // The specific keys generated is dependent on the order of the provided transaction receipts @@ -158,12 +205,13 @@ mod test { spending_condition.set_nonce(0); spending_condition.set_tx_fee(1000); let auth = TransactionAuth::Standard(spending_condition); - let stx_withdraw_event = + let mut stx_withdraw_event = StacksTransactionEvent::STXEvent(STXWithdrawEvent(STXWithdrawEventData { sender: user_addr.into(), amount: 1, + withdrawal_id: None, })); - let ft_withdraw_event = + let mut ft_withdraw_event = StacksTransactionEvent::FTEvent(FTWithdrawEvent(FTWithdrawEventData { asset_identifier: AssetIdentifier { contract_identifier: QualifiedContractIdentifier::new( @@ -172,10 +220,11 @@ mod test { ), asset_name: ClarityName::from("ft-token"), }, + withdrawal_id: None, sender: user_addr.into(), amount: 1, })); - let nft_withdraw_event = + let mut nft_withdraw_event = StacksTransactionEvent::NFTEvent(NFTWithdrawEvent(NFTWithdrawEventData { asset_identifier: AssetIdentifier { contract_identifier: QualifiedContractIdentifier::new( @@ -184,6 +233,7 @@ mod test { ), asset_name: ClarityName::from("nft-token"), }, + withdrawal_id: None, sender: user_addr.into(), value: Value::UInt(1), })); @@ -207,11 +257,13 @@ mod test { tx_index: 0, }; - let withdrawal_tree = create_withdrawal_merkle_tree(&vec![withdrawal_receipt]); + let mut receipts = vec![withdrawal_receipt]; + // supplying block height = 0 is okay in tests, because block height is only used for logging + let withdrawal_tree = create_withdrawal_merkle_tree(receipts.as_mut(), 0); let root_hash = withdrawal_tree.root(); // manually construct the expected Merkle tree - let stx_withdrawal_key = generate_key_from_event(&stx_withdraw_event, 0).unwrap(); + let stx_withdrawal_key = generate_key_from_event(&mut stx_withdraw_event, 0, 0).unwrap(); let stx_withdrawal_key_bytes = convert_withdrawal_key_to_bytes(&stx_withdrawal_key); let stx_withdrawal_leaf_hash = MerkleTree::::get_leaf_hash(stx_withdrawal_key_bytes.as_slice()); @@ -223,7 +275,7 @@ mod test { ] ); - let ft_withdrawal_key = generate_key_from_event(&ft_withdraw_event, 1).unwrap(); + let ft_withdrawal_key = generate_key_from_event(&mut ft_withdraw_event, 1, 0).unwrap(); let ft_withdrawal_key_bytes = convert_withdrawal_key_to_bytes(&ft_withdrawal_key); let ft_withdrawal_leaf_hash = MerkleTree::::get_leaf_hash(ft_withdrawal_key_bytes.as_slice()); @@ -235,7 +287,7 @@ mod test { ] ); - let nft_withdrawal_key = generate_key_from_event(&nft_withdraw_event, 2).unwrap(); + let nft_withdrawal_key = generate_key_from_event(&mut nft_withdraw_event, 2, 0).unwrap(); let nft_withdrawal_key_bytes = convert_withdrawal_key_to_bytes(&nft_withdrawal_key); let nft_withdrawal_leaf_hash = MerkleTree::::get_leaf_hash(nft_withdrawal_key_bytes.as_slice()); diff --git a/src/net/http.rs b/src/net/http.rs index 37432118a..309ebdf81 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -1803,7 +1803,7 @@ impl HttpRequestType { _protocol: &mut StacksHttp, preamble: &HttpRequestPreamble, captures: &Captures, - query: Option<&str>, + _query: Option<&str>, _fd: &mut R, ) -> Result { if preamble.get_content_length() != 0 { diff --git a/src/net/rpc.rs b/src/net/rpc.rs index 8c532b548..2162a939b 100644 --- a/src/net/rpc.rs +++ b/src/net/rpc.rs @@ -31,6 +31,7 @@ use clarity::util::hash::MerklePathOrder; use clarity::util::hash::MerklePathPoint; use clarity::util::hash::MerkleTree; use clarity::util::hash::Sha512Trunc256Sum; +use clarity::vm::types::AssetIdentifier; use clarity::vm::types::TupleData; use rand::prelude::*; use rand::thread_rng; @@ -937,6 +938,29 @@ impl ConversationHttp { withdrawal_id: u32, amount: u128, canonical_stacks_tip_height: u64, + ) -> Result<(), net_error> { + let withdrawal_key = withdrawal::make_key_for_stx_withdrawal(sender, withdrawal_id, amount); + Self::handle_get_generic_withdrawal_entry( + http, + fd, + req, + chainstate, + canonical_tip, + requested_block_height, + withdrawal_key, + canonical_stacks_tip_height, + ) + } + + fn handle_get_generic_withdrawal_entry( + http: &mut StacksHttp, + fd: &mut W, + req: &HttpRequestType, + chainstate: &mut StacksChainState, + canonical_tip: &StacksBlockId, + requested_block_height: u64, + withdrawal_key: String, + canonical_stacks_tip_height: u64, ) -> Result<(), net_error> { let response_metadata = HttpResponseMetadata::from_http_request_type(req, Some(canonical_stacks_tip_height)); @@ -976,8 +1000,6 @@ impl ConversationHttp { } }; - let withdrawal_key = withdrawal::make_key_for_stx_withdrawal(sender, withdrawal_id, amount); - let merkle_path = match withdrawal_tree.path(withdrawal_key.as_bytes()) { Some(path) => path, None => { diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index 3d6de7cb9..c1812a630 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -86,7 +86,7 @@ struct MicroblockMinerState { enum RelayerDirective { HandleNetResult(NetworkResult), ProcessTenure(ConsensusHash, BurnchainHeaderHash, BlockHeaderHash), - RunTenure(BlockSnapshot, u128), // (vrf key, chain tip, time of issuance in ms) + RunTenure, // (vrf key, chain tip, time of issuance in ms) RunMicroblockTenure(BlockSnapshot, u128), // time of issuance in ms Exit, } @@ -102,8 +102,8 @@ pub struct StacksNode { pub atlas_config: AtlasConfig, pub p2p_thread_handle: JoinHandle<()>, pub relayer_thread_handle: JoinHandle<()>, - /// Includes the data for the next `RunTenure` directive. Once the "wait for micro-blocks" nap is over. - next_run_tenure_data: Arc>>, + /// Lock used for the timer thread before issuing a tenure directive + is_tenure_timer_running: Arc>, } #[cfg(test)] @@ -727,7 +727,7 @@ fn spawn_peer( TrySendError::Full(directive) => { if let RelayerDirective::RunMicroblockTenure(..) = directive { // can drop this - } else if let RelayerDirective::RunTenure(..) = directive { + } else if let RelayerDirective::RunTenure = directive { // can drop this } else { // don't lose this data -- just try it again @@ -837,7 +837,6 @@ fn spawn_miner_relayer( let mut microblock_miner_state: Option = None; let mut miner_tip = None; // only set if we won the last sortition let mut last_microblock_tenure_time = 0; - let mut last_tenure_issue_time = 0; let relayer_handle = thread::Builder::new().name("relayer".to_string()).spawn(move || { let cost_estimator = config.make_cost_estimator() @@ -1014,7 +1013,7 @@ fn spawn_miner_relayer( ); } } - RelayerDirective::RunTenure(_last_burn_block, issue_timestamp_ms) => { + RelayerDirective::RunTenure => { let burn_chain_sn = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) .expect("FATAL: failed to query sortition DB for canonical burn chain tip"); @@ -1385,7 +1384,7 @@ impl StacksNode { atlas_config, p2p_thread_handle, relayer_thread_handle, - next_run_tenure_data: Arc::new(Mutex::new(None)), + is_tenure_timer_running: Arc::new(Mutex::new(false)), } } @@ -1402,12 +1401,50 @@ impl StacksNode { let wait_before_first_anchored_block = self.config.node.wait_before_first_anchored_block; - relay_channel - .send(RelayerDirective::RunTenure( - burnchain_tip, - get_epoch_time_ms(), - )) - .is_ok() + // Check if a thread to send the `RunTenure` directive is already running, and if not we should start one. + let start_new_thread = { + let mut tenure_timer_mutex = self.is_tenure_timer_running.lock().unwrap(); + + let thread_running = *tenure_timer_mutex; + + // Update the shared data for the `RunTenure` directive. + *tenure_timer_mutex = true; + + !thread_running + }; + + debug!( + "relayer_issue_tenure invoked"; + "will_spawn_new_issue" => start_new_thread, + "wait_ms" => wait_before_first_anchored_block, + "received_at_burn_hash" => %burnchain_tip.burn_header_hash, + "received_at_burn_height" => %burnchain_tip.block_height, + ); + + if start_new_thread { + let tenure_timer_mutex = self.is_tenure_timer_running.clone(); + thread::spawn(move || { + thread::sleep(time::Duration::from_millis( + wait_before_first_anchored_block, + )); + + { + let mut tenure_lock_handle = tenure_timer_mutex.lock().unwrap(); + *tenure_lock_handle = false; + } + + debug!( + "relayer_issue_tenure: Have waited {} ms and now will build off of chain tip", + wait_before_first_anchored_block, + ); + + // Send the signal. + let channel_accepted = relay_channel.send(RelayerDirective::RunTenure).is_ok(); + + channel_accepted + }); + } + true } else { warn!("Tenure: Do not know the last burn block. As a miner, this is bad."); true @@ -1424,7 +1461,7 @@ impl StacksNode { } if let Some(snapshot) = get_last_sortition(&self.last_sortition) { - info!( + debug!( "Tenure: Notify sortition!"; "consensus_hash" => %snapshot.consensus_hash, "burn_block_hash" => %snapshot.burn_header_hash, @@ -1443,7 +1480,7 @@ impl StacksNode { .is_ok(); } } else { - info!("Tenure: Notify sortition! No last burn block"); + debug!("Tenure: Notify sortition! No last burn block"); } true } diff --git a/testnet/stacks-node/src/tests/l1_observer_test.rs b/testnet/stacks-node/src/tests/l1_observer_test.rs index 3f2634662..0ec3c2bb3 100644 --- a/testnet/stacks-node/src/tests/l1_observer_test.rs +++ b/testnet/stacks-node/src/tests/l1_observer_test.rs @@ -4,7 +4,9 @@ use std::thread::{self, JoinHandle}; use crate::config::{EventKeyType, EventObserverConfig}; use crate::neon; -use crate::tests::neon_integrations::{get_account, submit_tx, test_observer}; +use crate::tests::neon_integrations::{ + filter_map_events, get_account, get_withdrawal_entry, submit_tx, test_observer, +}; use crate::tests::{make_contract_call, make_contract_publish, to_addr}; use clarity::types::chainstate::StacksAddress; use clarity::util::hash::{MerklePathOrder, MerkleTree, Sha512Trunc256Sum}; @@ -966,18 +968,20 @@ fn l1_deposit_and_withdraw_asset_integration_test() { spending_condition.set_nonce(l2_nonce - 1); spending_condition.set_tx_fee(1000); let auth = TransactionAuth::Standard(spending_condition); - let ft_withdraw_event = StacksTransactionEvent::FTEvent(FTWithdrawEvent(FTWithdrawEventData { - asset_identifier: AssetIdentifier { - contract_identifier: QualifiedContractIdentifier::new( - user_addr.into(), - ContractName::from("simple-ft"), - ), - asset_name: ClarityName::from("ft-token"), - }, - sender: user_addr.into(), - amount: 1, - })); - let nft_withdraw_event = + let mut ft_withdraw_event = + StacksTransactionEvent::FTEvent(FTWithdrawEvent(FTWithdrawEventData { + asset_identifier: AssetIdentifier { + contract_identifier: QualifiedContractIdentifier::new( + user_addr.into(), + ContractName::from("simple-ft"), + ), + asset_name: ClarityName::from("ft-token"), + }, + sender: user_addr.into(), + amount: 1, + withdrawal_id: None, + })); + let mut nft_withdraw_event = StacksTransactionEvent::NFTEvent(NFTWithdrawEvent(NFTWithdrawEventData { asset_identifier: AssetIdentifier { contract_identifier: QualifiedContractIdentifier::new( @@ -988,6 +992,7 @@ fn l1_deposit_and_withdraw_asset_integration_test() { }, sender: user_addr.into(), value: Value::UInt(1), + withdrawal_id: None, })); let withdrawal_receipt = StacksTransactionReceipt { transaction: TransactionOrigin::Stacks(StacksTransaction::new( @@ -1004,10 +1009,11 @@ fn l1_deposit_and_withdraw_asset_integration_test() { microblock_header: None, tx_index: 0, }; - let withdrawal_tree = create_withdrawal_merkle_tree(&vec![withdrawal_receipt]); + let mut receipts = vec![withdrawal_receipt]; + let withdrawal_tree = create_withdrawal_merkle_tree(&mut receipts, 0); let root_hash = withdrawal_tree.root().as_bytes().to_vec(); - let ft_withdrawal_key = generate_key_from_event(&ft_withdraw_event, 0).unwrap(); + let ft_withdrawal_key = generate_key_from_event(&mut ft_withdraw_event, 0, 0).unwrap(); let ft_withdrawal_key_bytes = convert_withdrawal_key_to_bytes(&ft_withdrawal_key); let ft_withdrawal_leaf_hash = MerkleTree::::get_leaf_hash(ft_withdrawal_key_bytes.as_slice()) @@ -1015,7 +1021,7 @@ fn l1_deposit_and_withdraw_asset_integration_test() { .to_vec(); let ft_path = withdrawal_tree.path(&ft_withdrawal_key_bytes).unwrap(); - let nft_withdrawal_key = generate_key_from_event(&nft_withdraw_event, 1).unwrap(); + let nft_withdrawal_key = generate_key_from_event(&mut nft_withdraw_event, 1, 0).unwrap(); let nft_withdrawal_key_bytes = convert_withdrawal_key_to_bytes(&nft_withdrawal_key); let nft_withdrawal_leaf_hash = MerkleTree::::get_leaf_hash(nft_withdrawal_key_bytes.as_slice()) @@ -1191,6 +1197,13 @@ fn l1_deposit_and_withdraw_stx_integration_test() { config.node.miner = true; + config.events_observers.push(EventObserverConfig { + endpoint: format!("localhost:{}", test_observer::EVENT_OBSERVER_PORT), + events_keys: vec![EventKeyType::AnyEvent], + }); + + test_observer::spawn(); + let mut run_loop = neon::RunLoop::new(config.clone()); let termination_switch = run_loop.get_termination_switch(); let run_loop_thread = thread::spawn(move || run_loop.start(None, 0)); @@ -1386,6 +1399,54 @@ fn l1_deposit_and_withdraw_stx_integration_test() { // Sleep to give the run loop time to mine a block thread::sleep(Duration::from_secs(25)); + + // TODO: here, read the withdrawal events to get the withdrawal ID, and figure out the + // block height to query. + let block_data = test_observer::get_blocks(); + let mut withdraw_events = filter_map_events(&block_data, |height, event| { + let ev_type = event.get("type").unwrap().as_str().unwrap(); + if ev_type == "stx_withdraw_event" { + Some((height, event.get("stx_withdraw_event").unwrap().clone())) + } else { + None + } + }); + + // should only be one withdrawal event + assert_eq!(withdraw_events.len(), 1); + let (withdrawal_height, withdrawal_json) = withdraw_events.pop().unwrap(); + + let withdrawal_id = withdrawal_json + .get("withdrawal_id") + .unwrap() + .as_u64() + .unwrap(); + let withdrawal_amount: u64 = withdrawal_json + .get("amount") + .unwrap() + .as_str() + .unwrap() + .parse() + .unwrap(); + let withdrawal_sender = withdrawal_json + .get("sender") + .unwrap() + .as_str() + .unwrap() + .to_string(); + + assert_eq!(withdrawal_id, 0); + assert_eq!(withdrawal_amount, 1); + assert_eq!(withdrawal_sender, user_addr.to_string()); + + let withdrawal_entry = get_withdrawal_entry( + &l2_rpc_origin, + withdrawal_height, + &user_addr, + withdrawal_id, + withdrawal_amount, + ); + // Check that the user does not own any additional STX anymore on the hyperchain now let account = get_account(&l2_rpc_origin, &user_addr); assert_eq!( @@ -1409,10 +1470,11 @@ fn l1_deposit_and_withdraw_stx_integration_test() { spending_condition.set_nonce(l2_nonce - 1); spending_condition.set_tx_fee(1000); let auth = TransactionAuth::Standard(spending_condition); - let stx_withdraw_event = + let mut stx_withdraw_event = StacksTransactionEvent::STXEvent(STXWithdrawEvent(STXWithdrawEventData { sender: user_addr.into(), amount: 1, + withdrawal_id: None, })); let withdrawal_receipt = StacksTransactionReceipt { @@ -1430,10 +1492,14 @@ fn l1_deposit_and_withdraw_stx_integration_test() { microblock_header: None, tx_index: 0, }; - let withdrawal_tree = create_withdrawal_merkle_tree(&vec![withdrawal_receipt]); + let mut receipts = vec![withdrawal_receipt]; + + // okay to pass a zero block height in tests: the block height parameter is only used for logging + let withdrawal_tree = create_withdrawal_merkle_tree(&mut receipts, 0); let root_hash = withdrawal_tree.root().as_bytes().to_vec(); - let stx_withdrawal_key = generate_key_from_event(&stx_withdraw_event, 0).unwrap(); + // okay to pass a zero block height in tests: the block height parameter is only used for logging + let stx_withdrawal_key = generate_key_from_event(&mut stx_withdraw_event, 0, 0).unwrap(); let stx_withdrawal_key_bytes = convert_withdrawal_key_to_bytes(&stx_withdrawal_key); let stx_withdrawal_leaf_hash = MerkleTree::::get_leaf_hash(stx_withdrawal_key_bytes.as_slice()) @@ -1454,6 +1520,25 @@ fn l1_deposit_and_withdraw_stx_integration_test() { stx_sib_data.push(sib_tuple); } + let root_hash_val = Value::buff_from(root_hash.clone()).unwrap(); + let leaf_hash_val = Value::buff_from(stx_withdrawal_leaf_hash).unwrap(); + let siblings_val = Value::list_from(stx_sib_data).unwrap(); + + assert_eq!( + &root_hash_val, &withdrawal_entry.root_hash, + "Root hash should match value returned via RPC" + ); + assert_eq!( + &leaf_hash_val, &withdrawal_entry.leaf_hash, + "Leaf hash should match value returned via RPC" + ); + assert_eq!( + &siblings_val, &withdrawal_entry.siblings, + "Sibling hashes should match value returned via RPC" + ); + + // test the result of our RPC call matches our constructed values + let l1_withdraw_stx_tx = make_contract_call( &MOCKNET_PRIVATE_KEY_1, l1_nonce, @@ -1464,9 +1549,9 @@ fn l1_deposit_and_withdraw_stx_integration_test() { &[ Value::UInt(1), Value::Principal(user_addr.into()), - Value::buff_from(root_hash.clone()).unwrap(), - Value::buff_from(stx_withdrawal_leaf_hash).unwrap(), - Value::list_from(stx_sib_data).unwrap(), + root_hash_val, + leaf_hash_val, + siblings_val, ], ); l1_nonce += 1; diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 548cdcb15..65aa5e1d8 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -8,7 +8,7 @@ use stacks::chainstate::burn::db::sortdb::SortitionDB; use stacks::chainstate::burn::ConsensusHash; use stacks::chainstate::stacks::TransactionPayload; use stacks::codec::StacksMessageCodec; -use stacks::net::{AccountEntryResponse, ContractSrcResponse, RPCPeerInfoData}; +use stacks::net::{AccountEntryResponse, ContractSrcResponse, RPCPeerInfoData, WithdrawalResponse}; use stacks::types::chainstate::{BlockHeaderHash, StacksAddress}; use stacks::util::get_epoch_time_secs; use stacks::util::hash::{hex_bytes, Hash160}; @@ -21,6 +21,8 @@ use stacks::{ net::RPCPoxInfoData, }; +use clarity::vm::Value as ClarityValue; + use crate::burnchains::mock_events::{reset_static_burnblock_simulator_channel, MockController}; use crate::config::{EventKeyType, EventObserverConfig}; use crate::neon; @@ -636,6 +638,13 @@ pub struct Account { pub nonce: u64, } +#[derive(Debug)] +pub struct WithdrawalEntry { + pub leaf_hash: ClarityValue, + pub root_hash: ClarityValue, + pub siblings: ClarityValue, +} + pub fn get_account(http_origin: &str, account: &F) -> Account { let client = reqwest::blocking::Client::new(); let path = format!("{}/v2/accounts/{}?proof=0", http_origin, account); @@ -652,6 +661,33 @@ pub fn get_account(http_origin: &str, account: &F) -> Acco } } +pub fn get_withdrawal_entry( + http_origin: &str, + block_height: u64, + sender: F, + withdrawal_id: u64, + amount: u64, +) -> WithdrawalEntry { + let client = reqwest::blocking::Client::new(); + let path = format!( + "{}/v2/withdrawal/stx/{}/{}/{}/{}", + http_origin, block_height, sender, withdrawal_id, amount + ); + + let res = client + .get(&path) + .send() + .unwrap() + .json::() + .unwrap(); + info!("Withdrawal response: {:#?}", res); + WithdrawalEntry { + leaf_hash: ClarityValue::try_deserialize_hex_untyped(&res.withdrawal_leaf_hash).unwrap(), + root_hash: ClarityValue::try_deserialize_hex_untyped(&res.withdrawal_root).unwrap(), + siblings: ClarityValue::try_deserialize_hex_untyped(&res.sibling_hashes).unwrap(), + } +} + fn get_pox_info(http_origin: &str) -> RPCPoxInfoData { let client = reqwest::blocking::Client::new(); let path = format!("{}/v2/pox", http_origin); @@ -1008,7 +1044,7 @@ fn assert_l2_l1_tip_heights(sortition_db: &SortitionDB, l2_height: u64, l1_heigh #[ignore] fn transactions_in_block_and_microblock() { reset_static_burnblock_simulator_channel(); - let (mut conf, _miner_account) = mockstack_test_conf(); + let (mut conf, miner_account) = mockstack_test_conf(); conf.node.microblock_frequency = 100; let contract_sk = StacksPrivateKey::from_hex(SK_1).unwrap(); let sk_2 = StacksPrivateKey::from_hex(SK_2).unwrap(); @@ -1171,6 +1207,26 @@ fn transactions_in_block_and_microblock() { channel.stop_chains_coordinator(); } +/// Iterate over all the events in supplied blocks, passing the block height and +/// event JSON to a filter_mapper `test_fn`. +pub fn filter_map_events(blocks: &Vec, test_fn: F) -> Vec +where + F: Fn(u64, &serde_json::Value) -> Option, +{ + let mut result = vec![]; + for block in blocks { + let height = block.get("block_height").unwrap().as_u64().unwrap(); + let events = block.get("events").unwrap().as_array().unwrap(); + for ev in events.iter() { + if let Some(v) = test_fn(height, ev) { + result.push(v); + } + } + } + + return result; +} + /// Deserializes the `StacksTransaction` objects from `blocks` and returns all those that /// match `test_fn`. fn select_transactions_where(