From f74816fb1e86e21a7d06b351e275fbc1d1b222ce Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 30 Sep 2022 14:08:02 -0500 Subject: [PATCH 1/3] feat: denormalize mempool/fee_estimates table to remove join, only deser txs actually considered --- src/core/mempool.rs | 118 +++++++++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 55588a713..27b847032 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -241,6 +241,12 @@ pub struct MemPoolTxInfo { pub metadata: MemPoolTxMetadata, } +#[derive(Debug, PartialEq, Clone)] +pub enum MemPoolTxInfoPartial { + NeedsNonces { addrs_needed: Vec }, + HasNonces(MemPoolTxInfo), +} + #[derive(Debug, PartialEq, Clone)] pub struct MemPoolTxMetadata { pub txid: Txid, @@ -258,6 +264,19 @@ pub struct MemPoolTxMetadata { pub accept_time: u64, } +impl MemPoolTxMetadata { + pub fn get_unknown_nonces(&self) -> Vec { + let mut needs_nonces = vec![]; + if self.last_known_origin_nonce.is_none() { + needs_nonces.push(self.origin_address); + } + if self.last_known_sponsor_nonce.is_none() { + needs_nonces.push(self.sponsor_address); + } + needs_nonces + } +} + #[derive(Debug, Clone)] pub struct MemPoolWalkSettings { /// Minimum transaction fee that will be considered @@ -346,6 +365,30 @@ impl FromRow for MemPoolTxInfo { } } +impl FromRow for MemPoolTxInfoPartial { + fn from_row<'a>(row: &'a Row) -> Result { + let md = MemPoolTxMetadata::from_row(row)?; + let needs_nonces = md.get_unknown_nonces(); + let consider = if !needs_nonces.is_empty() { + MemPoolTxInfoPartial::NeedsNonces { + addrs_needed: needs_nonces, + } + } else { + let tx_bytes: Vec = row.get_unwrap("tx"); + let tx = StacksTransaction::consensus_deserialize(&mut &tx_bytes[..]) + .map_err(|_e| db_error::ParseError)?; + + if tx.txid() != md.txid { + return Err(db_error::ParseError); + } + + MemPoolTxInfoPartial::HasNonces(MemPoolTxInfo { tx, metadata: md }) + }; + + Ok(consider) + } +} + impl FromRow<(u64, u64)> for (u64, u64) { fn from_row<'a>(row: &'a Row) -> Result<(u64, u64), db_error> { let t1: i64 = row.get_unwrap(0); @@ -377,6 +420,19 @@ const MEMPOOL_INITIAL_SCHEMA: &'static [&'static str] = &[r#" ); "#]; +const MEMPOOL_SCHEMA_5: &'static [&'static str] = &[ + r#" + ALTER TABLE mempool ADD COLUMN fee_rate NUMBER; + "#, + r#" + CREATE INDEX IF NOT EXISTS by_fee_rate ON mempool(fee_rate); + "#, + r#" + UPDATE mempool + SET fee_rate = (SELECT f.fee_rate FROM fee_estimates as f WHERE f.txid = mempool.txid); + "#, +]; + const MEMPOOL_SCHEMA_2_COST_ESTIMATOR: &'static [&'static str] = &[ r#" CREATE TABLE fee_estimates( @@ -744,6 +800,9 @@ impl MemPoolDB { MemPoolDB::instantiate_tx_blacklist(tx)?; } 4 => { + MemPoolDB::denormalize_fee_rate(tx)?; + } + 5 => { break; } _ => { @@ -788,6 +847,15 @@ impl MemPoolDB { Ok(()) } + /// Denormalize fee rate schema 5 + fn denormalize_fee_rate(tx: &DBTx) -> Result<(), db_error> { + for sql_exec in MEMPOOL_SCHEMA_5 { + tx.execute_batch(sql_exec)?; + } + + Ok(()) + } + /// Instantiate the tx blacklist schema fn instantiate_tx_blacklist(tx: &DBTx) -> Result<(), db_error> { for sql_exec in MEMPOOL_SCHEMA_4_BLACKLIST { @@ -925,11 +993,11 @@ impl MemPoolDB { /// whether or not the miner should propagate transaction receipts back to the estimator. fn get_next_tx_to_consider_no_estimate( &self, - ) -> Result, db_error> { - let select_no_estimate = "SELECT * FROM mempool LEFT JOIN fee_estimates as f ON mempool.txid = f.txid WHERE + ) -> Result, db_error> { + let select_no_estimate = "SELECT * FROM mempool WHERE ((origin_nonce = last_known_origin_nonce AND sponsor_nonce = last_known_sponsor_nonce) OR (last_known_origin_nonce is NULL) OR (last_known_sponsor_nonce is NULL)) - AND f.fee_rate IS NULL ORDER BY tx_fee DESC LIMIT 1"; + AND fee_rate IS NULL ORDER BY tx_fee DESC LIMIT 1"; query_row(&self.db, select_no_estimate, rusqlite::NO_PARAMS) .map(|opt_tx| opt_tx.map(|tx| (tx, true))) } @@ -939,11 +1007,11 @@ impl MemPoolDB { /// whether or not the miner should propagate transaction receipts back to the estimator. fn get_next_tx_to_consider_with_estimate( &self, - ) -> Result, db_error> { - let select_estimate = "SELECT * FROM mempool LEFT OUTER JOIN fee_estimates as f ON mempool.txid = f.txid WHERE + ) -> Result, db_error> { + let select_estimate = "SELECT * FROM mempool WHERE ((origin_nonce = last_known_origin_nonce AND sponsor_nonce = last_known_sponsor_nonce) OR (last_known_origin_nonce is NULL) OR (last_known_sponsor_nonce is NULL)) - AND f.fee_rate IS NOT NULL ORDER BY f.fee_rate DESC LIMIT 1"; + AND fee_rate IS NOT NULL ORDER BY fee_rate DESC LIMIT 1"; query_row(&self.db, select_estimate, rusqlite::NO_PARAMS) .map(|opt_tx| opt_tx.map(|tx| (tx, false))) } @@ -956,7 +1024,7 @@ impl MemPoolDB { &self, start_with_no_estimate: bool, ) -> Result { - let (next_tx, update_estimate): (MemPoolTxInfo, bool) = if start_with_no_estimate { + let (next_tx, update_estimate): (MemPoolTxInfoPartial, bool) = if start_with_no_estimate { match self.get_next_tx_to_consider_no_estimate()? { Some(result) => result, None => match self.get_next_tx_to_consider_with_estimate()? { @@ -974,21 +1042,16 @@ impl MemPoolDB { } }; - let mut needs_nonces = vec![]; - if next_tx.metadata.last_known_origin_nonce.is_none() { - needs_nonces.push(next_tx.metadata.origin_address); - } - if next_tx.metadata.last_known_sponsor_nonce.is_none() { - needs_nonces.push(next_tx.metadata.sponsor_address); - } - - if !needs_nonces.is_empty() { - Ok(ConsiderTransactionResult::UpdateNonces(needs_nonces)) - } else { - Ok(ConsiderTransactionResult::Consider(ConsiderTransaction { - tx: next_tx, - update_estimate, - })) + match next_tx { + MemPoolTxInfoPartial::NeedsNonces { addrs_needed } => { + Ok(ConsiderTransactionResult::UpdateNonces(addrs_needed)) + } + MemPoolTxInfoPartial::HasNonces(tx) => { + Ok(ConsiderTransactionResult::Consider(ConsiderTransaction { + tx, + update_estimate, + })) + } } } @@ -1027,8 +1090,7 @@ impl MemPoolDB { let sql_tx = tx_begin_immediate(&mut self.db)?; let txs: Vec = query_rows( &sql_tx, - "SELECT * FROM mempool as m LEFT OUTER JOIN fee_estimates as f ON - m.txid = f.txid WHERE f.fee_rate IS NULL LIMIT ?", + "SELECT * FROM mempool as m WHERE m.fee_rate IS NULL LIMIT ?", &[max_updates], )?; let mut updated = 0; @@ -1053,8 +1115,8 @@ impl MemPoolDB { }; sql_tx.execute( - "INSERT OR REPLACE INTO fee_estimates(txid, fee_rate) VALUES (?, ?)", - rusqlite::params![&txid, fee_rate_f64], + "UPDATE mempool SET fee_rate = ? WHERE txid = ?", + rusqlite::params![fee_rate_f64, &txid], )?; updated += 1; } @@ -1603,8 +1665,8 @@ impl MemPoolDB { mempool_tx .execute( - "INSERT OR REPLACE INTO fee_estimates(txid, fee_rate) VALUES (?, ?)", - rusqlite::params![&txid, fee_rate_estimate], + "UPDATE mempool SET fee_rate = ? WHERE txid = ?", + rusqlite::params![fee_rate_estimate, &txid], ) .map_err(db_error::from)?; From 3b4ca5f3cf1fe61d124243ad20db4d6625c48666 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 30 Sep 2022 14:26:53 -0500 Subject: [PATCH 2/3] set mempool schema version to 5 --- src/core/mempool.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 27b847032..db9b95f32 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -420,19 +420,6 @@ const MEMPOOL_INITIAL_SCHEMA: &'static [&'static str] = &[r#" ); "#]; -const MEMPOOL_SCHEMA_5: &'static [&'static str] = &[ - r#" - ALTER TABLE mempool ADD COLUMN fee_rate NUMBER; - "#, - r#" - CREATE INDEX IF NOT EXISTS by_fee_rate ON mempool(fee_rate); - "#, - r#" - UPDATE mempool - SET fee_rate = (SELECT f.fee_rate FROM fee_estimates as f WHERE f.txid = mempool.txid); - "#, -]; - const MEMPOOL_SCHEMA_2_COST_ESTIMATOR: &'static [&'static str] = &[ r#" CREATE TABLE fee_estimates( @@ -516,6 +503,22 @@ const MEMPOOL_SCHEMA_4_BLACKLIST: &'static [&'static str] = &[ "#, ]; +const MEMPOOL_SCHEMA_5: &'static [&'static str] = &[ + r#" + ALTER TABLE mempool ADD COLUMN fee_rate NUMBER; + "#, + r#" + CREATE INDEX IF NOT EXISTS by_fee_rate ON mempool(fee_rate); + "#, + r#" + UPDATE mempool + SET fee_rate = (SELECT f.fee_rate FROM fee_estimates as f WHERE f.txid = mempool.txid); + "#, + r#" + INSERT INTO schema_version (version) VALUES (5) + "#, +]; + const MEMPOOL_INDEXES: &'static [&'static str] = &[ "CREATE INDEX IF NOT EXISTS by_txid ON mempool(txid);", "CREATE INDEX IF NOT EXISTS by_height ON mempool(height);", From cff3d1e47b245d3e01dba9a9f279450e2887d984 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 12 Sep 2022 10:56:59 -0500 Subject: [PATCH 3/3] ci: use rust stable for code coverage tests --- .github/actions/bitcoin-int-tests/Dockerfile.code-cov | 7 +++---- .../bitcoin-int-tests/Dockerfile.generic.bitcoin-tests | 5 ++--- .github/actions/bitcoin-int-tests/Dockerfile.large-genesis | 5 ++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov index 733f879b7..209b80473 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov +++ b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov @@ -4,13 +4,12 @@ WORKDIR /build ENV CARGO_MANIFEST_DIR="$(pwd)" -RUN rustup override set nightly-2022-01-14 && \ - rustup component add llvm-tools-preview && \ +RUN rustup component add llvm-tools-preview && \ cargo install grcov -ENV RUSTFLAGS="-Zinstrument-coverage" \ +ENV RUSTFLAGS="-Cinstrument-coverage" \ LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" - + COPY . . RUN cargo build --workspace && \ diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests b/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests index 42a0235cf..2fd43a589 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests +++ b/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests @@ -6,11 +6,10 @@ COPY . . WORKDIR /src/testnet/stacks-node -RUN rustup override set nightly-2022-01-14 && \ - rustup component add llvm-tools-preview && \ +RUN rustup component add llvm-tools-preview && \ cargo install grcov -ENV RUSTFLAGS="-Zinstrument-coverage" \ +ENV RUSTFLAGS="-Cinstrument-coverage" \ LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" RUN cargo test --no-run && \ diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis b/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis index 4f96fd304..1350a6ed8 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis +++ b/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis @@ -9,11 +9,10 @@ RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/ -RUN rustup override set nightly-2022-01-14 && \ - rustup component add llvm-tools-preview && \ +RUN rustup component add llvm-tools-preview && \ cargo install grcov -ENV RUSTFLAGS="-Zinstrument-coverage" \ +ENV RUSTFLAGS="-Cinstrument-coverage" \ LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" RUN cargo test --no-run --workspace && \