kylefang/stacks-puppet-node
1
0
Fork 0
mirror of https://github.com/alexgo-io/stacks-puppet-node.git synced 2026-01-12 22:43:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3314 from stacks-network/feat/denorm-mempool

Browse Source 
Hotfix: denormalize mempool iteration, only deserialize considered txs
This commit is contained in:
Aaron Blankstein
2022-09-30 17:47:27 -05:00
committed by GitHub
parent 44e35d3158 cff3d1e47b
commit bd54bd758e
4 changed files with 100 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.github/actions/bitcoin-int-tests/Dockerfile.code-cov vendored
View File

@@ -4,13 +4,12 @@ WORKDIR /build
ENV CARGO_MANIFEST_DIR="$(pwd)"
RUN rustup override set nightly-2022-01-14 && \
rustup component add llvm-tools-preview && \
RUN rustup component add llvm-tools-preview && \
cargo install grcov
ENV RUSTFLAGS="-Zinstrument-coverage" \
ENV RUSTFLAGS="-Cinstrument-coverage" \
LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw"
COPY . .
RUN cargo build --workspace && \

5
.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests vendored
View File

@@ -6,11 +6,10 @@ COPY . .
WORKDIR /src/testnet/stacks-node
RUN rustup override set nightly-2022-01-14 && \
rustup component add llvm-tools-preview && \
RUN rustup component add llvm-tools-preview && \
cargo install grcov
ENV RUSTFLAGS="-Zinstrument-coverage" \
ENV RUSTFLAGS="-Cinstrument-coverage" \
LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw"
RUN cargo test --no-run && \

5
.github/actions/bitcoin-int-tests/Dockerfile.large-genesis vendored
View File

@@ -9,11 +9,10 @@ RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz
RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/
RUN rustup override set nightly-2022-01-14 && \
rustup component add llvm-tools-preview && \
RUN rustup component add llvm-tools-preview && \
cargo install grcov
ENV RUSTFLAGS="-Zinstrument-coverage" \
ENV RUSTFLAGS="-Cinstrument-coverage" \
LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw"
RUN cargo test --no-run --workspace && \

121
src/core/mempool.rs
View File

@@ -241,6 +241,12 @@ pub struct MemPoolTxInfo {
pub metadata: MemPoolTxMetadata,
}
#[derive(Debug, PartialEq, Clone)]
pub enum MemPoolTxInfoPartial {
NeedsNonces { addrs_needed: Vec<StacksAddress> },
HasNonces(MemPoolTxInfo),
}
#[derive(Debug, PartialEq, Clone)]
pub struct MemPoolTxMetadata {
pub txid: Txid,
@@ -258,6 +264,19 @@ pub struct MemPoolTxMetadata {
pub accept_time: u64,
}
impl MemPoolTxMetadata {
pub fn get_unknown_nonces(&self) -> Vec<StacksAddress> {
let mut needs_nonces = vec![];
if self.last_known_origin_nonce.is_none() {
needs_nonces.push(self.origin_address);
}
if self.last_known_sponsor_nonce.is_none() {
needs_nonces.push(self.sponsor_address);
}
needs_nonces
}
}
#[derive(Debug, Clone)]
pub struct MemPoolWalkSettings {
/// Minimum transaction fee that will be considered
@@ -346,6 +365,30 @@ impl FromRow<MemPoolTxInfo> for MemPoolTxInfo {
}
}
impl FromRow<MemPoolTxInfoPartial> for MemPoolTxInfoPartial {
fn from_row<'a>(row: &'a Row) -> Result<MemPoolTxInfoPartial, db_error> {
let md = MemPoolTxMetadata::from_row(row)?;
let needs_nonces = md.get_unknown_nonces();
let consider = if !needs_nonces.is_empty() {
MemPoolTxInfoPartial::NeedsNonces {
addrs_needed: needs_nonces,
}
} else {
let tx_bytes: Vec<u8> = row.get_unwrap("tx");
let tx = StacksTransaction::consensus_deserialize(&mut &tx_bytes[..])
.map_err(|_e| db_error::ParseError)?;
if tx.txid() != md.txid {
return Err(db_error::ParseError);
}
MemPoolTxInfoPartial::HasNonces(MemPoolTxInfo { tx, metadata: md })
};
Ok(consider)
}
}
impl FromRow<(u64, u64)> for (u64, u64) {
fn from_row<'a>(row: &'a Row) -> Result<(u64, u64), db_error> {
let t1: i64 = row.get_unwrap(0);
@@ -460,6 +503,22 @@ const MEMPOOL_SCHEMA_4_BLACKLIST: &'static [&'static str] = &[
"#,
];
const MEMPOOL_SCHEMA_5: &'static [&'static str] = &[
r#"
ALTER TABLE mempool ADD COLUMN fee_rate NUMBER;
"#,
r#"
CREATE INDEX IF NOT EXISTS by_fee_rate ON mempool(fee_rate);
"#,
r#"
UPDATE mempool
SET fee_rate = (SELECT f.fee_rate FROM fee_estimates as f WHERE f.txid = mempool.txid);
"#,
r#"
INSERT INTO schema_version (version) VALUES (5)
"#,
];
const MEMPOOL_INDEXES: &'static [&'static str] = &[
"CREATE INDEX IF NOT EXISTS by_txid ON mempool(txid);",
"CREATE INDEX IF NOT EXISTS by_height ON mempool(height);",
@@ -744,6 +803,9 @@ impl MemPoolDB {
MemPoolDB::instantiate_tx_blacklist(tx)?;
}
4 => {
MemPoolDB::denormalize_fee_rate(tx)?;
}
5 => {
break;
}
_ => {
@@ -788,6 +850,15 @@ impl MemPoolDB {
Ok(())
}
/// Denormalize fee rate schema 5
fn denormalize_fee_rate(tx: &DBTx) -> Result<(), db_error> {
for sql_exec in MEMPOOL_SCHEMA_5 {
tx.execute_batch(sql_exec)?;
}
Ok(())
}
/// Instantiate the tx blacklist schema
fn instantiate_tx_blacklist(tx: &DBTx) -> Result<(), db_error> {
for sql_exec in MEMPOOL_SCHEMA_4_BLACKLIST {
@@ -925,11 +996,11 @@ impl MemPoolDB {
/// whether or not the miner should propagate transaction receipts back to the estimator.
fn get_next_tx_to_consider_no_estimate(
&self,
) -> Result<Option<(MemPoolTxInfo, bool)>, db_error> {
let select_no_estimate = "SELECT * FROM mempool LEFT JOIN fee_estimates as f ON mempool.txid = f.txid WHERE
) -> Result<Option<(MemPoolTxInfoPartial, bool)>, db_error> {
let select_no_estimate = "SELECT * FROM mempool WHERE
((origin_nonce = last_known_origin_nonce AND
sponsor_nonce = last_known_sponsor_nonce) OR (last_known_origin_nonce is NULL) OR (last_known_sponsor_nonce is NULL))
AND f.fee_rate IS NULL ORDER BY tx_fee DESC LIMIT 1";
AND fee_rate IS NULL ORDER BY tx_fee DESC LIMIT 1";
query_row(&self.db, select_no_estimate, rusqlite::NO_PARAMS)
.map(|opt_tx| opt_tx.map(|tx| (tx, true)))
}
@@ -939,11 +1010,11 @@ impl MemPoolDB {
/// whether or not the miner should propagate transaction receipts back to the estimator.
fn get_next_tx_to_consider_with_estimate(
&self,
) -> Result<Option<(MemPoolTxInfo, bool)>, db_error> {
let select_estimate = "SELECT * FROM mempool LEFT OUTER JOIN fee_estimates as f ON mempool.txid = f.txid WHERE
) -> Result<Option<(MemPoolTxInfoPartial, bool)>, db_error> {
let select_estimate = "SELECT * FROM mempool WHERE
((origin_nonce = last_known_origin_nonce AND
sponsor_nonce = last_known_sponsor_nonce) OR (last_known_origin_nonce is NULL) OR (last_known_sponsor_nonce is NULL))
AND f.fee_rate IS NOT NULL ORDER BY f.fee_rate DESC LIMIT 1";
AND fee_rate IS NOT NULL ORDER BY fee_rate DESC LIMIT 1";
query_row(&self.db, select_estimate, rusqlite::NO_PARAMS)
.map(|opt_tx| opt_tx.map(|tx| (tx, false)))
}
@@ -956,7 +1027,7 @@ impl MemPoolDB {
&self,
start_with_no_estimate: bool,
) -> Result<ConsiderTransactionResult, db_error> {
let (next_tx, update_estimate): (MemPoolTxInfo, bool) = if start_with_no_estimate {
let (next_tx, update_estimate): (MemPoolTxInfoPartial, bool) = if start_with_no_estimate {
match self.get_next_tx_to_consider_no_estimate()? {
Some(result) => result,
None => match self.get_next_tx_to_consider_with_estimate()? {
@@ -974,21 +1045,16 @@ impl MemPoolDB {
}
};
let mut needs_nonces = vec![];
if next_tx.metadata.last_known_origin_nonce.is_none() {
needs_nonces.push(next_tx.metadata.origin_address);
}
if next_tx.metadata.last_known_sponsor_nonce.is_none() {
needs_nonces.push(next_tx.metadata.sponsor_address);
}
if !needs_nonces.is_empty() {
Ok(ConsiderTransactionResult::UpdateNonces(needs_nonces))
} else {
Ok(ConsiderTransactionResult::Consider(ConsiderTransaction {
tx: next_tx,
update_estimate,
}))
match next_tx {
MemPoolTxInfoPartial::NeedsNonces { addrs_needed } => {
Ok(ConsiderTransactionResult::UpdateNonces(addrs_needed))
}
MemPoolTxInfoPartial::HasNonces(tx) => {
Ok(ConsiderTransactionResult::Consider(ConsiderTransaction {
tx,
update_estimate,
}))
}
}
}
@@ -1027,8 +1093,7 @@ impl MemPoolDB {
let sql_tx = tx_begin_immediate(&mut self.db)?;
let txs: Vec<MemPoolTxInfo> = query_rows(
&sql_tx,
"SELECT * FROM mempool as m LEFT OUTER JOIN fee_estimates as f ON
m.txid = f.txid WHERE f.fee_rate IS NULL LIMIT ?",
"SELECT * FROM mempool as m WHERE m.fee_rate IS NULL LIMIT ?",
&[max_updates],
)?;
let mut updated = 0;
@@ -1053,8 +1118,8 @@ impl MemPoolDB {
};
sql_tx.execute(
"INSERT OR REPLACE INTO fee_estimates(txid, fee_rate) VALUES (?, ?)",
rusqlite::params![&txid, fee_rate_f64],
"UPDATE mempool SET fee_rate = ? WHERE txid = ?",
rusqlite::params![fee_rate_f64, &txid],
)?;
updated += 1;
}
@@ -1603,8 +1668,8 @@ impl MemPoolDB {
mempool_tx
.execute(
"INSERT OR REPLACE INTO fee_estimates(txid, fee_rate) VALUES (?, ?)",
rusqlite::params![&txid, fee_rate_estimate],
"UPDATE mempool SET fee_rate = ? WHERE txid = ?",
rusqlite::params![fee_rate_estimate, &txid],
)
.map_err(db_error::from)?;