kylefang/bitcoin-indexer
1
0
Fork 0
mirror of https://github.com/alexgo-io/bitcoin-indexer.git synced 2026-06-13 16:19:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: off-by-one in sats number resolution

Browse Source
This commit is contained in:
Ludo Galabru
2023-03-27 18:19:18 -04:00
parent d0714842a2
commit 42acbebcd5
4 changed files with 134 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
components/chainhook-cli/src/cli/mod.rs
View File

@@ -13,8 +13,9 @@ use chainhook_event_observer::chainhooks::types::{
StacksPrintEventBasedPredicate,
};
use chainhook_event_observer::hord::db::{
fetch_and_cache_blocks_in_hord_db, find_inscriptions_at_wached_outpoint,
find_latest_compacted_block_known, open_readonly_hord_db_conn, open_readwrite_hord_db_conn,
delete_data_in_hord_db, fetch_and_cache_blocks_in_hord_db,
find_inscriptions_at_wached_outpoint, find_latest_compacted_block_known,
open_readonly_hord_db_conn, open_readwrite_hord_db_conn,
retrieve_satoshi_point_using_local_storage,
};
use chainhook_event_observer::observer::BitcoinConfig;
@@ -185,6 +186,9 @@ enum DbCommand {
/// Update hord db
#[clap(name = "update", bin_name = "update")]
Update(UpdateHordDbCommand),
/// Rebuild inscriptions entries for a given block
#[clap(name = "drop", bin_name = "drop")]
Drop(DropHordDbCommand),
}
#[derive(Subcommand, PartialEq, Clone, Debug)]
@@ -265,6 +269,17 @@ struct UpdateHordDbCommand {
pub config_path: Option<String>,
}
#[derive(Parser, PartialEq, Clone, Debug)]
struct DropHordDbCommand {
/// Starting block
pub start_block: u64,
/// Starting block
pub end_block: u64,
/// Load config file path
#[clap(long = "config-path")]
pub config_path: Option<String>,
}
pub fn main() {
let logger = hiro_system_kit::log::setup_logger();
let _guard = hiro_system_kit::log::setup_global_logger(logger.clone());
@@ -454,11 +469,6 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
let config =
Config::default(cmd.devnet, cmd.testnet, cmd.mainnet, &cmd.config_path)?;
info!(
ctx.expect_logger(),
"Computing satoshi number for satoshi at offet 0 in 1st output of transaction {} (block ${})", cmd.txid, cmd.block_height
);
let hord_db_conn =
open_readonly_hord_db_conn(&config.expected_cache_path(), &ctx).unwrap();
@@ -536,6 +546,17 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
)
.await?;
}
DbCommand::Drop(cmd) => {
let config = Config::default(false, false, false, &cmd.config_path)?;
let rw_hord_db_conn =
open_readwrite_hord_db_conn(&config.expected_cache_path(), &ctx)?;
delete_data_in_hord_db(cmd.start_block, cmd.end_block, &rw_hord_db_conn, &ctx)?;
info!(
ctx.expect_logger(),
"Cleaning hord_db: {} blocks dropped",
cmd.end_block - cmd.start_block + 1
);
}
},
}
Ok(())
@@ -550,7 +571,7 @@ pub async fn perform_hord_db_update(
) -> Result<(), String> {
info!(
ctx.expect_logger(),
"Syncing hord_db: {} blocks to download ({start_block}: {end_block}), using {network_threads} network threads", end_block - start_block
"Syncing hord_db: {} blocks to download ({start_block}: {end_block}), using {network_threads} network threads", end_block - start_block + 1
);
let bitcoin_config = BitcoinConfig {

88
components/chainhook-event-observer/src/hord/db/mod.rs
View File

@@ -305,7 +305,7 @@ pub fn update_transfered_inscription(
pub fn find_latest_inscription_block_height(
hord_db_conn: &Connection,
_ctx: &Context,
) -> Result<u64, String> {
) -> Result<Option<u64>, String> {
let args: &[&dyn ToSql] = &[];
let mut stmt = hord_db_conn
.prepare("SELECT block_height FROM inscriptions ORDER BY block_height DESC LIMIT 1")
@@ -313,9 +313,9 @@ pub fn find_latest_inscription_block_height(
let mut rows = stmt.query(args).unwrap();
while let Ok(Some(row)) = rows.next() {
let block_height: u64 = row.get(0).unwrap();
return Ok(block_height);
return Ok(Some(block_height));
}
Ok(0)
Ok(None)
}
pub fn find_latest_inscription_number(
@@ -418,6 +418,34 @@ pub fn remove_entry_from_blocks(block_id: u32, hord_db_conn: &Connection, ctx: &
}
}
pub fn delete_blocks_in_block_range(
start_block: u32,
end_block: u32,
rw_hord_db_conn: &Connection,
ctx: &Context,
) {
if let Err(e) = rw_hord_db_conn.execute(
"DELETE FROM blocks WHERE id >= ?1 AND id <= ?2",
rusqlite::params![&start_block, &end_block],
) {
ctx.try_log(|logger| slog::error!(logger, "{}", e.to_string()));
}
}
pub fn delete_inscriptions_in_block_range(
start_block: u32,
end_block: u32,
rw_hord_db_conn: &Connection,
ctx: &Context,
) {
if let Err(e) = rw_hord_db_conn.execute(
"DELETE FROM inscriptions WHERE block_height >= ?1 AND block_height <= ?2",
rusqlite::params![&start_block, &end_block],
) {
ctx.try_log(|logger| slog::error!(logger, "{}", e.to_string()));
}
}
pub fn remove_entry_from_inscriptions(
inscription_id: &str,
hord_db_conn: &Connection,
@@ -513,6 +541,17 @@ pub fn remove_entry_from_inscriptions(
// Ok(())
// }
pub fn delete_data_in_hord_db(
start_block: u64,
end_block: u64,
rw_hord_db_conn: &Connection,
ctx: &Context,
) -> Result<(), String> {
delete_blocks_in_block_range(start_block as u32, end_block as u32, rw_hord_db_conn, &ctx);
delete_inscriptions_in_block_range(start_block as u32, end_block as u32, rw_hord_db_conn, &ctx);
Ok(())
}
pub async fn fetch_and_cache_blocks_in_hord_db(
bitcoin_config: &BitcoinConfig,
rw_hord_db_conn: &Connection,
@@ -521,6 +560,7 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
ctx: &Context,
network_thread: usize,
) -> Result<(), String> {
let number_of_blocks_to_process = end_block - start_block + 1;
let retrieve_block_hash_pool = ThreadPool::new(network_thread);
let (block_hash_tx, block_hash_rx) = crossbeam_channel::unbounded();
let retrieve_block_data_pool = ThreadPool::new(network_thread);
@@ -589,7 +629,7 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
.expect("unable to spawn thread");
let mut blocks_stored = 0;
let mut cursor = find_latest_inscription_block_height(&rw_hord_db_conn, &ctx)
let mut cursor = 1 + find_latest_inscription_block_height(&rw_hord_db_conn, &ctx)?
.unwrap_or(first_inscription_block_height) as usize;
let mut inbox = HashMap::new();
@@ -597,21 +637,24 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
ctx.try_log(|logger| slog::info!(logger, "Storing compacted block #{block_height}"));
insert_entry_in_blocks(block_height, &compacted_block, &rw_hord_db_conn, &ctx);
blocks_stored += 1;
// println!("{} < {}", raw_block.height, cursor);
// Early return, only considering blocks after 1st inscription
if raw_block.height < cursor {
continue;
}
let block_height = raw_block.height;
// if raw_block.height < cursor {
// continue;
// }
// let block_height = raw_block.height;
inbox.insert(raw_block.height, raw_block);
// In the context of ordinals, we're constrained to process blocks sequentially
// Blocks are processed by a threadpool and could be coming out of order.
// Inbox block for later if the current block is not the one we should be
// processing.
if block_height != cursor {
continue;
}
// if block_height != cursor {
// continue;
// }
// Is the action of processing a block allows us
// to process more blocks present in the inbox?
@@ -627,9 +670,12 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
}
};
if let Err(e) =
update_hord_db_and_augment_bitcoin_block(&mut new_block, &rw_hord_db_conn, &ctx)
{
if let Err(e) = update_hord_db_and_augment_bitcoin_block(
&mut new_block,
&rw_hord_db_conn,
&ctx,
false,
) {
ctx.try_log(|logger| {
slog::error!(logger, "Unable to augment bitcoin block with hord_db: {e}",)
});
@@ -638,8 +684,7 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
cursor += 1;
}
blocks_stored += 1;
if blocks_stored == end_block - start_block {
if blocks_stored == number_of_blocks_to_process {
let _ = block_data_tx.send(None);
let _ = block_hash_tx.send(None);
ctx.try_log(|logger| {
@@ -663,6 +708,15 @@ pub fn retrieve_satoshi_point_using_local_storage(
transaction_identifier: &TransactionIdentifier,
ctx: &Context,
) -> Result<(u64, u64, u64, u32), String> {
ctx.try_log(|logger| {
slog::info!(
logger,
"Computing Satoshi # for sat_point {}:0:0 (block #{})",
transaction_identifier.hash,
block_identifier.index
)
});
let mut ordinal_offset = 0;
let mut ordinal_block_number = block_identifier.index as u32;
let txid = {
@@ -676,7 +730,7 @@ pub fn retrieve_satoshi_point_using_local_storage(
let res = match find_compacted_block_at_block_height(ordinal_block_number, &hord_db_conn) {
Some(res) => res,
None => {
return Err(format!("unable to retrieve block ##{ordinal_block_number}"));
return Err(format!("unable to retrieve block #{ordinal_block_number}"));
}
};

57
components/chainhook-event-observer/src/hord/mod.rs
View File

@@ -65,8 +65,9 @@ pub fn update_hord_db_and_augment_bitcoin_block(
new_block: &mut BitcoinBlockData,
rw_hord_db_conn: &Connection,
ctx: &Context,
write_block: bool,
) -> Result<(), String> {
{
if write_block {
ctx.try_log(|logger| {
slog::info!(
logger,
@@ -156,9 +157,10 @@ pub fn update_hord_db_and_augment_bitcoin_block(
ctx.try_log(|logger| {
slog::info!(
logger,
"Transaction {} in block {} includes a new inscription {}",
"Transaction {} in block {} inscribed some content ({}) on Satoshi #{}",
new_tx.transaction_identifier.hash,
new_block.block_identifier.index,
inscription.content_type,
ordinal_number
);
});
@@ -177,7 +179,7 @@ pub fn update_hord_db_and_augment_bitcoin_block(
// Have inscriptions been transfered?
let mut sats_in_offset = 0;
let mut sats_out_offset = 0;
let mut sats_out_offset = new_tx.metadata.outputs[0].value;
for input in new_tx.metadata.inputs.iter() {
// input.previous_output.txid
@@ -192,27 +194,18 @@ pub fn update_hord_db_and_augment_bitcoin_block(
let entries =
find_inscriptions_at_wached_outpoint(&outpoint_pre_transfer, &rw_hord_db_conn);
ctx.try_log(|logger| {
slog::info!(
logger,
"Checking if {} is part of our watch outpoints set: {}",
outpoint_pre_transfer,
entries.len(),
)
});
// ctx.try_log(|logger| {
// slog::info!(
// logger,
// "Checking if {} is part of our watch outpoints set: {}",
// outpoint_pre_transfer,
// entries.len(),
// )
// });
for (inscription_id, inscription_number, ordinal_number, offset) in entries.into_iter()
{
let satpoint_pre_transfer = format!("{}:{}", outpoint_pre_transfer, offset);
// At this point we know that inscriptions are being moved.
ctx.try_log(|logger| {
slog::info!(
logger,
"Detected transaction {} involving txin {} that includes watched ordinals",
new_tx.transaction_identifier.hash,
satpoint_pre_transfer,
)
});
// Question is: are inscriptions moving to a new output,
// burnt or lost in fees and transfered to the miner?
@@ -220,11 +213,13 @@ pub fn update_hord_db_and_augment_bitcoin_block(
if sats_out_offset >= sats_in_offset + offset {
break Some(post_transfer_output_index);
}
if post_transfer_output_index >= new_tx.metadata.outputs.len() {
if post_transfer_output_index + 1 >= new_tx.metadata.outputs.len() {
break None;
} else {
post_transfer_output_index += 1;
sats_out_offset +=
new_tx.metadata.outputs[post_transfer_output_index].value;
}
sats_out_offset += new_tx.metadata.outputs[post_transfer_output_index].value;
post_transfer_output_index += 1;
};
let (outpoint_post_transfer, offset_post_transfer, updated_address) =
@@ -271,12 +266,24 @@ pub fn update_hord_db_and_augment_bitcoin_block(
}
};
// ctx.try_log(|logger| {
// slog::info!(
// logger,
// "Updating watched outpoint {} to outpoint {}",
// outpoint_post_transfer,
// outpoint_pre_transfer,
// )
// });
// At this point we know that inscriptions are being moved.
ctx.try_log(|logger| {
slog::info!(
logger,
"Updating watched outpoint {} to outpoint {}",
"Transaction {} in block {} moved inscribed Satoshi #{} from {} to {}",
new_tx.transaction_identifier.hash,
new_block.block_identifier.index,
ordinal_number,
satpoint_pre_transfer,
outpoint_post_transfer,
outpoint_pre_transfer,
)
});

2
components/chainhook-event-observer/src/observer/mod.rs
View File

@@ -554,6 +554,7 @@ pub async fn start_observer_commands_handler(
block,
&rw_hord_db_conn,
&ctx,
true,
) {
ctx.try_log(|logger| {
slog::error!(
@@ -663,6 +664,7 @@ pub async fn start_observer_commands_handler(
block,
&rw_hord_db_conn,
&ctx,
true,
) {
ctx.try_log(|logger| {
slog::error!(