kylefang/bitcoin-indexer
1
0
Fork 0
mirror of https://github.com/alexgo-io/bitcoin-indexer.git synced 2026-06-14 00:22:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: attempt transition to lazy model

Browse Source
This commit is contained in:
Ludo Galabru
2023-05-10 13:55:26 -04:00
parent 1468bbcff8
commit dda0b03ea3
8 changed files with 7327 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@@ -490,7 +490,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chainhook"
version = "0.12.0"
version = "0.13.0"
dependencies = [
"ansi_term",
"atty",

55
components/chainhook-cli/src/cli/mod.rs
View File

@@ -17,11 +17,13 @@ use chainhook_event_observer::hord::db::{
find_block_at_block_height, find_block_at_block_height_sqlite,
find_inscriptions_at_wached_outpoint, find_last_block_inserted,
find_watched_satpoint_for_inscription, initialize_hord_db, insert_entry_in_blocks,
open_readonly_hord_db_conn, open_readonly_hord_db_conn_rocks_db, open_readwrite_hord_db_conn,
open_readwrite_hord_db_conn_rocks_db, retrieve_satoshi_point_using_local_storage,
insert_entry_in_blocks_lazy_block, open_readonly_hord_db_conn,
open_readonly_hord_db_conn_rocks_db, open_readwrite_hord_db_conn,
open_readwrite_hord_db_conn_rocks_db, retrieve_satoshi_point_using_lazy_storage,
CompactedBlock, LazyBlock,
};
use chainhook_event_observer::hord::{
new_traversals_cache, retrieve_inscribed_satoshi_points_from_block,
new_traversals_cache, new_traversals_lazy_cache, retrieve_inscribed_satoshi_points_from_block,
update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, Storage,
};
use chainhook_event_observer::indexer;
@@ -211,15 +213,18 @@ enum DbCommand {
/// Rebuild inscriptions entries for a given block
#[clap(name = "drop", bin_name = "drop")]
Drop(DropHordDbCommand),
/// Patch DB
#[clap(name = "patch", bin_name = "patch")]
Patch(PatchHordDbCommand),
/// Check integrity
#[clap(name = "check", bin_name = "check")]
Check(CheckHordDbCommand),
/// Legacy command
#[clap(name = "init", bin_name = "init")]
Init(InitHordDbCommand),
/// Patch DB
#[clap(name = "patch", bin_name = "patch")]
Patch(PatchHordDbCommand),
/// Migrate
#[clap(name = "migrate", bin_name = "migrate")]
Migrate(MigrateHordDbCommand),
}
#[derive(Subcommand, PartialEq, Clone, Debug)]
@@ -346,6 +351,13 @@ struct PatchHordDbCommand {
pub config_path: Option<String>,
}
#[derive(Parser, PartialEq, Clone, Debug)]
struct MigrateHordDbCommand {
/// Load config file path
#[clap(long = "config-path")]
pub config_path: Option<String>,
}
#[derive(Parser, PartialEq, Clone, Debug)]
struct CheckHordDbCommand {
/// Load config file path
@@ -631,8 +643,8 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
};
let transaction_identifier = TransactionIdentifier { hash: txid.clone() };
let traversals_cache = new_traversals_cache();
let traversal = retrieve_satoshi_point_using_local_storage(
let traversals_cache = new_traversals_lazy_cache();
let traversal = retrieve_satoshi_point_using_lazy_storage(
&hord_db_conn,
&block_identifier,
&transaction_identifier,
@@ -652,7 +664,7 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
let block =
fetch_and_standardize_block(cmd.block_height, &bitcoin_config, &ctx)
.await?;
let traversals_cache = Arc::new(new_traversals_cache());
let traversals_cache = Arc::new(new_traversals_lazy_cache());
let _traversals = retrieve_inscribed_satoshi_points_from_block(
&block,
@@ -919,6 +931,31 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
}
}
}
DbCommand::Migrate(cmd) => {
let config = Config::default(false, false, false, &cmd.config_path)?;
let blocks_db_rw =
open_readwrite_hord_db_conn_rocks_db(&config.expected_cache_path(), &ctx)?;
let tip = find_last_block_inserted(&blocks_db_rw);
for i in 0..=tip {
match find_block_at_block_height(i, 5, &blocks_db_rw) {
Some(block) => {
let mut bytes = vec![];
block
.serialize_to_lazy_format(&mut bytes)
.expect("unable to convert to lazy block");
let lazy_block = LazyBlock::new(bytes);
insert_entry_in_blocks_lazy_block(i, &lazy_block, &blocks_db_rw, &ctx);
println!("Block #{} migrated to lazy block", i);
}
None => {
println!("Block #{} missing", i)
}
}
}
}
},
}
Ok(())

4
components/chainhook-event-observer/src/hord/db/mod.rs
View File

@@ -27,7 +27,7 @@ use crate::{
};
use super::{
new_traversals_cache,
new_traversals_cache, new_traversals_lazy_cache,
ord::{height::Height, sat::Sat},
update_hord_db_and_augment_bitcoin_block,
};
@@ -1009,7 +1009,7 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
let mut cursor = start_block as usize;
let mut inbox = HashMap::new();
let mut num_writes = 0;
let traversals_cache = Arc::new(new_traversals_cache());
let traversals_cache = Arc::new(new_traversals_lazy_cache());
while let Ok(Some((block_height, compacted_block, raw_block))) = block_compressed_rx.recv() {
insert_entry_in_blocks(block_height, &compacted_block, &blocks_db_rw, &ctx);

24
components/chainhook-event-observer/src/hord/mod.rs
View File

@@ -27,7 +27,7 @@ use crate::{
hord::{
db::{
find_inscription_with_ordinal_number, find_inscriptions_at_wached_outpoint,
insert_entry_in_blocks, retrieve_satoshi_point_using_local_storage,
insert_entry_in_blocks, retrieve_satoshi_point_using_lazy_storage,
store_new_inscription, update_transfered_inscription, CompactedBlock,
},
ord::height::Height,
@@ -38,7 +38,7 @@ use crate::{
use self::db::{
find_inscription_with_id, find_latest_inscription_number_at_block_height,
open_readonly_hord_db_conn_rocks_db, remove_entry_from_blocks, remove_entry_from_inscriptions,
TraversalResult, WatchedSatpoint,
LazyBlockTransaction, TraversalResult, WatchedSatpoint,
};
use self::inscription::InscriptionParser;
use self::ord::inscription_id::InscriptionId;
@@ -171,16 +171,18 @@ pub fn new_traversals_cache(
DashMap::with_hasher(hasher)
}
pub fn new_traversals_lazy_cache(
) -> DashMap<(u32, [u8; 8]), LazyBlockTransaction, BuildHasherDefault<FxHasher>> {
let hasher = FxBuildHasher::default();
DashMap::with_hasher(hasher)
}
pub fn retrieve_inscribed_satoshi_points_from_block(
block: &BitcoinBlockData,
inscriptions_db_conn: Option<&Connection>,
hord_db_path: &PathBuf,
traversals_cache: &Arc<
DashMap<
(u32, [u8; 8]),
(Vec<([u8; 8], u32, u16, u64)>, Vec<u64>),
BuildHasherDefault<FxHasher>,
>,
DashMap<(u32, [u8; 8]), LazyBlockTransaction, BuildHasherDefault<FxHasher>>,
>,
ctx: &Context,
) -> HashMap<TransactionIdentifier, TraversalResult> {
@@ -227,7 +229,7 @@ pub fn retrieve_inscribed_satoshi_points_from_block(
traversal_data_pool.execute(move || loop {
match open_readonly_hord_db_conn_rocks_db(&moved_hord_db_path, &moved_ctx) {
Ok(blocks_db) => {
let traversal = retrieve_satoshi_point_using_local_storage(
let traversal = retrieve_satoshi_point_using_lazy_storage(
&blocks_db,
&block_identifier,
&transaction_id,
@@ -293,11 +295,7 @@ pub fn update_hord_db_and_augment_bitcoin_block(
write_block: bool,
hord_db_path: &PathBuf,
traversals_cache: &Arc<
DashMap<
(u32, [u8; 8]),
(Vec<([u8; 8], u32, u16, u64)>, Vec<u64>),
BuildHasherDefault<FxHasher>,
>,
DashMap<(u32, [u8; 8]), LazyBlockTransaction, BuildHasherDefault<FxHasher>>,
>,
ctx: &Context,
) -> Result<(), String> {

18
tests/Chainhook.toml Normal file
View File

@@ -0,0 +1,18 @@
[storage]
driver = "redis"
redis_uri = "redis://localhost:6379/"
cache_path = "cache"
[chainhooks]
max_stacks_registrations = 500
max_bitcoin_registrations = 500
[network]
mode = "mainnet"
bitcoind_rpc_url = "http://localhost:8332"
bitcoind_rpc_username = "devnet"
bitcoind_rpc_password = "devnet"
stacks_node_rpc_url = "http://localhost:20443"
[[event_source]]
tsv_file_url = "https://archive.hiro.so/mainnet/stacks-blockchain-api/mainnet-stacks-blockchain-api-latest.gz"

4
tests/inscription_feed.json
View File

@@ -1,8 +1,8 @@
{
"uuid": "1",
"name": "Unwrap BTC",
"version": 1,
"name": "Hello Ordinals",
"chain": "bitcoin",
"version": 1,
"networks": {
"mainnet": {
"start_block": 777534,

7226
tests/mainnet-stacking-club.txt Normal file
View File

File diff suppressed because one or more lines are too long

21
tests/stacking-club.json Normal file
View File

@@ -0,0 +1,21 @@
{
"uuid": "1",
"name": "Unwrap BTC",
"chain": "stacks",
"version": 1,
"networks": {
"mainnet": {
"start_block": 1,
"end_block": 103672,
"if_this": {
"scope": "stx_event",
"actions": ["lock"]
},
"then_that": {
"file_append": {
"path": "mainnet-stacking-club.txt"
}
}
}
}
}