kylefang/bitcoin-indexer
1
0
Fork 0
mirror of https://github.com/alexgo-io/bitcoin-indexer.git synced 2026-06-14 00:22:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: build failing

Browse Source
This commit is contained in:
Ludo Galabru
2023-07-05 15:29:37 -04:00
parent 6ebeb77d6a
commit 83f14964a6
10 changed files with 168 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@@ -445,13 +445,11 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
name = "chainhook-sdk"
version = "0.5.0"
dependencies = [
"anyhow",
"base58 0.2.0",
"base64",
"bitcoincore-rpc",
"bitcoincore-rpc-json",
"chainhook-types 1.0.7",
"chrono",
"clarinet-files",
"clarinet-utils",
"crossbeam-channel 0.5.8",
@@ -464,8 +462,6 @@ dependencies = [
"rand 0.8.5",
"reqwest",
"rocket",
"rocksdb",
"rusqlite",
"schemars",
"serde",
"serde-hex",

2
components/hord-cli/Cargo.toml
View File

@@ -14,7 +14,7 @@ redis = "0.21.5"
serde-redis = "0.12.0"
hex = "0.4.3"
rand = "0.8.5"
chainhook-sdk = { version = "0.5.0", default-features = false, features = ["ordinals", "zeromq"], path = "../../../chainhook/components/chainhook-sdk" }
chainhook-sdk = { version = "0.5.0", default-features = false, features = ["zeromq"], path = "../../../chainhook/components/chainhook-sdk" }
chainhook-types = { version = "1.0.6", path = "../../../chainhook/components/chainhook-types-rs" }
clarinet-files = "1.0.1"
hiro-system-kit = "0.1.0"

72
components/hord-cli/src/cli/mod.rs
View File

@@ -1,35 +1,31 @@
use crate::config::generator::generate_config;
use crate::config::{Config, PredicatesApi};
use crate::scan::bitcoin::scan_bitcoin_chainstate_via_rpc_using_predicate;
use crate::service::Service;
use crate::storage::get_last_block_height_inserted;
use chainhook_sdk::chainhooks::types::ChainhookFullSpecification;
use chainhook_sdk::hord::db::{
use crate::db::{
delete_data_in_hord_db, find_last_block_inserted, find_lazy_block_at_block_height,
find_watched_satpoint_for_inscription, initialize_hord_db, open_readonly_hord_db_conn,
open_readonly_hord_db_conn_rocks_db, open_readwrite_hord_db_conn,
open_readwrite_hord_db_conn_rocks_db, retrieve_satoshi_point_using_lazy_storage,
};
use chainhook_sdk::hord::{
use crate::hord::{
new_traversals_lazy_cache, retrieve_inscribed_satoshi_points_from_block,
update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, Storage,
};
use chainhook_sdk::chainhooks::types::ChainhookFullSpecification;
use chainhook_sdk::indexer;
use chainhook_sdk::indexer::bitcoin::{
download_and_parse_block_with_retry, retrieve_block_hash_with_retry,
};
use chainhook_sdk::observer::BitcoinConfig;
use chainhook_sdk::utils::Context;
use chainhook_types::{BitcoinBlockData, BitcoinNetwork, BlockIdentifier, TransactionIdentifier};
use chainhook_types::{BitcoinBlockData, BlockIdentifier, TransactionIdentifier};
use clap::{Parser, Subcommand};
use ctrlc;
use hiro_system_kit;
use std::collections::BTreeMap;
use std::io::{BufReader, Read};
use std::path::PathBuf;
use std::process;
use std::sync::mpsc::Sender;
use std::sync::Arc;
#[derive(Parser, Debug)]
@@ -153,26 +149,6 @@ enum HordDbCommand {
Check(CheckDbCommand),
}
#[derive(Subcommand, PartialEq, Clone, Debug)]
enum StacksCommand {
/// Db maintenance related commands
#[clap(subcommand)]
Db(StacksDbCommand),
}
#[derive(Subcommand, PartialEq, Clone, Debug)]
enum StacksDbCommand {
/// Check integrity
#[clap(name = "check", bin_name = "check")]
Check(CheckDbCommand),
/// Update database using latest Stacks archive file
#[clap(name = "update", bin_name = "update")]
Update(UpdateDbCommand),
/// Retrieve a block from the Stacks db
#[clap(name = "get", bin_name = "get")]
GetBlock(GetBlockDbCommand),
}
#[derive(Subcommand, PartialEq, Clone, Debug)]
enum ScanCommand {
/// Compute ordinal number of the 1st satoshi of the 1st input of a given transaction
@@ -307,30 +283,6 @@ struct CheckDbCommand {
pub config_path: Option<String>,
}
#[derive(Parser, PartialEq, Clone, Debug)]
struct UpdateDbCommand {
/// Load config file path
#[clap(long = "config-path")]
pub config_path: Option<String>,
}
#[derive(Parser, PartialEq, Clone, Debug)]
struct GetBlockDbCommand {
/// Block index to retrieve
#[clap(long = "block-height")]
pub block_height: u64,
/// Load config file path
#[clap(long = "config-path")]
pub config_path: Option<String>,
}
#[derive(Parser, PartialEq, Clone, Debug)]
struct InitHordDbCommand {
/// Load config file path
#[clap(long = "config-path")]
pub config_path: Option<String>,
}
pub fn main() {
let logger = hiro_system_kit::log::setup_logger();
let _guard = hiro_system_kit::log::setup_global_logger(logger.clone());
@@ -444,6 +396,7 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
}
None => {
let event_observer_config = config.get_event_observer_config();
let hord_config = config.get_hord_config();
let bitcoin_config = event_observer_config.get_bitcoin_config();
let block =
fetch_and_standardize_block(cmd.block_height, &bitcoin_config, &ctx)
@@ -453,7 +406,7 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
let _traversals = retrieve_inscribed_satoshi_points_from_block(
&block,
None,
event_observer_config.hord_config.as_ref().unwrap(),
&hord_config,
&traversals_cache,
&ctx,
);
@@ -621,13 +574,12 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
"Cleaning hord_db: {} blocks dropped",
cmd.end_block - cmd.start_block + 1
);
}
// HordDbCommand::Patch(_cmd) => {
// unimplemented!()
// }
// HordDbCommand::Migrate(_cmd) => {
// unimplemented!()
// }
} // HordDbCommand::Patch(_cmd) => {
// unimplemented!()
// }
// HordDbCommand::Migrate(_cmd) => {
// unimplemented!()
// }
}
Ok(())
}

1
components/hord-cli/src/config/mod.rs
View File

@@ -136,7 +136,6 @@ impl Config {
cache_path: self.storage.working_dir.clone(),
bitcoin_network: self.network.bitcoin_network.clone(),
stacks_network: self.network.stacks_network.clone(),
hord_config: None,
}
}

17
components/hord-cli/src/hord/mod.rs
View File

@@ -977,3 +977,20 @@ fn test_identify_next_output_index_destination() {
SatPosition::Fee(0)
);
}
#[test]
fn test_ordinal_inscription_parsing() {
let bytes = hex::decode("208737bc46923c3e64c7e6768c0346879468bf3aba795a5f5f56efca288f50ed2aac0063036f7264010118746578742f706c61696e3b636861727365743d7574662d38004c9948656c6c6f2030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030300a68").unwrap();
let script = Script::from(bytes);
let parser = InscriptionParser {
instructions: script.instructions().peekable(),
};
let inscription = match parser.parse_script() {
Ok(inscription) => inscription,
Err(_) => panic!(),
};
println!("{:?}", inscription);
}

5
components/hord-cli/src/hord/ordinals.rs
View File

@@ -1,7 +1,4 @@
use std::{
collections::HashMap,
sync::mpsc::{channel, Sender},
};
use std::sync::mpsc::{channel, Sender};
use super::HordConfig;

18
components/hord-cli/src/scan/bitcoin.rs
View File

@@ -1,5 +1,13 @@
use crate::archive::download_ordinals_dataset_if_required;
use crate::config::{Config, PredicatesApi};
use crate::db::{
find_all_inscriptions_in_block, get_any_entry_in_ordinal_activities, open_readonly_hord_db_conn,
};
use crate::hord::{
get_inscriptions_revealed_in_block,
update_storage_and_augment_bitcoin_block_with_inscription_reveal_data,
update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, Storage,
};
use crate::service::{
open_readwrite_predicates_db_conn_or_panic, update_predicate_status, PredicateStatus,
ScanningData,
@@ -11,14 +19,6 @@ use chainhook_sdk::chainhooks::bitcoin::{
BitcoinChainhookOccurrence, BitcoinTriggerChainhook,
};
use chainhook_sdk::chainhooks::types::{BitcoinChainhookSpecification, BitcoinPredicateType};
use chainhook_sdk::hord::db::{
find_all_inscriptions_in_block, get_any_entry_in_ordinal_activities, open_readonly_hord_db_conn,
};
use chainhook_sdk::hord::{
get_inscriptions_revealed_in_block,
update_storage_and_augment_bitcoin_block_with_inscription_reveal_data,
update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, Storage,
};
use chainhook_sdk::indexer;
use chainhook_sdk::indexer::bitcoin::{
download_and_parse_block_with_retry, retrieve_block_hash_with_retry,
@@ -88,7 +88,7 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate(
let mut blocks_scanned = 0;
let mut actions_triggered = 0;
let mut occurrences_found = 0u64;
let occurrences_found = 0u64;
let mut err_count = 0;
let event_observer_config = config.get_event_observer_config();

164
components/hord-cli/src/service/mod.rs
View File

@@ -2,7 +2,11 @@ mod http_api;
mod runloops;
use crate::config::{Config, PredicatesApi, PredicatesApiConfig};
use crate::hord::should_sync_hord_db;
use crate::db::open_readwrite_hord_dbs;
use crate::hord::{
new_traversals_lazy_cache, revert_hord_db_with_augmented_bitcoin_block, should_sync_hord_db,
update_hord_db_and_augment_bitcoin_block,
};
use crate::scan::bitcoin::process_block_with_predicates;
use crate::service::http_api::{load_predicates_from_redis, start_predicate_api_server};
use crate::service::runloops::start_bitcoin_scan_runloop;
@@ -12,12 +16,15 @@ use chainhook_sdk::chainhooks::types::{
};
use chainhook_sdk::chainhooks::types::ChainhookSpecification;
use chainhook_sdk::chainhooks::types::{ChainhookConfig, ChainhookFullSpecification};
use chainhook_sdk::observer::{start_event_observer, ObserverEvent};
use chainhook_sdk::utils::Context;
use chainhook_types::BitcoinBlockSignaling;
use chainhook_types::{BitcoinBlockSignaling, BitcoinChainEvent};
use hiro_system_kit::slog;
use redis::{Commands, Connection};
use std::sync::mpsc::channel;
use std::sync::Arc;
pub struct Service {
config: Config,
@@ -102,32 +109,25 @@ impl Service {
let mut event_observer_config = self.config.get_event_observer_config();
event_observer_config.chainhook_config = Some(chainhook_config);
// Download and ingest a Ordinal dump, if hord is enabled
if !hord_disabled {
// TODO: add flag
// let _ = download_ordinals_dataset_if_required(&mut self.config, &self.ctx).await;
info!(
self.ctx.expect_logger(),
"Ordinal indexing is enabled by default, checking index... (use --no-hord to disable ordinals)"
);
let hord_config = self.config.get_hord_config();
if let Some((start_block, end_block)) = should_sync_hord_db(&self.config, &self.ctx)? {
if start_block == 0 {
info!(
self.ctx.expect_logger(),
"Initializing hord indexing from block #{}", start_block
);
} else {
info!(
self.ctx.expect_logger(),
"Resuming hord indexing from block #{}", start_block
);
}
if let Some((start_block, end_block)) = should_sync_hord_db(&self.config, &self.ctx)? {
if start_block == 0 {
info!(
self.ctx.expect_logger(),
"Initializing hord indexing from block #{}", start_block
);
} else {
info!(
self.ctx.expect_logger(),
"Resuming hord indexing from block #{}", start_block
);
}
let (tx, rx) = channel();
let (tx, rx) = channel();
let mut moved_event_observer_config = event_observer_config.clone();
let moved_ctx = self.ctx.clone();
let mut moved_event_observer_config = event_observer_config.clone();
let moved_ctx = self.ctx.clone();
let _ = hiro_system_kit::thread_named("Initial predicate processing")
.spawn(move || {
@@ -154,21 +154,23 @@ impl Service {
}
}
}
})
.expect("unable to spawn thread");
}
})
.expect("unable to spawn thread");
crate::hord::perform_hord_db_update(
start_block,
end_block,
&self.config.get_hord_config(),
&self.config,
Some(tx),
&self.ctx,
)
.await?;
}
crate::hord::perform_hord_db_update(
start_block,
end_block,
&self.config.get_hord_config(),
&self.config,
Some(tx),
&self.ctx,
)
.await?;
}
let traversals_cache = Arc::new(new_traversals_lazy_cache(hord_config.cache_size));
// Bitcoin scan operation threadpool
let (bitcoin_scan_op_tx, bitcoin_scan_op_rx) = crossbeam_channel::unbounded();
let ctx = self.ctx.clone();
@@ -346,8 +348,92 @@ impl Service {
}
}
}
ObserverEvent::BitcoinChainEvent((_chain_update, _report)) => {
debug!(self.ctx.expect_logger(), "Bitcoin update not stored");
ObserverEvent::BitcoinChainEvent((mut chain_update, _report)) => {
//
let (blocks_db, inscriptions_db_conn_rw) = match open_readwrite_hord_dbs(
&self.config.expected_cache_path(),
&self.ctx,
) {
Ok(dbs) => dbs,
Err(e) => {
self.ctx.try_log(|logger| {
slog::error!(logger, "Unable to open readwtite connection: {e}",)
});
continue;
}
};
// Update Chain event before propagation
match chain_update {
BitcoinChainEvent::ChainUpdatedWithBlocks(ref mut data) => {
for block in data.new_blocks.iter_mut() {
if let Err(e) = update_hord_db_and_augment_bitcoin_block(
block,
&blocks_db,
&inscriptions_db_conn_rw,
true,
&hord_config,
&traversals_cache,
&self.ctx,
) {
self.ctx.try_log(|logger| {
slog::error!(
logger,
"Unable to insert bitcoin block {} in hord_db: {e}",
block.block_identifier.index
)
});
}
}
}
BitcoinChainEvent::ChainUpdatedWithReorg(ref mut data) => {
for block in data.blocks_to_rollback.iter() {
if let Err(e) = revert_hord_db_with_augmented_bitcoin_block(
block,
&blocks_db,
&inscriptions_db_conn_rw,
&self.ctx,
) {
self.ctx.try_log(|logger| {
slog::error!(
logger,
"Unable to rollback bitcoin block {}: {e}",
block.block_identifier
)
});
}
}
for block in data.blocks_to_apply.iter_mut() {
if let Err(e) = update_hord_db_and_augment_bitcoin_block(
block,
&blocks_db,
&inscriptions_db_conn_rw,
true,
&hord_config,
&traversals_cache,
&self.ctx,
) {
self.ctx.try_log(|logger| {
slog::error!(
logger,
"Unable to apply bitcoin block {} with hord_db: {e}",
block.block_identifier.index
)
});
}
}
}
};
self.ctx.try_log(|logger| {
slog::info!(
logger,
"Flushing traversals_cache ({} entries)",
traversals_cache.len()
)
});
// traversals_cache.clear();
}
ObserverEvent::Terminate => {
info!(self.ctx.expect_logger(), "Terminating runloop");

4
components/hord-cli/src/service/runloops.rs
View File

@@ -1,9 +1,7 @@
use std::sync::mpsc::Sender;
use chainhook_sdk::{
chainhooks::types::{
BitcoinChainhookSpecification, ChainhookSpecification,
},
chainhooks::types::{BitcoinChainhookSpecification, ChainhookSpecification},
observer::ObserverCommand,
utils::Context,
};

3
components/hord-cli/src/storage/mod.rs
View File

@@ -1,7 +1,8 @@
use std::path::PathBuf;
use chainhook_sdk::{rocksdb::Options, rocksdb::DB, utils::Context};
use chainhook_sdk::utils::Context;
use chainhook_types::{BlockIdentifier, StacksBlockData, StacksBlockUpdate};
use rocksdb::{Options, DB};
fn get_db_default_options() -> Options {
let mut opts = Options::default();