kylefang/bitcoin-indexer
1
0
Fork 0
mirror of https://github.com/alexgo-io/bitcoin-indexer.git synced 2026-01-12 22:43:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: network, cascade changes

Browse Source
This commit is contained in:
Ludo Galabru
2023-03-02 02:16:45 -05:00
parent 42d34bf9b5
commit 1f45ec26da
15 changed files with 137 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@@ -415,7 +415,7 @@ dependencies = [
"ansi_term",
"atty",
"chainhook-event-observer",
"chainhook-types 1.0.2",
"chainhook-types 1.0.3",
"clap 3.2.23",
"clap_generate",
"clarinet-files",
@@ -450,7 +450,7 @@ dependencies = [
"base64 0.13.1",
"bitcoincore-rpc",
"bitcoincore-rpc-json",
"chainhook-types 1.0.2",
"chainhook-types 1.0.3",
"chrono",
"clap 3.2.23",
"clap_generate",
@@ -458,7 +458,10 @@ dependencies = [
"clarity-repl",
"crossbeam-channel",
"ctrlc",
"futures",
"hex",
"hiro-system-kit",
"hyper",
"redb",
"reqwest",
"rocket",
@@ -487,7 +490,7 @@ dependencies = [
[[package]]
name = "chainhook-types"
version = "1.0.2"
version = "1.0.3"
dependencies = [
"schemars",
"serde",
@@ -1692,9 +1695,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
[[package]]
name = "hyper"
version = "0.14.23"
version = "0.14.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c"
checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c"
dependencies = [
"bytes",
"futures-channel",

4
components/chainhook-cli/Cargo.toml
View File

@@ -15,8 +15,8 @@ redis = "0.21.5"
serde-redis = "0.12.0"
# tikv-client = { git = "https://github.com/tikv/client-rust.git", rev = "8f54e6114227718e256027df2577bbacdf425f86" }
# raft-proto = { git = "https://github.com/tikv/raft-rs", rev="f73766712a538c2f6eb135b455297ad6c03fc58d", version = "0.7.0"}
chainhook-event-observer = { version = "=1.0.4", default-features = false, path = "../chainhook-event-observer" }
chainhook-types = { version = "=1.0.2", path = "../chainhook-types-rs" }
chainhook-event-observer = { version = "=1.0.4", default-features = false, features = ["ordinals"], path = "../chainhook-event-observer" }
chainhook-types = { version = "=1.0.3", path = "../chainhook-types-rs" }
clarinet-files = "1"
hiro-system-kit = "0.1.0"
# clarinet-files = { path = "../../../clarinet/components/clarinet-files" }

24
components/chainhook-cli/src/block/ingestion.rs
View File

@@ -9,30 +9,6 @@ use std::{sync::mpsc::channel, thread};
use super::DigestingCommand;
#[derive(Debug, Deserialize)]
pub struct Record {
pub id: u64,
pub created_at: String,
pub kind: RecordKind,
pub raw_log: String,
}
#[derive(Debug, Deserialize)]
pub enum RecordKind {
#[serde(rename = "/new_block")]
StacksBlockReceived,
#[serde(rename = "/new_microblocks")]
StacksMicroblockReceived,
#[serde(rename = "/new_burn_block")]
BitcoinBlockReceived,
#[serde(rename = "/new_mempool_tx")]
TransactionAdmitted,
#[serde(rename = "/drop_mempool_tx")]
TransactionDropped,
#[serde(rename = "/attachments/new")]
AttachmentReceived,
}
pub fn start(
digestion_tx: Sender<DigestingCommand>,
config: &Config,

28
components/chainhook-cli/src/block/mod.rs
View File

@@ -1,5 +1,5 @@
pub mod digestion;
pub mod ingestion;
// pub mod digestion;
// pub mod ingestion;
use chainhook_types::BlockIdentifier;
#[derive(Debug, Clone, Eq, PartialEq)]
@@ -9,3 +9,27 @@ pub enum DigestingCommand {
Kill,
Terminate,
}
#[derive(Debug, Deserialize)]
pub struct Record {
pub id: u64,
pub created_at: String,
pub kind: RecordKind,
pub raw_log: String,
}
#[derive(Debug, Deserialize)]
pub enum RecordKind {
#[serde(rename = "/new_block")]
StacksBlockReceived,
#[serde(rename = "/new_microblocks")]
StacksMicroblockReceived,
#[serde(rename = "/new_burn_block")]
BitcoinBlockReceived,
#[serde(rename = "/new_mempool_tx")]
TransactionAdmitted,
#[serde(rename = "/drop_mempool_tx")]
TransactionDropped,
#[serde(rename = "/attachments/new")]
AttachmentReceived,
}

1
components/chainhook-cli/src/node/mod.rs
View File

@@ -91,6 +91,7 @@ impl Node {
operators: HashSet::new(),
display_logs: false,
cache_path: self.config.storage.cache_path.clone(),
bitcoin_network: self.config.network.bitcoin_network.clone(),
};
info!(
self.ctx.expect_logger(),

45
components/chainhook-cli/src/scan/bitcoin.rs
View File

@@ -5,9 +5,14 @@ use chainhook_event_observer::chainhooks::bitcoin::{
handle_bitcoin_hook_action, BitcoinChainhookOccurrence, BitcoinTriggerChainhook,
};
use chainhook_event_observer::chainhooks::types::BitcoinChainhookFullSpecification;
use chainhook_event_observer::indexer;
use chainhook_event_observer::indexer::ordinals::indexing::updater::OrdinalIndexUpdater;
use chainhook_event_observer::indexer::ordinals::initialize_ordinal_index;
use chainhook_event_observer::indexer::{self, BitcoinChainContext};
use chainhook_event_observer::observer::{
EventObserverConfig, DEFAULT_CONTROL_PORT, DEFAULT_INGESTION_PORT,
};
use chainhook_event_observer::utils::{file_append, send_request, Context};
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::time::Duration;
pub async fn scan_bitcoin_chain_with_predicate(
@@ -133,8 +138,40 @@ pub async fn scan_bitcoin_chain_with_predicate(
.result::<indexer::bitcoin::Block>()
.map_err(|e| format!("unable to parse response ({})", e))?;
let block =
indexer::bitcoin::standardize_bitcoin_block(&config.network, cursor, raw_block, ctx)?;
let event_observer_config = EventObserverConfig {
normalization_enabled: true,
grpc_server_enabled: false,
hooks_enabled: true,
bitcoin_rpc_proxy_enabled: true,
event_handlers: vec![],
chainhook_config: None,
ingestion_port: DEFAULT_INGESTION_PORT,
control_port: DEFAULT_CONTROL_PORT,
bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(),
bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(),
bitcoin_node_rpc_url: config.network.bitcoin_node_rpc_url.clone(),
stacks_node_rpc_url: config.network.stacks_node_rpc_url.clone(),
operators: HashSet::new(),
display_logs: false,
cache_path: config.storage.cache_path.clone(),
bitcoin_network: config.network.bitcoin_network.clone(),
};
let ordinal_index = initialize_ordinal_index(&event_observer_config).unwrap();
match OrdinalIndexUpdater::update(&ordinal_index) {
Ok(_r) => {}
Err(e) => {}
}
let mut bitcoin_context = BitcoinChainContext::new(ordinal_index);
let block = indexer::bitcoin::standardize_bitcoin_block(
&config.network,
cursor,
raw_block,
&mut bitcoin_context,
ctx,
)?;
let mut hits = vec![];
for tx in block.transactions.iter() {

29
components/chainhook-cli/src/scan/stacks.rs
View File

@@ -1,17 +1,19 @@
use std::{
collections::{HashMap, VecDeque},
collections::{HashMap, HashSet, VecDeque},
process,
};
use crate::{
archive,
block::ingestion::{Record, RecordKind},
block::{Record, RecordKind},
config::Config,
};
use chainhook_event_observer::{
chainhooks::stacks::{
handle_stacks_hook_action, StacksChainhookOccurrence, StacksTriggerChainhook,
},
indexer::ordinals::{indexing::updater::OrdinalIndexUpdater, initialize_ordinal_index},
observer::{EventObserverConfig, DEFAULT_CONTROL_PORT, DEFAULT_INGESTION_PORT},
utils::{file_append, send_request, AbstractStacksBlock},
};
use chainhook_event_observer::{
@@ -82,7 +84,28 @@ pub async fn scan_stacks_chain_with_predicate(
let _ = record_tx.send(None);
});
let mut indexer = Indexer::new(config.network.clone());
let event_observer_config = EventObserverConfig {
normalization_enabled: true,
grpc_server_enabled: false,
hooks_enabled: true,
bitcoin_rpc_proxy_enabled: true,
event_handlers: vec![],
chainhook_config: None,
ingestion_port: DEFAULT_INGESTION_PORT,
control_port: DEFAULT_CONTROL_PORT,
bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(),
bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(),
bitcoin_node_rpc_url: config.network.bitcoin_node_rpc_url.clone(),
stacks_node_rpc_url: config.network.stacks_node_rpc_url.clone(),
operators: HashSet::new(),
display_logs: false,
cache_path: config.storage.cache_path.clone(),
bitcoin_network: config.network.bitcoin_network.clone(),
};
let ordinal_index = initialize_ordinal_index(&event_observer_config).unwrap();
let mut indexer = Indexer::new(config.network.clone(), ordinal_index);
let mut canonical_fork = {
let mut cursor = BlockIdentifier::default();

3
components/chainhook-event-observer/Cargo.toml
View File

@@ -41,6 +41,9 @@ crossbeam-channel = "0.5.6"
redb = { version = "0.13.0", optional = true }
chrono = { version = "0.4.19", optional = true }
anyhow = { version = "1.0.56", features = ["backtrace"], optional = true }
futures = "0.3.21"
hyper = { version = "0.14.24", features = ["http1", "client"] }
hex = "0.4.3"
[replace]
"jsonrpc:0.13.0" = { git = 'https://github.com/apoelstra/rust-jsonrpc', rev = "1063671f122a8985c1b7c29030071253da515839" }

26
components/chainhook-event-observer/src/indexer/bitcoin/mod.rs
View File

@@ -27,6 +27,7 @@ use rocket::serde::json::Value as JsonValue;
use super::ordinals::indexing::updater::OrdinalIndexUpdater;
use super::ordinals::indexing::OrdinalIndex;
use super::ordinals::inscription::InscriptionParser;
use super::ordinals::inscription_id::InscriptionId;
use super::BitcoinChainContext;
#[derive(Clone, PartialEq, Debug, Deserialize, Serialize)]
@@ -115,7 +116,15 @@ pub fn standardize_bitcoin_block(
let mut transactions = vec![];
match OrdinalIndexUpdater::update(&mut bitcoin_context.ordinal_index) {
Ok(_r) => {}
Ok(_) => {
ctx.try_log(|logger| {
slog::info!(
logger,
"Ordinal index updated (block count: {:?})",
bitcoin_context.ordinal_index.block_count()
)
});
}
Err(e) => {
ctx.try_log(|logger| slog::error!(logger, "{}", e.to_string()));
}
@@ -235,22 +244,15 @@ fn try_parse_ordinal_operation(
Err(_) => continue,
};
let outpoint = bitcoin::OutPoint {
let inscription_id = InscriptionId {
txid: tx.txid.clone(),
vout: 0,
index: 0,
};
let entries = ordinal_index.get_feed_inscriptions(3).unwrap();
ctx.try_log(|logger| slog::info!(logger, "Feed: {:?}", entries));
let inscription_ids = ordinal_index.get_inscriptions_on_output(outpoint).unwrap();
if inscription_ids.is_empty() {
ctx.try_log(|logger| slog::info!(logger, "No inscriptions found in index, despite inscription detected in transaction"));
return None;
}
let inscription_entry = match ordinal_index
.get_inscription_entry(inscription_ids[0])
.get_inscription_entry(inscription_id.clone())
{
Ok(Some(entry)) => entry,
_ => {
@@ -270,7 +272,7 @@ fn try_parse_ordinal_operation(
.to_string(),
content_bytes: format!("0x{}", to_hex(&inscription_content_bytes)),
content_length: inscription_content_bytes.len(),
inscription_id: "".into(),
inscription_id: inscription_id.to_string(),
inscription_number: inscription_entry.number,
inscription_author: "".into(),
inscription_fee: inscription_entry.fee,

2
components/chainhook-event-observer/src/indexer/bitcoin/tests.rs
View File

@@ -1,6 +1,6 @@
use bitcoincore_rpc::bitcoin::Script;
use crate::indexer::bitcoin::ordinal::InscriptionParser;
use crate::indexer::ordinals::inscription::InscriptionParser;
use super::super::tests::{helpers, process_bitcoin_blocks_and_check_expectations};

13
components/chainhook-event-observer/src/indexer/ordinals/mod.rs
View File

@@ -5,7 +5,7 @@ mod epoch;
mod height;
pub mod indexing;
pub mod inscription;
mod inscription_id;
pub mod inscription_id;
mod sat;
mod sat_point;
@@ -13,6 +13,8 @@ use std::time::Duration;
type Result<T = (), E = anyhow::Error> = std::result::Result<T, E>;
use chainhook_types::BitcoinNetwork;
use crate::observer::EventObserverConfig;
const DIFFCHANGE_INTERVAL: u64 =
@@ -24,12 +26,17 @@ const CYCLE_EPOCHS: u64 = 6;
pub fn initialize_ordinal_index(
config: &EventObserverConfig,
) -> Result<self::indexing::OrdinalIndex, String> {
let chain = match &config.bitcoin_network {
BitcoinNetwork::Mainnet => chain::Chain::Mainnet,
BitcoinNetwork::Testnet => chain::Chain::Testnet,
BitcoinNetwork::Regtest => chain::Chain::Regtest,
};
let index_options = self::indexing::Options {
rpc_username: config.bitcoin_node_username.clone(),
rpc_password: config.bitcoin_node_password.clone(),
data_dir: config.cache_path.clone().into(),
chain: chain::Chain::Mainnet,
first_inscription_height: None,
chain: chain,
first_inscription_height: Some(chain.first_inscription_height()),
height_limit: None,
index: None,
rpc_url: config.bitcoin_node_rpc_url.clone(),

1
components/chainhook-event-observer/src/main.rs
View File

@@ -147,6 +147,7 @@ impl EventObserverConfig {
operators,
display_logs: true,
cache_path: config_file.cache_path.unwrap_or("cache".into()),
bitcoin_network: chainhook_types::BitcoinNetwork::Mainnet, // todo(lgalabru)
};
config
}

3
components/chainhook-event-observer/src/observer/mod.rs
View File

@@ -126,6 +126,7 @@ pub struct EventObserverConfig {
pub operators: HashSet<String>,
pub display_logs: bool,
pub cache_path: String,
pub bitcoin_network: BitcoinNetwork,
}
#[derive(Deserialize, Debug)]
@@ -227,7 +228,7 @@ pub async fn start_event_observer(
ctx.try_log(|logger| {
slog::info!(
logger,
"Initializing ordinals index in file {}",
"Initializing ordinals index in dir `{}`",
config.cache_path
)
});

1
components/chainhook-event-observer/src/observer/tests/mod.rs
View File

@@ -40,6 +40,7 @@ fn generate_test_config() -> (EventObserverConfig, ChainhookStore) {
operators,
display_logs: false,
cache_path: "cache".into(),
bitcoin_network: BitcoinNetwork::Regtest,
};
let mut entries = HashMap::new();
entries.insert(ApiKey(None), ChainhookConfig::new());

2
components/chainhook-types-rs/Cargo.toml
View File

@@ -2,7 +2,7 @@
name = "chainhook-types"
description = "Bitcoin and Stacks data schemas, based on the Rosetta specification"
license = "MIT"
version = "1.0.2"
version = "1.0.3"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html