kylefang/bitcoin-indexer
1
0
Fork 0
mirror of https://github.com/alexgo-io/bitcoin-indexer.git synced 2026-06-14 00:22:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: improve onboarding

Browse Source
This commit is contained in:
Ludo Galabru
2023-08-05 00:21:02 +02:00
parent fa802fae7a
commit deaa739bdd
6 changed files with 106 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
components/hord-cli/src/cli/mod.rs
View File

@@ -13,7 +13,7 @@ use crate::db::{
find_all_transfers_in_block, find_inscription_with_id, find_last_block_inserted,
find_latest_inscription_block_height, find_lazy_block_at_block_height,
open_readonly_hord_db_conn, open_readonly_hord_db_conn_rocks_db, open_readwrite_hord_db_conn,
open_readwrite_hord_db_conn_rocks_db, initialize_hord_db,
open_readwrite_hord_db_conn_rocks_db, initialize_hord_db, get_default_hord_db_file_path,
};
use chainhook_sdk::bitcoincore_rpc::{Auth, Client, RpcApi};
use chainhook_sdk::chainhooks::types::HttpHook;
@@ -261,6 +261,9 @@ struct StartCommand {
#[derive(Subcommand, PartialEq, Clone, Debug)]
enum HordDbCommand {
/// Initialize a new hord db
#[clap(name = "new", bin_name = "new")]
New(SyncHordDbCommand),
/// Catch-up hord db
#[clap(name = "sync", bin_name = "sync")]
Sync(SyncHordDbCommand),
@@ -476,21 +479,25 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
.await?;
} else {
let _ = download_ordinals_dataset_if_required(&config, ctx).await;
let mut total_inscriptions = 0;
let mut total_transfers = 0;
let inscriptions_db_conn =
open_readonly_hord_db_conn(&config.expected_cache_path(), &ctx)?;
initialize_hord_db(&config.expected_cache_path(), &ctx);
while let Some(block_height) = block_range.pop_front() {
let mut total_transfers = 0;
let inscriptions =
find_all_inscriptions_in_block(&block_height, &inscriptions_db_conn, &ctx);
let mut locations =
find_all_transfers_in_block(&block_height, &inscriptions_db_conn, &ctx);
let mut total_transfers_in_block = 0;
for (_, inscription) in inscriptions.iter() {
println!("Inscription {} revealed at block #{} (inscription_number {}, ordinal_number {})", inscription.get_inscription_id(), block_height, inscription.inscription_number, inscription.ordinal_number);
if let Some(transfers) = locations.remove(&inscription.get_inscription_id())
{
for t in transfers.iter().skip(1) {
total_transfers += 1;
total_transfers_in_block += 1;
println!(
"\t→ Transferred in transaction {}",
t.transaction_identifier_location.hash
@@ -501,21 +508,28 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
for (inscription_id, transfers) in locations.iter() {
println!("Inscription {}", inscription_id);
for t in transfers.iter() {
total_transfers += 1;
total_transfers_in_block += 1;
println!(
"\t→ Transferred in transaction {}",
t.transaction_identifier_location.hash
);
}
}
if total_transfers > 0 && inscriptions.len() > 0 {
if total_transfers_in_block > 0 && inscriptions.len() > 0 {
println!(
"Inscriptions revealed: {}, inscriptions transferred: {total_transfers}",
"Inscriptions revealed: {}, inscriptions transferred: {total_transfers_in_block}",
inscriptions.len()
);
println!("-----");
}
total_inscriptions += inscriptions.len();
total_transfers += total_transfers_in_block;
}
if total_transfers == 0 && total_inscriptions == 0 {
let db_file_path = get_default_hord_db_file_path(&config.expected_cache_path());
warn!(ctx.expect_logger(), "No data available. Check the validity of the range being scanned and the validity of your local database {}", db_file_path.display());
}
}
}
Command::Scan(ScanCommand::Inscription(cmd)) => {
@@ -620,6 +634,10 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
println!("Created file Hord.toml");
}
},
Command::Db(HordDbCommand::New(cmd)) => {
let config = Config::default(false, false, false, &cmd.config_path)?;
initialize_hord_db(&config.expected_cache_path(), &ctx);
},
Command::Db(HordDbCommand::Sync(_cmd)) => unimplemented!(),
Command::Db(HordDbCommand::Repair(subcmd)) => match subcmd {
RepairCommand::Blocks(cmd) => {

11
components/hord-cli/src/config/file.rs
View File

@@ -2,10 +2,10 @@
pub struct ConfigFile {
pub storage: StorageConfigFile,
pub http_api: Option<PredicatesApiConfigFile>,
pub event_source: Option<Vec<EventSourceConfigFile>>,
pub limits: LimitsConfigFile,
pub network: NetworkConfigFile,
pub logs: Option<LogConfigFile>,
pub bootstrap: Option<BootstrapConfigFile>,
}
#[derive(Deserialize, Debug, Clone)]
@@ -28,13 +28,8 @@ pub struct PredicatesApiConfigFile {
}
#[derive(Deserialize, Debug, Clone)]
pub struct EventSourceConfigFile {
pub source_type: Option<String>,
pub stacks_node_url: Option<String>,
pub chainhook_node_url: Option<String>,
pub polling_delay: Option<u32>,
pub tsv_file_path: Option<String>,
pub tsv_file_url: Option<String>,
pub struct BootstrapConfigFile {
pub download_url: Option<String>,
}
#[derive(Deserialize, Debug, Clone)]

5
components/hord-cli/src/config/generator.rs
View File

@@ -34,6 +34,11 @@ max_number_of_processing_threads = 16
bitcoin_concurrent_http_requests_max = 16
max_caching_memory_size_mb = 32000
# Disable the following section if the state
# must be built locally
[bootstrap]
download_url = "https://archive.hiro.so/mainnet/chainhooks/hord.sqlite"
[logs]
ordinals_internals = true
chainhook_internals = true

96
components/hord-cli/src/config/mod.rs
View File

@@ -27,9 +27,9 @@ pub const BITCOIN_MAX_PREDICATE_REGISTRATION: usize = 50;
pub struct Config {
pub storage: StorageConfig,
pub http_api: PredicatesApi,
pub event_sources: Vec<EventSourceConfig>,
pub limits: LimitsConfig,
pub network: IndexerConfig,
pub bootstrap: BootstrapConfig,
pub logs: LogConfig,
}
@@ -58,9 +58,9 @@ pub struct PredicatesApiConfig {
}
#[derive(Clone, Debug)]
pub enum EventSourceConfig {
OrdinalsSqlitePath(PathConfig),
OrdinalsSqliteUrl(UrlConfig),
pub enum BootstrapConfig {
Build,
Download(String),
}
#[derive(Clone, Debug)]
@@ -152,21 +152,13 @@ impl Config {
_ => return Err("network.mode not supported".to_string()),
};
let mut event_sources = vec![];
for source in config_file.event_source.unwrap_or(vec![]).iter_mut() {
if let Some(dst) = source.tsv_file_path.take() {
let mut file_path = PathBuf::new();
file_path.push(dst);
event_sources.push(EventSourceConfig::OrdinalsSqlitePath(PathConfig {
file_path,
}));
continue;
let bootstrap = match config_file.bootstrap {
Some(bootstrap) => match bootstrap.download_url {
Some(ref url) => BootstrapConfig::Download(url.to_string()),
None => BootstrapConfig::Build
}
if let Some(file_url) = source.tsv_file_url.take() {
event_sources.push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig { file_url }));
continue;
}
}
None => BootstrapConfig::Build
};
let config = Config {
storage: StorageConfig {
@@ -185,7 +177,7 @@ impl Config {
}),
},
},
event_sources,
bootstrap,
limits: LimitsConfig {
max_number_of_stacks_predicates: config_file
.limits
@@ -248,28 +240,11 @@ impl Config {
Ok(config)
}
pub fn is_initial_ingestion_required(&self) -> bool {
for source in self.event_sources.iter() {
match source {
EventSourceConfig::OrdinalsSqlitePath(_)
| EventSourceConfig::OrdinalsSqliteUrl(_) => return true,
}
pub fn should_bootstrap_through_download(&self) -> bool {
match &self.bootstrap {
BootstrapConfig::Build => false,
BootstrapConfig::Download(_) => true
}
return false;
}
pub fn add_ordinals_sqlite_remote_source_url(&mut self, file_url: &str) {
self.event_sources
.push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
file_url: file_url.to_string(),
}));
}
pub fn add_local_ordinals_sqlite_source(&mut self, file_path: &PathBuf) {
self.event_sources
.push(EventSourceConfig::OrdinalsSqlitePath(PathConfig {
file_path: file_path.clone(),
}));
}
pub fn expected_api_database_uri(&self) -> &str {
@@ -289,13 +264,11 @@ impl Config {
destination_path
}
fn expected_remote_ordinals_sqlite_base_url(&self) -> &String {
for source in self.event_sources.iter() {
if let EventSourceConfig::OrdinalsSqliteUrl(config) = source {
return &config.file_url;
}
fn expected_remote_ordinals_sqlite_base_url(&self) -> &str {
match &self.bootstrap {
BootstrapConfig::Build => unreachable!(),
BootstrapConfig::Download(url) => &url
}
panic!("expected remote-tsv source")
}
pub fn expected_remote_ordinals_sqlite_sha256(&self) -> String {
@@ -306,29 +279,6 @@ impl Config {
format!("{}.gz", self.expected_remote_ordinals_sqlite_base_url())
}
pub fn rely_on_remote_ordinals_sqlite(&self) -> bool {
for source in self.event_sources.iter() {
if let EventSourceConfig::OrdinalsSqliteUrl(_config) = source {
return true;
}
}
false
}
pub fn should_download_remote_ordinals_sqlite(&self) -> bool {
let mut rely_on_remote_tsv = false;
let mut remote_tsv_present_locally = false;
for source in self.event_sources.iter() {
if let EventSourceConfig::OrdinalsSqliteUrl(_config) = source {
rely_on_remote_tsv = true;
}
if let EventSourceConfig::OrdinalsSqlitePath(_config) = source {
remote_tsv_present_locally = true;
}
}
rely_on_remote_tsv == true && remote_tsv_present_locally == false
}
pub fn default(
devnet: bool,
testnet: bool,
@@ -351,7 +301,7 @@ impl Config {
working_dir: default_cache_path(),
},
http_api: PredicatesApi::Off,
event_sources: vec![],
bootstrap: BootstrapConfig::Build,
limits: LimitsConfig {
max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION,
max_number_of_concurrent_bitcoin_scans: BITCOIN_SCAN_THREAD_POOL_SIZE,
@@ -384,7 +334,7 @@ impl Config {
working_dir: default_cache_path(),
},
http_api: PredicatesApi::Off,
event_sources: vec![],
bootstrap: BootstrapConfig::Build,
limits: LimitsConfig {
max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION,
max_number_of_concurrent_bitcoin_scans: BITCOIN_SCAN_THREAD_POOL_SIZE,
@@ -417,9 +367,7 @@ impl Config {
working_dir: default_cache_path(),
},
http_api: PredicatesApi::Off,
event_sources: vec![EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
file_url: DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE.into(),
})],
bootstrap: BootstrapConfig::Download(DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE.to_string()),
limits: LimitsConfig {
max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION,
max_number_of_concurrent_bitcoin_scans: BITCOIN_SCAN_THREAD_POOL_SIZE,

4
components/hord-cli/src/db/mod.rs
View File

@@ -23,7 +23,7 @@ use crate::{
core::protocol::inscription_parsing::get_inscriptions_revealed_in_block, ord::sat::Sat,
};
fn get_default_hord_db_file_path(base_dir: &PathBuf) -> PathBuf {
pub fn get_default_hord_db_file_path(base_dir: &PathBuf) -> PathBuf {
let mut destination_path = base_dir.clone();
destination_path.push("hord.sqlite");
destination_path
@@ -124,7 +124,7 @@ pub fn initialize_hord_db(path: &PathBuf, ctx: &Context) -> Connection {
conn
}
fn create_or_open_readwrite_db(cache_path: &PathBuf, ctx: &Context) -> Connection {
pub fn create_or_open_readwrite_db(cache_path: &PathBuf, ctx: &Context) -> Connection {
let path = get_default_hord_db_file_path(&cache_path);
let open_flags = match std::fs::metadata(&path) {
Err(e) => {

107
components/hord-cli/src/download/mod.rs
View File

@@ -132,71 +132,62 @@ impl Read for ChannelRead {
}
pub async fn download_ordinals_dataset_if_required(config: &Config, ctx: &Context) -> bool {
if config.is_initial_ingestion_required() {
// Download default tsv.
if config.rely_on_remote_ordinals_sqlite()
&& config.should_download_remote_ordinals_sqlite()
{
let url = config.expected_remote_ordinals_sqlite_url();
let mut sqlite_file_path = config.expected_cache_path();
sqlite_file_path.push(default_sqlite_file_path(&config.network.bitcoin_network));
let mut sqlite_sha_file_path = config.expected_cache_path();
sqlite_sha_file_path.push(default_sqlite_sha_file_path(
&config.network.bitcoin_network,
));
if config.should_bootstrap_through_download() {
let url = config.expected_remote_ordinals_sqlite_url();
let mut sqlite_file_path = config.expected_cache_path();
sqlite_file_path.push(default_sqlite_file_path(&config.network.bitcoin_network));
let mut sqlite_sha_file_path = config.expected_cache_path();
sqlite_sha_file_path.push(default_sqlite_sha_file_path(
&config.network.bitcoin_network,
));
// Download archive if not already present in cache
// Load the local
let local_sha_file = read_file_content_at_path(&sqlite_sha_file_path);
let sha_url = config.expected_remote_ordinals_sqlite_sha256();
// Download archive if not already present in cache
// Load the local
let local_sha_file = read_file_content_at_path(&sqlite_sha_file_path);
let sha_url = config.expected_remote_ordinals_sqlite_sha256();
let remote_sha_file = match reqwest::get(&sha_url).await {
Ok(response) => response.bytes().await,
Err(e) => Err(e),
};
let should_download = match (local_sha_file, remote_sha_file) {
(Ok(local), Ok(remote_response)) => {
let cache_not_expired = remote_response.starts_with(&local[0..32]) == false;
if cache_not_expired {
info!(ctx.expect_logger(), "More recent hord.sqlite file detected");
}
cache_not_expired == false
let remote_sha_file = match reqwest::get(&sha_url).await {
Ok(response) => response.bytes().await,
Err(e) => Err(e),
};
let should_download = match (local_sha_file, remote_sha_file) {
(Ok(local), Ok(remote_response)) => {
let cache_not_expired = remote_response.starts_with(&local[0..32]) == false;
if cache_not_expired {
info!(ctx.expect_logger(), "More recent hord.sqlite file detected");
}
(_, _) => match std::fs::metadata(&sqlite_file_path) {
Ok(_) => false,
_ => {
info!(
ctx.expect_logger(),
"Unable to retrieve hord.sqlite file locally"
);
true
}
},
};
if should_download {
info!(ctx.expect_logger(), "Downloading {}", url);
match download_sqlite_file(&config, &ctx).await {
Ok(_) => {}
Err(e) => {
error!(ctx.expect_logger(), "{}", e);
std::process::exit(1);
}
}
} else {
info!(
ctx.expect_logger(),
"Basing ordinals evaluation on database {}",
sqlite_file_path.display()
);
cache_not_expired == false
}
// config.add_local_ordinals_sqlite_source(&sqlite_file_path);
(_, _) => match std::fs::metadata(&sqlite_file_path) {
Ok(_) => false,
_ => {
info!(
ctx.expect_logger(),
"Unable to retrieve hord.sqlite file locally"
);
true
}
},
};
if should_download {
info!(ctx.expect_logger(), "Downloading {}", url);
match download_sqlite_file(&config, &ctx).await {
Ok(_) => {}
Err(e) => {
error!(ctx.expect_logger(), "{}", e);
std::process::exit(1);
}
}
} else {
info!(
ctx.expect_logger(),
"Basing ordinals evaluation on database {}",
sqlite_file_path.display()
);
}
// config.add_local_ordinals_sqlite_source(&sqlite_file_path);
true
} else {
info!(
ctx.expect_logger(),
"Streaming blocks from bitcoind {}", config.network.bitcoind_rpc_url
);
false
}
}