kylefang/bitcoin-indexer
1
0
Fork 0
mirror of https://github.com/alexgo-io/bitcoin-indexer.git synced 2026-06-14 16:39:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: polish first impression

Browse Source
This commit is contained in:
Ludo Galabru
2023-07-20 17:31:52 -04:00
parent 74ae98c5bf
commit 3c2b00ce38
6 changed files with 79 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Cargo.lock generated
View File

@@ -1720,6 +1720,7 @@ dependencies = [
"hiro-system-kit",
"num_cpus",
"pprof",
"progressing",
"rand 0.8.5",
"redis",
"reqwest",
@@ -2790,6 +2791,15 @@ dependencies = [
"yansi",
]
[[package]]
name = "progressing"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97b7db19a74ba7c34de36558abed080568491d2b8999a34de914b1793b0b4b1b"
dependencies = [
"log",
]
[[package]]
name = "psm"
version = "0.1.21"

1
components/hord-cli/Cargo.toml
View File

@@ -39,6 +39,7 @@ rusqlite = { version = "0.27.0", features = ["bundled"] }
anyhow = { version = "1.0.56", features = ["backtrace"] }
schemars = { version = "0.8.10", git = "https://github.com/hirosystems/schemars.git", branch = "feat-chainhook-fixes" }
pprof = { version = "0.12", features = ["flamegraph"], optional = true }
progressing = '3'
[dependencies.rocksdb]
version = "0.20.1"

73
components/hord-cli/src/archive/mod.rs
View File

@@ -1,5 +1,5 @@
use crate::config::Config;
use crate::utils::{read_file_content_at_path, write_file_content_at_path};
use crate::utils::read_file_content_at_path;
use chainhook_sdk::types::BitcoinNetwork;
use chainhook_sdk::utils::Context;
use flate2::read::GzDecoder;
@@ -7,6 +7,8 @@ use futures_util::StreamExt;
use std::fs;
use std::io::{self, Cursor};
use std::io::{Read, Write};
use progressing::Baring;
use progressing::mapping::Bar as MappingBar;
pub fn default_sqlite_file_path(_network: &BitcoinNetwork) -> String {
format!("hord.sqlite").to_lowercase()
@@ -16,28 +18,28 @@ pub fn default_sqlite_sha_file_path(_network: &BitcoinNetwork) -> String {
format!("hord.sqlite.sha256").to_lowercase()
}
pub async fn download_sqlite_file(config: &Config) -> Result<(), String> {
pub async fn download_sqlite_file(config: &Config, ctx: &Context) -> Result<(), String> {
let mut destination_path = config.expected_cache_path();
std::fs::create_dir_all(&destination_path).unwrap_or_else(|e| {
println!("{}", e.to_string());
});
let remote_sha_url = config.expected_remote_ordinals_sqlite_sha256();
let res = reqwest::get(&remote_sha_url)
.await
.or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?
.bytes()
.await
.or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?;
// let remote_sha_url = config.expected_remote_ordinals_sqlite_sha256();
// let res = reqwest::get(&remote_sha_url)
// .await
// .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?
// .bytes()
// .await
// .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?;
let mut local_sha_file_path = destination_path.clone();
local_sha_file_path.push(default_sqlite_sha_file_path(
&config.network.bitcoin_network,
));
write_file_content_at_path(&local_sha_file_path, &res.to_vec())?;
// let mut local_sha_file_path = destination_path.clone();
// local_sha_file_path.push(default_sqlite_sha_file_path(
// &config.network.bitcoin_network,
// ));
// write_file_content_at_path(&local_sha_file_path, &res.to_vec())?;
let file_url = config.expected_remote_ordinals_sqlite_url();
println!("=> {file_url}");
let res = reqwest::get(&file_url)
.await
.or(Err(format!("Failed to GET from '{}'", &file_url)))?;
@@ -59,9 +61,25 @@ pub async fn download_sqlite_file(config: &Config) -> Result<(), String> {
});
if res.status() == reqwest::StatusCode::OK {
let mut progress_bar = MappingBar::with_range(0i64, 4_800_001_704);
progress_bar.set_len(40);
info!(
ctx.expect_logger(),
"{}", progress_bar
);
let mut stream = res.bytes_stream();
let mut progress = 0;
while let Some(item) = stream.next().await {
let chunk = item.or(Err(format!("Error while downloading file")))?;
progress += chunk.len() as i64;
progress_bar.set(progress);
if progress_bar.has_progressed_significantly() {
progress_bar.remember_significant_progress();
info!(
ctx.expect_logger(),
"{}", progress_bar
);
}
tx.send_async(chunk.to_vec())
.await
.map_err(|e| format!("unable to download stacks event: {}", e.to_string()))?;
@@ -116,14 +134,14 @@ pub async fn download_ordinals_dataset_if_required(config: &Config, ctx: &Contex
let url = config.expected_remote_ordinals_sqlite_url();
let mut sqlite_file_path = config.expected_cache_path();
sqlite_file_path.push(default_sqlite_file_path(&config.network.bitcoin_network));
let mut tsv_sha_file_path = config.expected_cache_path();
tsv_sha_file_path.push(default_sqlite_sha_file_path(
let mut sqlite_sha_file_path = config.expected_cache_path();
sqlite_sha_file_path.push(default_sqlite_sha_file_path(
&config.network.bitcoin_network,
));
// Download archive if not already present in cache
// Load the local
let local_sha_file = read_file_content_at_path(&tsv_sha_file_path);
let local_sha_file = read_file_content_at_path(&sqlite_sha_file_path);
let sha_url = config.expected_remote_ordinals_sqlite_sha256();
let remote_sha_file = match reqwest::get(&sha_url).await {
@@ -136,22 +154,27 @@ pub async fn download_ordinals_dataset_if_required(config: &Config, ctx: &Contex
if cache_not_expired {
info!(
ctx.expect_logger(),
"More recent Stacks archive file detected"
"More recent hord.sqlite file detected"
);
}
cache_not_expired == false
}
(_, _) => {
info!(
ctx.expect_logger(),
"Unable to retrieve Stacks archive file locally"
);
true
match std::fs::metadata(&sqlite_file_path) {
Ok(_) => false,
_ => {
info!(
ctx.expect_logger(),
"Unable to retrieve hord.sqlite file locally"
);
true
}
}
}
};
if should_download {
info!(ctx.expect_logger(), "Downloading {}", url);
match download_sqlite_file(&config).await {
match download_sqlite_file(&config, &ctx).await {
Ok(_) => {}
Err(e) => {
error!(ctx.expect_logger(), "{}", e);

22
components/hord-cli/src/cli/mod.rs
View File

@@ -1,3 +1,4 @@
use crate::archive::download_ordinals_dataset_if_required;
use crate::config::generator::generate_config;
use crate::config::Config;
use crate::scan::bitcoin::scan_bitcoin_chainstate_via_rpc_using_predicate;
@@ -455,9 +456,13 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
.get_sorted_entries();
if let Some(ref post_to) = cmd.post_to {
info!(ctx.expect_logger(), "A fully synchronized bitcoind node is required for retrieving inscriptions content.");
info!(ctx.expect_logger(), "Checking {}...", config.network.bitcoind_rpc_url);
let tip = check_bitcoind_connection(&config).await?;
if tip < cmd.end_block {
error!(ctx.expect_logger(), "unable to scan block range [{}, {}]: underlying bitcoind synchronized until block {} ", cmd.start_block, cmd.end_block, tip);
error!(ctx.expect_logger(), "Unable to scan block range [{}, {}]: underlying bitcoind synchronized until block {} ", cmd.start_block, cmd.end_block, tip);
} else {
info!(ctx.expect_logger(), "Starting scan");
}
let predicate_spec = build_predicate_from_cli(
@@ -470,6 +475,8 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
scan_bitcoin_chainstate_via_rpc_using_predicate(&predicate_spec, &config, &ctx)
.await?;
} else {
let _ = download_ordinals_dataset_if_required(&config, ctx).await;
let inscriptions_db_conn =
open_readonly_hord_db_conn(&config.expected_cache_path(), &ctx)?;
while let Some(block_height) = block_range.pop_front() {
@@ -501,12 +508,13 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
);
}
}
println!(
"Inscriptions revealed: {}, inscriptions transferred: {total_transfers}",
inscriptions.len()
);
println!("-----");
if total_transfers > 0 && inscriptions.len() > 0 {
println!(
"Inscriptions revealed: {}, inscriptions transferred: {total_transfers}",
inscriptions.len()
);
println!("-----");
}
}
}
}

78
components/hord-cli/src/config/mod.rs
View File

@@ -12,12 +12,8 @@ use std::fs::File;
use std::io::{BufReader, Read};
use std::path::PathBuf;
const DEFAULT_MAINNET_STACKS_TSV_ARCHIVE: &str =
"https://archive.hiro.so/mainnet/stacks-blockchain-api/mainnet-stacks-blockchain-api-latest";
const DEFAULT_TESTNET_STACKS_TSV_ARCHIVE: &str =
"https://archive.hiro.so/testnet/stacks-blockchain-api/testnet-stacks-blockchain-api-latest";
const DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE: &str =
"https://archive.hiro.so/mainnet/chainhooks/hord-latest.sqlite";
"https://archive.hiro.so/mainnet/chainhooks/hord.sqlite";
const DEFAULT_REDIS_URI: &str = "redis://localhost:6379/";
pub const DEFAULT_INGESTION_PORT: u16 = 20455;
@@ -56,8 +52,6 @@ pub struct PredicatesApiConfig {
#[derive(Clone, Debug)]
pub enum EventSourceConfig {
StacksTsvPath(PathConfig),
StacksTsvUrl(UrlConfig),
OrdinalsSqlitePath(PathConfig),
OrdinalsSqliteUrl(UrlConfig),
}
@@ -154,11 +148,11 @@ impl Config {
if let Some(dst) = source.tsv_file_path.take() {
let mut file_path = PathBuf::new();
file_path.push(dst);
event_sources.push(EventSourceConfig::StacksTsvPath(PathConfig { file_path }));
event_sources.push(EventSourceConfig::OrdinalsSqlitePath(PathConfig { file_path }));
continue;
}
if let Some(file_url) = source.tsv_file_url.take() {
event_sources.push(EventSourceConfig::StacksTsvUrl(UrlConfig { file_url }));
event_sources.push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig { file_url }));
continue;
}
}
@@ -234,22 +228,14 @@ impl Config {
pub fn is_initial_ingestion_required(&self) -> bool {
for source in self.event_sources.iter() {
match source {
EventSourceConfig::StacksTsvUrl(_) | EventSourceConfig::StacksTsvPath(_) => {
EventSourceConfig::OrdinalsSqlitePath(_) | EventSourceConfig::OrdinalsSqliteUrl(_) => {
return true
}
_ => {}
}
}
return false;
}
pub fn add_local_stacks_tsv_source(&mut self, file_path: &PathBuf) {
self.event_sources
.push(EventSourceConfig::StacksTsvPath(PathConfig {
file_path: file_path.clone(),
}));
}
pub fn add_ordinals_sqlite_remote_source_url(&mut self, file_url: &str) {
self.event_sources
.push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
@@ -275,15 +261,6 @@ impl Config {
}
}
pub fn expected_local_stacks_tsv_file(&self) -> &PathBuf {
for source in self.event_sources.iter() {
if let EventSourceConfig::StacksTsvPath(config) = source {
return &config.file_path;
}
}
panic!("expected local-tsv source")
}
pub fn expected_cache_path(&self) -> PathBuf {
let mut destination_path = PathBuf::new();
destination_path.push(&self.storage.working_dir);
@@ -299,23 +276,6 @@ impl Config {
panic!("expected remote-tsv source")
}
fn expected_remote_stacks_tsv_base_url(&self) -> &String {
for source in self.event_sources.iter() {
if let EventSourceConfig::StacksTsvUrl(config) = source {
return &config.file_url;
}
}
panic!("expected remote-tsv source")
}
pub fn expected_remote_stacks_tsv_sha256(&self) -> String {
format!("{}.sha256", self.expected_remote_stacks_tsv_base_url())
}
pub fn expected_remote_stacks_tsv_url(&self) -> String {
format!("{}.gz", self.expected_remote_stacks_tsv_base_url())
}
pub fn expected_remote_ordinals_sqlite_sha256(&self) -> String {
format!("{}.sha256", self.expected_remote_ordinals_sqlite_base_url())
}
@@ -324,15 +284,6 @@ impl Config {
format!("{}.gz", self.expected_remote_ordinals_sqlite_base_url())
}
pub fn rely_on_remote_stacks_tsv(&self) -> bool {
for source in self.event_sources.iter() {
if let EventSourceConfig::StacksTsvUrl(_config) = source {
return true;
}
}
false
}
pub fn rely_on_remote_ordinals_sqlite(&self) -> bool {
for source in self.event_sources.iter() {
if let EventSourceConfig::OrdinalsSqliteUrl(_config) = source {
@@ -342,20 +293,6 @@ impl Config {
false
}
pub fn should_download_remote_stacks_tsv(&self) -> bool {
let mut rely_on_remote_tsv = false;
let mut remote_tsv_present_locally = false;
for source in self.event_sources.iter() {
if let EventSourceConfig::StacksTsvUrl(_config) = source {
rely_on_remote_tsv = true;
}
if let EventSourceConfig::StacksTsvPath(_config) = source {
remote_tsv_present_locally = true;
}
}
rely_on_remote_tsv == true && remote_tsv_present_locally == false
}
pub fn should_download_remote_ordinals_sqlite(&self) -> bool {
let mut rely_on_remote_tsv = false;
let mut remote_tsv_present_locally = false;
@@ -421,9 +358,7 @@ impl Config {
working_dir: default_cache_path(),
},
http_api: PredicatesApi::Off,
event_sources: vec![EventSourceConfig::StacksTsvUrl(UrlConfig {
file_url: DEFAULT_TESTNET_STACKS_TSV_ARCHIVE.into(),
})],
event_sources: vec![],
limits: LimitsConfig {
max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION,
max_number_of_concurrent_bitcoin_scans: BITCOIN_SCAN_THREAD_POOL_SIZE,
@@ -453,9 +388,6 @@ impl Config {
},
http_api: PredicatesApi::Off,
event_sources: vec![
EventSourceConfig::StacksTsvUrl(UrlConfig {
file_url: DEFAULT_MAINNET_STACKS_TSV_ARCHIVE.into(),
}),
EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
file_url: DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE.into(),
}),

1
components/hord-cli/src/db/mod.rs
View File

@@ -1,6 +1,5 @@
use std::{
collections::{BTreeMap, HashMap},
fs::File,
hash::BuildHasherDefault,
path::PathBuf,
sync::{mpsc::Sender, Arc},