feat(runes): add runes indexer (#453)

* rename ordhook-cli to cli

* rename ordhook-cli

* update configs

* update config to future support runehook integration

* new code

* add ci

* files

* standard

* add files

* rename binary to bitcoin-indexer and commands to ordinals from ordhook

* config component

* integration

* runes build

* fix runes

* indexer toml

* add runes tests to ci

* rename dockerfile

* fix: doctest

---------

Co-authored-by: ASuciuX <asuciu@hiro.so>

components/config/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "config"
+version.workspace = true
+edition = "2021"
+
+[dependencies]
+bitcoin = { workspace = true }
+serde = "1"
+serde_json = "1"
+serde_derive = "1"
+hiro-system-kit = { workspace = true }
+num_cpus = "1.16.0"
+toml = { version = "0.5.11", features = ["preserve_order"] }

components/config/src/config.rs

@@ -0,0 +1,216 @@
+use std::path::PathBuf;
+
+use bitcoin::Network;
+
+use crate::toml::ConfigToml;
+
+pub const DEFAULT_WORKING_DIR: &str = "data";
+pub const DEFAULT_ULIMIT: usize = 2048;
+pub const DEFAULT_MEMORY_AVAILABLE: usize = 8;
+pub const DEFAULT_BITCOIND_RPC_THREADS: usize = 4;
+pub const DEFAULT_BITCOIND_RPC_TIMEOUT: u32 = 15;
+pub const DEFAULT_LRU_CACHE_SIZE: usize = 50_000;
+
+#[derive(Clone, Debug)]
+pub struct Config {
+    pub bitcoind: BitcoindConfig,
+    pub ordinals: Option<OrdinalsConfig>,
+    pub runes: Option<RunesConfig>,
+    pub resources: ResourcesConfig,
+    pub storage: StorageConfig,
+    pub metrics: Option<MetricsConfig>,
+}
+
+#[derive(Clone, Debug)]
+pub struct OrdinalsConfig {
+    pub db: PgDatabaseConfig,
+    pub meta_protocols: Option<OrdinalsMetaProtocolsConfig>,
+}
+
+#[derive(Clone, Debug)]
+pub struct OrdinalsMetaProtocolsConfig {
+    pub brc20: Option<OrdinalsBrc20Config>,
+}
+
+#[derive(Clone, Debug)]
+pub struct OrdinalsBrc20Config {
+    pub enabled: bool,
+    pub lru_cache_size: usize,
+    pub db: PgDatabaseConfig,
+}
+
+#[derive(Clone, Debug)]
+pub struct RunesConfig {
+    pub lru_cache_size: usize,
+    pub db: PgDatabaseConfig,
+}
+
+#[derive(Clone, Debug)]
+pub struct BitcoindConfig {
+    pub network: Network,
+    pub rpc_url: String,
+    pub rpc_username: String,
+    pub rpc_password: String,
+    pub zmq_url: String,
+}
+
+/// A Postgres configuration for a single database.
+#[derive(Clone, Debug)]
+pub struct PgDatabaseConfig {
+    pub dbname: String,
+    pub host: String,
+    pub port: u16,
+    pub user: String,
+    pub password: Option<String>,
+    pub search_path: Option<String>,
+    pub pool_max_size: Option<usize>,
+}
+
+#[derive(Clone, Debug)]
+pub struct StorageConfig {
+    pub working_dir: String,
+}
+
+#[derive(Clone, Debug)]
+pub struct MetricsConfig {
+    pub enabled: bool,
+    pub prometheus_port: u16,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct ResourcesConfig {
+    pub ulimit: usize,
+    pub cpu_core_available: usize,
+    pub memory_available: usize,
+    pub bitcoind_rpc_threads: usize,
+    pub bitcoind_rpc_timeout: u32,
+}
+
+impl ResourcesConfig {
+    pub fn get_optimal_thread_pool_capacity(&self) -> usize {
+        // Generally speaking when dealing a pool, we need one thread for
+        // feeding the thread pool and eventually another thread for
+        // handling the "reduce" step.
+        self.cpu_core_available.saturating_sub(2).max(1)
+    }
+}
+
+impl Config {
+    pub fn from_file_path(file_path: &str) -> Result<Config, String> {
+        ConfigToml::config_from_file_path(file_path)
+    }
+
+    pub fn expected_cache_path(&self) -> PathBuf {
+        let mut destination_path = PathBuf::new();
+        destination_path.push(&self.storage.working_dir);
+        destination_path
+    }
+
+    pub fn devnet_default() -> Config {
+        Config {
+            storage: StorageConfig {
+                working_dir: default_cache_path(),
+            },
+            resources: ResourcesConfig {
+                cpu_core_available: num_cpus::get(),
+                memory_available: DEFAULT_MEMORY_AVAILABLE,
+                ulimit: DEFAULT_ULIMIT,
+                bitcoind_rpc_threads: DEFAULT_BITCOIND_RPC_THREADS,
+                bitcoind_rpc_timeout: DEFAULT_BITCOIND_RPC_TIMEOUT,
+            },
+            bitcoind: BitcoindConfig {
+                rpc_url: "http://0.0.0.0:18443".into(),
+                rpc_username: "devnet".into(),
+                rpc_password: "devnet".into(),
+                network: Network::Regtest,
+                zmq_url: "http://0.0.0.0:18543".into(),
+            },
+            ordinals: Some(OrdinalsConfig {
+                db: PgDatabaseConfig {
+                    dbname: "ordinals".to_string(),
+                    host: "localhost".to_string(),
+                    port: 5432,
+                    user: "postgres".to_string(),
+                    password: Some("postgres".to_string()),
+                    search_path: None,
+                    pool_max_size: None,
+                },
+                meta_protocols: None,
+            }),
+            runes: Some(RunesConfig {
+                lru_cache_size: DEFAULT_LRU_CACHE_SIZE,
+                db: PgDatabaseConfig {
+                    dbname: "runes".to_string(),
+                    host: "localhost".to_string(),
+                    port: 5432,
+                    user: "postgres".to_string(),
+                    password: Some("postgres".to_string()),
+                    search_path: None,
+                    pool_max_size: None,
+                },
+            }),
+            metrics: Some(MetricsConfig {
+                enabled: true,
+                prometheus_port: 9153,
+            }),
+        }
+    }
+
+    pub fn testnet_default() -> Config {
+        let mut default = Config::devnet_default();
+        default.bitcoind.network = Network::Testnet;
+        default
+    }
+
+    pub fn mainnet_default() -> Config {
+        let mut default = Config::devnet_default();
+        default.bitcoind.rpc_url = "http://localhost:8332".into();
+        default.bitcoind.network = Network::Bitcoin;
+        default
+    }
+
+    // TODO: Move this to a shared test utils component
+    pub fn test_default() -> Config {
+        let mut config = Self::mainnet_default();
+        config.storage.working_dir = "tmp".to_string();
+        config.resources.bitcoind_rpc_threads = 1;
+        config.resources.cpu_core_available = 1;
+        config
+    }
+
+    pub fn ordinals_brc20_config(&self) -> Option<&OrdinalsBrc20Config> {
+        if let Some(OrdinalsConfig {
+            meta_protocols:
+                Some(OrdinalsMetaProtocolsConfig {
+                    brc20: Some(brc20), ..
+                }),
+            ..
+        }) = &self.ordinals
+        {
+            if brc20.enabled {
+                return Some(brc20);
+            }
+        }
+        None
+    }
+
+    pub fn assert_ordinals_config(&self) -> Result<(), String> {
+        if self.ordinals.is_none() {
+            return Err(format!("Config entry for `ordinals` not found in config file."));
+        }
+        Ok(())
+    }
+
+    pub fn assert_runes_config(&self) -> Result<(), String> {
+        if self.runes.is_none() {
+            return Err(format!("Config entry for `runes` not found in config file."));
+        }
+        Ok(())
+    }
+}
+
+pub fn default_cache_path() -> String {
+    let mut cache_path = std::env::current_dir().expect("unable to get current dir");
+    cache_path.push("data");
+    format!("{}", cache_path.display())
+}

components/config/src/generator.rs

@@ -0,0 +1,55 @@
+pub fn generate_toml_config(network: &str) -> String {
+    let conf = format!(
+        r#"[storage]
+working_dir = "tmp"
+
+[metrics]
+enabled = true
+prometheus_port = 9153
+
+[ordinals.db]
+database = "ordinals"
+host = "localhost"
+port = 5432
+username = "postgres"
+password = "postgres"
+
+[ordinals.meta_protocols.brc20]
+enabled = true
+lru_cache_size = 10000
+
+[ordinals.meta_protocols.brc20.db]
+database = "brc20"
+host = "localhost"
+port = 5432
+username = "postgres"
+password = "postgres"
+
+[runes]
+lru_cache_size = 10000
+
+[runes.db]
+database = "runes"
+host = "localhost"
+port = 5432
+username = "postgres"
+password = "postgres"
+
+[bitcoind]
+network = "{network}"
+rpc_url = "http://localhost:8332"
+rpc_username = "devnet"
+rpc_password = "devnet"
+zmq_url = "tcp://0.0.0.0:18543"
+
+[resources]
+ulimit = 2048
+cpu_core_available = 6
+memory_available = 16
+bitcoind_rpc_threads = 2
+bitcoind_rpc_timeout = 15
+"#,
+        network = network.to_lowercase(),
+    );
+    conf
+}

components/config/src/lib.rs

@@ -0,0 +1,8 @@
+#[macro_use]
+extern crate serde_derive;
+
+pub mod toml;
+pub mod generator;
+
+mod config;
+pub use config::*;

components/config/src/toml.rs

@@ -0,0 +1,199 @@
+use std::fs::File;
+use std::io::{BufReader, Read};
+
+use bitcoin::Network;
+
+use crate::{
+    BitcoindConfig, Config, MetricsConfig, OrdinalsBrc20Config, OrdinalsConfig,
+    OrdinalsMetaProtocolsConfig, PgDatabaseConfig, ResourcesConfig, RunesConfig, StorageConfig,
+    DEFAULT_BITCOIND_RPC_THREADS, DEFAULT_BITCOIND_RPC_TIMEOUT, DEFAULT_LRU_CACHE_SIZE,
+    DEFAULT_MEMORY_AVAILABLE, DEFAULT_ULIMIT, DEFAULT_WORKING_DIR,
+};
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct PgDatabaseConfigToml {
+    pub database: String,
+    pub host: String,
+    pub port: u16,
+    pub username: String,
+    pub password: Option<String>,
+    pub search_path: Option<String>,
+    pub pool_max_size: Option<usize>,
+}
+
+impl PgDatabaseConfigToml {
+    fn to_config(self) -> PgDatabaseConfig {
+        PgDatabaseConfig {
+            dbname: self.database,
+            host: self.host,
+            port: self.port,
+            user: self.username,
+            password: self.password,
+            search_path: self.search_path,
+            pool_max_size: self.pool_max_size,
+        }
+    }
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct OrdinalsConfigToml {
+    pub db: PgDatabaseConfigToml,
+    pub meta_protocols: Option<OrdinalsMetaProtocolsConfigToml>,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct OrdinalsMetaProtocolsConfigToml {
+    pub brc20: Option<OrdinalsBrc20ConfigToml>,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct OrdinalsBrc20ConfigToml {
+    pub enabled: bool,
+    pub lru_cache_size: Option<usize>,
+    pub db: PgDatabaseConfigToml,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct RunesConfigToml {
+    pub lru_cache_size: Option<usize>,
+    pub db: PgDatabaseConfigToml,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct StorageConfigToml {
+    pub working_dir: Option<String>,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct ResourcesConfigToml {
+    pub ulimit: Option<usize>,
+    pub cpu_core_available: Option<usize>,
+    pub memory_available: Option<usize>,
+    pub bitcoind_rpc_threads: Option<usize>,
+    pub bitcoind_rpc_timeout: Option<u32>,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct BitcoindConfigToml {
+    pub network: String,
+    pub rpc_url: String,
+    pub rpc_username: String,
+    pub rpc_password: String,
+    pub zmq_url: String,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct MetricsConfigToml {
+    pub enabled: bool,
+    pub prometheus_port: u16,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct ConfigToml {
+    pub storage: StorageConfigToml,
+    pub ordinals: Option<OrdinalsConfigToml>,
+    pub runes: Option<RunesConfigToml>,
+    pub bitcoind: BitcoindConfigToml,
+    pub resources: ResourcesConfigToml,
+    pub metrics: Option<MetricsConfigToml>,
+}
+
+impl ConfigToml {
+    pub fn config_from_file_path(file_path: &str) -> Result<Config, String> {
+        let file = File::open(file_path)
+            .map_err(|e| format!("unable to read file {}\n{:?}", file_path, e))?;
+        let mut file_reader = BufReader::new(file);
+        let mut file_buffer = vec![];
+        file_reader
+            .read_to_end(&mut file_buffer)
+            .map_err(|e| format!("unable to read file {}\n{:?}", file_path, e))?;
+
+        let config_file: ConfigToml = match toml::from_slice(&file_buffer) {
+            Ok(s) => s,
+            Err(e) => {
+                return Err(format!("Config file malformatted {}", e));
+            }
+        };
+        ConfigToml::config_from_toml(config_file)
+    }
+
+    fn config_from_toml(toml: ConfigToml) -> Result<Config, String> {
+        let bitcoin_network = match toml.bitcoind.network.as_str() {
+            "devnet" => Network::Regtest,
+            "testnet" => Network::Testnet,
+            "mainnet" => Network::Bitcoin,
+            "signet" => Network::Signet,
+            _ => return Err("bitcoind.network not supported".to_string()),
+        };
+        let ordinals = match toml.ordinals {
+            Some(ordinals) => Some(OrdinalsConfig {
+                db: ordinals.db.to_config(),
+                meta_protocols: match ordinals.meta_protocols {
+                    Some(meta_protocols) => Some(OrdinalsMetaProtocolsConfig {
+                        brc20: match meta_protocols.brc20 {
+                            Some(brc20) => Some(OrdinalsBrc20Config {
+                                enabled: brc20.enabled,
+                                lru_cache_size: brc20
+                                    .lru_cache_size
+                                    .unwrap_or(DEFAULT_LRU_CACHE_SIZE),
+                                db: brc20.db.to_config(),
+                            }),
+                            None => None,
+                        },
+                    }),
+                    None => None,
+                },
+            }),
+            None => None,
+        };
+        let runes = match toml.runes {
+            Some(runes) => Some(RunesConfig {
+                lru_cache_size: runes.lru_cache_size.unwrap_or(DEFAULT_LRU_CACHE_SIZE),
+                db: runes.db.to_config(),
+            }),
+            None => None,
+        };
+        let metrics = match toml.metrics {
+            Some(metrics) => Some(MetricsConfig {
+                enabled: metrics.enabled,
+                prometheus_port: metrics.prometheus_port,
+            }),
+            None => None,
+        };
+        let config = Config {
+            storage: StorageConfig {
+                working_dir: toml
+                    .storage
+                    .working_dir
+                    .unwrap_or(DEFAULT_WORKING_DIR.into()),
+            },
+            ordinals,
+            runes,
+            resources: ResourcesConfig {
+                ulimit: toml.resources.ulimit.unwrap_or(DEFAULT_ULIMIT),
+                cpu_core_available: toml.resources.cpu_core_available.unwrap_or(num_cpus::get()),
+                memory_available: toml
+                    .resources
+                    .memory_available
+                    .unwrap_or(DEFAULT_MEMORY_AVAILABLE),
+                bitcoind_rpc_threads: toml
+                    .resources
+                    .bitcoind_rpc_threads
+                    .unwrap_or(DEFAULT_BITCOIND_RPC_THREADS),
+                bitcoind_rpc_timeout: toml
+                    .resources
+                    .bitcoind_rpc_timeout
+                    .unwrap_or(DEFAULT_BITCOIND_RPC_TIMEOUT),
+            },
+            bitcoind: BitcoindConfig {
+                rpc_url: toml.bitcoind.rpc_url.to_string(),
+                rpc_username: toml.bitcoind.rpc_username.to_string(),
+                rpc_password: toml.bitcoind.rpc_password.to_string(),
+                network: bitcoin_network,
+                zmq_url: toml.bitcoind.zmq_url,
+            },
+            metrics,
+        };
+        Ok(config)
+    }
+}