From bc313fad5c4d4af23b4f34bf27b7ea5048180c98 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Sun, 4 Jun 2023 23:55:18 -0400 Subject: [PATCH] feat: move thread pool size to config --- components/chainhook-cli/src/block/mod.rs | 2 +- components/chainhook-cli/src/config/file.rs | 6 ++- .../chainhook-cli/src/config/generator.rs | 8 ++-- components/chainhook-cli/src/config/mod.rs | 46 ++++++++++++++----- 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/components/chainhook-cli/src/block/mod.rs b/components/chainhook-cli/src/block/mod.rs index 16e504d..c0e9c27 100644 --- a/components/chainhook-cli/src/block/mod.rs +++ b/components/chainhook-cli/src/block/mod.rs @@ -15,7 +15,7 @@ pub struct Record { pub id: u64, pub created_at: String, pub kind: RecordKind, - pub raw_log: String, + pub blob: Option, } #[derive(Debug, Deserialize)] diff --git a/components/chainhook-cli/src/config/file.rs b/components/chainhook-cli/src/config/file.rs index 076e764..b447760 100644 --- a/components/chainhook-cli/src/config/file.rs +++ b/components/chainhook-cli/src/config/file.rs @@ -25,8 +25,10 @@ pub struct EventSourceConfigFile { #[derive(Deserialize, Debug, Clone)] pub struct ChainhooksConfigFile { - pub max_stacks_registrations: Option, - pub max_bitcoin_registrations: Option, + pub max_stacks_registrations: Option, + pub max_bitcoin_registrations: Option, + pub max_stacks_concurrent_scans: Option, + pub max_bitcoin_concurrent_scans: Option, } #[derive(Deserialize, Debug, Clone)] diff --git a/components/chainhook-cli/src/config/generator.rs b/components/chainhook-cli/src/config/generator.rs index 04b06eb..935f86d 100644 --- a/components/chainhook-cli/src/config/generator.rs +++ b/components/chainhook-cli/src/config/generator.rs @@ -6,8 +6,10 @@ redis_uri = "redis://localhost:6379/" cache_path = "cache" [chainhooks] -max_stacks_registrations = 500 -max_bitcoin_registrations = 500 +max_stacks_registrations = 100 +max_bitcoin_registrations = 100 +max_stacks_concurrent_scans = 10 +max_bitcoin_concurrent_scans = 10 [network] mode = "mainnet" @@ -17,7 +19,7 @@ bitcoind_rpc_password = "devnet" stacks_node_rpc_url = "http://localhost:20443" [[event_source]] -tsv_file_url = "https://archive.hiro.so/mainnet/stacks-blockchain-api/mainnet-stacks-blockchain-api-latest.gz" +tsv_file_url = "https://archive.hiro.so/mainnet/stacks-blockchain-api/mainnet-stacks-blockchain-api-latest" "# ); return conf; diff --git a/components/chainhook-cli/src/config/mod.rs b/components/chainhook-cli/src/config/mod.rs index 96867d5..a97191d 100644 --- a/components/chainhook-cli/src/config/mod.rs +++ b/components/chainhook-cli/src/config/mod.rs @@ -10,8 +10,6 @@ use std::fs::File; use std::io::{BufReader, Read}; use std::path::PathBuf; -use crate::service::{DEFAULT_CONTROL_PORT, DEFAULT_INGESTION_PORT}; - const DEFAULT_MAINNET_STACKS_TSV_ARCHIVE: &str = "https://archive.hiro.so/mainnet/stacks-blockchain-api/mainnet-stacks-blockchain-api-latest"; const DEFAULT_TESTNET_STACKS_TSV_ARCHIVE: &str = @@ -19,6 +17,13 @@ const DEFAULT_TESTNET_STACKS_TSV_ARCHIVE: &str = const DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE: &str = "https://archive.hiro.so/mainnet/chainhooks/hord-latest.sqlite"; +pub const DEFAULT_INGESTION_PORT: u16 = 20455; +pub const DEFAULT_CONTROL_PORT: u16 = 20456; +pub const STACKS_SCAN_THREAD_POOL_SIZE: usize = 10; +pub const BITCOIN_SCAN_THREAD_POOL_SIZE: usize = 10; +pub const STACKS_MAX_PREDICATE_REGISTRATION: usize = 50; +pub const BITCOIN_MAX_PREDICATE_REGISTRATION: usize = 50; + #[derive(Clone, Debug)] pub struct Config { pub storage: StorageConfig, @@ -70,8 +75,10 @@ pub struct UrlConfig { #[derive(Clone, Debug)] pub struct ChainhooksConfig { - pub max_stacks_registrations: u16, - pub max_bitcoin_registrations: u16, + pub max_stacks_registrations: usize, + pub max_bitcoin_registrations: usize, + pub max_stacks_concurrent_scans: usize, + pub max_bitcoin_concurrent_scans: usize, pub enable_http_api: bool, } @@ -151,11 +158,20 @@ impl Config { max_stacks_registrations: config_file .chainhooks .max_stacks_registrations - .unwrap_or(100), + .unwrap_or(STACKS_MAX_PREDICATE_REGISTRATION), max_bitcoin_registrations: config_file .chainhooks .max_bitcoin_registrations - .unwrap_or(100), + .unwrap_or(BITCOIN_MAX_PREDICATE_REGISTRATION), + max_stacks_concurrent_scans: config_file + .chainhooks + .max_stacks_registrations + .unwrap_or(STACKS_SCAN_THREAD_POOL_SIZE), + max_bitcoin_concurrent_scans: config_file + .chainhooks + .max_bitcoin_registrations + .unwrap_or(BITCOIN_SCAN_THREAD_POOL_SIZE), + enable_http_api: true, }, network: IndexerConfig { @@ -344,8 +360,10 @@ impl Config { }, event_sources: vec![], chainhooks: ChainhooksConfig { - max_stacks_registrations: 50, - max_bitcoin_registrations: 50, + max_stacks_registrations: STACKS_MAX_PREDICATE_REGISTRATION, + max_bitcoin_registrations: BITCOIN_MAX_PREDICATE_REGISTRATION, + max_stacks_concurrent_scans: STACKS_SCAN_THREAD_POOL_SIZE, + max_bitcoin_concurrent_scans: BITCOIN_SCAN_THREAD_POOL_SIZE, enable_http_api: true, }, network: IndexerConfig { @@ -374,8 +392,10 @@ impl Config { file_url: DEFAULT_TESTNET_STACKS_TSV_ARCHIVE.into(), })], chainhooks: ChainhooksConfig { - max_stacks_registrations: 10, - max_bitcoin_registrations: 10, + max_stacks_registrations: STACKS_MAX_PREDICATE_REGISTRATION, + max_bitcoin_registrations: BITCOIN_MAX_PREDICATE_REGISTRATION, + max_stacks_concurrent_scans: STACKS_SCAN_THREAD_POOL_SIZE, + max_bitcoin_concurrent_scans: BITCOIN_SCAN_THREAD_POOL_SIZE, enable_http_api: true, }, network: IndexerConfig { @@ -409,8 +429,10 @@ impl Config { }), ], chainhooks: ChainhooksConfig { - max_stacks_registrations: 10, - max_bitcoin_registrations: 10, + max_stacks_registrations: STACKS_MAX_PREDICATE_REGISTRATION, + max_bitcoin_registrations: BITCOIN_MAX_PREDICATE_REGISTRATION, + max_stacks_concurrent_scans: STACKS_SCAN_THREAD_POOL_SIZE, + max_bitcoin_concurrent_scans: BITCOIN_SCAN_THREAD_POOL_SIZE, enable_http_api: true, }, network: IndexerConfig {