kylefang/bitcoin-indexer
1
0
Fork 0
mirror of https://github.com/alexgo-io/bitcoin-indexer.git synced 2026-01-12 16:52:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: revisit caching strategy

Browse Source
This commit is contained in:
Ludo Galabru
2023-06-05 10:45:11 -04:00
parent e794542072
commit 2705b9501b
2 changed files with 20 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
components/chainhook-event-observer/src/hord/db/mod.rs
View File

@@ -714,7 +714,7 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
let mut cursor = start_block as usize;
let mut inbox = HashMap::new();
let mut num_writes = 0;
let traversals_cache = Arc::new(new_traversals_lazy_cache());
let traversals_cache = Arc::new(new_traversals_lazy_cache(hord_config.cache_size));
while let Ok(Some((block_height, compacted_block, raw_block))) = block_compressed_rx.recv() {
insert_entry_in_blocks(block_height, &compacted_block, &blocks_db_rw, &ctx);
@@ -792,18 +792,20 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
return Ok(());
}
if num_writes % 24 == 0 {
ctx.try_log(|logger| {
slog::info!(
logger,
"Flushing traversals cache (#{} entries)",
traversals_cache.len()
);
});
traversals_cache.clear();
if !traversals_cache.is_empty() {
if num_writes % 128 == 0 {
ctx.try_log(|logger| {
slog::info!(
logger,
"Flushing traversals cache (#{} entries)",
traversals_cache.len()
);
});
traversals_cache.shrink_to_fit();
}
}
if num_writes % 128 == 0 {
if num_writes % 512 == 0 {
ctx.try_log(|logger| {
slog::info!(logger, "Flushing DB to disk ({num_writes} inserts)");
});

8
components/chainhook-event-observer/src/hord/mod.rs
View File

@@ -18,6 +18,7 @@ use rocksdb::DB;
use rusqlite::Connection;
use std::collections::{BTreeMap, HashMap, VecDeque};
use std::hash::BuildHasherDefault;
use std::ops::Div;
use std::path::PathBuf;
use std::sync::mpsc::channel;
use std::sync::Arc;
@@ -208,9 +209,14 @@ pub fn new_traversals_cache(
}
pub fn new_traversals_lazy_cache(
cache_size: usize,
) -> DashMap<(u32, [u8; 8]), LazyBlockTransaction, BuildHasherDefault<FxHasher>> {
let hasher = FxBuildHasher::default();
DashMap::with_hasher(hasher)
DashMap::with_capacity_and_hasher(
((cache_size.saturating_sub(500)) * 1000 * 1000)
.div(LazyBlockTransaction::get_average_bytes_size()),
hasher,
)
}
pub fn retrieve_inscribed_satoshi_points_from_block(