feat: draft sha256 verification (wip)

components/chainhook-cli/src/archive/mod.rs

@@ -1,5 +1,6 @@
 use crate::config::Config;
 use chainhook_types::StacksNetwork;
+use clarinet_files::FileLocation;
 use flate2::read::GzDecoder;
 use futures_util::StreamExt;
 use std::fs;
@@ -10,16 +11,35 @@ pub fn default_tsv_file_path(network: &StacksNetwork) -> String {
     format!("{:?}-stacks-events.tsv", network).to_lowercase()
 }
 
+pub fn default_tsv_sha_file_path(network: &StacksNetwork) -> String {
+    format!("{:?}-stacks-events.sha256", network).to_lowercase()
+}
+
 pub async fn download_tsv_file(config: &Config) -> Result<(), String> {
     let mut destination_path = config.expected_cache_path();
     std::fs::create_dir_all(&destination_path).unwrap_or_else(|e| {
         println!("{}", e.to_string());
     });
 
-    let url = config.expected_remote_tsv_url();
-    let res = reqwest::get(url)
+    let remote_sha_url = config.expected_remote_tsv_sha256();
+    let res = reqwest::get(&remote_sha_url)
         .await
-        .or(Err(format!("Failed to GET from '{}'", &url)))?;
+        .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?
+        .bytes()
+        .await
+        .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?;
+
+    let mut local_sha_file_path = destination_path.clone();
+    local_sha_file_path.push(default_tsv_sha_file_path(&config.network.stacks_network));
+
+    println!("1");
+    let local_sha_file = FileLocation::from_path(local_sha_file_path);
+    let _ = local_sha_file.write_content(&res.to_vec());
+
+    let file_url = config.expected_remote_tsv_url();
+    let res = reqwest::get(&file_url)
+        .await
+        .or(Err(format!("Failed to GET from '{}'", &file_url)))?;
 
     // Download chunks
     let (tx, rx) = flume::bounded(0);

components/chainhook-cli/src/config/mod.rs

@@ -13,9 +13,9 @@ use std::path::PathBuf;
 use crate::service::{DEFAULT_CONTROL_PORT, DEFAULT_INGESTION_PORT};
 
 const DEFAULT_MAINNET_TSV_ARCHIVE: &str =
-    "https://archive.hiro.so/mainnet/stacks-blockchain-api/mainnet-stacks-blockchain-api-latest.gz";
+    "https://archive.hiro.so/mainnet/stacks-blockchain-api/mainnet-stacks-blockchain-api-latest";
 const DEFAULT_TESTNET_TSV_ARCHIVE: &str =
-    "https://archive.hiro.so/testnet/stacks-blockchain-api/testnet-stacks-blockchain-api-latest.gz";
+    "https://archive.hiro.so/testnet/stacks-blockchain-api/testnet-stacks-blockchain-api-latest";
 
 #[derive(Clone, Debug)]
 pub struct Config {
@@ -216,7 +216,7 @@ impl Config {
         destination_path
     }
 
-    pub fn expected_remote_tsv_url(&self) -> &String {
+    fn expected_remote_tsv_base_url(&self) -> &String {
         for source in self.event_sources.iter() {
             if let EventSourceConfig::TsvUrl(config) = source {
                 return &config.file_url;
@@ -225,6 +225,14 @@ impl Config {
         panic!("expected remote-tsv source")
     }
 
+    pub fn expected_remote_tsv_sha256(&self) -> String {
+        format!("{}.sha256", self.expected_remote_tsv_base_url())
+    }
+
+    pub fn expected_remote_tsv_url(&self) -> String {
+        format!("{}.gz", self.expected_remote_tsv_base_url())
+    }
+
     pub fn rely_on_remote_tsv(&self) -> bool {
         for source in self.event_sources.iter() {
             if let EventSourceConfig::TsvUrl(_config) = source {

components/chainhook-cli/src/scan/stacks.rs

@@ -21,6 +21,7 @@ use chainhook_event_observer::{
     utils::{file_append, send_request, AbstractStacksBlock},
 };
 use chainhook_types::BlockIdentifier;
+use clarinet_files::FileLocation;
 
 pub async fn scan_stacks_chainstate_via_csv_using_predicate(
     predicate_spec: &StacksChainhookSpecification,
@@ -201,12 +202,40 @@ async fn download_dataset_if_required(config: &mut Config, ctx: &Context) -> boo
         // Download default tsv.
         if config.rely_on_remote_tsv() && config.should_download_remote_tsv() {
             let url = config.expected_remote_tsv_url();
-            let mut destination_path = config.expected_cache_path();
-            destination_path.push(archive::default_tsv_file_path(
+            let mut tsv_file_path = config.expected_cache_path();
+            tsv_file_path.push(archive::default_tsv_file_path(
                 &config.network.stacks_network,
             ));
+            let mut tsv_sha_file_path = config.expected_cache_path();
+            tsv_sha_file_path.push(archive::default_tsv_sha_file_path(
+                &config.network.stacks_network,
+            ));
+
             // Download archive if not already present in cache
-            if !destination_path.exists() {
+            // Load the local
+            let local_sha_file = FileLocation::from_path(tsv_sha_file_path).read_content();
+            let sha_url = config.expected_remote_tsv_sha256();
+
+            let remote_sha_file = match reqwest::get(&sha_url).await {
+                Ok(response) => response.bytes().await,
+                Err(e) => Err(e),
+            };
+            match (local_sha_file, remote_sha_file) {
+                (Ok(local), Ok(remote_response)) => {
+                    println!("{:?}", local);
+                    println!("{:?}", remote_response);
+                }
+                (Ok(local), _) => {
+                    // println!("Local: {:?}", local)
+                    println!("Here 2");
+                }
+                (_, _) => {
+                    // We will download the latest file
+                    println!("error reading local / remote");
+                }
+            }
+
+            if !tsv_file_path.exists() {
                 info!(ctx.expect_logger(), "Downloading {}", url);
                 match archive::download_tsv_file(&config).await {
                     Ok(_) => {}
@@ -219,10 +248,10 @@ async fn download_dataset_if_required(config: &mut Config, ctx: &Context) -> boo
                 info!(
                     ctx.expect_logger(),
                     "Building in-memory chainstate from file {}",
-                    destination_path.display()
+                    tsv_file_path.display()
                 );
             }
-            config.add_local_tsv_source(&destination_path);
+            config.add_local_tsv_source(&tsv_file_path);
         }
         true
     } else {