Port burnchain-neon-controller

.cargo/config

@@ -1,2 +1,3 @@
 [alias]
-testnet = "run --package stacks-testnet --"
+testnet = "run --package stacks-node --"
+bitcoin-neon-controller = "run --package bitcoin-neon-controller --"

Cargo.toml

@@ -88,4 +88,7 @@ default = ["developer-mode"]
 sha2-asm = "0.5.3"
 
 [workspace]
-members = [".", "testnet/"]
+members = [
+    ".", 
+    "testnet/stacks-node",
+    "testnet/bitcoin-neon-controller"]

testnet/bitcoin-neon-controller/.dockerignore

@@ -0,0 +1,2 @@
+Dockerfile
+target

testnet/bitcoin-neon-controller/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "bitcoin-neon-controller"
+version = "0.1.0"
+authors = ["Ludo Galabru <ludovic@blockstack.com>"]
+edition = "2018"
+
+[dependencies]
+async-h1 = "1.0.2"
+async-std = { version = "1.4.0", features = ["attributes"] }
+base64 = "0.12.0"
+http-types = "1.0.0"
+serde = "1"
+serde_derive = "1"
+serde_json = { version = "1.0", features = ["arbitrary_precision"] }
+toml = "0.5"

testnet/bitcoin-neon-controller/Dockerfile

@@ -0,0 +1,24 @@
+FROM rust as builder
+WORKDIR /src/bitcoin-neon-controller
+COPY . .
+RUN cd /src/bitcoin-neon-controller && cargo build --target x86_64-unknown-linux-gnu --release
+RUN cargo install --target x86_64-unknown-linux-gnu --path /src/bitcoin-neon-controller
+
+FROM alpine
+ARG GLIBC_VERSION="2.31-r0"
+ARG GLIBC_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk"
+ARG GLIBC_BIN_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-bin-${GLIBC_VERSION}.apk"
+ARG GLIBC_I18N_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-i18n-${GLIBC_VERSION}.apk"
+WORKDIR /
+RUN apk --no-cache add --update ca-certificates curl libgcc \
+    && curl -L -s -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \
+    && curl -L -s -o /tmp/glibc-${GLIBC_VERSION}.apk ${GLIBC_URL} \
+    && curl -L -s -o /tmp/glibc-bin-${GLIBC_VERSION}.apk ${GLIBC_BIN_URL} \
+    && curl -L -s -o /tmp/glibc-i18n-${GLIBC_VERSION}.apk ${GLIBC_I18N_URL} \
+    && apk --no-cache add /tmp/glibc-${GLIBC_VERSION}.apk /tmp/glibc-bin-${GLIBC_VERSION}.apk /tmp/glibc-i18n-${GLIBC_VERSION}.apk \
+    && /usr/glibc-compat/sbin/ldconfig /usr/lib /lib \
+    && rm /tmp/glibc-${GLIBC_VERSION}.apk /tmp/glibc-bin-${GLIBC_VERSION}.apk /tmp/glibc-i18n-${GLIBC_VERSION}.apk
+COPY --from=builder /usr/local/cargo/bin/bitcoin-neon-controller /usr/local/bin/bitcoin-neon-controller
+COPY config.toml.default /etc/bitcoin-neon-controller/Config.toml
+EXPOSE 3000
+CMD ["/usr/local/bin/bitcoin-neon-controller /etc/bitcoin-neon-controller/Config.toml"]

testnet/bitcoin-neon-controller/README.md

@@ -0,0 +1,6 @@
+# Neon master node
+
+Neon master node is responsible for:
+- forwarding authorized RPC calls (such as `sendrawtransaction`, `importpubkey` and `listunspent`) to a centralized bitcoind chain, running in regtest mode.
+- mining bitcoin blocks (every 7 secs)
+- seed BTC faucet.

testnet/bitcoin-neon-controller/config.toml.default

@@ -0,0 +1,7 @@
+[neon]
+rpc_bind = "0.0.0.0:18443"
+block_time = 7000
+miner_address = "mx2uds6sgnn9znABQ6iDSSmXY9K5D4SHF9"
+bitcoind_rpc_host = "127.0.0.1:18443"
+bitcoind_rpc_user = "helium-node"
+bitcoind_rpc_pass = "secret"

testnet/bitcoin-neon-controller/local-leader.toml.default

@@ -0,0 +1,7 @@
+[neon]
+rpc_bind = "0.0.0.0:28443"
+block_time = 7000
+miner_address = "mtFzK54XtpktHj7fKonFExEPEGkUMsiXdy"
+bitcoind_rpc_host = "127.0.0.1:18443"
+bitcoind_rpc_user = "helium-node"
+bitcoind_rpc_pass = "secret"

testnet/bitcoin-neon-controller/src/main.rs

@@ -0,0 +1,287 @@
+#[macro_use] extern crate serde_derive;
+
+use std::io::{BufReader, Read};
+use std::fs::File;
+use std::env;
+use std::thread;
+use std::time::Duration;
+
+use async_h1::{client};
+use async_std::net::{TcpListener, TcpStream};
+use async_std::prelude::*;
+use async_std::task;
+use base64::{encode};
+use http_types::{
+    Response, 
+    StatusCode, 
+    Method, 
+    headers,
+    Request, 
+    Url
+};
+use serde::Deserialize;
+use serde_json::Deserializer;
+use toml;
+
+#[async_std::main]
+async fn main() -> http_types::Result<()> {
+    let argv : Vec<String> = env::args().collect();
+
+    // Guard: config missing
+    if argv.len() != 2 {
+        panic!("Config argument missing");
+    }
+
+    let config = ConfigFile::from_path(&argv[1]);
+
+    if is_bootstrap_chain_required(&config).await? {
+        println!("Bootstrapping chain");
+        generate_blocks(200, &config).await;
+    }
+
+    // Start a loop in a separate thread, generating new blocks
+    // on a given frequence (coming from config).
+    let conf = config.clone();
+    thread::spawn(move || {
+        let block_time = Duration::from_millis(conf.neon.block_time);
+    
+        loop {
+            println!("Generating block");
+            async_std::task::block_on(async {
+                generate_blocks(1, &conf).await;
+            });
+            thread::sleep(block_time);
+        }
+    });
+
+    // Open up a TCP connection and create a URL.
+    let bind_addr = config.neon.rpc_bind.clone();
+    let listener = TcpListener::bind(bind_addr).await?;
+    let addr = format!("http://{}", listener.local_addr()?);
+    println!("Listening on {}", addr);
+
+    // For each incoming TCP connection, spawn a task and call `accept`.
+    let mut incoming = listener.incoming();
+    while let Some(stream) = incoming.next().await {
+        let stream = stream?;
+        let addr = addr.clone();
+        let config = config.clone();
+
+        task::spawn(async move {
+            if let Err(err) = accept(addr, stream, &config).await {
+                eprintln!("{}", err);
+            }
+        });
+    }
+    Ok(())
+}
+
+// Take a TCP stream, and convert it into sequential HTTP request / response pairs.
+async fn accept(addr: String, stream: TcpStream, config: &ConfigFile) -> http_types::Result<()> {
+    println!("starting new connection from {}", stream.peer_addr()?);
+    async_h1::accept(&addr, stream.clone(), |mut req| async {
+        match (req.method(), req.url().path(), req.header(&headers::CONTENT_TYPE)) {
+            (Method::Get, "/ping", Some(_content_type)) => {
+                Ok(Response::new(StatusCode::Ok))
+            },
+            (Method::Post, "/", Some(_content_types)) => {
+                
+                let (res, buffer) = async_std::task::block_on(async move {
+                    let mut buffer = Vec::new();
+                    let mut body = req.take_body();
+                    let res = body.read_to_end(&mut buffer).await;
+                    (res, buffer)
+                });
+
+                // Guard: can't be read 
+                if res.is_err() {
+                    return Ok(Response::new(StatusCode::MethodNotAllowed))
+                }
+
+                let mut deserializer = Deserializer::from_slice(&buffer);
+
+                // Guard: can't be parsed
+                let rpc_req: RPCRequest = match RPCRequest::deserialize(&mut deserializer) {
+                    Ok(rpc_req) => rpc_req,
+                    _ => return Ok(Response::new(StatusCode::MethodNotAllowed))
+                };
+
+                println!("{:?}", rpc_req);
+
+                let authorized_methods = vec![
+                    "listunspent",
+                    "importaddress",
+                    "sendrawtransaction"];
+                
+                // Guard: unauthorized method
+                if !authorized_methods.contains(&rpc_req.method.as_str()) {
+                    return Ok(Response::new(StatusCode::MethodNotAllowed))
+                }
+
+                // Forward the request
+                let stream = TcpStream::connect(config.neon.bitcoind_rpc_host.clone()).await?;
+                let body = serde_json::to_vec(&rpc_req).unwrap();
+                let req = build_request(&config, body);
+                client::connect(stream.clone(), req).await
+            },
+            _ => {
+                Ok(Response::new(StatusCode::MethodNotAllowed))
+            }
+        }
+    }).await?;
+
+    Ok(())
+}
+
+async fn is_bootstrap_chain_required(config: &ConfigFile) -> http_types::Result<bool> {
+
+    let req = RPCRequest::is_chain_bootstrapped();
+    let stream = TcpStream::connect(config.neon.bitcoind_rpc_host.clone()).await?;
+    let body = serde_json::to_vec(&req).unwrap();
+    let mut resp = client::connect(stream.clone(), build_request(&config, body)).await?;
+
+    let (res, buffer) = async_std::task::block_on(async move {
+        let mut buffer = Vec::new();
+        let mut body = resp.take_body();
+        let res = body.read_to_end(&mut buffer).await;
+        (res, buffer)
+    });
+
+    // Guard: can't be read 
+    if res.is_err() {
+        panic!("Chain height could not be determined")
+    }
+    // let mut deserializer = Deserializer::from_slice(&buffer);
+
+    let mut deserializer = Deserializer::from_slice(&buffer);
+
+    // Guard: can't be parsed
+    let rpc_resp: RPCResult = match RPCResult::deserialize(&mut deserializer) {
+        Ok(rpc_req) => rpc_req,
+        _ => panic!("Chain height could not be determined")
+    };
+
+    match (rpc_resp.result, rpc_resp.error) {
+        (Some(_), None) => return Ok(false),
+        (None, Some(error)) => {
+            if let Some(keys) = error.as_object() {
+                if let Some(message) = keys.get("message") {
+                    if let Some(message) = message.as_str() {
+                        if message == "Block height out of range" {
+                            return Ok(true)
+                        }
+                    }
+                }
+            }
+        }
+        (_, _) => {}
+    }
+
+    panic!("Chain height could not be determined")
+}
+
+async fn generate_blocks(blocks_count: u64, config: &ConfigFile) {
+    let rpc_addr = config.neon.bitcoind_rpc_host.clone();
+    let miner_address = config.neon.miner_address.clone();
+
+    let rpc_req = RPCRequest::generate_next_block_req(blocks_count, miner_address.clone());
+
+    let stream = TcpStream::connect(rpc_addr).await.unwrap();
+    let body = serde_json::to_vec(&rpc_req).unwrap();
+    let req = build_request(&config, body);
+    client::connect(stream.clone(), req).await.unwrap();
+}
+
+fn build_request(config: &ConfigFile, body: Vec<u8>) -> Request {
+    let url = Url::parse(&format!("http://{}/", config.neon.bitcoind_rpc_host)).unwrap();
+    let mut req = Request::new(Method::Post, url);
+    req.append_header("Authorization", config.neon.authorization_token()).unwrap();
+    req.append_header("Content-Type", "application/json").unwrap();
+    req.append_header("Content-Length", format!("{}", body.len())).unwrap();
+    req.append_header("Host", format!("{}", config.neon.bitcoind_rpc_host)).unwrap();
+    req.set_body(body);
+    req
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+/// JSONRPC Request
+pub struct RPCRequest {
+    /// The name of the RPC call
+    pub method: String,
+    /// Parameters to the RPC call
+    pub params: Vec<serde_json::Value>,
+    /// Identifier for this Request, which should appear in the response
+    pub id: serde_json::Value,
+    /// jsonrpc field, MUST be "2.0"
+    pub jsonrpc: Option<String>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct RPCResult {
+    /// The error returned by the RPC call
+    pub error: Option<serde_json::Value>,
+    /// The value returned by the RPC call
+    pub result: Option<serde_json::Value>,
+}
+
+impl RPCRequest {
+
+    pub fn generate_next_block_req(blocks_count: u64, address: String) -> RPCRequest {
+        RPCRequest {
+            method: "generatetoaddress".to_string(),
+            params: vec![blocks_count.into(), address.into()],
+            id: 0.into(),
+            jsonrpc: Some("2.0".to_string())
+        }
+    }
+
+    pub fn is_chain_bootstrapped() -> RPCRequest {
+        RPCRequest {
+            method: "getblockhash".to_string(),
+            params: vec![200.into()],
+            id: 0.into(),
+            jsonrpc: Some("2.0".to_string())
+        }
+    }
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct ConfigFile {
+    /// Regtest node
+    neon: RegtestConfig,
+}
+
+impl ConfigFile {
+
+    pub fn from_path(path: &str) -> ConfigFile {
+        let path = File::open(path).unwrap();
+        let mut config_reader = BufReader::new(path);
+        let mut config = vec![];
+        config_reader.read_to_end(&mut config).unwrap();    
+        toml::from_slice(&config[..]).unwrap()
+    }
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct RegtestConfig {
+    /// Proxy's port
+    rpc_bind: String,
+    /// Duration between blocks
+    block_time: u64,
+    /// Address receiving coinbases and mining fee
+    miner_address: String,
+    /// RPC address used by bitcoind
+    bitcoind_rpc_host: String,
+    /// Credential - username
+    bitcoind_rpc_user: String,
+    /// Credential - password
+    bitcoind_rpc_pass: String,
+}
+
+impl RegtestConfig {
+
+    pub fn authorization_token(&self) -> String {
+        let token = encode(format!("{}:{}", self.bitcoind_rpc_user, self.bitcoind_rpc_pass));
+        format!("Basic {}", token)
+    }
+}

testnet/stacks-node/Cargo.toml

@@ -14,9 +14,14 @@ secp256k1 = { version = "0.11.5" }
 serde = "1"
 serde_derive = "1"
 serde_json = { version = "1.0", features = ["arbitrary_precision"] }
-stacks = { package = "blockstack-core", path = "../." }
+stacks = { package = "blockstack-core", path = "../../." }
 toml = "0.5.6"
+prometheus = { version = "0.8", optional = true }
 
 [[bin]]
 name = "stacks-node"
 path = "src/main.rs"
+
+[features]
+default = []
+prometheus_monitoring = ["prometheus"]

testnet/stacks-node/src/main.rs

@@ -2,6 +2,9 @@ extern crate rand;
 extern crate mio;
 extern crate serde;
 
+#[cfg(feature = "prometheus_monitoring")]
+extern crate prometheus;
+
 #[macro_use] extern crate lazy_static;
 #[macro_use] extern crate serde_derive;
 #[macro_use] extern crate serde_json;
@@ -19,6 +22,9 @@ pub mod operations;
 pub mod burnchains;
 pub mod neon_node;
 
+#[cfg(feature = "prometheus_monitoring")]
+pub mod metrics;
+
 pub use self::keychain::{Keychain};
 pub use self::node::{Node, ChainTip};
 pub use self::neon_node::{InitializedNeonNode, NeonGenesisNode};

testnet/stacks-node/src/run_loop/mod.rs

@@ -70,6 +70,18 @@ impl RunLoopCallbacks {
             burnchain_tip.block_snapshot.burn_header_hash, 
             burnchain_tip.block_snapshot.sortition_hash);
 
+        #[cfg(feature = "prometheus_monitoring")]
+        let counter_opts = prometheus::Opts::new("test_counter", "test counter help");
+    
+        #[cfg(feature = "prometheus_monitoring")]
+        let counter = prometheus::Counter::with_opts(counter_opts).unwrap();
+        
+        // if cfg!(feature = "prometheus_monitoring") {
+
+        //     let counter_opts = prometheus::Opts::new("test_counter", "test counter help");
+        //     let counter = prometheus::Counter::with_opts(counter_opts).unwrap();
+        // }
+
         if let Some(cb) = self.on_new_burn_chain_state {
             cb(round, burnchain_tip, chain_tip);
         }
@@ -80,6 +92,11 @@ impl RunLoopCallbacks {
             chain_tip.metadata.block_height, 
             chain_tip.metadata.index_block_hash(), 
             chain_tip.block.txs.len());
+
+        if cfg!(feature = "prometheus_monitoring") {
+        
+        }
+
         for tx in chain_tip.block.txs.iter() {
             match &tx.auth {            
                 TransactionAuth::Standard(TransactionSpendingCondition::Singlesig(auth)) => println!("-> Tx issued by {:?} (fee: {}, nonce: {})", auth.signer, auth.fee_rate, auth.nonce),
@@ -99,6 +116,10 @@ impl RunLoopCallbacks {
     }
 
     pub fn invoke_new_tenure(&self, round: u64, burnchain_tip: &BurnchainTip, chain_tip: &ChainTip, tenure: &mut Tenure) {
+        if cfg!(feature = "prometheus_monitoring") {
+        
+        }
+
         if let Some(cb) = self.on_new_tenure {
             cb(round, burnchain_tip, chain_tip, tenure);
         }