From 451694434b10cec576248a713a612e4e2514edef Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 24 Jul 2020 15:29:43 -0500 Subject: [PATCH 1/8] feat: add gh workflow for bitcoind integration tests --- .../bitcoin-int-tests/Dockerfile.bitcoin-tests | 15 +++++++++++++++ .github/workflows/bitcoin-tests.yml | 14 ++++++++++++++ testnet/stacks-node/src/tests/bitcoin_regtest.rs | 4 ++++ 3 files changed, 33 insertions(+) create mode 100644 .github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests create mode 100644 .github/workflows/bitcoin-tests.yml diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests b/.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests new file mode 100644 index 000000000..4ab3c8a51 --- /dev/null +++ b/.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests @@ -0,0 +1,15 @@ +FROM rust:buster + +WORKDIR /src + +COPY . . + +RUN cargo test --no-run --workspace + +RUN cd / && wget https://bitcoin.org/bin/bitcoin-core-0.20.0/bitcoin-0.20.0-x86_64-linux-gnu.tar.gz +RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz + +RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/ + +ENV BITCOIND_TEST 1 +RUN cargo test --workspace -- --ignored diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml new file mode 100644 index 000000000..01f158137 --- /dev/null +++ b/.github/workflows/bitcoin-tests.yml @@ -0,0 +1,14 @@ +name: Run bitcoind/stacks-node integration tests + +on: pull_request + +jobs: + bitcoin-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Run bitcoin tests as a docker script + env: + DOCKER_BUILDKIT: 1 + run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests . diff --git a/testnet/stacks-node/src/tests/bitcoin_regtest.rs b/testnet/stacks-node/src/tests/bitcoin_regtest.rs index 8a5f3f21b..d0dfdac75 100644 --- a/testnet/stacks-node/src/tests/bitcoin_regtest.rs +++ b/testnet/stacks-node/src/tests/bitcoin_regtest.rs @@ -6,6 +6,7 @@ use crate::helium::RunLoop; use stacks::chainstate::burn::operations::BlockstackOperationType::{LeaderBlockCommit, LeaderKeyRegister}; use stacks::util::hash::{hex_bytes}; +use std::env; use std::io::{BufReader, BufRead}; use super::{PUBLISH_CONTRACT}; @@ -101,6 +102,9 @@ impl Drop for BitcoinCoreController { #[test] #[ignore] fn bitcoind_integration_test() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return + } let mut conf = super::new_test_conf(); conf.burnchain.commit_anchor_block_within = 2000; From 4502f7a2a48172b6757185bd76675ed9364f8ccd Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 24 Jul 2020 15:33:14 -0500 Subject: [PATCH 2/8] clean up the circle.yml from the :stretch branch --- circle.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/circle.yml b/circle.yml index 690b55fba..e16cc64d7 100644 --- a/circle.yml +++ b/circle.yml @@ -114,8 +114,6 @@ jobs: docker push $IMAGE_NAME:latest docker push $IMAGE_NAME:master docker push $IMAGE_NAME:stretch - elif [ "${CIRCLE_BRANCH}" = "feat/docker-stretch" ]; then - docker push $IMAGE_NAME:stretch fi workflows: version: 2 @@ -141,4 +139,3 @@ workflows: branches: only: - master - - feat/docker-stretch From 532efbec551ad865ad935712c92b3625e817cfef Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 24 Jul 2020 15:55:45 -0500 Subject: [PATCH 3/8] correct the yaml --- .github/workflows/bitcoin-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 01f158137..982993909 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -11,4 +11,4 @@ jobs: - name: Run bitcoin tests as a docker script env: DOCKER_BUILDKIT: 1 - run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests . + run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests . From 2bcabc7f45fbdda2c4de85332b7a062d27bcbbad Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 24 Jul 2020 16:03:13 -0500 Subject: [PATCH 4/8] only run stacks-node tests --- .github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests b/.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests index 4ab3c8a51..005c21b43 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests +++ b/.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests @@ -12,4 +12,4 @@ RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/ ENV BITCOIND_TEST 1 -RUN cargo test --workspace -- --ignored +RUN cargo test --workspace --bin=stacks-node -- --test-threads 1 --ignored From 652081e306056f3aaf1a56e3bbc9823bb7fb0bc7 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 24 Jul 2020 18:23:09 -0500 Subject: [PATCH 5/8] more aggressively rotating VRF keys solves the flakiness in the microblock integration test --- testnet/stacks-node/src/keychain.rs | 7 ++++++- testnet/stacks-node/src/tests/bitcoin_regtest.rs | 12 ++++++------ testnet/stacks-node/src/tests/neon_integrations.rs | 9 +++++++++ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/testnet/stacks-node/src/keychain.rs b/testnet/stacks-node/src/keychain.rs index 665037c0c..c004bca43 100644 --- a/testnet/stacks-node/src/keychain.rs +++ b/testnet/stacks-node/src/keychain.rs @@ -18,6 +18,7 @@ pub struct Keychain { microblocks_secret_keys: Vec, vrf_secret_keys: Vec, vrf_map: HashMap, + rotations: u64, } impl Keychain { @@ -38,6 +39,7 @@ impl Keychain { microblocks_secret_keys: vec![], secret_keys, threshold, + rotations: 0, vrf_secret_keys: vec![], vrf_map: HashMap::new(), } @@ -60,9 +62,12 @@ impl Keychain { } pub fn rotate_vrf_keypair(&mut self, block_height: u64) -> VRFPublicKey { + self.rotations = self.rotations.checked_add(1) + .expect("Exhausted VRF keypairs"); // this would require quite the hash power... let mut seed = { let mut secret_state = self.hashed_secret_state.to_bytes().to_vec(); - secret_state.extend_from_slice(&block_height.to_be_bytes()[..]); + secret_state.extend_from_slice(&self.rotations.to_be_bytes()); + secret_state.extend_from_slice(&block_height.to_be_bytes()); Sha256Sum::from_data(&secret_state) }; diff --git a/testnet/stacks-node/src/tests/bitcoin_regtest.rs b/testnet/stacks-node/src/tests/bitcoin_regtest.rs index d0dfdac75..83f31eb55 100644 --- a/testnet/stacks-node/src/tests/bitcoin_regtest.rs +++ b/testnet/stacks-node/src/tests/bitcoin_regtest.rs @@ -145,7 +145,7 @@ fn bitcoind_integration_test() { for op in &state_transition.accepted_ops { match op { LeaderKeyRegister(op) => { - assert!(op.public_key.to_hex() == "99fe9d43bbb0d36a23e4102cef59accfa983a342ae1e5acedc1b8dcb06b17cd4"); + assert_eq!(op.public_key.to_hex(), "325dfe3ba0b52e6b800f5cea1283dd2cd0aee88be0270e45dc8a6e01d19218b4"); }, LeaderBlockCommit(op) => { assert!(op.parent_block_ptr == 0); @@ -170,7 +170,7 @@ fn bitcoind_integration_test() { for op in &state_transition.accepted_ops { match op { LeaderKeyRegister(op) => { - assert!(op.public_key.to_hex() == "f6fb508bdbeb8c64faf4a376d773c4b6514874d8e508a30aa9ee4db86c6b7e8e"); + assert_eq!(op.public_key.to_hex(), "e131b0305f0146e2ce8bd7d6b5004200bf19538b32e29c20d2002e9cd8d7907f"); }, LeaderBlockCommit(op) => { assert!(op.parent_block_ptr == 203); @@ -196,7 +196,7 @@ fn bitcoind_integration_test() { for op in &state_transition.accepted_ops { match op { LeaderKeyRegister(op) => { - assert!(op.public_key.to_hex() == "e89fc82ea3b5cfeab082e3a3294a6fbc6b9bb9a18d8179898898c2acafe21ab0"); + assert_eq!(op.public_key.to_hex(), "1840db439989068a9bfe32beaab4a2ce1ab46b0149f54b20c330333a5de51b3d"); }, LeaderBlockCommit(op) => { assert!(op.parent_block_ptr == 204); @@ -222,7 +222,7 @@ fn bitcoind_integration_test() { for op in &state_transition.accepted_ops { match op { LeaderKeyRegister(op) => { - assert!(op.public_key.to_hex() == "c44604de9b87ee911db176c8d416c53c6aa046aa4be88333f5bc74d7f7c5a561"); + assert_eq!(op.public_key.to_hex(), "d5f8569cc5aeadfb508b93f8a5740144bb52ef409d9f0afbe5a2e1d11a57f4a3"); }, LeaderBlockCommit(op) => { assert!(op.parent_block_ptr == 205); @@ -248,7 +248,7 @@ fn bitcoind_integration_test() { for op in &state_transition.accepted_ops { match op { LeaderKeyRegister(op) => { - assert!(op.public_key.to_hex() == "cb58759fcb51972f98b53d755f5980695e5a0abd0e307002f32e8d73b40c2019"); + assert_eq!(op.public_key.to_hex(), "81a60590f859300f994f725cd35e510d2ffa05789d0e2b2dc71bf7bc09eca576"); }, LeaderBlockCommit(op) => { assert!(op.parent_block_ptr == 206); @@ -274,7 +274,7 @@ fn bitcoind_integration_test() { for op in &state_transition.accepted_ops { match op { LeaderKeyRegister(op) => { - assert!(op.public_key.to_hex() == "848a6ec4da123bd44b79e05dd8eb90fcfb92b985812cb54c886987f7dd54ac90"); + assert_eq!(op.public_key.to_hex(), "e8de363ffd6baec1ddf629eab0a6eff6a56385d34e4d7f0f2caff8febe74acc4"); }, LeaderBlockCommit(op) => { assert!(op.parent_block_ptr == 207); diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index f04a24097..39ddd436f 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -306,6 +306,15 @@ fn microblock_integration_test() { // which *should* have also confirmed the microblock. next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + // I guess let's push another block for good measure? + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // microblock must have bumped our nonce + let path = format!("{}/v2/accounts/{}?proof=0", + &http_origin, &spender_addr); + let res = client.get(&path).send().unwrap().json::().unwrap(); + assert_eq!(res.nonce, 1); + // push another transaction that is marked microblock only let recipient = StacksAddress::from_string(ADDR_4).unwrap(); let unconfirmed_tx_bytes = make_stacks_transfer_mblock_only(&spender_sk, 1, 1000, &recipient.into(), 1000); From bf1c1f1223d9691ed316f7094d09f9e49b70def5 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 27 Jul 2020 11:24:01 -0500 Subject: [PATCH 6/8] add self-canceling to the build --- .github/workflows/bitcoin-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 982993909..6d9a580eb 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -6,6 +6,11 @@ jobs: bitcoin-test: runs-on: ubuntu-latest steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.4.1 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 - name: Run bitcoin tests as a docker script From c1374d20c95a88880d88c95ac01b8589f17f1dd6 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 27 Jul 2020 11:24:59 -0500 Subject: [PATCH 7/8] test the cancelation --- .github/workflows/bitcoin-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 6d9a580eb..3c9f2b7d9 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -6,7 +6,7 @@ jobs: bitcoin-test: runs-on: ubuntu-latest steps: - - name: Cancel Previous Runs + - name: Cancel prior runs uses: styfle/cancel-workflow-action@0.4.1 with: access_token: ${{ github.token }} From 103ad459358e213bf9b7f8dd0f13d2ef4c6cca50 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 27 Jul 2020 11:28:53 -0500 Subject: [PATCH 8/8] add self-canceling to workflows/deploy as well --- .github/workflows/deploy.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 10a369ab3..18243dd46 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -15,6 +15,11 @@ jobs: platform: [ windows-x64, macos-x64, linux-x64, linux-musl-x64, linux-armv7, linux-arm64 ] steps: + - name: Cancel prior runs + uses: styfle/cancel-workflow-action@0.4.1 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 - name: Build distributable