From 203c04c18b42be71946de48bb2b7815a22d6dd07 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 29 Sep 2021 13:50:08 -0400 Subject: [PATCH 01/92] feat: clarinet demo --- core-contracts/Clarinet.toml | 8 + core-contracts/settings/Devnet.toml | 126 ++++ .../tests/bns/name_register_test.ts | 566 ++++++++++++++++++ 3 files changed, 700 insertions(+) create mode 100644 core-contracts/Clarinet.toml create mode 100644 core-contracts/settings/Devnet.toml create mode 100644 core-contracts/tests/bns/name_register_test.ts diff --git a/core-contracts/Clarinet.toml b/core-contracts/Clarinet.toml new file mode 100644 index 000000000..f2e6a218b --- /dev/null +++ b/core-contracts/Clarinet.toml @@ -0,0 +1,8 @@ + +[project] +name = "core-contracts" +costs_version = 1 + +[contracts.bns] +path = "../src/chainstate/stacks/boot/bns.clar" +depends_on = [] diff --git a/core-contracts/settings/Devnet.toml b/core-contracts/settings/Devnet.toml new file mode 100644 index 000000000..6f364bc47 --- /dev/null +++ b/core-contracts/settings/Devnet.toml @@ -0,0 +1,126 @@ +[network] +name = "devnet" + +[accounts.deployer] +mnemonic = "twice kind fence tip hidden tilt action fragile skin nothing glory cousin green tomorrow spring wrist shed math olympic multiply hip blue scout claw" +balance = 100_000_000_000_000 +# secret_key: 753b7cc01a1a2e86221266a154af739463fce51219d97e4f856cd7200c3bd2a601 +# stx_address: ST1PQHQKV0RJXZFY1DGX8MNSNYVE3VGZJSRTPGZGM +# btc_address: mqVnk6NPRdhntvfm4hh9vvjiRkFDUuSYsH + +[accounts.wallet_1] +mnemonic = "sell invite acquire kitten bamboo drastic jelly vivid peace spawn twice guilt pave pen trash pretty park cube fragile unaware remain midnight betray rebuild" +balance = 100_000_000_000_000 +# secret_key: 7287ba251d44a4d3fd9276c88ce34c5c52a038955511cccaf77e61068649c17801 +# stx_address: ST1SJ3DTE5DN7X54YDH5D64R3BCB6A2AG2ZQ8YPD5 +# btc_address: mr1iPkD9N3RJZZxXRk7xF9d36gffa6exNC + +[accounts.wallet_2] +mnemonic = "hold excess usual excess ring elephant install account glad dry fragile donkey gaze humble truck breeze nation gasp vacuum limb head keep delay hospital" +balance = 100_000_000_000_000 +# secret_key: 530d9f61984c888536871c6573073bdfc0058896dc1adfe9a6a10dfacadc209101 +# stx_address: ST2CY5V39NHDPWSXMW9QDT3HC3GD6Q6XX4CFRK9AG +# btc_address: muYdXKmX9bByAueDe6KFfHd5Ff1gdN9ErG + +[accounts.wallet_3] +mnemonic = "cycle puppy glare enroll cost improve round trend wrist mushroom scorpion tower claim oppose clever elephant dinosaur eight problem before frozen dune wagon high" +balance = 100_000_000_000_000 +# secret_key: d655b2523bcd65e34889725c73064feb17ceb796831c0e111ba1a552b0f31b3901 +# stx_address: ST2JHG361ZXG51QTKY2NQCVBPPRRE2KZB1HR05NNC +# btc_address: mvZtbibDAAA3WLpY7zXXFqRa3T4XSknBX7 + +[accounts.wallet_4] +mnemonic = "board list obtain sugar hour worth raven scout denial thunder horse logic fury scorpion fold genuine phrase wealth news aim below celery when cabin" +balance = 100_000_000_000_000 +# secret_key: f9d7206a47f14d2870c163ebab4bf3e70d18f5d14ce1031f3902fbbc894fe4c701 +# stx_address: ST2NEB84ASENDXKYGJPQW86YXQCEFEX2ZQPG87ND +# btc_address: mg1C76bNTutiCDV3t9nWhZs3Dc8LzUufj8 + +[accounts.wallet_5] +mnemonic = "hurry aunt blame peanut heavy update captain human rice crime juice adult scale device promote vast project quiz unit note reform update climb purchase" +balance = 100_000_000_000_000 +# secret_key: 3eccc5dac8056590432db6a35d52b9896876a3d5cbdea53b72400bc9c2099fe801 +# stx_address: ST2REHHS5J3CERCRBEPMGH7921Q6PYKAADT7JP2VB +# btc_address: mweN5WVqadScHdA81aATSdcVr4B6dNokqx + +[accounts.wallet_6] +mnemonic = "area desk dutch sign gold cricket dawn toward giggle vibrant indoor bench warfare wagon number tiny universe sand talk dilemma pottery bone trap buddy" +balance = 100_000_000_000_000 +# secret_key: 7036b29cb5e235e5fd9b09ae3e8eec4404e44906814d5d01cbca968a60ed4bfb01 +# stx_address: ST3AM1A56AK2C1XAFJ4115ZSV26EB49BVQ10MGCS0 +# btc_address: mzxXgV6e4BZSsz8zVHm3TmqbECt7mbuErt + +[accounts.wallet_7] +mnemonic = "prevent gallery kind limb income control noise together echo rival record wedding sense uncover school version force bleak nuclear include danger skirt enact arrow" +balance = 100_000_000_000_000 +# secret_key: b463f0df6c05d2f156393eee73f8016c5372caa0e9e29a901bb7171d90dc4f1401 +# stx_address: ST3PF13W7Z0RRM42A8VZRVFQ75SV1K26RXEP8YGKJ +# btc_address: n37mwmru2oaVosgfuvzBwgV2ysCQRrLko7 + +[accounts.wallet_8] +mnemonic = "female adjust gallery certain visit token during great side clown fitness like hurt clip knife warm bench start reunion globe detail dream depend fortune" +balance = 100_000_000_000_000 +# secret_key: 6a1a754ba863d7bab14adbbc3f8ebb090af9e871ace621d3e5ab634e1422885e01 +# stx_address: ST3NBRSFKX28FQ2ZJ1MAKX58HKHSDGNV5N7R21XCP +# btc_address: n2v875jbJ4RjBnTjgbfikDfnwsDV5iUByw + +[accounts.wallet_9] +mnemonic = "shadow private easily thought say logic fault paddle word top book during ignore notable orange flight clock image wealth health outside kitten belt reform" +balance = 100_000_000_000_000 +# secret_key: de433bdfa14ec43aa1098d5be594c8ffb20a31485ff9de2923b2689471c401b801 +# stx_address: STNHKEPYEPJ8ET55ZZ0M5A34J0R3N5FM2CMMMAZ6 +# btc_address: mjSrB3wS4xab3kYqFktwBzfTdPg367ZJ2d + +[devnet] +disable_bitcoin_explorer = true +# disable_stacks_explorer = true +# disable_stacks_api = true +# working_dir = "tmp/devnet" +# stacks_node_events_observers = ["host.docker.internal:8002"] +# miner_mnemonic = "twice kind fence tip hidden tilt action fragile skin nothing glory cousin green tomorrow spring wrist shed math olympic multiply hip blue scout claw" +# miner_derivation_path = "m/44'/5757'/0'/0/0" +# orchestrator_port = 20445 +# bitcoin_node_p2p_port = 18444 +# bitcoin_node_rpc_port = 18443 +# bitcoin_node_username = "devnet" +# bitcoin_node_password = "devnet" +# bitcoin_controller_port = 18442 +# bitcoin_controller_block_time = 30_000 +# stacks_node_rpc_port = 20443 +# stacks_node_p2p_port = 20444 +# stacks_api_port = 3999 +# stacks_api_events_port = 3700 +# bitcoin_explorer_port = 8001 +# stacks_explorer_port = 8000 +# postgres_port = 5432 +# postgres_username = "postgres" +# postgres_password = "postgres" +# postgres_database = "postgres" +# bitcoin_node_image_url = "quay.io/hirosystems/bitcoind:devnet" +# stacks_node_image_url = "localhost:5000/stacks-node:devnet" +# stacks_api_image_url = "blockstack/stacks-blockchain-api:latest" +# stacks_explorer_image_url = "blockstack/explorer:latest" +# bitcoin_explorer_image_url = "quay.io/hirosystems/bitcoin-explorer:devnet" +# postgres_image_url = "postgres:alpine" + +# Send some stacking orders +[[devnet.pox_stacking_orders]] +start_at_cycle = 3 +duration = 12 +wallet = "wallet_1" +slots = 2 +btc_address = "mr1iPkD9N3RJZZxXRk7xF9d36gffa6exNC" + +[[devnet.pox_stacking_orders]] +start_at_cycle = 3 +duration = 12 +wallet = "wallet_2" +slots = 1 +btc_address = "muYdXKmX9bByAueDe6KFfHd5Ff1gdN9ErG" + +[[devnet.pox_stacking_orders]] +start_at_cycle = 3 +duration = 12 +wallet = "wallet_3" +slots = 1 +btc_address = "mvZtbibDAAA3WLpY7zXXFqRa3T4XSknBX7" diff --git a/core-contracts/tests/bns/name_register_test.ts b/core-contracts/tests/bns/name_register_test.ts new file mode 100644 index 000000000..bda230aad --- /dev/null +++ b/core-contracts/tests/bns/name_register_test.ts @@ -0,0 +1,566 @@ +// import { Clarinet, Tx, Chain, Account, Contract, types } from 'https://deno.land/x/clarinet@v0.13.0/index.ts'; +import { Clarinet, Tx, Chain, Account, Contract, types } from '/Users/ludovic/Coding/clarinet/clarinet-cli/deno/index.ts'; +import { assertEquals } from "https://deno.land/std@0.90.0/testing/asserts.ts"; +import { createHash } from "https://deno.land/std@0.107.0/hash/mod.ts"; + +Clarinet.test({ + name: "Ensure that counter can be incremented multiples per block, accross multiple blocks", + async fn(chain: Chain, accounts: Map, contracts: Map) { + + const alice = accounts.get("wallet_1")!; + const bob = accounts.get("wallet_2")!; + const charlie = accounts.get("wallet_3")!; + const dave = accounts.get("wallet_4")!; + + const cases = [{ + namespace: "blockstack", + version: 1, + salt: "0000", + value: 640000000, + namespaceOwner: alice, + nameOwner: bob, + priceFunction: [ + types.uint(4), // base + types.uint(250), // coeff + types.uint(7), // bucket 1 + types.uint(6), // bucket 2 + types.uint(5), // bucket 3 + types.uint(4), // bucket 4 + types.uint(3), // bucket 5 + types.uint(2), // bucket 6 + types.uint(1), // bucket 7 + types.uint(1), // bucket 8 + types.uint(1), // bucket 9 + types.uint(1), // bucket 10 + types.uint(1), // bucket 11 + types.uint(1), // bucket 12 + types.uint(1), // bucket 13 + types.uint(1), // bucket 14 + types.uint(1), // bucket 15 + types.uint(1), // bucket 16+ + types.uint(4), // nonAlphaDiscount + types.uint(4), // noVowelDiscount + ], + renewalRule: 10, + nameImporter: alice, + zonefile: "0000", + }, { + namespace: "id", + version: 1, + salt: "0000", + value: 64000000000, + namespaceOwner: alice, + nameOwner: bob, + priceFunction: [ + types.uint(4), // base + types.uint(250), // coeff + types.uint(6), // bucket 1 + types.uint(5), // bucket 2 + types.uint(4), // bucket 3 + types.uint(3), // bucket 4 + types.uint(2), // bucket 5 + types.uint(1), // bucket 6 + types.uint(0), // bucket 7 + types.uint(0), // bucket 8 + types.uint(0), // bucket 9 + types.uint(0), // bucket 10 + types.uint(0), // bucket 11 + types.uint(0), // bucket 12 + types.uint(0), // bucket 13 + types.uint(0), // bucket 14 + types.uint(0), // bucket 15 + types.uint(0), // bucket 16+ + types.uint(20), // nonAlphaDiscount + types.uint(20), // noVowelDiscount + ], + renewalRule: 20, + nameImporter: alice, + zonefile: "1111", + }]; + + let call = chain.callReadOnlyFn("bns", "resolve-principal", [types.principal(bob.address)], alice.address) + let error:any = call.result + .expectErr() + .expectTuple(); + error['code'].expectInt(2013); + + // Registering a name at this point should fail, namespace have not been registered yet + let block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[1].namespace), + types.buff("bob"), + types.buff(cases[1].salt), + types.buff(cases[1].zonefile) + ], + cases[0].nameOwner.address), + ]); + assertEquals(block.height, 2); + block.receipts[0].result + .expectErr() + .expectInt(1005); + + // Preorder a namespace + let merged = new TextEncoder().encode(`${cases[1].namespace}${cases[1].salt}`); + let sha256 = createHash("sha256") + .update(merged) + .digest(); + let ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "namespace-preorder", + [ + types.buff(ripemd160), + types.uint(cases[1].value) + ], + cases[1].namespaceOwner.address), + ]); + assertEquals(block.height, 3); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + // Reveal the namespace + block = chain.mineBlock([ + Tx.contractCall("bns", "namespace-reveal", + [ + types.buff(cases[1].namespace), + types.buff(cases[1].salt), + ...cases[1].priceFunction, + types.uint(cases[1].renewalRule), + types.principal(cases[1].nameImporter.address), + ], + cases[1].namespaceOwner.address), + ]); + assertEquals(block.height, 4); + block.receipts[0].result + .expectOk() + .expectBool(true); + + // Bob can now preorder a name + let name = "baobab"; + merged = new TextEncoder().encode(`${name}.${cases[1].namespace}${cases[1].salt}`); + sha256 = createHash("sha256") + .update(merged) + .digest(); + ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "name-preorder", + [ + types.buff(ripemd160), + types.uint(100), + ], + bob.address), + ]); + assertEquals(block.height, 5); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + // But revealing the name should fail - the namespace was not launched yet + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[1].namespace), + types.buff(name), + types.buff(cases[1].salt), + types.buff(cases[1].zonefile), + ], + bob.address), + ]); + assertEquals(block.height, 6); + block.receipts[0].result + .expectErr() + .expectInt(2004); + + // // Given a launched namespace 'blockstack', owned by Alice + merged = new TextEncoder().encode(`${cases[0].namespace}${cases[0].salt}`); + sha256 = createHash("sha256") + .update(merged) + .digest(); + ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "namespace-preorder", + [ + types.buff(ripemd160), + types.uint(cases[0].value) + ], + cases[0].namespaceOwner.address), + ]); + assertEquals(block.height, 7); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + // Reveal the namespace + block = chain.mineBlock([ + Tx.contractCall("bns", "namespace-reveal", + [ + types.buff(cases[0].namespace), + types.buff(cases[0].salt), + ...cases[0].priceFunction, + types.uint(cases[0].renewalRule), + types.principal(cases[0].nameImporter.address), + ], + cases[0].namespaceOwner.address), + ]); + assertEquals(block.height, 8); + block.receipts[0].result + .expectOk() + .expectBool(true); + + // Launch the namespace + block = chain.mineBlock([ + Tx.contractCall("bns", "namespace-ready", + [ + types.buff(cases[0].namespace), + ], + cases[0].namespaceOwner.address), + ]); + assertEquals(block.height, 9); + block.receipts[0].result + .expectOk() + .expectBool(true); + + // Revealing the name 'bob.blockstack' + // should fail if no matching pre-order can be found + // But revealing the name should fail - the namespace was not launched yet + name = "bob"; + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[0].namespace), + types.buff(name), + types.buff(cases[0].salt), + types.buff(cases[0].zonefile), + ], + bob.address), + ]); + assertEquals(block.height, 10); + block.receipts[0].result + .expectErr() + .expectInt(2001); + + // Bob can now preorder a name + name = "bub"; + merged = new TextEncoder().encode(`${name}.${cases[0].namespace}${cases[0].salt}`); + sha256 = createHash("sha256") + .update(merged) + .digest(); + ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "name-preorder", + [ + types.buff(ripemd160), + types.uint(2559999), + ], + bob.address), + ]); + assertEquals(block.height, 11); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + // should fail + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[0].namespace), + types.buff("bub"), + types.buff(cases[0].salt), + types.buff(cases[0].zonefile), + ], + bob.address), + ]); + assertEquals(block.height, 12); + block.receipts[0].result + .expectErr() + .expectInt(2007); + + // Given an existing pre-order of the name 'Bob.blockstack' + name = "Bob"; + merged = new TextEncoder().encode(`${name}.${cases[0].namespace}${cases[0].salt}`); + sha256 = createHash("sha256") + .update(merged) + .digest(); + ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "name-preorder", + [ + types.buff(ripemd160), + types.uint(2560000), + ], + bob.address), + ]); + assertEquals(block.height, 13); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + // Bob registering the name 'Bob.blockstack' should fail + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[0].namespace), + types.buff(name), + types.buff(cases[0].salt), + types.buff(cases[0].zonefile), + ], + bob.address), + ]); + assertEquals(block.height, 14); + block.receipts[0].result + .expectErr() + .expectInt(2022); + + // Given an existing pre-order of the name 'bob.blockstack' + name = "bob"; + merged = new TextEncoder().encode(`${name}.${cases[0].namespace}${cases[0].salt}`); + sha256 = createHash("sha256") + .update(merged) + .digest(); + ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "name-preorder", + [ + types.buff(ripemd160), + types.uint(2560000), + ], + bob.address), + ]); + assertEquals(block.height, 15); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + // Bob registering the name 'bob.blockstack' should succeed + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[0].namespace), + types.buff(name), + types.buff(cases[0].salt), + types.buff(cases[0].zonefile), + ], + bob.address), + ]); + assertEquals(block.height, 16); + block.receipts[0].result + .expectOk() + .expectBool(true); + + call = chain.callReadOnlyFn("bns", "resolve-principal", [types.principal(bob.address)], alice.address) + let response:any = call.result + .expectOk() + .expectTuple(); + response["name"].expectBuff("bob"); + response["namespace"].expectBuff("blockstack"); + + call = chain.callReadOnlyFn("bns", "name-resolve", [types.buff(cases[0].namespace), types.buff(name)], alice.address) + response = call.result + .expectOk() + .expectTuple(); + response["owner"].expectPrincipal(bob.address); + response["zonefile-hash"].expectBuff(cases[0].zonefile); + + // should fail registering twice + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[0].namespace), + types.buff(name), + types.buff(cases[0].salt), + types.buff(cases[0].zonefile), + ], + bob.address), + ]); + assertEquals(block.height, 17); + block.receipts[0].result + .expectErr() + .expectInt(2004); + + // Charlie registering 'bob.blockstack' + // should fail + name = "bob"; + let salt = "1111" + merged = new TextEncoder().encode(`${name}.${cases[0].namespace}${salt}`); + sha256 = createHash("sha256") + .update(merged) + .digest(); + ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "name-preorder", + [ + types.buff(ripemd160), + types.uint(2560000), + ], + charlie.address), + ]); + assertEquals(block.height, 18); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + // Bob registering the name 'bob.blockstack' should succeed + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[0].namespace), + types.buff(name), + types.buff(salt), + types.buff(cases[0].zonefile), + ], + charlie.address), + ]); + assertEquals(block.height, 19); + block.receipts[0].result + .expectErr() + .expectInt(2004); + + // Bob registering a second name 'bobby.blockstack' + // should fail if 'bob.blockstack' is not expired + name = "bobby"; + salt = "1111" + merged = new TextEncoder().encode(`${name}.${cases[0].namespace}${salt}`); + sha256 = createHash("sha256") + .update(merged) + .digest(); + ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "name-preorder", + [ + types.buff(ripemd160), + types.uint(2560000), + ], + bob.address), + ]); + assertEquals(block.height, 20); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + // Bob registering the name 'bob.blockstack' should succeed + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[0].namespace), + types.buff(name), + types.buff(salt), + types.buff(cases[0].zonefile), + ], + bob.address), + ]); + assertEquals(block.height, 21); + block.receipts[0].result + .expectErr() + .expectInt(3001); + + // should succeed once 'bob.blockstack' is expired + chain.mineEmptyBlock(cases[0].renewalRule + 5000); + + call = chain.callReadOnlyFn("bns", "resolve-principal", [types.principal(bob.address)], alice.address) + response = call.result + .expectErr() + .expectTuple(); + response["code"].expectInt("2008"); // Indicates ERR_NAME_EXPIRED + let inner:any = response["name"].expectSome().expectTuple(); + inner["name"].expectBuff("bob"); + inner["namespace"].expectBuff("blockstack"); + + + name = "bobby"; + salt = "1111" + merged = new TextEncoder().encode(`${name}.${cases[0].namespace}${salt}`); + sha256 = createHash("sha256") + .update(merged) + .digest(); + ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "name-preorder", + [ + types.buff(ripemd160), + types.uint(2560000), + ], + bob.address), + ]); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + // Bob registering the name 'bobby.blockstack' should succeed + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[0].namespace), + types.buff(name), + types.buff(salt), + types.buff(cases[0].zonefile), + ], + bob.address), + ]); + block.receipts[0].result + .expectOk() + .expectBool(true); + + // Charlie registering 'bob.blockstack' + // should succeed once 'bob.blockstack' is expired + name = "bob"; + salt = "2222" + merged = new TextEncoder().encode(`${name}.${cases[0].namespace}${salt}`); + sha256 = createHash("sha256") + .update(merged) + .digest(); + ripemd160 = createHash("ripemd160") + .update(sha256) + .digest(); + block = chain.mineBlock([ + Tx.contractCall("bns", "name-preorder", + [ + types.buff(ripemd160), + types.uint(2560000), + ], + charlie.address), + ]); + block.receipts[0].result + .expectOk() + .expectUint(144 + block.height - 1); + + block = chain.mineBlock([ + Tx.contractCall("bns", "name-register", + [ + types.buff(cases[0].namespace), + types.buff(name), + types.buff(salt), + types.buff("CHARLIE"), + ], + charlie.address), + ]); + block.receipts[0].result + .expectOk() + .expectBool(true); + + call = chain.callReadOnlyFn("bns", "name-resolve", [types.buff(cases[0].namespace), types.buff(name)], alice.address) + response = call.result + .expectOk() + .expectTuple(); + response["owner"].expectPrincipal(charlie.address); + response["zonefile-hash"].expectBuff("CHARLIE"); + }, +}); From 738dd16e08c5128d6d2883f80a75d3b2c4ce21ca Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 29 Sep 2021 13:57:45 -0400 Subject: [PATCH 02/92] chore: remove default settings --- core-contracts/settings/Devnet.toml | 53 ------------------- .../tests/bns/name_register_test.ts | 5 +- 2 files changed, 2 insertions(+), 56 deletions(-) diff --git a/core-contracts/settings/Devnet.toml b/core-contracts/settings/Devnet.toml index 6f364bc47..bb941fddc 100644 --- a/core-contracts/settings/Devnet.toml +++ b/core-contracts/settings/Devnet.toml @@ -71,56 +71,3 @@ balance = 100_000_000_000_000 # stx_address: STNHKEPYEPJ8ET55ZZ0M5A34J0R3N5FM2CMMMAZ6 # btc_address: mjSrB3wS4xab3kYqFktwBzfTdPg367ZJ2d -[devnet] -disable_bitcoin_explorer = true -# disable_stacks_explorer = true -# disable_stacks_api = true -# working_dir = "tmp/devnet" -# stacks_node_events_observers = ["host.docker.internal:8002"] -# miner_mnemonic = "twice kind fence tip hidden tilt action fragile skin nothing glory cousin green tomorrow spring wrist shed math olympic multiply hip blue scout claw" -# miner_derivation_path = "m/44'/5757'/0'/0/0" -# orchestrator_port = 20445 -# bitcoin_node_p2p_port = 18444 -# bitcoin_node_rpc_port = 18443 -# bitcoin_node_username = "devnet" -# bitcoin_node_password = "devnet" -# bitcoin_controller_port = 18442 -# bitcoin_controller_block_time = 30_000 -# stacks_node_rpc_port = 20443 -# stacks_node_p2p_port = 20444 -# stacks_api_port = 3999 -# stacks_api_events_port = 3700 -# bitcoin_explorer_port = 8001 -# stacks_explorer_port = 8000 -# postgres_port = 5432 -# postgres_username = "postgres" -# postgres_password = "postgres" -# postgres_database = "postgres" -# bitcoin_node_image_url = "quay.io/hirosystems/bitcoind:devnet" -# stacks_node_image_url = "localhost:5000/stacks-node:devnet" -# stacks_api_image_url = "blockstack/stacks-blockchain-api:latest" -# stacks_explorer_image_url = "blockstack/explorer:latest" -# bitcoin_explorer_image_url = "quay.io/hirosystems/bitcoin-explorer:devnet" -# postgres_image_url = "postgres:alpine" - -# Send some stacking orders -[[devnet.pox_stacking_orders]] -start_at_cycle = 3 -duration = 12 -wallet = "wallet_1" -slots = 2 -btc_address = "mr1iPkD9N3RJZZxXRk7xF9d36gffa6exNC" - -[[devnet.pox_stacking_orders]] -start_at_cycle = 3 -duration = 12 -wallet = "wallet_2" -slots = 1 -btc_address = "muYdXKmX9bByAueDe6KFfHd5Ff1gdN9ErG" - -[[devnet.pox_stacking_orders]] -start_at_cycle = 3 -duration = 12 -wallet = "wallet_3" -slots = 1 -btc_address = "mvZtbibDAAA3WLpY7zXXFqRa3T4XSknBX7" diff --git a/core-contracts/tests/bns/name_register_test.ts b/core-contracts/tests/bns/name_register_test.ts index bda230aad..3fb378949 100644 --- a/core-contracts/tests/bns/name_register_test.ts +++ b/core-contracts/tests/bns/name_register_test.ts @@ -1,10 +1,9 @@ -// import { Clarinet, Tx, Chain, Account, Contract, types } from 'https://deno.land/x/clarinet@v0.13.0/index.ts'; -import { Clarinet, Tx, Chain, Account, Contract, types } from '/Users/ludovic/Coding/clarinet/clarinet-cli/deno/index.ts'; +import { Clarinet, Tx, Chain, Account, Contract, types } from 'https://deno.land/x/clarinet@v0.16.0/index.ts'; import { assertEquals } from "https://deno.land/std@0.90.0/testing/asserts.ts"; import { createHash } from "https://deno.land/std@0.107.0/hash/mod.ts"; Clarinet.test({ - name: "Ensure that counter can be incremented multiples per block, accross multiple blocks", + name: "Ensure that name can be registered", async fn(chain: Chain, accounts: Map, contracts: Map) { const alice = accounts.get("wallet_1")!; From b32d1b98ca9557bce8b53e55b295b90ed7d20bef Mon Sep 17 00:00:00 2001 From: Pavitthra Pandurangan Date: Thu, 28 Apr 2022 15:22:25 -0400 Subject: [PATCH 03/92] cleanup: added tests + updated docs for burn-related Clarity functions --- .../vm/analysis/arithmetic_checker/tests.rs | 4 + clarity/src/vm/docs/mod.rs | 27 ++-- clarity/src/vm/functions/assets.rs | 2 +- clarity/src/vm/tests/assets.rs | 149 +++++++++++------- 4 files changed, 109 insertions(+), 73 deletions(-) diff --git a/clarity/src/vm/analysis/arithmetic_checker/tests.rs b/clarity/src/vm/analysis/arithmetic_checker/tests.rs index 3bfaee10f..59cdbdd24 100644 --- a/clarity/src/vm/analysis/arithmetic_checker/tests.rs +++ b/clarity/src/vm/analysis/arithmetic_checker/tests.rs @@ -138,9 +138,13 @@ fn test_functions() { ("(define-private (foo (a principal)) (ft-mint? stackaroo u100 'SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR))", FunctionNotPermitted(NativeFunctions::MintToken)), + ("(ft-burn? stackaroo u100 'SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR)", + FunctionNotPermitted(NativeFunctions::BurnToken)), ("(define-private (foo (a principal)) (nft-mint? stackaroo \"Roo\" 'SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR))", FunctionNotPermitted(NativeFunctions::MintAsset)), + ("(nft-burn? stackaroo \"Roo\" 'SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR)", + FunctionNotPermitted(NativeFunctions::BurnAsset)), ("(nft-transfer? stackaroo \"Roo\" 'SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR 'SPAXYA5XS51713FDTQ8H94EJ4V579CXMTRNBZKSF)", FunctionNotPermitted(NativeFunctions::TransferAsset)), ("(nft-get-owner? stackaroo \"Roo\")", diff --git a/clarity/src/vm/docs/mod.rs b/clarity/src/vm/docs/mod.rs index b80088c2f..7899ed679 100644 --- a/clarity/src/vm/docs/mod.rs +++ b/clarity/src/vm/docs/mod.rs @@ -1528,8 +1528,11 @@ const BURN_TOKEN: SpecialAPI = SpecialAPI { type defined using `define-fungible-token`. The decreased token balance is _not_ transfered to another principal, but rather destroyed, reducing the circulating supply. -If a non-positive amount is provided to burn, this function returns `(err 1)`. Otherwise, on successfuly burn, it -returns `(ok true)`. +On a successful burn, it returns `(ok true)`. In the event of an unsuccessful burn it +returns one of the following error codes: + +`(err u1)` -- `sender` does not have enough balance to burn this amount +`(err u3)` -- the amount specified is not positive ", example: " (define-fungible-token stackaroo) @@ -1541,22 +1544,22 @@ returns `(ok true)`. const BURN_ASSET: SpecialAPI = SpecialAPI { input_type: "AssetName, A, principal", output_type: "(response bool uint)", - signature: "(nft-burn? asset-class asset-identifier recipient)", - description: "`nft-burn?` is used to burn an asset and remove that asset's owner from the `recipient` principal. -The asset must have been defined using `define-non-fungible-token`, and the supplied `asset-identifier` must be of the same type specified in -that definition. + signature: "(nft-burn? asset-class asset-identifier sender)", + description: "`nft-burn?` is used to burn an asset that the `sender` principal owns. +The asset must have been defined using `define-non-fungible-token`, and the supplied +`asset-identifier` must be of the same type specified in that definition. -If an asset identified by `asset-identifier` _doesn't exist_, this function will return an error with the following error code: +On a successful burn, it returns `(ok true)`. In the event of an unsuccessful burn it +returns one of the following error codes: -`(err u1)` - -Otherwise, on successfuly burn, it returns `(ok true)`. +`(err u1)` -- `sender` does not own the specified asset +`(err u3)` -- the asset specified by `asset-identifier` does not exist ", example: " (define-non-fungible-token stackaroo (string-ascii 40)) (nft-mint? stackaroo \"Roo\" 'SPAXYA5XS51713FDTQ8H94EJ4V579CXMTRNBZKSF) ;; Returns (ok true) (nft-burn? stackaroo \"Roo\" 'SPAXYA5XS51713FDTQ8H94EJ4V579CXMTRNBZKSF) ;; Returns (ok true) -" +", }; const STX_GET_BALANCE: SimpleFunctionAPI = SimpleFunctionAPI { @@ -1598,7 +1601,7 @@ one of the following error codes: const STX_BURN: SimpleFunctionAPI = SimpleFunctionAPI { name: None, signature: "(stx-burn? amount sender)", - description: "`stx-burn?` debits the `sender` principal's STX holdings by `amount`, destroying + description: "`stx-burn?` decreases the `sender` principal's STX holdings by `amount`, destroying the STX. The `sender` principal _must_ be equal to the current context's `tx-sender`. This function returns (ok true) if the transfer is successful. In the event of an unsuccessful transfer it returns diff --git a/clarity/src/vm/functions/assets.rs b/clarity/src/vm/functions/assets.rs index 6226e530c..f3e21c3be 100644 --- a/clarity/src/vm/functions/assets.rs +++ b/clarity/src/vm/functions/assets.rs @@ -840,7 +840,7 @@ pub fn special_burn_token( if let (Value::UInt(amount), Value::Principal(ref burner)) = (amount, from) { if amount == 0 { - return clarity_ecode!(MintTokenErrorCodes::NON_POSITIVE_AMOUNT); + return clarity_ecode!(BurnTokenErrorCodes::NON_POSITIVE_AMOUNT); } let burner_bal = env.global_context.database.get_ft_balance( diff --git a/clarity/src/vm/tests/assets.rs b/clarity/src/vm/tests/assets.rs index 7a5327d27..0ce9a5868 100644 --- a/clarity/src/vm/tests/assets.rs +++ b/clarity/src/vm/tests/assets.rs @@ -42,8 +42,8 @@ const FIRST_CLASS_TOKENS: &str = "(define-fungible-token stackaroos) (if (>= block-height block-to-release) (faucet) (err \"must be in the future\"))) - (define-public (burn (amount uint)) - (ft-burn? stackaroos amount tx-sender)) + (define-public (burn (amount uint) (p principal)) + (ft-burn? stackaroos amount p)) (begin (ft-mint? stackaroos u10000 'SZ2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQ9H6DPR) (ft-mint? stackaroos u200 'SM2J6ZY48GV1EZ5V2V5RB9MP66SW86PYKKQVX8X0G) (ft-mint? stackaroos u4 .tokens))"; @@ -75,8 +75,8 @@ const ASSET_NAMES: &str = (define-public (force-mint (name int)) (nft-mint? names name tx-sender)) - (define-public (force-burn (name int)) - (nft-burn? names name tx-sender)) + (define-public (force-burn (name int) (p principal)) + (nft-burn? names name p)) (define-public (try-bad-transfers) (begin (contract-call? .tokens my-token-transfer burn-address u50000) @@ -339,12 +339,8 @@ fn test_native_stx_ops(owned_env: &mut OwnedEnvironment) { assert!(is_committed(&result)); let table = asset_map.to_table(); assert_eq!( - table - .get(&p2_principal) - .unwrap() - .get(&AssetIdentifier::STX_burned()) - .unwrap(), - &AssetMapEntry::Burn(10) + table[&p2_principal][&AssetIdentifier::STX_burned()], + AssetMapEntry::Burn(10) ); let (result, asset_map, _events) = execute_transaction( @@ -359,12 +355,8 @@ fn test_native_stx_ops(owned_env: &mut OwnedEnvironment) { assert!(is_committed(&result)); let table = asset_map.to_table(); assert_eq!( - table - .get(&p2_principal) - .unwrap() - .get(&AssetIdentifier::STX()) - .unwrap(), - &AssetMapEntry::STX(500) + table[&p2_principal][&AssetIdentifier::STX()], + AssetMapEntry::STX(500) ); let (result, asset_map, _events) = execute_transaction( @@ -379,12 +371,8 @@ fn test_native_stx_ops(owned_env: &mut OwnedEnvironment) { assert!(is_committed(&result)); let table = asset_map.to_table(); assert_eq!( - table - .get(&p3_principal) - .unwrap() - .get(&AssetIdentifier::STX()) - .unwrap(), - &AssetMapEntry::STX(1) + table[&p3_principal][&AssetIdentifier::STX()], + AssetMapEntry::STX(1) ); // let's try a user -> contract transfer @@ -401,12 +389,8 @@ fn test_native_stx_ops(owned_env: &mut OwnedEnvironment) { assert!(is_committed(&result)); let table = asset_map.to_table(); assert_eq!( - table - .get(&p2_principal) - .unwrap() - .get(&AssetIdentifier::STX()) - .unwrap(), - &AssetMapEntry::STX(10) + table[&p2_principal.clone()][&AssetIdentifier::STX()], + AssetMapEntry::STX(10) ); // now check contract balance with stx-get-balance @@ -446,12 +430,8 @@ fn test_native_stx_ops(owned_env: &mut OwnedEnvironment) { let contract_principal = token_contract_id.clone().into(); assert_eq!( - table - .get(&contract_principal) - .unwrap() - .get(&AssetIdentifier::STX()) - .unwrap(), - &AssetMapEntry::STX(10) + table[&contract_principal][&AssetIdentifier::STX()], + AssetMapEntry::STX(10) ); // now let's do a contract -> contract transfer @@ -476,12 +456,8 @@ fn test_native_stx_ops(owned_env: &mut OwnedEnvironment) { assert_eq!(table.len(), 1); assert_eq!( - table - .get(&second_contract_principal) - .unwrap() - .get(&AssetIdentifier::STX()) - .unwrap(), - &AssetMapEntry::STX(500) + table[&second_contract_principal][&AssetIdentifier::STX()], + AssetMapEntry::STX(500) ); // now, let's send some back @@ -500,12 +476,8 @@ fn test_native_stx_ops(owned_env: &mut OwnedEnvironment) { assert_eq!(table.len(), 1); assert_eq!( - table - .get(&contract_principal) - .unwrap() - .get(&AssetIdentifier::STX()) - .unwrap(), - &AssetMapEntry::STX(100) + table[&contract_principal][&AssetIdentifier::STX()], + AssetMapEntry::STX(100) ); // and, one more time for good measure @@ -524,12 +496,8 @@ fn test_native_stx_ops(owned_env: &mut OwnedEnvironment) { assert_eq!(table.len(), 1); assert_eq!( - table - .get(&second_contract_principal) - .unwrap() - .get(&AssetIdentifier::STX()) - .unwrap(), - &AssetMapEntry::STX(100) + table[&second_contract_principal][&AssetIdentifier::STX()], + AssetMapEntry::STX(100) ); } @@ -735,13 +703,12 @@ fn test_simple_token_system(owned_env: &mut OwnedEnvironment) { p2_principal.clone(), &token_contract_id.clone(), "burn", - &symbols_from_values(vec![Value::UInt(100)]), + &symbols_from_values(vec![Value::UInt(100), p2.clone()]), ) .unwrap(); let asset_map = asset_map.to_table(); assert!(is_committed(&result)); - println!("{:?}", asset_map); assert_eq!( asset_map[&p2_principal][&token_identifier], AssetMapEntry::Token(100) @@ -776,13 +743,44 @@ fn test_simple_token_system(owned_env: &mut OwnedEnvironment) { p2_principal.clone(), &token_contract_id.clone(), "burn", - &symbols_from_values(vec![Value::UInt(9101)]), + &symbols_from_values(vec![Value::UInt(9101), p2.clone()]), ) .unwrap(); assert!(!is_committed(&result)); assert!(is_err_code(&result, 1)); + // Try to burn 0 tokens from p2's balance - Should fail with error code 3 + let (result, _asset_map, _events) = execute_transaction( + owned_env, + p2_principal.clone(), + &token_contract_id.clone(), + "burn", + &symbols_from_values(vec![Value::UInt(0), p2.clone()]), + ) + .unwrap(); + + assert!(!is_committed(&result)); + assert!(is_err_code(&result, 3)); + + // Try to burn 1 tokens from p2's balance (out of 9100) - Should pass even though + // sender != tx sender + let (result, asset_map, _events) = execute_transaction( + owned_env, + p1_principal.clone(), + &token_contract_id.clone(), + "burn", + &symbols_from_values(vec![Value::UInt(1), p2.clone()]), + ) + .unwrap(); + + let asset_map = asset_map.to_table(); + assert!(is_committed(&result)); + assert_eq!( + asset_map[&p2_principal][&token_identifier], + AssetMapEntry::Token(1) + ); + let (result, asset_map, _events) = execute_transaction( owned_env, p1_principal.clone(), @@ -1214,26 +1212,57 @@ fn test_simple_naming_system(owned_env: &mut OwnedEnvironment) { // preorder must exist! assert!(is_err_code(&result, 5)); - // p1 burning 5 should fail (not owner anymore). + // p1 burning 5 should fail since p1 is not the owner of that asset let (result, asset_map, _events) = execute_transaction( owned_env, p1_principal.clone(), &names_contract_id, "force-burn", - &symbols_from_values(vec![Value::Int(5)]), + &symbols_from_values(vec![Value::Int(5), p1.clone()]), ) .unwrap(); assert!(!is_committed(&result)); assert!(is_err_code(&result, 1)); + // p1 minting 8 should succeed + let (result, asset_map, _events) = execute_transaction( + owned_env, + p1_principal.clone(), + &names_contract_id, + "force-mint", + &symbols_from_values(vec![Value::Int(8)]), + ) + .unwrap(); + + assert!(is_committed(&result)); + assert_eq!(asset_map.to_table().len(), 0); + + // p2 burning 8 (which belongs to p1) should succeed even though sender != tx_sender. + let (result, asset_map, _events) = execute_transaction( + owned_env, + p2_principal.clone(), + &names_contract_id, + "force-burn", + &symbols_from_values(vec![Value::Int(8), p1.clone()]), + ) + .unwrap(); + + let asset_map = asset_map.to_table(); + + assert!(is_committed(&result)); + assert_eq!( + asset_map[&p1_principal][&names_identifier], + AssetMapEntry::Asset(vec![Value::Int(8)]) + ); + // p2 burning 5 should succeed. let (result, asset_map, _events) = execute_transaction( owned_env, p2_principal.clone(), &names_contract_id, "force-burn", - &symbols_from_values(vec![Value::Int(5)]), + &symbols_from_values(vec![Value::Int(5), p2.clone()]), ) .unwrap(); @@ -1245,13 +1274,13 @@ fn test_simple_naming_system(owned_env: &mut OwnedEnvironment) { AssetMapEntry::Asset(vec![Value::Int(5)]) ); - // p2 re-burning 5 should succeed. + // p2 re-burning 5 should not succeed since the asset does not exist let (result, asset_map, _events) = execute_transaction( owned_env, p2_principal.clone(), &names_contract_id, "force-burn", - &symbols_from_values(vec![Value::Int(5)]), + &symbols_from_values(vec![Value::Int(5), p2.clone()]), ) .unwrap(); assert!(!is_committed(&result)); From 4271ea1593e07f7b6e6b8a85aeca77e21091a6e3 Mon Sep 17 00:00:00 2001 From: Pavitthra Pandurangan Date: Mon, 2 May 2022 10:49:42 -0400 Subject: [PATCH 04/92] Changed BurnTokenErrorCodes to avoid consensus-breaking issue --- clarity/src/vm/docs/mod.rs | 3 +-- clarity/src/vm/functions/assets.rs | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/clarity/src/vm/docs/mod.rs b/clarity/src/vm/docs/mod.rs index 7899ed679..1bb96b234 100644 --- a/clarity/src/vm/docs/mod.rs +++ b/clarity/src/vm/docs/mod.rs @@ -1531,8 +1531,7 @@ rather destroyed, reducing the circulating supply. On a successful burn, it returns `(ok true)`. In the event of an unsuccessful burn it returns one of the following error codes: -`(err u1)` -- `sender` does not have enough balance to burn this amount -`(err u3)` -- the amount specified is not positive +`(err u1)` -- `sender` does not have enough balance to burn this amount or the amount specified is not positive ", example: " (define-fungible-token stackaroo) diff --git a/clarity/src/vm/functions/assets.rs b/clarity/src/vm/functions/assets.rs index f3e21c3be..867d165e4 100644 --- a/clarity/src/vm/functions/assets.rs +++ b/clarity/src/vm/functions/assets.rs @@ -54,8 +54,7 @@ enum BurnAssetErrorCodes { DOES_NOT_EXIST = 3, } enum BurnTokenErrorCodes { - NOT_ENOUGH_BALANCE = 1, - NON_POSITIVE_AMOUNT = 3, + NOT_ENOUGH_BALANCE_OR_NON_POSITIVE = 1, } enum StxErrorCodes { @@ -840,7 +839,7 @@ pub fn special_burn_token( if let (Value::UInt(amount), Value::Principal(ref burner)) = (amount, from) { if amount == 0 { - return clarity_ecode!(BurnTokenErrorCodes::NON_POSITIVE_AMOUNT); + return clarity_ecode!(BurnTokenErrorCodes::NOT_ENOUGH_BALANCE_OR_NON_POSITIVE); } let burner_bal = env.global_context.database.get_ft_balance( @@ -851,7 +850,7 @@ pub fn special_burn_token( )?; if amount > burner_bal { - return clarity_ecode!(BurnTokenErrorCodes::NOT_ENOUGH_BALANCE); + return clarity_ecode!(BurnTokenErrorCodes::NOT_ENOUGH_BALANCE_OR_NON_POSITIVE); } env.global_context.database.checked_decrease_token_supply( From 141840871a3a7285ccf51c68a1cff3fb4821a362 Mon Sep 17 00:00:00 2001 From: Pavitthra Pandurangan Date: Tue, 3 May 2022 15:05:29 -0400 Subject: [PATCH 05/92] update test assertion --- clarity/src/vm/tests/assets.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clarity/src/vm/tests/assets.rs b/clarity/src/vm/tests/assets.rs index 0ce9a5868..0fdb8ba82 100644 --- a/clarity/src/vm/tests/assets.rs +++ b/clarity/src/vm/tests/assets.rs @@ -750,7 +750,7 @@ fn test_simple_token_system(owned_env: &mut OwnedEnvironment) { assert!(!is_committed(&result)); assert!(is_err_code(&result, 1)); - // Try to burn 0 tokens from p2's balance - Should fail with error code 3 + // Try to burn 0 tokens from p2's balance - Should fail with error code 1 let (result, _asset_map, _events) = execute_transaction( owned_env, p2_principal.clone(), @@ -761,7 +761,7 @@ fn test_simple_token_system(owned_env: &mut OwnedEnvironment) { .unwrap(); assert!(!is_committed(&result)); - assert!(is_err_code(&result, 3)); + assert!(is_err_code(&result, 1)); // Try to burn 1 tokens from p2's balance (out of 9100) - Should pass even though // sender != tx sender From 199017a7f890298f5a2167c41158a6936f5159dc Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Wed, 4 May 2022 12:13:01 -0500 Subject: [PATCH 06/92] feat: add gh action for clarinet tests, move clarinet definitions to contrib/ --- .github/workflows/ci.yml | 14 ++++++++++++++ .../core-contract-tests}/Clarinet.toml | 2 ++ .../core-contract-tests}/settings/Devnet.toml | 0 .../tests/bns/name_register_test.ts | 0 4 files changed, 16 insertions(+) rename {core-contracts => contrib/core-contract-tests}/Clarinet.toml (94%) rename {core-contracts => contrib/core-contract-tests}/settings/Devnet.toml (100%) rename {core-contracts => contrib/core-contract-tests}/tests/bns/name_register_test.ts (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b20a74ac2..3a48626df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,20 @@ jobs: DOCKER_BUILDKIT: 1 run: docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.net-tests . + core-contracts-clarinet-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: "Execute core contract unit tests in Clarinet" + uses: docker://hirosystems/clarinet:latest + with: + args: test --coverage --manifest-path=./contrib/core-contract-tests/Clarinet.toml + - name: "Export code coverage" + uses: codecov/codecov-action@v1 + with: + files: ./coverage.lcov + verbose: true + # rustfmt checking rustfmt: runs-on: ubuntu-latest diff --git a/core-contracts/Clarinet.toml b/contrib/core-contract-tests/Clarinet.toml similarity index 94% rename from core-contracts/Clarinet.toml rename to contrib/core-contract-tests/Clarinet.toml index f2e6a218b..b470f963f 100644 --- a/core-contracts/Clarinet.toml +++ b/contrib/core-contract-tests/Clarinet.toml @@ -1,6 +1,8 @@ [project] name = "core-contracts" + +[repl] costs_version = 1 [contracts.bns] diff --git a/core-contracts/settings/Devnet.toml b/contrib/core-contract-tests/settings/Devnet.toml similarity index 100% rename from core-contracts/settings/Devnet.toml rename to contrib/core-contract-tests/settings/Devnet.toml diff --git a/core-contracts/tests/bns/name_register_test.ts b/contrib/core-contract-tests/tests/bns/name_register_test.ts similarity index 100% rename from core-contracts/tests/bns/name_register_test.ts rename to contrib/core-contract-tests/tests/bns/name_register_test.ts From 75c266563f7e6eb7c0382112ce6a824a163c7886 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Wed, 4 May 2022 14:13:41 -0500 Subject: [PATCH 07/92] oops, fix pathing change --- contrib/core-contract-tests/Clarinet.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/core-contract-tests/Clarinet.toml b/contrib/core-contract-tests/Clarinet.toml index b470f963f..d872af640 100644 --- a/contrib/core-contract-tests/Clarinet.toml +++ b/contrib/core-contract-tests/Clarinet.toml @@ -6,5 +6,5 @@ name = "core-contracts" costs_version = 1 [contracts.bns] -path = "../src/chainstate/stacks/boot/bns.clar" +path = "../../src/chainstate/stacks/boot/bns.clar" depends_on = [] From d09ea40a07f19c331f42f6bf22f5edb727d13091 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Mon, 16 May 2022 16:03:33 -0500 Subject: [PATCH 08/92] fix: add logging for some tx outcomes that were missing --- src/chainstate/stacks/miner.rs | 50 +++++++++++++++++++++++++--------- src/core/mempool.rs | 9 ++++-- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index cf32c28c8..b3151ec9c 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -890,11 +890,12 @@ impl<'a> StacksMicroblockBuilder<'a> { "Microblock miner deadline exceeded ({} ms)", self.settings.max_miner_time_ms ); - return Ok(false); + return Ok(None); } if considered.contains(&mempool_tx.tx.txid()) { - return Ok(true); + return Ok(Some(TransactionResult::skipped( + &mempool_tx.tx, "Transaction already considered.".to_string()).convert_to_event())); } else { considered.insert(mempool_tx.tx.txid()); } @@ -907,6 +908,7 @@ impl<'a> StacksMicroblockBuilder<'a> { &block_limit_hit, ) { Ok(tx_result) => { + let result_event = tx_result.convert_to_event(); tx_events.push(tx_result.convert_to_event()); match tx_result { TransactionResult::Success(TransactionSuccess { @@ -937,7 +939,7 @@ impl<'a> StacksMicroblockBuilder<'a> { num_txs += 1; num_added += 1; num_selected += 1; - Ok(true) + Ok(Some(result_event)) } TransactionResult::Skipped(TransactionSkipped { error, @@ -963,7 +965,7 @@ impl<'a> StacksMicroblockBuilder<'a> { debug!("Block budget exceeded while mining microblock"; "tx" => %mempool_tx.tx.txid(), "next_behavior" => "Stop mining microblock"); block_limit_hit = BlockLimitFunction::LIMIT_REACHED; - return Ok(false); + return Ok(None); } } Error::TransactionTooBigError => { @@ -971,7 +973,7 @@ impl<'a> StacksMicroblockBuilder<'a> { } _ => {} } - return Ok(true) + return Ok(Some(result_event)) } } } @@ -1995,28 +1997,49 @@ impl StacksBlockBuilder { let update_estimator = to_consider.update_estimate; if block_limit_hit == BlockLimitFunction::LIMIT_REACHED { - return Ok(false); + return Ok(None); } if get_epoch_time_ms() >= deadline { debug!("Miner mining time exceeded ({} ms)", max_miner_time_ms); - return Ok(false); + return Ok(None); } // skip transactions early if we can if considered.contains(&txinfo.tx.txid()) { - return Ok(true); + return Ok(Some( + TransactionResult::skipped( + &txinfo.tx, + "Transaction already considered.".to_string(), + ) + .convert_to_event(), + )); } if let Some(nonce) = mined_origin_nonces.get(&txinfo.tx.origin_address()) { if *nonce >= txinfo.tx.get_origin_nonce() { - return Ok(true); + let message = format!( + "Bad origin nonce, tx nonce {} versus {}.", + txinfo.tx.get_origin_nonce(), + *nonce + ); + return Ok(Some( + TransactionResult::skipped(&txinfo.tx, message) + .convert_to_event(), + )); } } if let Some(sponsor_addr) = txinfo.tx.sponsor_address() { if let Some(nonce) = mined_sponsor_nonces.get(&sponsor_addr) { if let Some(sponsor_nonce) = txinfo.tx.get_sponsor_nonce() { if *nonce >= sponsor_nonce { - return Ok(true); + let message = format!( + "Bad sponsor nonce, tx nonce {} versus {}.", + sponsor_nonce, *nonce + ); + return Ok(Some( + TransactionResult::skipped(&txinfo.tx, message) + .convert_to_event(), + )); } } } @@ -2033,6 +2056,7 @@ impl StacksBlockBuilder { ); tx_events.push(tx_result.convert_to_event()); + let result_event = tx_result.convert_to_event(); match tx_result { TransactionResult::Success(TransactionSuccess { receipt, .. }) => { num_txs += 1; @@ -2079,7 +2103,7 @@ impl StacksBlockBuilder { "Stop mining anchored block due to limit exceeded" ); block_limit_hit = BlockLimitFunction::LIMIT_REACHED; - return Ok(false); + return Ok(Some(result_event)); } } Error::TransactionTooBigError => { @@ -2090,13 +2114,13 @@ impl StacksBlockBuilder { } e => { warn!("Failed to apply tx {}: {:?}", &txinfo.tx.txid(), &e); - return Ok(true); + return Ok(Some(result_event)); } } } } - Ok(true) + Ok(Some(result_event)) }, ); diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 7b6a21ea1..c21b6da8c 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -1019,7 +1019,11 @@ impl MemPoolDB { ) -> Result where C: ClarityConnection, - F: FnMut(&mut C, &ConsiderTransaction, &mut dyn CostEstimator) -> Result, + F: FnMut( + &mut C, + &ConsiderTransaction, + &mut dyn CostEstimator, + ) -> Result, E>, E: From + From, { let start_time = Instant::now(); @@ -1079,7 +1083,8 @@ impl MemPoolDB { "size" => consider.tx.metadata.len); total_considered += 1; - if !todo(clarity_tx, &consider, self.cost_estimator.as_mut())? { + let todo_result = todo(clarity_tx, &consider, self.cost_estimator.as_mut())?; + if !todo_result.is_some() { debug!("Mempool iteration early exit from iterator"); break; } From 148294ab9577151cfa73c731f1f69472b4caf041 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Mon, 16 May 2022 17:04:09 -0500 Subject: [PATCH 09/92] fix tests --- src/core/tests/mod.rs | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/src/core/tests/mod.rs b/src/core/tests/mod.rs index ea883437d..84484346d 100644 --- a/src/core/tests/mod.rs +++ b/src/core/tests/mod.rs @@ -25,6 +25,7 @@ use crate::chainstate::stacks::db::test::chainstate_path; use crate::chainstate::stacks::db::test::instantiate_chainstate; use crate::chainstate::stacks::db::test::instantiate_chainstate_with_balances; use crate::chainstate::stacks::db::StreamCursor; +use crate::chainstate::stacks::miner::TransactionResult; use crate::chainstate::stacks::test::codec_all_transactions; use crate::chainstate::stacks::{ db::blocks::MemPoolRejection, db::StacksChainState, index::MarfTrieId, CoinbasePayload, @@ -287,7 +288,13 @@ fn mempool_walk_over_fork() { mempool_settings.clone(), |_, available_tx, _| { count_txs += 1; - Ok(true) + Ok(Some( + TransactionResult::skipped( + &available_tx.tx.tx, + "event not relevant to test".to_string(), + ) + .convert_to_event(), + )) }, ) .unwrap(); @@ -312,7 +319,13 @@ fn mempool_walk_over_fork() { mempool_settings.clone(), |_, available_tx, _| { count_txs += 1; - Ok(true) + Ok(Some( + TransactionResult::skipped( + &available_tx.tx.tx, + "event not relevant to test".to_string(), + ) + .convert_to_event(), + )) }, ) .unwrap(); @@ -336,7 +349,13 @@ fn mempool_walk_over_fork() { mempool_settings.clone(), |_, available_tx, _| { count_txs += 1; - Ok(true) + Ok(Some( + TransactionResult::skipped( + &available_tx.tx.tx, + "event not relevant to test".to_string(), + ) + .convert_to_event(), + )) }, ) .unwrap(); @@ -365,7 +384,13 @@ fn mempool_walk_over_fork() { mempool_settings.clone(), |_, available_tx, _| { count_txs += 1; - Ok(true) + Ok(Some( + TransactionResult::skipped( + &available_tx.tx.tx, + "event not relevant to test".to_string(), + ) + .convert_to_event(), + )) }, ) .unwrap(); @@ -392,7 +417,13 @@ fn mempool_walk_over_fork() { mempool_settings.clone(), |_, available_tx, _| { count_txs += 1; - Ok(true) + Ok(Some( + TransactionResult::skipped( + &available_tx.tx.tx, + "event not relevant to test".to_string(), + ) + .convert_to_event(), + )) }, ) .unwrap(); From 368ab2c7c3c0332b086a20c61caeaec63d093ea0 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Tue, 17 May 2022 11:11:02 -0500 Subject: [PATCH 10/92] have `iterate_candidates` manage the adding of events --- src/chainstate/stacks/miner.rs | 6 +++--- src/core/mempool.rs | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index b3151ec9c..a61a4671e 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -879,6 +879,7 @@ impl<'a> StacksMicroblockBuilder<'a> { let mut num_added = 0; intermediate_result = mem_pool.iterate_candidates( &mut clarity_tx, + &mut tx_events, self.anchor_block_height, mempool_settings.clone(), |clarity_tx, to_consider, estimator| { @@ -908,8 +909,7 @@ impl<'a> StacksMicroblockBuilder<'a> { &block_limit_hit, ) { Ok(tx_result) => { - let result_event = tx_result.convert_to_event(); - tx_events.push(tx_result.convert_to_event()); + let result_event = tx_result.convert_to_event(); match tx_result { TransactionResult::Success(TransactionSuccess { receipt, @@ -1990,6 +1990,7 @@ impl StacksBlockBuilder { let mut num_considered = 0; intermediate_result = mempool.iterate_candidates( &mut epoch_tx, + &mut tx_events, tip_height, mempool_settings.clone(), |epoch_tx, to_consider, estimator| { @@ -2054,7 +2055,6 @@ impl StacksBlockBuilder { txinfo.metadata.len, &block_limit_hit, ); - tx_events.push(tx_result.convert_to_event()); let result_event = tx_result.convert_to_event(); match tx_result { diff --git a/src/core/mempool.rs b/src/core/mempool.rs index c21b6da8c..b7ae1c61e 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -1009,10 +1009,14 @@ impl MemPoolDB { /// highest-fee-first order. This method is interruptable -- in the `settings` struct, the /// caller may choose how long to spend iterating before this method stops. /// - /// `todo` returns a boolean representing whether or not to keep iterating. + /// `todo` returns an option to a `TransactionEvent` representing the outcome, or None if we + /// hit an error that wasn't transaction specific. + /// + /// `output_events` is modified in place, adding all substantive transaction events output by `todo`. pub fn iterate_candidates( &mut self, clarity_tx: &mut C, + output_events: &mut Vec, _tip_height: u64, settings: MemPoolWalkSettings, mut todo: F, @@ -1083,10 +1087,15 @@ impl MemPoolDB { "size" => consider.tx.metadata.len); total_considered += 1; - let todo_result = todo(clarity_tx, &consider, self.cost_estimator.as_mut())?; - if !todo_result.is_some() { - debug!("Mempool iteration early exit from iterator"); - break; + // Run `todo` on the transaction. + match todo(clarity_tx, &consider, self.cost_estimator.as_mut())? { + Some(tx_event) => { + output_events.push(tx_event); + } + None => { + debug!("Mempool iteration early exit from iterator"); + break; + } } self.bump_last_known_nonces(&consider.tx.metadata.origin_address)?; From e3dedef6031b24b261c8547cf33a4c59abde08eb Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Tue, 17 May 2022 16:44:02 -0500 Subject: [PATCH 11/92] fix build errors in tests --- src/core/mempool.rs | 2 +- src/core/tests/mod.rs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index b7ae1c61e..3bc4e2c5e 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -1011,7 +1011,7 @@ impl MemPoolDB { /// /// `todo` returns an option to a `TransactionEvent` representing the outcome, or None if we /// hit an error that wasn't transaction specific. - /// + /// /// `output_events` is modified in place, adding all substantive transaction events output by `todo`. pub fn iterate_candidates( &mut self, diff --git a/src/core/tests/mod.rs b/src/core/tests/mod.rs index 84484346d..805e05601 100644 --- a/src/core/tests/mod.rs +++ b/src/core/tests/mod.rs @@ -275,7 +275,7 @@ fn mempool_walk_over_fork() { let mut mempool_settings = MemPoolWalkSettings::default(); mempool_settings.min_tx_fee = 10; - + let mut tx_events = Vec::new(); chainstate.with_read_only_clarity_tx( &TEST_BURN_STATE_DB, &StacksBlockHeader::make_index_block_hash(&b_2.0, &b_2.1), @@ -284,6 +284,7 @@ fn mempool_walk_over_fork() { mempool .iterate_candidates::<_, ChainstateError, _>( clarity_conn, + &mut tx_events, 2, mempool_settings.clone(), |_, available_tx, _| { @@ -315,6 +316,7 @@ fn mempool_walk_over_fork() { mempool .iterate_candidates::<_, ChainstateError, _>( clarity_conn, + &mut tx_events, 2, mempool_settings.clone(), |_, available_tx, _| { @@ -345,6 +347,7 @@ fn mempool_walk_over_fork() { mempool .iterate_candidates::<_, ChainstateError, _>( clarity_conn, + &mut tx_events, 3, mempool_settings.clone(), |_, available_tx, _| { @@ -380,6 +383,7 @@ fn mempool_walk_over_fork() { mempool .iterate_candidates::<_, ChainstateError, _>( clarity_conn, + &mut tx_events, 2, mempool_settings.clone(), |_, available_tx, _| { @@ -413,6 +417,7 @@ fn mempool_walk_over_fork() { mempool .iterate_candidates::<_, ChainstateError, _>( clarity_conn, + &mut tx_events, 3, mempool_settings.clone(), |_, available_tx, _| { From eb8f237c0c4b98ab3dda231860caef0c4aa36271 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 24 May 2022 17:29:21 +0200 Subject: [PATCH 12/92] chore: update vscode rust-analyzer extension ID The `matklad.rust-analyzer` vscode rust extension changed to `rust-lang.rust-analyzer` in Feb 2022, see https://blog.rust-lang.org/2022/02/21/rust-analyzer-joins-rust-org.html -- this fixes the vscode error prompt due to the no longer existent plugin ID being specified --- .vscode/extensions.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index deb8143f7..be7e11c2a 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,8 +1,8 @@ { "recommendations": [ - "matklad.rust-analyzer", + "rust-lang.rust-analyzer", "vadimcn.vscode-lldb", "serayuzgur.crates", "editorconfig.editorconfig", ] -} \ No newline at end of file +} From dd2c17ad549563b30296c5365b17d69a42689f11 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Wed, 25 May 2022 15:44:20 -0500 Subject: [PATCH 13/92] fix Some(None) bug, and reformat messages --- src/chainstate/stacks/miner.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index a61a4671e..2ba60c735 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -2018,14 +2018,16 @@ impl StacksBlockBuilder { if let Some(nonce) = mined_origin_nonces.get(&txinfo.tx.origin_address()) { if *nonce >= txinfo.tx.get_origin_nonce() { - let message = format!( - "Bad origin nonce, tx nonce {} versus {}.", - txinfo.tx.get_origin_nonce(), - *nonce - ); return Ok(Some( - TransactionResult::skipped(&txinfo.tx, message) - .convert_to_event(), + TransactionResult::skipped( + &txinfo.tx, + format!( + "Bad origin nonce, tx nonce {} versus {}.", + txinfo.tx.get_origin_nonce(), + *nonce + ), + ) + .convert_to_event(), )); } } @@ -2033,13 +2035,15 @@ impl StacksBlockBuilder { if let Some(nonce) = mined_sponsor_nonces.get(&sponsor_addr) { if let Some(sponsor_nonce) = txinfo.tx.get_sponsor_nonce() { if *nonce >= sponsor_nonce { - let message = format!( - "Bad sponsor nonce, tx nonce {} versus {}.", - sponsor_nonce, *nonce - ); return Ok(Some( - TransactionResult::skipped(&txinfo.tx, message) - .convert_to_event(), + TransactionResult::skipped( + &txinfo.tx, + format!( + "Bad sponsor nonce, tx nonce {} versus {}.", + sponsor_nonce, *nonce + ), + ) + .convert_to_event(), )); } } @@ -2103,7 +2107,7 @@ impl StacksBlockBuilder { "Stop mining anchored block due to limit exceeded" ); block_limit_hit = BlockLimitFunction::LIMIT_REACHED; - return Ok(Some(result_event)); + return Ok(None); } } Error::TransactionTooBigError => { From e570d394ec329ece6f453192c59b5d8fb5fc385c Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 25 May 2022 16:57:27 -0400 Subject: [PATCH 14/92] fix: use latest clarinet test lib --- contrib/core-contract-tests/tests/bns/name_register_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/core-contract-tests/tests/bns/name_register_test.ts b/contrib/core-contract-tests/tests/bns/name_register_test.ts index 3fb378949..6fd5d9df1 100644 --- a/contrib/core-contract-tests/tests/bns/name_register_test.ts +++ b/contrib/core-contract-tests/tests/bns/name_register_test.ts @@ -1,4 +1,4 @@ -import { Clarinet, Tx, Chain, Account, Contract, types } from 'https://deno.land/x/clarinet@v0.16.0/index.ts'; +import { Clarinet, Tx, Chain, Account, Contract, types } from 'https://deno.land/x/clarinet@v0.31.0/index.ts'; import { assertEquals } from "https://deno.land/std@0.90.0/testing/asserts.ts"; import { createHash } from "https://deno.land/std@0.107.0/hash/mod.ts"; From 74cc4169524696a4246f7a05ade0ff76b97f27f1 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 00:11:32 -0400 Subject: [PATCH 15/92] chore: convert immunefi bug report into test vectors and update the indexer's find_bitcoin_reorg() method to check the original and reorg chain's total work in order to decide whether to move forward with reorg processing. --- src/burnchains/bitcoin/indexer.rs | 435 ++++++++++++++++++++++++++++-- 1 file changed, 416 insertions(+), 19 deletions(-) diff --git a/src/burnchains/bitcoin/indexer.rs b/src/burnchains/bitcoin/indexer.rs index 1ffd8ed1b..07814ab5c 100644 --- a/src/burnchains/bitcoin/indexer.rs +++ b/src/burnchains/bitcoin/indexer.rs @@ -385,41 +385,111 @@ impl BitcoinIndexer { .and_then(|_r| Ok(spv_client.end_block_height.unwrap())) } + #[cfg(test)] + fn new_reorg_spv_client( + reorg_headers_path: &str, + start_block: u64, + end_block: Option, + network_id: BitcoinNetworkType, + ) -> Result { + SpvClient::new_without_migration( + &reorg_headers_path, + start_block, + end_block, + network_id, + true, + true, + ) + } + + #[cfg(not(test))] + fn new_reorg_spv_client( + reorg_headers_path: &str, + start_block: u64, + end_block: Option, + network_id: BitcoinNetworkType, + ) -> Result { + SpvClient::new( + &reorg_headers_path, + start_block, + end_block, + network_id, + true, + true, + ) + } + /// Create a SPV client for starting reorg processing fn setup_reorg_headers( &mut self, canonical_spv_client: &SpvClient, reorg_headers_path: &str, start_block: u64, + remove_old: bool, ) -> Result { - if PathBuf::from(&reorg_headers_path).exists() { - fs::remove_file(&reorg_headers_path).map_err(|e| { - error!("Failed to remove {}", reorg_headers_path); - btc_error::Io(e) - })?; + if remove_old { + if PathBuf::from(&reorg_headers_path).exists() { + fs::remove_file(&reorg_headers_path).map_err(|e| { + error!("Failed to remove {}", reorg_headers_path); + btc_error::Io(e) + })?; + } } // bootstrap reorg client - let mut reorg_spv_client = SpvClient::new( - &reorg_headers_path, + let mut reorg_spv_client = BitcoinIndexer::new_reorg_spv_client( + reorg_headers_path, start_block, Some(start_block + REORG_BATCH_SIZE), self.runtime.network_id, - true, - true, )?; + if start_block > 0 { - let start_header = canonical_spv_client - .read_block_header(start_block)? - .expect(&format!("BUG: missing block header for {}", start_block)); - reorg_spv_client.insert_block_headers_before(start_block - 1, vec![start_header])?; + if start_block > BLOCK_DIFFICULTY_CHUNK_SIZE { + if remove_old { + let interval_start_block = start_block / BLOCK_DIFFICULTY_CHUNK_SIZE - 2; + let base_block = interval_start_block * BLOCK_DIFFICULTY_CHUNK_SIZE; + let interval_headers = + canonical_spv_client.read_block_headers(base_block, start_block)?; + assert!( + interval_headers.len() == (start_block - base_block) as usize, + "BUG: missing headers for {}-{}", + base_block, + start_block + ); + + test_debug!( + "Copy headers {}-{}", + base_block, + base_block + interval_headers.len() as u64 + ); + reorg_spv_client + .insert_block_headers_before(base_block - 1, interval_headers)?; + + let last_interval = canonical_spv_client.find_highest_work_score_interval()?; + + // copy over the relevant difficulty intervals as well + for interval in interval_start_block..(last_interval + 1) { + test_debug!("Copy interval {} to {}", interval, &reorg_headers_path); + let work_score = canonical_spv_client + .find_interval_work(interval)? + .expect(&format!("FATAL: no work score for interval {}", interval)); + reorg_spv_client.store_interval_work(interval, work_score)?; + } + } + } else { + // no full difficulty intervals yet + let interval_headers = canonical_spv_client.read_block_headers(1, start_block)?; + + reorg_spv_client.insert_block_headers_before(0, interval_headers)?; + } } Ok(reorg_spv_client) } /// Search for a bitcoin reorg. Return the offset into the canonical bitcoin headers where - /// the reorg starts. Returns the hight of the highest common ancestor, and its block hash. + /// the reorg starts. Returns the hight of the highest common ancestor. /// Note that under certain testnet settings, the bitcoin chain itself can shrink. pub fn find_bitcoin_reorg( &mut self, @@ -454,7 +524,7 @@ impl BitcoinIndexer { // bootstrap reorg client let mut start_block = canonical_end_block.saturating_sub(REORG_BATCH_SIZE); let mut reorg_spv_client = - self.setup_reorg_headers(&orig_spv_client, reorg_headers_path, start_block)?; + self.setup_reorg_headers(&orig_spv_client, reorg_headers_path, start_block, true)?; let mut discontiguous_header_error_count = 0; while !found_common_ancestor { @@ -493,6 +563,7 @@ impl BitcoinIndexer { &orig_spv_client, reorg_headers_path, start_block, + false, )?; continue; } @@ -600,10 +671,27 @@ impl BitcoinIndexer { // try again start_block = start_block.saturating_sub(REORG_BATCH_SIZE); reorg_spv_client = - self.setup_reorg_headers(&orig_spv_client, reorg_headers_path, start_block)?; + self.setup_reorg_headers(&orig_spv_client, reorg_headers_path, start_block, false)?; } - debug!("Bitcoin headers history is consistent up to {}", new_tip); + let reorg_total_work = reorg_spv_client.update_chain_work()?; + let orig_total_work = orig_spv_client.get_chain_work()?; + + debug!("Bitcoin headers history is consistent up to {}. Orig chainwork: {}, reorg chainwork: {}", new_tip, orig_total_work, reorg_total_work); + if orig_total_work < reorg_total_work { + let reorg_tip = reorg_spv_client.get_headers_height()?; + let hdr_reorg = reorg_spv_client + .read_block_header(reorg_tip - 1)? + .expect("FATAL: no tip hash for existing chain tip"); + info!( + "New canonical Bitcoin chain found! New tip is {}", + &hdr_reorg.header.bitcoin_hash() + ); + } else { + // ignore the reorg + test_debug!("Reorg chain does not overtake original Bitcoin chain"); + new_tip = orig_spv_client.get_headers_height()?; + } let hdr_reorg = reorg_spv_client.read_block_header(new_tip)?; let hdr_canonical = orig_spv_client.read_block_header(new_tip)?; @@ -1224,8 +1312,8 @@ mod test { peer_port: port, rpc_port: port + 1, // ignored rpc_ssl: false, - username: None, - password: None, + username: Some("blockstack".to_string()), + password: Some("blockstacksystem".to_string()), timeout: 30, spv_headers_path: "/tmp/test_indexer_sync_headers.sqlite".to_string(), first_block: 0, @@ -1241,4 +1329,313 @@ mod test { let last_block = indexer.sync_headers(0, None).unwrap(); eprintln!("sync'ed to block {}", last_block); } + + #[test] + fn test_spv_check_work_reorg_ignored() { + if !env::var("BLOCKSTACK_SPV_HEADERS_DB").is_ok() { + eprintln!("Skipping test_spv_check_work_reorg_ignored -- no BLOCKSTACK_SPV_HEADERS_DB envar set"); + return; + } + let db_path_source = env::var("BLOCKSTACK_SPV_HEADERS_DB").unwrap(); + let db_path = "/tmp/test_spv_check_work_reorg_ignored.dat".to_string(); + let reorg_db_path = "/tmp/test_spv_check_work_ignored.dat.reorg".to_string(); + + if fs::metadata(&db_path).is_ok() { + fs::remove_file(&db_path).unwrap(); + } + + if fs::metadata(&reorg_db_path).is_ok() { + fs::remove_file(&reorg_db_path).unwrap(); + } + + fs::copy(&db_path_source, &db_path).unwrap(); + + { + // set up SPV client so we don't have chain work at first + let mut spv_client = SpvClient::new_without_migration( + &db_path, + 0, + None, + BitcoinNetworkType::Mainnet, + true, + false, + ) + .unwrap(); + + assert!( + spv_client.get_headers_height().unwrap() >= 40322, + "This test needs headers up to 40320" + ); + spv_client.drop_headers(40320).unwrap(); + } + + let mut spv_client = + SpvClient::new(&db_path, 0, None, BitcoinNetworkType::Mainnet, true, false).unwrap(); + + assert_eq!(spv_client.get_headers_height().unwrap(), 40321); + let total_work_before = spv_client.update_chain_work().unwrap(); + assert_eq!(total_work_before, spv_client.get_chain_work().unwrap()); + + let total_work_before_idempotent = spv_client.update_chain_work().unwrap(); + assert_eq!(total_work_before, total_work_before_idempotent); + + // fake block headers for mainnet 40319-40320, which is on a difficulty adjustment boundary + let bad_headers = vec![ + LoneBlockHeader { + header: BlockHeader { + version: 1, + prev_blockhash: Sha256dHash::from_hex( + "000000000683a474ef810000fd22f0edde4cf33ae76ae506b220e57aeeafeaa4", + ) + .unwrap(), + merkle_root: Sha256dHash::from_hex( + "b4d736ca74838036ebd19b085c3eeb9ffec2307f6452347cdd8ddaa249686f39", + ) + .unwrap(), + time: 1716199659, + bits: 486575299, + nonce: 201337507, + }, + tx_count: VarInt(0), + }, + LoneBlockHeader { + header: BlockHeader { + version: 1, + prev_blockhash: Sha256dHash::from_hex( + "000000006f403731d720174cd6875e331ac079b438cf53aa685f9cd068fd4ca8", + ) + .unwrap(), + merkle_root: Sha256dHash::from_hex( + "a86b3c149f204d4cb47c67bf9bfeea2719df101dd6e6fc3f0e60d86efeba22a8", + ) + .unwrap(), + time: 1716161259, + bits: 486604799, + nonce: 144574511, + }, + tx_count: VarInt(0), + }, + ]; + + let mut indexer = BitcoinIndexer::new( + BitcoinIndexerConfig::test_default(db_path.to_string()), + BitcoinIndexerRuntime::new(BitcoinNetworkType::Mainnet), + ); + + let mut inserted_bad_header = false; + + let new_tip = indexer + .find_bitcoin_reorg( + &db_path, + &reorg_db_path, + |ref mut indexer, ref mut reorg_spv_client, start_block, end_block_opt| { + let end_block = + end_block_opt.unwrap_or(start_block + BLOCK_DIFFICULTY_CHUNK_SIZE); + + let mut ret = vec![]; + for block_height in start_block..end_block { + if block_height > 40320 { + break; + } + if block_height >= 40319 && block_height <= 40320 { + test_debug!("insert bad header {}", block_height); + ret.push(bad_headers[(block_height - 40319) as usize].clone()); + inserted_bad_header = true; + } else { + let orig_spv_client = SpvClient::new_without_migration( + &db_path, + 0, + None, + BitcoinNetworkType::Mainnet, + true, + false, + ) + .unwrap(); + let hdr = orig_spv_client.read_block_header(block_height)?.unwrap(); + ret.push(hdr); + } + } + + test_debug!( + "add headers after {} (bad header: {})", + start_block, + inserted_bad_header + ); + reorg_spv_client + .insert_block_headers_after(start_block - 1, ret) + .unwrap(); + Ok(()) + }, + ) + .unwrap(); + + assert!(inserted_bad_header); + + // reorg is ignored + assert_eq!(new_tip, 40321); + let hdr = spv_client.read_block_header(new_tip - 1).unwrap().unwrap(); + eprintln!("{}", &hdr.header.bitcoin_hash()); + let total_work_after = spv_client.update_chain_work().unwrap(); + assert_eq!(total_work_after, total_work_before); + } + + #[test] + fn test_spv_check_work_reorg_accepted() { + if !env::var("BLOCKSTACK_SPV_HEADERS_DB").is_ok() { + eprintln!("Skipping test_spv_check_work_reorg_accepted -- no BLOCKSTACK_SPV_HEADERS_DB envar set"); + return; + } + let db_path_source = env::var("BLOCKSTACK_SPV_HEADERS_DB").unwrap(); + let db_path = "/tmp/test_spv_check_work_reorg_accepted.dat".to_string(); + let reorg_db_path = "/tmp/test_spv_check_work_reorg_accepted.dat.reorg".to_string(); + + if fs::metadata(&db_path).is_ok() { + fs::remove_file(&db_path).unwrap(); + } + + if fs::metadata(&reorg_db_path).is_ok() { + fs::remove_file(&reorg_db_path).unwrap(); + } + + fs::copy(&db_path_source, &db_path).unwrap(); + + // set up SPV client so we don't have chain work at first + let mut spv_client = SpvClient::new_without_migration( + &db_path, + 0, + None, + BitcoinNetworkType::Mainnet, + true, + false, + ) + .unwrap(); + + assert!( + spv_client.get_headers_height().unwrap() >= 40322, + "This test needs headers up to 40320" + ); + spv_client.drop_headers(40320).unwrap(); + + assert_eq!(spv_client.get_headers_height().unwrap(), 40321); + + // fake block headers for mainnet 40319-40320, which is on a difficulty adjustment boundary + let bad_headers = vec![ + LoneBlockHeader { + header: BlockHeader { + version: 1, + prev_blockhash: Sha256dHash::from_hex( + "000000000683a474ef810000fd22f0edde4cf33ae76ae506b220e57aeeafeaa4", + ) + .unwrap(), + merkle_root: Sha256dHash::from_hex( + "b4d736ca74838036ebd19b085c3eeb9ffec2307f6452347cdd8ddaa249686f39", + ) + .unwrap(), + time: 1716199659, + bits: 486575299, + nonce: 201337507, + }, + tx_count: VarInt(0), + }, + LoneBlockHeader { + header: BlockHeader { + version: 1, + prev_blockhash: Sha256dHash::from_hex( + "000000006f403731d720174cd6875e331ac079b438cf53aa685f9cd068fd4ca8", + ) + .unwrap(), + merkle_root: Sha256dHash::from_hex( + "a86b3c149f204d4cb47c67bf9bfeea2719df101dd6e6fc3f0e60d86efeba22a8", + ) + .unwrap(), + time: 1716161259, + bits: 486604799, + nonce: 144574511, + }, + tx_count: VarInt(0), + }, + ]; + + // get the canonical chain's headers for this range + let good_headers = spv_client.read_block_headers(40319, 40321).unwrap(); + assert_eq!(good_headers.len(), 2); + assert_eq!( + good_headers[0].header.prev_blockhash, + bad_headers[0].header.prev_blockhash + ); + assert!(good_headers[0].header != bad_headers[0].header); + assert!(good_headers[1].header != bad_headers[1].header); + + // put these bad headers into the "main" chain + spv_client + .insert_block_headers_after(40318, bad_headers.clone()) + .unwrap(); + + // *now* calculate main chain work + SpvClient::test_db_migrate(spv_client.conn_mut()).unwrap(); + let total_work_before = spv_client.update_chain_work().unwrap(); + assert_eq!(total_work_before, spv_client.get_chain_work().unwrap()); + + let total_work_before_idempotent = spv_client.update_chain_work().unwrap(); + assert_eq!(total_work_before, total_work_before_idempotent); + + let mut indexer = BitcoinIndexer::new( + BitcoinIndexerConfig::test_default(db_path.to_string()), + BitcoinIndexerRuntime::new(BitcoinNetworkType::Mainnet), + ); + + let mut inserted_good_header = false; + + let new_tip = indexer + .find_bitcoin_reorg( + &db_path, + &reorg_db_path, + |ref mut indexer, ref mut reorg_spv_client, start_block, end_block_opt| { + let end_block = + end_block_opt.unwrap_or(start_block + BLOCK_DIFFICULTY_CHUNK_SIZE); + + let mut ret = vec![]; + for block_height in start_block..end_block { + if block_height > 40320 { + break; + } + if block_height >= 40319 && block_height <= 40320 { + test_debug!("insert good header {}", block_height); + ret.push(good_headers[(block_height - 40319) as usize].clone()); + inserted_good_header = true; + } else { + let orig_spv_client = SpvClient::new_without_migration( + &db_path, + 0, + None, + BitcoinNetworkType::Mainnet, + true, + false, + ) + .unwrap(); + let hdr = orig_spv_client.read_block_header(block_height)?.unwrap(); + ret.push(hdr); + } + } + + test_debug!( + "add headers after {} (good header: {})", + start_block, + inserted_good_header + ); + reorg_spv_client + .insert_block_headers_after(start_block - 1, ret) + .unwrap(); + Ok(()) + }, + ) + .unwrap(); + + assert!(inserted_good_header); + + // chain reorg detected! + assert_eq!(new_tip, 40318); + let total_work_after = spv_client.update_chain_work().unwrap(); + assert_eq!(total_work_after, total_work_before); + } } From 04d2275aa05527b9f7f536280d04f4323113bd91 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 00:12:26 -0400 Subject: [PATCH 16/92] chore: add InvalidDifficulty variant --- src/burnchains/bitcoin/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/burnchains/bitcoin/mod.rs b/src/burnchains/bitcoin/mod.rs index 3ce099dc6..c9b7ab5d3 100644 --- a/src/burnchains/bitcoin/mod.rs +++ b/src/burnchains/bitcoin/mod.rs @@ -79,6 +79,8 @@ pub enum Error { MissingHeader, /// Invalid target InvalidPoW, + /// Bad difficulty + InvalidDifficulty, /// Wrong number of bytes for constructing an address InvalidByteSequence, /// Configuration error @@ -107,6 +109,7 @@ impl fmt::Display for Error { Error::NoncontiguousHeader => write!(f, "Non-contiguous header"), Error::MissingHeader => write!(f, "Missing header"), Error::InvalidPoW => write!(f, "Invalid proof of work"), + Error::InvalidDifficulty => write!(f, "Chain difficulty cannot decrease"), Error::InvalidByteSequence => write!(f, "Invalid sequence of bytes"), Error::ConfigError(ref e_str) => fmt::Display::fmt(e_str, f), Error::BlockchainHeight => write!(f, "Value is beyond the end of the blockchain"), @@ -133,6 +136,7 @@ impl error::Error for Error { Error::NoncontiguousHeader => None, Error::MissingHeader => None, Error::InvalidPoW => None, + Error::InvalidDifficulty => None, Error::InvalidByteSequence => None, Error::ConfigError(ref _e_str) => None, Error::BlockchainHeight => None, From 998b9b73d8db73e7cd584fdba6d48c0203b352e3 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 00:12:39 -0400 Subject: [PATCH 17/92] feat: update the SPV client to track the total chain work over time. BUT, right now the work calculation for a difficulty adjustment interval is getting the wrong answers and I don't know why. --- src/burnchains/bitcoin/spv.rs | 452 +++++++++++++++++++++++++++++++--- 1 file changed, 421 insertions(+), 31 deletions(-) diff --git a/src/burnchains/bitcoin/spv.rs b/src/burnchains/bitcoin/spv.rs index 7631aae25..c86d8b8df 100644 --- a/src/burnchains/bitcoin/spv.rs +++ b/src/burnchains/bitcoin/spv.rs @@ -39,13 +39,14 @@ use crate::burnchains::bitcoin::Error as btc_error; use crate::burnchains::bitcoin::PeerMessage; use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ToSql, ToSqlOutput, ValueRef}; +use rusqlite::OptionalExtension; use rusqlite::Row; use rusqlite::Transaction; use rusqlite::{Connection, OpenFlags, NO_PARAMS}; use crate::util_lib::db::{ - query_row, query_rows, sqlite_open, tx_begin_immediate, tx_busy_handler, u64_to_sql, DBConn, - DBTx, Error as db_error, FromColumn, FromRow, + query_int, query_row, query_rows, sqlite_open, tx_begin_immediate, tx_busy_handler, u64_to_sql, + DBConn, DBTx, Error as db_error, FromColumn, FromRow, }; use stacks_common::util::get_epoch_time_secs; use stacks_common::util::hash::{hex_bytes, to_hex}; @@ -67,7 +68,7 @@ const BITCOIN_GENESIS_BLOCK_HASH_REGTEST: &'static str = pub const BLOCK_DIFFICULTY_CHUNK_SIZE: u64 = 2016; const BLOCK_DIFFICULTY_INTERVAL: u32 = 14 * 24 * 60 * 60; // two weeks, in seconds -pub const SPV_DB_VERSION: &'static str = "1"; +pub const SPV_DB_VERSION: &'static str = "2"; const SPV_INITIAL_SCHEMA: &[&'static str] = &[ r#" @@ -84,6 +85,13 @@ const SPV_INITIAL_SCHEMA: &[&'static str] = &[ "CREATE TABLE db_config(version TEXT NOT NULL);", ]; +const SPV_SCHEMA_2: &[&'static str] = &[r#" + CREATE TABLE chain_work( + interval INTEGER PRIMARY KEY, + work TEXT NOT NULL -- 32-byte (256-bit) integer + ); + "#]; + pub struct SpvClient { pub headers_path: String, pub start_block_height: u64, @@ -93,6 +101,9 @@ pub struct SpvClient { readwrite: bool, reverse_order: bool, headers_db: DBConn, + + // only writeable in #[cfg(test)] + ignore_work_checks: bool, } impl FromColumn for Sha256dHash { @@ -130,7 +141,7 @@ impl SpvClient { readwrite: bool, reverse_order: bool, ) -> Result { - let conn = SpvClient::db_open(headers_path, readwrite)?; + let conn = SpvClient::db_open(headers_path, readwrite, true)?; let mut client = SpvClient { headers_path: headers_path.to_owned(), start_block_height: start_block, @@ -140,19 +151,59 @@ impl SpvClient { readwrite: readwrite, reverse_order: reverse_order, headers_db: conn, + ignore_work_checks: false, }; if readwrite { - client.init_block_headers()?; + client.init_block_headers(true)?; } Ok(client) } + #[cfg(test)] + pub fn new_without_migration( + headers_path: &str, + start_block: u64, + end_block: Option, + network_id: BitcoinNetworkType, + readwrite: bool, + reverse_order: bool, + ) -> Result { + let conn = SpvClient::db_open(headers_path, readwrite, false)?; + let mut client = SpvClient { + headers_path: headers_path.to_owned(), + start_block_height: start_block, + end_block_height: end_block, + cur_block_height: start_block, + network_id: network_id, + readwrite: readwrite, + reverse_order: reverse_order, + headers_db: conn, + ignore_work_checks: true, + }; + + if readwrite { + client.init_block_headers(false)?; + } + + Ok(client) + } + + #[cfg(test)] + pub fn set_ignore_work_checks(&mut self, ignore: bool) { + self.ignore_work_checks = ignore; + } + pub fn conn(&self) -> &DBConn { &self.headers_db } + #[cfg(test)] + pub fn conn_mut(&mut self) -> &mut DBConn { + &mut self.headers_db + } + pub fn tx_begin<'a>(&'a mut self) -> Result, btc_error> { if !self.readwrite { return Err(db_error::ReadOnly.into()); @@ -169,6 +220,9 @@ impl SpvClient { for row_text in SPV_INITIAL_SCHEMA { tx.execute_batch(row_text).map_err(db_error::SqliteError)?; } + for row_text in SPV_SCHEMA_2 { + tx.execute_batch(row_text).map_err(db_error::SqliteError)?; + } tx.execute( "INSERT INTO db_config (version) VALUES (?1)", @@ -180,7 +234,57 @@ impl SpvClient { Ok(()) } - fn db_open(headers_path: &str, readwrite: bool) -> Result { + fn db_get_version(conn: &DBConn) -> Result { + let version_str = conn + .query_row("SELECT MAX(version) FROM db_config", NO_PARAMS, |row| { + let version: String = row.get_unwrap(0); + Ok(version) + }) + .optional() + .map_err(db_error::SqliteError)? + .unwrap_or("0".to_string()); + Ok(version_str) + } + + fn db_set_version(tx: &Transaction, version: &str) -> Result<(), btc_error> { + tx.execute("UPDATE db_config SET version = ?1", &[version]) + .map_err(db_error::SqliteError) + .map_err(|e| e.into()) + .and_then(|_| Ok(())) + } + + #[cfg(test)] + pub fn test_db_migrate(conn: &mut DBConn) -> Result<(), btc_error> { + SpvClient::db_migrate(conn) + } + + fn db_migrate(conn: &mut DBConn) -> Result<(), btc_error> { + let version = SpvClient::db_get_version(conn)?; + while version != SPV_DB_VERSION { + let version = SpvClient::db_get_version(conn)?; + match version.as_str() { + "1" => { + debug!("Migrate SPV DB from schema 1 to 2"); + let tx = tx_begin_immediate(conn)?; + for row_text in SPV_SCHEMA_2 { + tx.execute_batch(row_text).map_err(db_error::SqliteError)?; + } + + SpvClient::db_set_version(&tx, "2")?; + tx.commit().map_err(db_error::SqliteError)?; + } + SPV_DB_VERSION => { + break; + } + _ => { + panic!("Unrecognized SPV version {}", &version); + } + } + } + Ok(()) + } + + fn db_open(headers_path: &str, readwrite: bool, migrate: bool) -> Result { let mut create_flag = false; let open_flags = if fs::metadata(headers_path).is_err() { // need to create @@ -205,6 +309,9 @@ impl SpvClient { if create_flag { SpvClient::db_instantiate(&mut conn)?; } + if readwrite && migrate { + SpvClient::db_migrate(&mut conn)?; + } Ok(conn) } @@ -229,6 +336,234 @@ impl SpvClient { indexer.peer_communicate(self, true) } + /// Calculate the work of a single header given the first and last header in the interval + fn get_expected_work_in_range( + first_header: &LoneBlockHeader, + last_header: &LoneBlockHeader, + ) -> Uint256 { + let (_, target) = SpvClient::get_target_between_headers(&first_header, &last_header); + let work = + (Uint256::max() - target) / (target + Uint256::from_u64(1)) + Uint256::from_u64(1); + test_debug!("{}, {}", &work, &target); + work + } + + /// Calculate the total work over a full interval of headers. + fn get_full_interval_work(interval_headers: &Vec) -> Uint256 { + assert_eq!(interval_headers.len() as u64, BLOCK_DIFFICULTY_CHUNK_SIZE); + let first_header = interval_headers + .first() + .expect("FATAL: no first header in non-empty list of headers"); + let last_header = interval_headers + .last() + .expect("FATAL: no last header in non-empty list of headers"); + SpvClient::get_expected_work_in_range(first_header, last_header) + * Uint256::from_u64(BLOCK_DIFFICULTY_CHUNK_SIZE) + } + + /// Calculate a partial interval's work, given the last full interval before it + fn get_partial_interval_work( + &self, + last_full_interval: u64, + partial_interval_len: usize, + ) -> Result, btc_error> { + let last_interval_work = self.get_interval_header_work(last_full_interval)?; + if let Some(last_interval_work) = last_interval_work { + let work = last_interval_work * Uint256::from_u64(partial_interval_len as u64); + Ok(Some(work)) + } else { + Ok(None) + } + } + + /// Calculate the work done by a single header in `interval`, if we have the headers for that + /// interval + pub fn get_interval_header_work(&self, interval: u64) -> Result, btc_error> { + let first_header = + match self.read_block_header((interval - 1) * BLOCK_DIFFICULTY_CHUNK_SIZE)? { + Some(res) => res, + None => { + test_debug!( + "No header at height {}", + (interval - 1) * BLOCK_DIFFICULTY_CHUNK_SIZE + ); + return Ok(None); + } + }; + + let last_header = + match self.read_block_header(interval * BLOCK_DIFFICULTY_CHUNK_SIZE - 1)? { + Some(res) => res, + None => { + test_debug!( + "No header at height {}", + interval * BLOCK_DIFFICULTY_CHUNK_SIZE - 1 + ); + return Ok(None); + } + }; + + Ok(Some(SpvClient::get_expected_work_in_range( + &first_header, + &last_header, + ))) + } + + /// Find the highest interval for which we have a chain work score. + /// The interval corresponds to blocks (interval - 1) * 2016 ... interval * 2016 + pub fn find_highest_work_score_interval(&self) -> Result { + let max_interval_opt: Option = self + .conn() + .query_row( + "SELECT interval FROM chain_work ORDER BY interval DESC LIMIT 1", + NO_PARAMS, + |row| row.get(0), + ) + .optional() + .map_err(db_error::SqliteError)?; + + Ok(max_interval_opt.map(|x| x as u64).unwrap_or(0)) + } + + /// Find the total work score for an interval, if it has been calculated + pub fn find_interval_work(&self, interval: u64) -> Result, btc_error> { + let work_hex: Option = self + .conn() + .query_row( + "SELECT work FROM chain_work WHERE interval = ?1", + &[&u64_to_sql(interval)?], + |row| row.get(0), + ) + .optional() + .map_err(db_error::SqliteError)?; + Ok(work_hex.map(|x| Uint256::from_hex_le(&x).expect("FATAL: work is not a uint256"))) + } + + /// Store an interval's running total work. + /// The interval must not yet have an interval work score, or must be less than or equal to the + /// currently-stored interval. + pub fn store_interval_work(&mut self, interval: u64, work: Uint256) -> Result<(), btc_error> { + if let Some(cur_work) = self.find_interval_work(interval)? { + if cur_work > work && !self.ignore_work_checks { + error!( + "Tried to store work {} to interval {}, which has work {} already", + work, interval, cur_work + ); + return Err(btc_error::InvalidDifficulty); + } + } + + let tx = self.tx_begin()?; + let args: &[&dyn ToSql] = &[&u64_to_sql(interval)?, &work.to_hex_le()]; + tx.execute( + "INSERT OR REPLACE INTO chain_work (interval,work) VALUES (?1,?2)", + args, + ) + .map_err(db_error::SqliteError)?; + + tx.commit().map_err(db_error::SqliteError)?; + Ok(()) + } + + /// Update the total chain work table up to a given interval (even if partial). + /// Returns the total work + pub fn update_chain_work(&mut self) -> Result { + let highest_interval = self.find_highest_work_score_interval()?; + let mut work_so_far = if highest_interval > 0 { + self.find_interval_work(highest_interval - 1)? + .expect("FATAL: no work score for highest known interval") + } else { + Uint256::from_u64(0) + }; + + let last_interval = self.get_headers_height()? / BLOCK_DIFFICULTY_CHUNK_SIZE + 1; + + debug!( + "Highest work-calculation interval is {} (height {}), work {}; update to {}", + highest_interval, + highest_interval * BLOCK_DIFFICULTY_CHUNK_SIZE, + work_so_far, + last_interval + ); + for interval in (highest_interval + 1)..(last_interval + 1) { + let mut partial = false; + let interval_headers = self.read_block_headers( + (interval - 1) * BLOCK_DIFFICULTY_CHUNK_SIZE, + interval * BLOCK_DIFFICULTY_CHUNK_SIZE, + )?; + let interval_work = if interval_headers.len() == BLOCK_DIFFICULTY_CHUNK_SIZE as usize { + // full interval + let work = SpvClient::get_full_interval_work(&interval_headers); + work_so_far = work_so_far + work; + self.store_interval_work(interval - 1, work_so_far)?; + work + } else { + // partial (and last) interval + let work = if interval > 2 { + let work = self + .get_partial_interval_work(interval - 2, interval_headers.len())? + .expect(&format!( + "FATAL: do not have work score for interval {}", + interval - 2 + )); + + work_so_far = work_so_far + work; + work + } else { + Uint256::from_u64(0) + }; + + partial = true; + work + }; + + debug!( + "Chain work in {} interval {} ({}-{}) is {}, total is {}", + if partial { "partial" } else { "full" }, + interval - 1, + (interval - 1) * BLOCK_DIFFICULTY_CHUNK_SIZE, + (interval - 1) * BLOCK_DIFFICULTY_CHUNK_SIZE + (interval_headers.len() as u64), + interval_work, + work_so_far + ); + if partial { + break; + } + } + + Ok(work_so_far) + } + + /// Get the total chain work. + /// You will have needed to call update_chain_work() prior to this after inserting new headers. + pub fn get_chain_work(&self) -> Result { + let highest_full_interval = self.find_highest_work_score_interval()?; + if highest_full_interval == 0 { + return Ok(Uint256::from_u64(0)); + } + + let highest_interval_work = self + .find_interval_work(highest_full_interval)? + .expect("FATAL: have interval but no work"); + + let partial_interval = highest_full_interval + 1; + let partial_interval_headers = self.read_block_headers( + partial_interval * BLOCK_DIFFICULTY_CHUNK_SIZE, + (partial_interval + 1) * BLOCK_DIFFICULTY_CHUNK_SIZE, + )?; + assert!(partial_interval_headers.len() < BLOCK_DIFFICULTY_CHUNK_SIZE as usize); + + let partial_interval_work = self + .get_partial_interval_work(highest_full_interval, partial_interval_headers.len())? + .expect(&format!( + "FATAL: no work score for interval {}", + highest_full_interval + )); + + debug!("Chain work: highest work-calculated interval is {} with total work {} partial {} ({} headers)", &highest_full_interval, &highest_interval_work, &partial_interval_work, partial_interval_headers.len()); + Ok(highest_interval_work + partial_interval_work) + } + /// Validate a headers message we requested /// * must have at least one header /// * headers must be contiguous @@ -292,6 +627,24 @@ impl SpvClient { Some(res) => res.header, }; + // each header's timestamp must exceed the median of the past 11 blocks + if block_height > 11 { + let past_11_headers = + self.read_block_headers(block_height - 11, block_height)?; + let mut past_timestamps: Vec = + past_11_headers.iter().map(|hdr| hdr.header.time).collect(); + past_timestamps.sort(); + + if header_i.time < past_timestamps[5] { + error!( + "Block {} timestamp {} < {} (median of {:?})", + block_height, header_i.time, past_timestamps[5], &past_timestamps + ); + return Err(btc_error::InvalidPoW); + } + } + + // header difficulty must not change in a difficulty interval let (bits, difficulty) = match self.get_target(block_height, &header_i, &headers, i)? { Some(x) => x, @@ -307,7 +660,7 @@ impl SpvClient { return Err(btc_error::InvalidPoW); } let header_hash = header_i.bitcoin_hash().into_le(); - if difficulty <= header_hash { + if difficulty < header_hash { error!( "block {} hash {} has less work than difficulty {} in {}", block_height, @@ -429,8 +782,9 @@ impl SpvClient { .and_then(|_x| Ok(())) } - /// Initialize the block headers file with the genesis block hash - fn init_block_headers(&mut self) -> Result<(), btc_error> { + /// Initialize the block headers file with the genesis block hash. + /// Optionally sip migration for testing. + fn init_block_headers(&mut self, migrate: bool) -> Result<(), btc_error> { assert!(self.readwrite, "SPV header DB is open read-only"); let (genesis_block, genesis_block_hash_str) = match self.network_id { BitcoinNetworkType::Mainnet => ( @@ -464,6 +818,10 @@ impl SpvClient { tx.commit().map_err(db_error::SqliteError)?; debug!("Initialized block headers at {}", self.headers_path); + + if migrate { + self.update_chain_work()?; + } return Ok(()); } @@ -471,7 +829,7 @@ impl SpvClient { /// -- validate them /// -- store them /// Can error if there has been a reorg, or if the headers don't correspond to headers we asked - /// for. + /// for, or if the new chain has less total work than the old chain. fn handle_headers( &mut self, insert_height: u64, @@ -482,6 +840,7 @@ impl SpvClient { let num_headers = block_headers.len(); let first_header_hash = block_headers[0].header.bitcoin_hash(); let last_header_hash = block_headers[block_headers.len() - 1].header.bitcoin_hash(); + let total_work_before = self.get_chain_work()?; if !self.reverse_order { // fetching headers in ascending order @@ -530,6 +889,15 @@ impl SpvClient { } if num_headers > 0 { + let total_work_after = self.update_chain_work()?; + if total_work_after < total_work_before { + error!( + "New headers represent less work than the old headers ({} < {})", + total_work_before, total_work_after + ); + return Err(btc_error::InvalidDifficulty); + } + debug!( "Handled {} Headers: {}-{}", num_headers, first_header_hash, last_header_hash @@ -707,8 +1075,44 @@ impl SpvClient { Ok(()) } + /// Determine the (bits, target) between two headers + pub fn get_target_between_headers( + first_header: &LoneBlockHeader, + last_header: &LoneBlockHeader, + ) -> (u32, Uint256) { + let max_target = Uint256([ + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x00000000ffff0000, + ]); + + // find actual timespan as being clamped between +/- 4x of the target timespan + let mut actual_timespan = (last_header.header.time - first_header.header.time) as u64; + let target_timespan = BLOCK_DIFFICULTY_INTERVAL as u64; + if actual_timespan < (target_timespan / 4) { + actual_timespan = target_timespan / 4; + } + if actual_timespan > (target_timespan * 4) { + actual_timespan = target_timespan * 4; + } + + let last_target = last_header.header.target(); + let new_target = + (last_target * Uint256::from_u64(actual_timespan)) / Uint256::from_u64(target_timespan); + let target = cmp::min(new_target, max_target); + + let bits = BlockHeader::compact_target_from_u256(&target); + let target = BlockHeader::compact_target_to_u256(bits); + + (bits, target) + } + /// Determine the target difficult over a given difficulty adjustment interval /// the `interval` parameter is the difficulty interval -- a 2016-block interval. + /// * On mainnet, `headers_in_range` can be empty. If it's not empty, then the 0th element is + /// treated as the parent of `current_header`. On testnet, `headers_in_range` must be a range + /// of headers in the given `interval`. /// Returns (new bits, new target) pub fn get_target( &self, @@ -758,7 +1162,7 @@ impl SpvClient { if current_header_height % BLOCK_DIFFICULTY_CHUNK_SIZE != 0 && self.network_id == BitcoinNetworkType::Testnet { - // In Testnet mode, if the new block's timestamp is more than 2* 10 minutes + // In Testnet mode, if the new block's timestamp is more than 2 * 60 * 10 minutes // then allow mining of a min-difficulty block. if current_header.time > parent_header.time + 10 * 60 * 2 { return Ok(Some((max_target_bits, max_target))); @@ -775,34 +1179,20 @@ impl SpvClient { let first_header = match self.read_block_header((interval - 1) * BLOCK_DIFFICULTY_CHUNK_SIZE)? { - Some(res) => res.header, + Some(res) => res, None => return Ok(None), }; let last_header = match self.read_block_header(interval * BLOCK_DIFFICULTY_CHUNK_SIZE - 1)? { - Some(res) => res.header, + Some(res) => res, None => return Ok(None), }; - // find actual timespan as being clamped between +/- 4x of the target timespan - let mut actual_timespan = (last_header.time - first_header.time) as u64; - let target_timespan = BLOCK_DIFFICULTY_INTERVAL as u64; - if actual_timespan < (target_timespan / 4) { - actual_timespan = target_timespan / 4; - } - if actual_timespan > (target_timespan * 4) { - actual_timespan = target_timespan * 4; - } - - let last_target = last_header.target(); - let new_target = - last_target * Uint256::from_u64(actual_timespan) / Uint256::from_u64(target_timespan); - let target = cmp::min(new_target, max_target); - - let bits = BlockHeader::compact_target_from_u256(&target); - - Ok(Some((bits, target))) + Ok(Some(SpvClient::get_target_between_headers( + &first_header, + &last_header, + ))) } /// Ask for the next batch of headers (note that this will return the maximal size of headers) From 11f6d7ee16ff313180774b656def93d3384fe7a2 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 00:13:11 -0400 Subject: [PATCH 18/92] chore: unused mut --- src/chainstate/burn/db/sortdb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chainstate/burn/db/sortdb.rs b/src/chainstate/burn/db/sortdb.rs index 09f3e0ebb..cdfafa945 100644 --- a/src/chainstate/burn/db/sortdb.rs +++ b/src/chainstate/burn/db/sortdb.rs @@ -7891,7 +7891,7 @@ pub mod tests { // drop descendancy information { - let mut db_tx = db.tx_begin().unwrap(); + let db_tx = db.tx_begin().unwrap(); db_tx .execute("DELETE FROM block_commit_parents", NO_PARAMS) .unwrap(); From 9425a39ff76a32996650fecc0f3afee066907124 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 00:13:24 -0400 Subject: [PATCH 19/92] feat: add static method to convert a bitcoin target to a compact target, which is what gets used in the difficulty comparison --- .../src/deps_common/bitcoin/blockdata/block.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/stacks-common/src/deps_common/bitcoin/blockdata/block.rs b/stacks-common/src/deps_common/bitcoin/blockdata/block.rs index 9ff4d9f73..40a103771 100644 --- a/stacks-common/src/deps_common/bitcoin/blockdata/block.rs +++ b/stacks-common/src/deps_common/bitcoin/blockdata/block.rs @@ -74,20 +74,17 @@ pub struct LoneBlockHeader { impl BlockHeader { /// Computes the target [0, T] that a blockhash must land in to be valid - pub fn target(&self) -> Uint256 { + pub fn compact_target_to_u256(bits: u32) -> Uint256 { // This is a floating-point "compact" encoding originally used by // OpenSSL, which satoshi put into consensus code, so we're stuck // with it. The exponent needs to have 3 subtracted from it, hence // this goofy decoding code: let (mant, expt) = { - let unshifted_expt = self.bits >> 24; + let unshifted_expt = bits >> 24; if unshifted_expt <= 3 { - ( - (self.bits & 0xFFFFFF) >> (8 * (3 - unshifted_expt as usize)), - 0, - ) + ((bits & 0xFFFFFF) >> (8 * (3 - unshifted_expt as usize)), 0) } else { - (self.bits & 0xFFFFFF, 8 * ((self.bits >> 24) - 3)) + (bits & 0xFFFFFF, 8 * ((bits >> 24) - 3)) } }; @@ -99,6 +96,11 @@ impl BlockHeader { } } + /// Computes the target [0, T] that a blockhash must land in to be valid + pub fn target(&self) -> Uint256 { + BlockHeader::compact_target_to_u256(self.bits) + } + /// Computes the target value in float format from Uint256 format. pub fn compact_target_from_u256(value: &Uint256) -> u32 { let mut size = (value.bits() + 7) / 8; From e8b2cecee96ff93d46e470b79a10849dfebdf840 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 00:13:45 -0400 Subject: [PATCH 20/92] chore: add uint256 codec to/from hex strings --- stacks-common/src/util/uint.rs | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/stacks-common/src/util/uint.rs b/stacks-common/src/util/uint.rs index 466c8d3ab..bcf69dfe1 100644 --- a/stacks-common/src/util/uint.rs +++ b/stacks-common/src/util/uint.rs @@ -19,6 +19,7 @@ //! Implementation of a various large-but-fixed sized unsigned integer types. //! The functions here are designed to be fast. //! +use crate::util::hash::{hex_bytes, to_hex}; /// Borrowed with gratitude from Andrew Poelstra's rust-bitcoin library use std::fmt; @@ -141,6 +142,31 @@ macro_rules! construct_uint { } ret } + + /// from a little-endian hex string + /// padding is expected + pub fn from_hex_le(hex: &str) -> Option<$name> { + let bytes = hex_bytes(hex).ok()?; + if bytes.len() % 8 != 0 { + return None; + } + if bytes.len() / 8 != $n_words { + return None; + } + let mut ret = [0u64; $n_words]; + for i in 0..(bytes.len() / 8) { + let mut next_bytes = [0u8; 8]; + next_bytes.copy_from_slice(&bytes[8 * i..(8 * (i + 1))]); + let next = u64::from_le_bytes(next_bytes); + ret[i] = next; + } + Some($name(ret)) + } + + /// to a little-endian hex string + pub fn to_hex_le(&self) -> String { + to_hex(&self.to_u8_slice()) + } } impl ::std::ops::Add<$name> for $name { @@ -671,4 +697,14 @@ mod tests { Uint256([0, 0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0]) ); } + + #[test] + pub fn hex_codec() { + let init = Uint256::from_u64(0xDEADBEEFDEADBEEF); + // little-endian representation + let hex_init = "efbeaddeefbeadde000000000000000000000000000000000000000000000000"; + assert_eq!(Uint256::from_hex_le(&hex_init).unwrap(), init); + assert_eq!(&init.to_hex_le(), hex_init); + assert_eq!(Uint256::from_hex_le(&init.to_hex_le()).unwrap(), init); + } } From c6ac355f3475b021b1b0ca9484b326cd0c7de2f1 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 13:55:54 -0400 Subject: [PATCH 21/92] feat: have find_bitcoin_reorg() actually merge the reorg headers and chainwork into the original SPV database, instead of re-downloading the same headers and hoping for the best. Also, update the mainnet unit test to compare chainwork against known chainwork from bitcoind --- src/burnchains/bitcoin/indexer.rs | 1612 ++++++++++++++++++++++++++++- 1 file changed, 1580 insertions(+), 32 deletions(-) diff --git a/src/burnchains/bitcoin/indexer.rs b/src/burnchains/bitcoin/indexer.rs index 07814ab5c..c35d9616b 100644 --- a/src/burnchains/bitcoin/indexer.rs +++ b/src/burnchains/bitcoin/indexer.rs @@ -15,6 +15,7 @@ // along with this program. If not, see . use rand::{thread_rng, Rng}; +use std::cmp; use std::fs; use std::net; use std::net::Shutdown; @@ -447,6 +448,10 @@ impl BitcoinIndexer { if start_block > 0 { if start_block > BLOCK_DIFFICULTY_CHUNK_SIZE { if remove_old { + // set up a .reorg db + // * needs the last difficulty interval of headers (note that the current + // interval is `start_block / BLOCK_DIFFICULTY_CHUNK_SIZE - 1). + // * needs the last interval's chain work calculation let interval_start_block = start_block / BLOCK_DIFFICULTY_CHUNK_SIZE - 2; let base_block = interval_start_block * BLOCK_DIFFICULTY_CHUNK_SIZE; let interval_headers = @@ -479,8 +484,8 @@ impl BitcoinIndexer { } } else { // no full difficulty intervals yet - let interval_headers = canonical_spv_client.read_block_headers(1, start_block)?; - + let interval_headers = + canonical_spv_client.read_block_headers(1, start_block + 1)?; reorg_spv_client.insert_block_headers_before(0, interval_headers)?; } } @@ -492,10 +497,29 @@ impl BitcoinIndexer { /// the reorg starts. Returns the hight of the highest common ancestor. /// Note that under certain testnet settings, the bitcoin chain itself can shrink. pub fn find_bitcoin_reorg( + &mut self, + canonical_headers_path: &str, + reorg_headers_path: &str, + load_reorg_headers: F, + ) -> Result + where + F: FnMut(&mut BitcoinIndexer, &mut SpvClient, u64, Option) -> Result<(), btc_error>, + { + // always check chain work, except in testing + self.inner_find_bitcoin_reorg( + canonical_headers_path, + reorg_headers_path, + load_reorg_headers, + true, + ) + } + + fn inner_find_bitcoin_reorg( &mut self, canonical_headers_path: &str, reorg_headers_path: &str, mut load_reorg_headers: F, + check_chain_work: bool, ) -> Result where F: FnMut(&mut BitcoinIndexer, &mut SpvClient, u64, Option) -> Result<(), btc_error>, @@ -674,23 +698,62 @@ impl BitcoinIndexer { self.setup_reorg_headers(&orig_spv_client, reorg_headers_path, start_block, false)?; } - let reorg_total_work = reorg_spv_client.update_chain_work()?; - let orig_total_work = orig_spv_client.get_chain_work()?; + if check_chain_work { + let reorg_total_work = reorg_spv_client.update_chain_work()?; + let orig_total_work = orig_spv_client.get_chain_work()?; - debug!("Bitcoin headers history is consistent up to {}. Orig chainwork: {}, reorg chainwork: {}", new_tip, orig_total_work, reorg_total_work); - if orig_total_work < reorg_total_work { - let reorg_tip = reorg_spv_client.get_headers_height()?; - let hdr_reorg = reorg_spv_client - .read_block_header(reorg_tip - 1)? - .expect("FATAL: no tip hash for existing chain tip"); - info!( - "New canonical Bitcoin chain found! New tip is {}", - &hdr_reorg.header.bitcoin_hash() - ); - } else { - // ignore the reorg - test_debug!("Reorg chain does not overtake original Bitcoin chain"); - new_tip = orig_spv_client.get_headers_height()?; + debug!("Bitcoin headers history is consistent up to {}. Orig chainwork: {}, reorg chainwork: {}", new_tip, orig_total_work, reorg_total_work); + + if orig_total_work < reorg_total_work { + let reorg_tip = reorg_spv_client.get_headers_height()?; + let hdr_reorg = reorg_spv_client + .read_block_header(reorg_tip - 1)? + .expect("FATAL: no tip hash for existing chain tip"); + info!( + "New canonical Bitcoin chain found! New tip is {}", + &hdr_reorg.header.bitcoin_hash() + ); + + // merge the new headers and chain difficulty to the original headers + let mut orig_spv_client = SpvClient::new( + canonical_headers_path, + 0, + None, + self.runtime.network_id, + true, + false, + )?; + + // copy over new headers + if new_tip > 0 { + let new_headers = + reorg_spv_client.read_block_headers(new_tip, reorg_tip + 1)?; + orig_spv_client.drop_headers(new_tip)?; + orig_spv_client.insert_block_headers_after(new_tip - 1, new_headers)?; + } + + // copy over new chain work + let orig_highest_interval = orig_spv_client.find_highest_work_score_interval()?; + let reorg_highest_interval = reorg_spv_client.find_highest_work_score_interval()?; + for interval in cmp::min(orig_highest_interval, reorg_highest_interval) + ..(cmp::max(orig_highest_interval, reorg_highest_interval) + 1) + { + if let Some(work_score) = reorg_spv_client.find_interval_work(interval)? { + test_debug!( + "Copy work score for interval {} ({}) to original SPV client DB", + interval, + &work_score + ); + orig_spv_client + .store_interval_work(interval, work_score) + .expect("FATAL: failed to store better chain work"); + } + } + } else { + // ignore the reorg + test_debug!("Reorg chain does not overtake original Bitcoin chain"); + new_tip = orig_spv_client.get_headers_height()?; + } } let hdr_reorg = reorg_spv_client.read_block_header(new_tip)?; @@ -925,6 +988,7 @@ mod test { deserialize, serialize, BitcoinHash, }; use stacks_common::deps_common::bitcoin::util::hash::Sha256dHash; + use stacks_common::util::uint::Uint256; use std::env; @@ -940,6 +1004,9 @@ mod test { if fs::metadata(path_2).is_ok() { fs::remove_file(path_2).unwrap(); } + if fs::metadata(path_reorg).is_ok() { + fs::remove_file(path_reorg).unwrap(); + } // two header sets -- both of which build off of the genesis block let headers_1 = vec![ @@ -1062,6 +1129,9 @@ mod test { .insert_block_headers_after(0, headers_2.clone()) .unwrap(); + spv_client.update_chain_work().unwrap(); + spv_client_reorg.update_chain_work().unwrap(); + assert_eq!(spv_client.read_block_headers(0, 10).unwrap().len(), 4); assert_eq!(spv_client_reorg.read_block_headers(0, 10).unwrap().len(), 4); @@ -1072,7 +1142,7 @@ mod test { BitcoinIndexerRuntime::new(BitcoinNetworkType::Regtest), ); let common_ancestor_height = indexer - .find_bitcoin_reorg( + .inner_find_bitcoin_reorg( path_1, path_reorg, |ref mut indexer, ref mut spv_client, start_block, end_block_opt| { @@ -1081,18 +1151,20 @@ mod test { let hdrs = spv_client_reorg .read_block_headers(start_block, end_block) .unwrap(); + if start_block > 0 { + test_debug!("insert at {}: {:?}", start_block - 1, &hdrs); spv_client .insert_block_headers_before(start_block - 1, hdrs) .unwrap(); } else if hdrs.len() > 0 { - spv_client - .insert_block_headers_before(0, hdrs[1..].to_vec()) - .unwrap(); + test_debug!("insert at {}: {:?}", 0, &hdrs); + spv_client.test_write_block_headers(0, hdrs).unwrap(); } Ok(()) }, + false, ) .unwrap(); @@ -1113,7 +1185,7 @@ mod test { fs::remove_file(path_2).unwrap(); } - // two header sets -- both of which build off of the genesis block + // two header sets -- both of which build off of same first block let headers_1 = vec![ LoneBlockHeader { header: BlockHeader { @@ -1244,7 +1316,7 @@ mod test { BitcoinIndexerRuntime::new(BitcoinNetworkType::Regtest), ); let common_ancestor_height = indexer - .find_bitcoin_reorg( + .inner_find_bitcoin_reorg( path_1, path_reorg, |ref mut indexer, ref mut spv_client, start_block, end_block_opt| { @@ -1258,13 +1330,12 @@ mod test { .insert_block_headers_before(start_block - 1, hdrs) .unwrap(); } else if hdrs.len() > 0 { - spv_client - .insert_block_headers_before(0, hdrs[1..].to_vec()) - .unwrap(); + test_debug!("insert at {}: {:?}", 0, &hdrs); + spv_client.test_write_block_headers(0, hdrs).unwrap(); } - Ok(()) }, + false, ) .unwrap(); @@ -1307,6 +1378,7 @@ mod test { } }; + let db_path = "/tmp/test_indexer_sync_headers.sqlite"; let indexer_conf = BitcoinIndexerConfig { peer_host: host, peer_port: port, @@ -1315,7 +1387,7 @@ mod test { username: Some("blockstack".to_string()), password: Some("blockstacksystem".to_string()), timeout: 30, - spv_headers_path: "/tmp/test_indexer_sync_headers.sqlite".to_string(), + spv_headers_path: db_path.to_string(), first_block: 0, magic_bytes: MagicBytes([105, 100]), epochs: None, @@ -1328,6 +1400,1482 @@ mod test { let mut indexer = BitcoinIndexer::new(indexer_conf, BitcoinIndexerRuntime::new(mode)); let last_block = indexer.sync_headers(0, None).unwrap(); eprintln!("sync'ed to block {}", last_block); + + // compare against known-good chain work + let chain_work: Vec<(u64, &str)> = vec![ + ( + 0, + "000000000000000000000000000000000000000000000000000007e007e007e0", + ), + ( + 1, + "00000000000000000000000000000000000000000000000000000fc00fc00fc0", + ), + ( + 2, + "000000000000000000000000000000000000000000000000000017a017a017a0", + ), + ( + 3, + "00000000000000000000000000000000000000000000000000001f801f801f80", + ), + ( + 4, + "0000000000000000000000000000000000000000000000000000276027602760", + ), + ( + 5, + "00000000000000000000000000000000000000000000000000002f402f402f40", + ), + ( + 6, + "0000000000000000000000000000000000000000000000000000372037203720", + ), + ( + 7, + "00000000000000000000000000000000000000000000000000003f003f003f00", + ), + ( + 8, + "000000000000000000000000000000000000000000000000000046e046e046e0", + ), + ( + 9, + "00000000000000000000000000000000000000000000000000004ec04ec04ec0", + ), + ( + 10, + "000000000000000000000000000000000000000000000000000056a056a056a0", + ), + ( + 11, + "00000000000000000000000000000000000000000000000000005e805e805e80", + ), + ( + 12, + "0000000000000000000000000000000000000000000000000000666066606660", + ), + ( + 13, + "00000000000000000000000000000000000000000000000000006e406e406e40", + ), + ( + 14, + "0000000000000000000000000000000000000000000000000000762076207620", + ), + ( + 15, + "00000000000000000000000000000000000000000000000000007e007e007e00", + ), + ( + 16, + "00000000000000000000000000000000000000000000000000008751410913c0", + ), + ( + 17, + "000000000000000000000000000000000000000000000000000091984ca8a7c0", + ), + ( + 18, + "00000000000000000000000000000000000000000000000000009c2e4c600dc0", + ), + ( + 19, + "0000000000000000000000000000000000000000000000000000aa80bfeea100", + ), + ( + 20, + "0000000000000000000000000000000000000000000000000000be68bf6b8cc0", + ), + ( + 21, + "0000000000000000000000000000000000000000000000000000dc2fb8af3b80", + ), + ( + 22, + "0000000000000000000000000000000000000000000000000000ffde8588bce0", + ), + ( + 23, + "000000000000000000000000000000000000000000000000000123d207cd7780", + ), + ( + 24, + "000000000000000000000000000000000000000000000000000153be8a040220", + ), + ( + 25, + "000000000000000000000000000000000000000000000000000191537d8be600", + ), + ( + 26, + "0000000000000000000000000000000000000000000000000001eb9be75bf700", + ), + ( + 27, + "000000000000000000000000000000000000000000000000000250cc4092ede0", + ), + ( + 28, + "0000000000000000000000000000000000000000000000000002ae169cd3d9a0", + ), + ( + 29, + "000000000000000000000000000000000000000000000000000330f72fc5b200", + ), + ( + 30, + "0000000000000000000000000000000000000000000000000003b9d8cd2a7b60", + ), + ( + 31, + "000000000000000000000000000000000000000000000000000452a977bf36e0", + ), + ( + 32, + "00000000000000000000000000000000000000000000000000050bbcb9ab7b40", + ), + ( + 33, + "00000000000000000000000000000000000000000000000000067127f0749ce0", + ), + ( + 34, + "000000000000000000000000000000000000000000000000000c06d4cb992b40", + ), + ( + 35, + "00000000000000000000000000000000000000000000000000138a0a2a644e00", + ), + ( + 36, + "000000000000000000000000000000000000000000000000001e5f59ff0f0e00", + ), + ( + 37, + "000000000000000000000000000000000000000000000000002e1da12f45c380", + ), + ( + 38, + "00000000000000000000000000000000000000000000000000414ae078f5d1e0", + ), + ( + 39, + "000000000000000000000000000000000000000000000000005738ee4a11f0e0", + ), + ( + 40, + "000000000000000000000000000000000000000000000000007374f54c5c30a0", + ), + ( + 41, + "000000000000000000000000000000000000000000000000009c05a4af3fcdc0", + ), + ( + 42, + "00000000000000000000000000000000000000000000000000c669c7db3fed80", + ), + ( + 43, + "00000000000000000000000000000000000000000000000001088595f1a953e0", + ), + ( + 44, + "0000000000000000000000000000000000000000000000000167a1629fa7a960", + ), + ( + 45, + "00000000000000000000000000000000000000000000000001f32db747272760", + ), + ( + 46, + "00000000000000000000000000000000000000000000000002c66b5e31f1f5c0", + ), + ( + 47, + "00000000000000000000000000000000000000000000000003beec205689a020", + ), + ( + 48, + "0000000000000000000000000000000000000000000000000537d218c0d68ea0", + ), + ( + 49, + "00000000000000000000000000000000000000000000000006f5629da3560ee0", + ), + ( + 50, + "00000000000000000000000000000000000000000000000008eb0983e6ec8ee0", + ), + ( + 51, + "0000000000000000000000000000000000000000000000000b22382e2dcefd60", + ), + ( + 52, + "0000000000000000000000000000000000000000000000000dc75e541af84d60", + ), + ( + 53, + "00000000000000000000000000000000000000000000000010e71ec1cb23ca20", + ), + ( + 54, + "0000000000000000000000000000000000000000000000001548b4bf6b9d3100", + ), + ( + 55, + "0000000000000000000000000000000000000000000000001bf6c2e204f41b40", + ), + ( + 56, + "000000000000000000000000000000000000000000000000251e9cea79c2cce0", + ), + ( + 57, + "0000000000000000000000000000000000000000000000002d688542329dbac0", + ), + ( + 58, + "000000000000000000000000000000000000000000000000374da719dc958d00", + ), + ( + 59, + "0000000000000000000000000000000000000000000000004266777a08f8ce80", + ), + ( + 60, + "0000000000000000000000000000000000000000000000004f9428f4722a17c0", + ), + ( + 61, + "000000000000000000000000000000000000000000000000627ea20909250840", + ), + ( + 62, + "0000000000000000000000000000000000000000000000007fd41135d2b41520", + ), + ( + 63, + "000000000000000000000000000000000000000000000000b415d6336051fce0", + ), + ( + 64, + "000000000000000000000000000000000000000000000000f84049eaa2bdc920", + ), + ( + 65, + "00000000000000000000000000000000000000000000000161a153ee991e8a80", + ), + ( + 66, + "000000000000000000000000000000000000000000000002075c4ceea37a38c0", + ), + ( + 67, + "000000000000000000000000000000000000000000000002c32e7638f85db9e0", + ), + ( + 68, + "0000000000000000000000000000000000000000000000038e5e1ddb9420fbc0", + ), + ( + 69, + "00000000000000000000000000000000000000000000000471555420c8491da0", + ), + ( + 70, + "0000000000000000000000000000000000000000000000054a50a331db8feba0", + ), + ( + 71, + "0000000000000000000000000000000000000000000000061ff0deddce4307e0", + ), + ( + 72, + "000000000000000000000000000000000000000000000006f2e198344ff63d80", + ), + ( + 73, + "000000000000000000000000000000000000000000000007bde137a39a5782a0", + ), + ( + 74, + "0000000000000000000000000000000000000000000000086e4e1f0dc8b7fa60", + ), + ( + 75, + "000000000000000000000000000000000000000000000008feeb3e567cff41c0", + ), + ( + 76, + "0000000000000000000000000000000000000000000000098e37156a82413240", + ), + ( + 77, + "00000000000000000000000000000000000000000000000a1147e2764b0a21a0", + ), + ( + 78, + "00000000000000000000000000000000000000000000000a9c1364231203dde0", + ), + ( + 79, + "00000000000000000000000000000000000000000000000b27755c4f71b45e40", + ), + ( + 80, + "00000000000000000000000000000000000000000000000bbdc167cddde49e60", + ), + ( + 81, + "00000000000000000000000000000000000000000000000c5ae5fdc96e314540", + ), + ( + 82, + "00000000000000000000000000000000000000000000000d00aef727dc4d2a40", + ), + ( + 83, + "00000000000000000000000000000000000000000000000da61108e5fd222a00", + ), + ( + 84, + "00000000000000000000000000000000000000000000000e59f35f37e5c50260", + ), + ( + 85, + "00000000000000000000000000000000000000000000000f0dfe2f5e261117c0", + ), + ( + 86, + "00000000000000000000000000000000000000000000000fd172877cda20ea20", + ), + ( + 87, + "0000000000000000000000000000000000000000000000108f0e99cc0c40f240", + ), + ( + 88, + "00000000000000000000000000000000000000000000001144561ebe70d48900", + ), + ( + 89, + "000000000000000000000000000000000000000000000012149b602f9d5e4e40", + ), + ( + 90, + "000000000000000000000000000000000000000000000012d3cc52b3a56e4f80", + ), + ( + 91, + "000000000000000000000000000000000000000000000013920a567e1baf5720", + ), + ( + 92, + "00000000000000000000000000000000000000000000001461834d9e685448a0", + ), + ( + 93, + "00000000000000000000000000000000000000000000001533f9e08c3a70f180", + ), + ( + 94, + "00000000000000000000000000000000000000000000001614402859652da9e0", + ), + ( + 95, + "00000000000000000000000000000000000000000000001708fc9de8a9016820", + ), + ( + 96, + "000000000000000000000000000000000000000000000018104072b037fd6840", + ), + ( + 97, + "0000000000000000000000000000000000000000000000193587f47f44b318c0", + ), + ( + 98, + "00000000000000000000000000000000000000000000001a7942c3db3c544e00", + ), + ( + 99, + "00000000000000000000000000000000000000000000001bd16dfe8636359a80", + ), + ( + 100, + "00000000000000000000000000000000000000000000001d407d055b91205080", + ), + ( + 101, + "00000000000000000000000000000000000000000000001eb1ac5c2ea8ef52e0", + ), + ( + 102, + "0000000000000000000000000000000000000000000000203ebd97d829576860", + ), + ( + 103, + "000000000000000000000000000000000000000000000021d38c3de21fde2be0", + ), + ( + 104, + "00000000000000000000000000000000000000000000002370c89b2e2b749be0", + ), + ( + 105, + "00000000000000000000000000000000000000000000002505c2c5d3ae324400", + ), + ( + 106, + "0000000000000000000000000000000000000000000000266bceea3b91dfc7a0", + ), + ( + 107, + "000000000000000000000000000000000000000000000027f24a2bb126d7cfc0", + ), + ( + 108, + "0000000000000000000000000000000000000000000000295708322ca3f160e0", + ), + ( + 109, + "00000000000000000000000000000000000000000000002ae0a0a7639d5382c0", + ), + ( + 110, + "00000000000000000000000000000000000000000000002c9759c2b432e2cbc0", + ), + ( + 111, + "00000000000000000000000000000000000000000000002ea4372f1351e945c0", + ), + ( + 112, + "000000000000000000000000000000000000000000000030eabb6aea1e3372a0", + ), + ( + 113, + "0000000000000000000000000000000000000000000000340f55af7e1992dda0", + ), + ( + 114, + "000000000000000000000000000000000000000000000037a95bf3e36b001820", + ), + ( + 115, + "00000000000000000000000000000000000000000000003bdfc0ef666a1293c0", + ), + ( + 116, + "0000000000000000000000000000000000000000000000409a91c0ac3435e780", + ), + ( + 117, + "000000000000000000000000000000000000000000000045dae2457ed37e1a60", + ), + ( + 118, + "00000000000000000000000000000000000000000000004b8f4bcf1f459655e0", + ), + ( + 119, + "000000000000000000000000000000000000000000000052e28b37bc272455e0", + ), + ( + 120, + "00000000000000000000000000000000000000000000005bf6711e872f9c9c40", + ), + ( + 121, + "000000000000000000000000000000000000000000000065fa32870e624f9bc0", + ), + ( + 122, + "000000000000000000000000000000000000000000000072420dd4e9bfc326c0", + ), + ( + 123, + "000000000000000000000000000000000000000000000080ee0a56a1701d7e40", + ), + ( + 124, + "0000000000000000000000000000000000000000000000927b55a53fe0b5f960", + ), + ( + 125, + "0000000000000000000000000000000000000000000000aa54f2dade69a01dc0", + ), + ( + 126, + "0000000000000000000000000000000000000000000000c931ca9362b0377b20", + ), + ( + 127, + "0000000000000000000000000000000000000000000000f200146c9f43cd6f60", + ), + ( + 128, + "000000000000000000000000000000000000000000000126de11075b399a25c0", + ), + ( + 129, + "00000000000000000000000000000000000000000000016cb8e540a683fba740", + ), + ( + 130, + "0000000000000000000000000000000000000000000001c591d6a7ae7afa8d20", + ), + ( + 131, + "0000000000000000000000000000000000000000000002433db5b93a1c218940", + ), + ( + 132, + "0000000000000000000000000000000000000000000002fabd96a3c1683667a0", + ), + ( + 133, + "0000000000000000000000000000000000000000000003ea915b5e66b2ba4640", + ), + ( + 134, + "000000000000000000000000000000000000000000000508a7b83ce27d6e0d80", + ), + ( + 135, + "000000000000000000000000000000000000000000000654b54aef7d013eec60", + ), + ( + 136, + "0000000000000000000000000000000000000000000007ff151710fa2c0766a0", + ), + ( + 137, + "000000000000000000000000000000000000000000000a29667c9507de4f5860", + ), + ( + 138, + "000000000000000000000000000000000000000000000cc33a042440e69953e0", + ), + ( + 139, + "00000000000000000000000000000000000000000000100b3a9024583bf28b80", + ), + ( + 140, + "00000000000000000000000000000000000000000000141101d9154085911fe0", + ), + ( + 141, + "0000000000000000000000000000000000000000000018df7a6211abc5ab0f00", + ), + ( + 142, + "000000000000000000000000000000000000000000001e9c7ae8df8f81f56640", + ), + ( + 143, + "00000000000000000000000000000000000000000000259b8e9646e7349c0c00", + ), + ( + 144, + "000000000000000000000000000000000000000000002d66952994737e0a63e0", + ), + ( + 145, + "000000000000000000000000000000000000000000003694c58d08d508cc8300", + ), + ( + 146, + "0000000000000000000000000000000000000000000041cd5532605cb88f6a60", + ), + ( + 147, + "000000000000000000000000000000000000000000004e992868fd1d93ec6400", + ), + ( + 148, + "000000000000000000000000000000000000000000005d44b796f30b5b47bae0", + ), + ( + 149, + "000000000000000000000000000000000000000000006d8074912a6737d3d380", + ), + ( + 150, + "0000000000000000000000000000000000000000000080ac4e0f3e76ba089b80", + ), + ( + 151, + "00000000000000000000000000000000000000000000963ac1bd3bc314c0d7a0", + ), + ( + 152, + "00000000000000000000000000000000000000000000aeea01f39ddc8c90f040", + ), + ( + 153, + "00000000000000000000000000000000000000000000cdc07cf49ac256735280", + ), + ( + 154, + "00000000000000000000000000000000000000000000ed8a0bf93786bc4ea1c0", + ), + ( + 155, + "000000000000000000000000000000000000000000010fe4d0ad93ec88d58a20", + ), + ( + 156, + "000000000000000000000000000000000000000000013411c99602e0779512c0", + ), + ( + 157, + "000000000000000000000000000000000000000000015fca5387f865e1609380", + ), + ( + 158, + "00000000000000000000000000000000000000000001921527684f8e18e0f120", + ), + ( + 159, + "00000000000000000000000000000000000000000001c8c70b3ef33636f10d20", + ), + ( + 160, + "000000000000000000000000000000000000000000020854e6788dc151fee520", + ), + ( + 161, + "000000000000000000000000000000000000000000024882d8a223b780bebf20", + ), + ( + 162, + "000000000000000000000000000000000000000000028a7e47ce725d7d426340", + ), + ( + 163, + "00000000000000000000000000000000000000000002d31bfe56e2b1739d6bc0", + ), + ( + 164, + "000000000000000000000000000000000000000000031d00935207d1ab495d20", + ), + ( + 165, + "00000000000000000000000000000000000000000003665bd4e1aba42c7dd8c0", + ), + ( + 166, + "00000000000000000000000000000000000000000003aeb503f622705470cc20", + ), + ( + 167, + "00000000000000000000000000000000000000000003f939a016b21b1b395760", + ), + ( + 168, + "0000000000000000000000000000000000000000000449d9a5f3dbacdbb93960", + ), + ( + 169, + "000000000000000000000000000000000000000000049586e07bd6f20810b960", + ), + ( + 170, + "00000000000000000000000000000000000000000004e709f889ae74fa318c40", + ), + ( + 171, + "000000000000000000000000000000000000000000053ca35329505af64851c0", + ), + ( + 172, + "00000000000000000000000000000000000000000005939985b1e73e86585920", + ), + ( + 173, + "00000000000000000000000000000000000000000005e9427295b0327510f160", + ), + ( + 174, + "0000000000000000000000000000000000000000000643ec461b119e93fa0120", + ), + ( + 175, + "000000000000000000000000000000000000000000069b385ff2430bd50d39c0", + ), + ( + 176, + "00000000000000000000000000000000000000000006f293e337e48534b58620", + ), + ( + 177, + "000000000000000000000000000000000000000000074c11d1095634524084a0", + ), + ( + 178, + "00000000000000000000000000000000000000000007a354129e16951771cac0", + ), + ( + 179, + "00000000000000000000000000000000000000000007fe715e2872e96c5294a0", + ), + ( + 180, + "0000000000000000000000000000000000000000000859065d467171f99cd620", + ), + ( + 181, + "00000000000000000000000000000000000000000008b6ad4a7c5e93761ed960", + ), + ( + 182, + "0000000000000000000000000000000000000000000916886665dd85cb9e37c0", + ), + ( + 183, + "00000000000000000000000000000000000000000009772960493504b307b5c0", + ), + ( + 184, + "00000000000000000000000000000000000000000009daa5194766250ba1e4e0", + ), + ( + 185, + "0000000000000000000000000000000000000000000a4314a99165a339d76940", + ), + ( + 186, + "0000000000000000000000000000000000000000000aafe04e07a0cc76908780", + ), + ( + 187, + "0000000000000000000000000000000000000000000b1f61a6c72823bc6f7cc0", + ), + ( + 188, + "0000000000000000000000000000000000000000000b8f0423557c7834c9c440", + ), + ( + 189, + "0000000000000000000000000000000000000000000c0129c4864d86d6937540", + ), + ( + 190, + "0000000000000000000000000000000000000000000c79e686c513ee1711d700", + ), + ( + 191, + "0000000000000000000000000000000000000000000cff3e24f98a31a9513bc0", + ), + ( + 192, + "0000000000000000000000000000000000000000000d90484e8d690d207cb3e0", + ), + ( + 193, + "0000000000000000000000000000000000000000000e3ba087263ab2bf5acbe0", + ), + ( + 194, + "0000000000000000000000000000000000000000000efa194f42d4866a387da0", + ), + ( + 195, + "0000000000000000000000000000000000000000000fc9f11bbc39959b21b000", + ), + ( + 196, + "00000000000000000000000000000000000000000010a60801ccdc23faa49280", + ), + ( + 197, + "00000000000000000000000000000000000000000011ae475d9025c6286edae0", + ), + ( + 198, + "00000000000000000000000000000000000000000012da0d5328636c44f7bb20", + ), + ( + 199, + "00000000000000000000000000000000000000000013fc8a1001c47b4dec7d80", + ), + ( + 200, + "000000000000000000000000000000000000000000152bfd3dacde2eb7fd1260", + ), + ( + 201, + "000000000000000000000000000000000000000000165dec4bf88a5938102cc0", + ), + ( + 202, + "00000000000000000000000000000000000000000017a58ac69e578aeff74d60", + ), + ( + 203, + "00000000000000000000000000000000000000000018ed2050238fb72a6adb60", + ), + ( + 204, + "0000000000000000000000000000000000000000001a514e4f44f2f7b58cce20", + ), + ( + 205, + "0000000000000000000000000000000000000000001bbec2257da9ba542e3dc0", + ), + ( + 206, + "0000000000000000000000000000000000000000001d264026c89fc3a561ff20", + ), + ( + 207, + "0000000000000000000000000000000000000000001ea64c2728a2c3bd62bf20", + ), + ( + 208, + "000000000000000000000000000000000000000000202d9445cb5993709c9940", + ), + ( + 209, + "00000000000000000000000000000000000000000021b50850f1b264bd8ee400", + ), + ( + 210, + "000000000000000000000000000000000000000000232737b9bae704d658e980", + ), + ( + 211, + "00000000000000000000000000000000000000000024b5ca6a95511e529b9a60", + ), + ( + 212, + "000000000000000000000000000000000000000000264a8fdb9737cec5270360", + ), + ( + 213, + "00000000000000000000000000000000000000000027e8a464ee3e6441e33ba0", + ), + ( + 214, + "00000000000000000000000000000000000000000029a2f2ee951b390851d020", + ), + ( + 215, + "0000000000000000000000000000000000000000002b7cf7e446f67a01521e40", + ), + ( + 216, + "0000000000000000000000000000000000000000002d4dfeb582d570cb6ec4c0", + ), + ( + 217, + "0000000000000000000000000000000000000000002f20dbd4bde0279e863f60", + ), + ( + 218, + "00000000000000000000000000000000000000000031258f6adfa6b4147044c0", + ), + ( + 219, + "00000000000000000000000000000000000000000033335d7927c4d1cc706340", + ), + ( + 220, + "000000000000000000000000000000000000000000356c0dc0666c9a25e31d60", + ), + ( + 221, + "00000000000000000000000000000000000000000037b28eb2ad32e4eb0725a0", + ), + ( + 222, + "0000000000000000000000000000000000000000003a1c496adb7a0fa510f440", + ), + ( + 223, + "0000000000000000000000000000000000000000003ceccfe9ad4acc1bac8580", + ), + ( + 224, + "0000000000000000000000000000000000000000003ff2e4225485aa755b79a0", + ), + ( + 225, + "000000000000000000000000000000000000000000431b177600a43a49c8ff20", + ), + ( + 226, + "0000000000000000000000000000000000000000004667f1b695192e96aa5e00", + ), + ( + 227, + "00000000000000000000000000000000000000000049d02ec230291e1ed89fe0", + ), + ( + 228, + "0000000000000000000000000000000000000000004d644cbc7c8dac48b042e0", + ), + ( + 229, + "000000000000000000000000000000000000000000511f3d1a5d2ee6dddf2c60", + ), + ( + 230, + "00000000000000000000000000000000000000000054dc50acc5ee22163a87e0", + ), + ( + 231, + "00000000000000000000000000000000000000000058df0f81b00e65e31d9fc0", + ), + ( + 232, + "0000000000000000000000000000000000000000005d23b986246a80e2a66160", + ), + ( + 233, + "0000000000000000000000000000000000000000006200474547413007eb54e0", + ), + ( + 234, + "0000000000000000000000000000000000000000006719397aeed92cea73c0c0", + ), + ( + 235, + "0000000000000000000000000000000000000000006c2c99cc24de404ac4f6c0", + ), + ( + 236, + "00000000000000000000000000000000000000000071efc0e32e8d53c3437520", + ), + ( + 237, + "000000000000000000000000000000000000000000781907b3b129168140d360", + ), + ( + 238, + "0000000000000000000000000000000000000000007eb5d786594edfb7192580", + ), + ( + 239, + "00000000000000000000000000000000000000000085125dd58b787822420060", + ), + ( + 240, + "0000000000000000000000000000000000000000008bae3f082d510ef55e75a0", + ), + ( + 241, + "00000000000000000000000000000000000000000093956885c724768b3e4220", + ), + ( + 242, + "0000000000000000000000000000000000000000009ba216e9c83e948399d3e0", + ), + ( + 243, + "000000000000000000000000000000000000000000a4347de9712a3d299897c0", + ), + ( + 244, + "000000000000000000000000000000000000000000ae9c5fcf35f61f498146e0", + ), + ( + 245, + "000000000000000000000000000000000000000000b86222fe3501a784060ac0", + ), + ( + 246, + "000000000000000000000000000000000000000000c207f50841bc71fbf34200", + ), + ( + 247, + "000000000000000000000000000000000000000000cd6cfa174358d251800c40", + ), + ( + 248, + "000000000000000000000000000000000000000000dad77213452f0444c351e0", + ), + ( + 249, + "000000000000000000000000000000000000000000e8ac5170255ea89b74f900", + ), + ( + 250, + "000000000000000000000000000000000000000000f8a13b2c589aeeb23ffba0", + ), + ( + 251, + "0000000000000000000000000000000000000000010b46275cd6a0d8d647dcc0", + ), + ( + 252, + "0000000000000000000000000000000000000000011fdd1173e9b175a204fbc0", + ), + ( + 253, + "0000000000000000000000000000000000000000013567509d0940b8bba28240", + ), + ( + 254, + "0000000000000000000000000000000000000000014cf8de771e406fcb574e00", + ), + ( + 255, + "00000000000000000000000000000000000000000165c5ae302bc30be69eb9a0", + ), + ( + 256, + "0000000000000000000000000000000000000000017eeb74084c207738949880", + ), + ( + 257, + "0000000000000000000000000000000000000000019a6b1b59c384990c32ece0", + ), + ( + 258, + "000000000000000000000000000000000000000001b739d4c259343246ef1ee0", + ), + ( + 259, + "000000000000000000000000000000000000000001d4e7eb5fde62f143663aa0", + ), + ( + 260, + "000000000000000000000000000000000000000001f3c1028afece7ae8982120", + ), + ( + 261, + "0000000000000000000000000000000000000000021724227cc1eca0a316fde0", + ), + ( + 262, + "0000000000000000000000000000000000000000023b8214bfc487e587047c20", + ), + ( + 263, + "00000000000000000000000000000000000000000261ecc1d79b256c651d81c0", + ), + ( + 264, + "0000000000000000000000000000000000000000028704359f9c7d6769226240", + ), + ( + 265, + "000000000000000000000000000000000000000002b1a0ea483b571304264320", + ), + ( + 266, + "000000000000000000000000000000000000000002df642ba14be8dd6aa4de80", + ), + ( + 267, + "0000000000000000000000000000000000000000030f9301272cdfb2ac437b80", + ), + ( + 268, + "00000000000000000000000000000000000000000341d93154f4bdb6a5c457a0", + ), + ( + 269, + "00000000000000000000000000000000000000000375140aa6d3469564e40d20", + ), + ( + 270, + "000000000000000000000000000000000000000003aa793e4456d51fee079d20", + ), + ( + 271, + "000000000000000000000000000000000000000003ddeb802da8e6b18d7f2440", + ), + ( + 272, + "00000000000000000000000000000000000000000411609ae24d1bf31e937fa0", + ), + ( + 273, + "0000000000000000000000000000000000000000044107e5ba926026f20196c0", + ), + ( + 274, + "0000000000000000000000000000000000000000046978f859a2d324a4f423a0", + ), + ( + 275, + "0000000000000000000000000000000000000000048e0bf34e00b79c6e0c9cc0", + ), + ( + 276, + "000000000000000000000000000000000000000004b64a09060ec73d90f77520", + ), + ( + 277, + "000000000000000000000000000000000000000004e06ebc5bfb6b016e590e80", + ), + ( + 278, + "0000000000000000000000000000000000000000050a145245ab90a8067ccd40", + ), + ( + 279, + "000000000000000000000000000000000000000005357e89442872e853f88fe0", + ), + ( + 280, + "00000000000000000000000000000000000000000560fbafcacfef7b2141bde0", + ), + ( + 281, + "0000000000000000000000000000000000000000058c736b7d94f11ac4af8820", + ), + ( + 282, + "000000000000000000000000000000000000000005ba243ec2581be932e72bc0", + ), + ( + 283, + "000000000000000000000000000000000000000005e7ee4c12541090941dbe60", + ), + ( + 284, + "000000000000000000000000000000000000000006156f04d90982240b8b39c0", + ), + ( + 285, + "000000000000000000000000000000000000000006456fe96f932a6a69ce1de0", + ), + ( + 286, + "0000000000000000000000000000000000000000067575520b861045f8089f80", + ), + ( + 287, + "000000000000000000000000000000000000000006aae3297a3f9d93d55e9ce0", + ), + ( + 288, + "000000000000000000000000000000000000000006dff4cf1a2437365611d5c0", + ), + ( + 289, + "00000000000000000000000000000000000000000718c9a7d0e51cfd930a0b20", + ), + ( + 290, + "00000000000000000000000000000000000000000759b56bb260925290180080", + ), + ( + 291, + "0000000000000000000000000000000000000000079a44d2dcddadcd50c16380", + ), + ( + 292, + "000000000000000000000000000000000000000007e1c9a6b59653827fadcba0", + ), + ( + 293, + "0000000000000000000000000000000000000000082ab9c86e527cfd7dffdc40", + ), + ( + 294, + "00000000000000000000000000000000000000000877e0fc3b665d3187c61ce0", + ), + ( + 295, + "000000000000000000000000000000000000000008cd0b371205d869e58815e0", + ), + ( + 296, + "000000000000000000000000000000000000000009286f3a6c1469a93569fda0", + ), + ( + 297, + "000000000000000000000000000000000000000009859a773846d18d99e33c40", + ), + ( + 298, + "000000000000000000000000000000000000000009e7aabe3dbb65d04b436960", + ), + ( + 299, + "00000000000000000000000000000000000000000a42c5c116143a6675fe4c40", + ), + ( + 300, + "00000000000000000000000000000000000000000a9fb114d65d94f00168f6a0", + ), + ( + 301, + "00000000000000000000000000000000000000000afbeba9e7b19fc8c09584c0", + ), + ( + 302, + "00000000000000000000000000000000000000000b58a9ce9935920487232a80", + ), + ( + 303, + "00000000000000000000000000000000000000000bbb7ed558b66d4d2e1c0d60", + ), + ( + 304, + "00000000000000000000000000000000000000000c255453c47c551c36aa3540", + ), + ( + 305, + "00000000000000000000000000000000000000000c941a7dca358fb9521e03e0", + ), + ( + 306, + "00000000000000000000000000000000000000000d037486edebab30d3c6c0e0", + ), + ( + 307, + "00000000000000000000000000000000000000000d7260db2663c608c7f7a7c0", + ), + ( + 308, + "00000000000000000000000000000000000000000de8efca09e80642843d4be0", + ), + ( + 309, + "00000000000000000000000000000000000000000e4c955dbc140174f247f260", + ), + ( + 310, + "00000000000000000000000000000000000000000eb5fabb18e954747a74b000", + ), + ( + 311, + "00000000000000000000000000000000000000000f284806995597f3cd0bfd80", + ), + ( + 312, + "00000000000000000000000000000000000000000f9ba14e6a962918bf2127c0", + ), + ( + 313, + "000000000000000000000000000000000000000010080df526f4ff21960f1a40", + ), + ( + 314, + "0000000000000000000000000000000000000000106a692d441a6aad1cace9e0", + ), + ( + 315, + "000000000000000000000000000000000000000010db77996e285750cd8b9c80", + ), + ( + 316, + "0000000000000000000000000000000000000000114c850e564cfaa41534e5a0", + ), + ( + 317, + "000000000000000000000000000000000000000011c8c20e26e90338310d8b40", + ), + ( + 318, + "000000000000000000000000000000000000000012416d3a1b42c5f9e33e93e0", + ), + ( + 319, + "000000000000000000000000000000000000000012bad0326db68dfbabe3a0a0", + ), + ( + 320, + "0000000000000000000000000000000000000000133891fe722cd8f8d46b71e0", + ), + ( + 321, + "000000000000000000000000000000000000000013b4cf153adbbd38ac356200", + ), + ( + 322, + "0000000000000000000000000000000000000000143f25d8093643758a467060", + ), + ( + 323, + "000000000000000000000000000000000000000014c95e395adc5f2947c38f60", + ), + ( + 324, + "0000000000000000000000000000000000000000155898b9afe71b5bc6234aa0", + ), + ( + 325, + "000000000000000000000000000000000000000015d0d64858237f52b67fbd80", + ), + ( + 326, + "0000000000000000000000000000000000000000164edf3c9afdff38aca62a40", + ), + ( + 327, + "000000000000000000000000000000000000000016d815351bf2448270b26bc0", + ), + ( + 328, + "0000000000000000000000000000000000000000175dce415adf182a317efee0", + ), + ( + 329, + "000000000000000000000000000000000000000017e305e5e5ebe1b42a2283c0", + ), + ( + 330, + "000000000000000000000000000000000000000018769f070c2824c962eee160", + ), + ( + 331, + "0000000000000000000000000000000000000000190bc46a36b8e7956e861a40", + ), + ( + 332, + "000000000000000000000000000000000000000019a549dd7f975730622fb1e0", + ), + ( + 333, + "00000000000000000000000000000000000000001a40e2926d569536de587200", + ), + ( + 334, + "00000000000000000000000000000000000000001ada8179bddb7efbe43bb160", + ), + ( + 335, + "00000000000000000000000000000000000000001b771d7dcf1fac50373e6440", + ), + ( + 336, + "00000000000000000000000000000000000000001c1cd59725d81e6e21d5cae0", + ), + ( + 337, + "00000000000000000000000000000000000000001cc5bcc99d2c4a90357ad360", + ), + ( + 338, + "00000000000000000000000000000000000000001d595888caa6d458e6efa260", + ), + ( + 339, + "00000000000000000000000000000000000000001e0cbd014668d1e4d9ba8e40", + ), + ( + 340, + "00000000000000000000000000000000000000001ea37d7a3f2552a2f909f620", + ), + ( + 341, + "00000000000000000000000000000000000000001f3241a19347d4dd02c83d00", + ), + ( + 342, + "00000000000000000000000000000000000000001f99213be9ee53bddf9ee5c0", + ), + ( + 343, + "00000000000000000000000000000000000000001ffb0ee21327a85c0b6cd6c0", + ), + ( + 344, + "00000000000000000000000000000000000000002062e31d9a89a510058f0680", + ), + ( + 345, + "000000000000000000000000000000000000000020d24e4a9d0743295882b380", + ), + ( + 346, + "0000000000000000000000000000000000000000215078acda07c153babfb140", + ), + ( + 347, + "000000000000000000000000000000000000000021d45e243085daf592b0bbe0", + ), + ( + 348, + "0000000000000000000000000000000000000000225c6fa2067f24b11235b6a0", + ), + ( + 349, + "000000000000000000000000000000000000000022eaeae8d7274e795d554f80", + ), + ( + 350, + "0000000000000000000000000000000000000000237ac17de8cd15067a6a0fc0", + ), + ( + 351, + "00000000000000000000000000000000000000002415e366c94c1e34c7f72b20", + ), + ( + 352, + "000000000000000000000000000000000000000024b84a0606d0a6eff7d24240", + ), + ( + 353, + "000000000000000000000000000000000000000025584400aa7a24ab60f95da0", + ), + ( + 354, + "000000000000000000000000000000000000000026058fbce96b8fd898fb4440", + ), + ( + 355, + "000000000000000000000000000000000000000026b368bd9b25fad76f8f80e0", + ), + ( + 356, + "00000000000000000000000000000000000000002761f842fb541ec705fbab80", + ), + ( + 357, + "00000000000000000000000000000000000000002820cc635abe2ef6bb03bfa0", + ), + ( + 358, + "000000000000000000000000000000000000000028dff750d76099ef8067b5e0", + ), + ( + 359, + "000000000000000000000000000000000000000029a847072a5004727d7bb6c0", + ), + ( + 360, + "00000000000000000000000000000000000000002a6d9a7894891e6c8d042a60", + ), + ( + 361, + "00000000000000000000000000000000000000002b323ae9b7f6eaaec69c08a0", + ), + ( + 362, + "00000000000000000000000000000000000000002bfefb71afd13545bc2444e0", + ), + ( + 363, + "00000000000000000000000000000000000000002cc925a3aa907374b5f0b6c0", + ), + ( + 364, + "00000000000000000000000000000000000000002d9e8bc0134624c9d3ce88c0", + ), + ( + 365, + "00000000000000000000000000000000000000002e7e60cf6c8d3d2643d9ed00", + ), + ]; + + let spv_client = + SpvClient::new(db_path, 0, None, BitcoinNetworkType::Mainnet, false, false).unwrap(); + for (interval, work_str) in chain_work.iter() { + let calculated_work = spv_client.find_interval_work(*interval).unwrap().unwrap(); + let expected_work = Uint256::from_hex_be(work_str).unwrap(); + assert_eq!(calculated_work, expected_work); + } } #[test] @@ -1473,8 +3021,6 @@ mod test { // reorg is ignored assert_eq!(new_tip, 40321); - let hdr = spv_client.read_block_header(new_tip - 1).unwrap().unwrap(); - eprintln!("{}", &hdr.header.bitcoin_hash()); let total_work_after = spv_client.update_chain_work().unwrap(); assert_eq!(total_work_after, total_work_before); } @@ -1635,7 +3181,9 @@ mod test { // chain reorg detected! assert_eq!(new_tip, 40318); + + // total work increased let total_work_after = spv_client.update_chain_work().unwrap(); - assert_eq!(total_work_after, total_work_before); + assert!(total_work_after > total_work_before); } } From fcbc6606083d6cebef3083d4cbb4322ee02ec674 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 13:56:55 -0400 Subject: [PATCH 22/92] chore: s/InvalidDifficulty/InvalidChainWork/g --- src/burnchains/bitcoin/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/burnchains/bitcoin/mod.rs b/src/burnchains/bitcoin/mod.rs index c9b7ab5d3..3822ea0de 100644 --- a/src/burnchains/bitcoin/mod.rs +++ b/src/burnchains/bitcoin/mod.rs @@ -77,10 +77,10 @@ pub enum Error { NoncontiguousHeader, /// Missing header MissingHeader, - /// Invalid target + /// Invalid header proof-of-work (i.e. due to a bad timestamp or a bad `bits` field) InvalidPoW, - /// Bad difficulty - InvalidDifficulty, + /// Chainwork would decrease by including a given header + InvalidChainWork, /// Wrong number of bytes for constructing an address InvalidByteSequence, /// Configuration error @@ -109,7 +109,7 @@ impl fmt::Display for Error { Error::NoncontiguousHeader => write!(f, "Non-contiguous header"), Error::MissingHeader => write!(f, "Missing header"), Error::InvalidPoW => write!(f, "Invalid proof of work"), - Error::InvalidDifficulty => write!(f, "Chain difficulty cannot decrease"), + Error::InvalidChainWork => write!(f, "Chain difficulty cannot decrease"), Error::InvalidByteSequence => write!(f, "Invalid sequence of bytes"), Error::ConfigError(ref e_str) => fmt::Display::fmt(e_str, f), Error::BlockchainHeight => write!(f, "Value is beyond the end of the blockchain"), @@ -136,7 +136,7 @@ impl error::Error for Error { Error::NoncontiguousHeader => None, Error::MissingHeader => None, Error::InvalidPoW => None, - Error::InvalidDifficulty => None, + Error::InvalidChainWork => None, Error::InvalidByteSequence => None, Error::ConfigError(ref _e_str) => None, Error::BlockchainHeight => None, From 4721f670789bbe35ae16b183c6d2ac52798f28b2 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 13:57:14 -0400 Subject: [PATCH 23/92] fix: use the correct chainwork calculation by summing over individual headers --- src/burnchains/bitcoin/spv.rs | 220 ++++++++++------------------------ 1 file changed, 66 insertions(+), 154 deletions(-) diff --git a/src/burnchains/bitcoin/spv.rs b/src/burnchains/bitcoin/spv.rs index c86d8b8df..fa21c5559 100644 --- a/src/burnchains/bitcoin/spv.rs +++ b/src/burnchains/bitcoin/spv.rs @@ -85,6 +85,10 @@ const SPV_INITIAL_SCHEMA: &[&'static str] = &[ "CREATE TABLE db_config(version TEXT NOT NULL);", ]; +// store the running chain work totals for each difficulty interval. +// unlike the `headers` table, this table will never be deleted from, since we use it to determine +// whether or not newly-arrived headers represent a better chain than the best-known chain. The +// only way to _replace_ a row is to find a header difficulty interval with a _higher_ work score. const SPV_SCHEMA_2: &[&'static str] = &[r#" CREATE TABLE chain_work( interval INTEGER PRIMARY KEY, @@ -101,9 +105,6 @@ pub struct SpvClient { readwrite: bool, reverse_order: bool, headers_db: DBConn, - - // only writeable in #[cfg(test)] - ignore_work_checks: bool, } impl FromColumn for Sha256dHash { @@ -151,7 +152,6 @@ impl SpvClient { readwrite: readwrite, reverse_order: reverse_order, headers_db: conn, - ignore_work_checks: false, }; if readwrite { @@ -180,7 +180,6 @@ impl SpvClient { readwrite: readwrite, reverse_order: reverse_order, headers_db: conn, - ignore_work_checks: true, }; if readwrite { @@ -190,11 +189,6 @@ impl SpvClient { Ok(client) } - #[cfg(test)] - pub fn set_ignore_work_checks(&mut self, ignore: bool) { - self.ignore_work_checks = ignore; - } - pub fn conn(&self) -> &DBConn { &self.headers_db } @@ -336,77 +330,13 @@ impl SpvClient { indexer.peer_communicate(self, true) } - /// Calculate the work of a single header given the first and last header in the interval - fn get_expected_work_in_range( - first_header: &LoneBlockHeader, - last_header: &LoneBlockHeader, - ) -> Uint256 { - let (_, target) = SpvClient::get_target_between_headers(&first_header, &last_header); - let work = - (Uint256::max() - target) / (target + Uint256::from_u64(1)) + Uint256::from_u64(1); - test_debug!("{}, {}", &work, &target); - work - } - - /// Calculate the total work over a full interval of headers. - fn get_full_interval_work(interval_headers: &Vec) -> Uint256 { - assert_eq!(interval_headers.len() as u64, BLOCK_DIFFICULTY_CHUNK_SIZE); - let first_header = interval_headers - .first() - .expect("FATAL: no first header in non-empty list of headers"); - let last_header = interval_headers - .last() - .expect("FATAL: no last header in non-empty list of headers"); - SpvClient::get_expected_work_in_range(first_header, last_header) - * Uint256::from_u64(BLOCK_DIFFICULTY_CHUNK_SIZE) - } - - /// Calculate a partial interval's work, given the last full interval before it - fn get_partial_interval_work( - &self, - last_full_interval: u64, - partial_interval_len: usize, - ) -> Result, btc_error> { - let last_interval_work = self.get_interval_header_work(last_full_interval)?; - if let Some(last_interval_work) = last_interval_work { - let work = last_interval_work * Uint256::from_u64(partial_interval_len as u64); - Ok(Some(work)) - } else { - Ok(None) + /// Calculate the total work over a given interval of headers. + fn get_interval_work(interval_headers: &Vec) -> Uint256 { + let mut work = Uint256::from_u64(0); + for hdr in interval_headers.iter() { + work = work + hdr.header.work(); } - } - - /// Calculate the work done by a single header in `interval`, if we have the headers for that - /// interval - pub fn get_interval_header_work(&self, interval: u64) -> Result, btc_error> { - let first_header = - match self.read_block_header((interval - 1) * BLOCK_DIFFICULTY_CHUNK_SIZE)? { - Some(res) => res, - None => { - test_debug!( - "No header at height {}", - (interval - 1) * BLOCK_DIFFICULTY_CHUNK_SIZE - ); - return Ok(None); - } - }; - - let last_header = - match self.read_block_header(interval * BLOCK_DIFFICULTY_CHUNK_SIZE - 1)? { - Some(res) => res, - None => { - test_debug!( - "No header at height {}", - interval * BLOCK_DIFFICULTY_CHUNK_SIZE - 1 - ); - return Ok(None); - } - }; - - Ok(Some(SpvClient::get_expected_work_in_range( - &first_header, - &last_header, - ))) + work } /// Find the highest interval for which we have a chain work score. @@ -436,7 +366,7 @@ impl SpvClient { ) .optional() .map_err(db_error::SqliteError)?; - Ok(work_hex.map(|x| Uint256::from_hex_le(&x).expect("FATAL: work is not a uint256"))) + Ok(work_hex.map(|x| Uint256::from_hex_be(&x).expect("FATAL: work is not a uint256"))) } /// Store an interval's running total work. @@ -444,17 +374,17 @@ impl SpvClient { /// currently-stored interval. pub fn store_interval_work(&mut self, interval: u64, work: Uint256) -> Result<(), btc_error> { if let Some(cur_work) = self.find_interval_work(interval)? { - if cur_work > work && !self.ignore_work_checks { + if cur_work > work { error!( "Tried to store work {} to interval {}, which has work {} already", work, interval, cur_work ); - return Err(btc_error::InvalidDifficulty); + return Err(btc_error::InvalidChainWork); } } let tx = self.tx_begin()?; - let args: &[&dyn ToSql] = &[&u64_to_sql(interval)?, &work.to_hex_le()]; + let args: &[&dyn ToSql] = &[&u64_to_sql(interval)?, &work.to_hex_be()]; tx.execute( "INSERT OR REPLACE INTO chain_work (interval,work) VALUES (?1,?2)", args, @@ -466,7 +396,8 @@ impl SpvClient { } /// Update the total chain work table up to a given interval (even if partial). - /// Returns the total work + /// This method is idempotent. + /// Returns the total work. pub fn update_chain_work(&mut self) -> Result { let highest_interval = self.find_highest_work_score_interval()?; let mut work_so_far = if highest_interval > 0 { @@ -478,7 +409,7 @@ impl SpvClient { let last_interval = self.get_headers_height()? / BLOCK_DIFFICULTY_CHUNK_SIZE + 1; - debug!( + test_debug!( "Highest work-calculation interval is {} (height {}), work {}; update to {}", highest_interval, highest_interval * BLOCK_DIFFICULTY_CHUNK_SIZE, @@ -491,33 +422,16 @@ impl SpvClient { (interval - 1) * BLOCK_DIFFICULTY_CHUNK_SIZE, interval * BLOCK_DIFFICULTY_CHUNK_SIZE, )?; - let interval_work = if interval_headers.len() == BLOCK_DIFFICULTY_CHUNK_SIZE as usize { - // full interval - let work = SpvClient::get_full_interval_work(&interval_headers); - work_so_far = work_so_far + work; + let interval_work = SpvClient::get_interval_work(&interval_headers); + work_so_far = work_so_far + interval_work; + + if interval_headers.len() == BLOCK_DIFFICULTY_CHUNK_SIZE as usize { self.store_interval_work(interval - 1, work_so_far)?; - work } else { - // partial (and last) interval - let work = if interval > 2 { - let work = self - .get_partial_interval_work(interval - 2, interval_headers.len())? - .expect(&format!( - "FATAL: do not have work score for interval {}", - interval - 2 - )); - - work_so_far = work_so_far + work; - work - } else { - Uint256::from_u64(0) - }; - partial = true; - work - }; + } - debug!( + test_debug!( "Chain work in {} interval {} ({}-{}) is {}, total is {}", if partial { "partial" } else { "full" }, interval - 1, @@ -538,27 +452,30 @@ impl SpvClient { /// You will have needed to call update_chain_work() prior to this after inserting new headers. pub fn get_chain_work(&self) -> Result { let highest_full_interval = self.find_highest_work_score_interval()?; - if highest_full_interval == 0 { - return Ok(Uint256::from_u64(0)); - } + let highest_interval_work = if highest_full_interval == 0 { + Uint256::from_u64(0) + } else { + self.find_interval_work(highest_full_interval)? + .expect("FATAL: have interval but no work") + }; - let highest_interval_work = self - .find_interval_work(highest_full_interval)? - .expect("FATAL: have interval but no work"); + let partial_interval = if highest_full_interval == 0 { + 0 + } else { + highest_full_interval + 1 + }; - let partial_interval = highest_full_interval + 1; let partial_interval_headers = self.read_block_headers( partial_interval * BLOCK_DIFFICULTY_CHUNK_SIZE, (partial_interval + 1) * BLOCK_DIFFICULTY_CHUNK_SIZE, )?; - assert!(partial_interval_headers.len() < BLOCK_DIFFICULTY_CHUNK_SIZE as usize); + assert!( + partial_interval_headers.len() < BLOCK_DIFFICULTY_CHUNK_SIZE as usize, + "interval {} is not partial", + partial_interval + ); - let partial_interval_work = self - .get_partial_interval_work(highest_full_interval, partial_interval_headers.len())? - .expect(&format!( - "FATAL: no work score for interval {}", - highest_full_interval - )); + let partial_interval_work = SpvClient::get_interval_work(&partial_interval_headers); debug!("Chain work: highest work-calculated interval is {} with total work {} partial {} ({} headers)", &highest_full_interval, &highest_interval_work, &partial_interval_work, partial_interval_headers.len()); Ok(highest_interval_work + partial_interval_work) @@ -635,9 +552,9 @@ impl SpvClient { past_11_headers.iter().map(|hdr| hdr.header.time).collect(); past_timestamps.sort(); - if header_i.time < past_timestamps[5] { + if header_i.time <= past_timestamps[5] { error!( - "Block {} timestamp {} < {} (median of {:?})", + "Block {} timestamp {} <= {} (median of {:?})", block_height, header_i.time, past_timestamps[5], &past_timestamps ); return Err(btc_error::InvalidPoW); @@ -840,10 +757,11 @@ impl SpvClient { let num_headers = block_headers.len(); let first_header_hash = block_headers[0].header.bitcoin_hash(); let last_header_hash = block_headers[block_headers.len() - 1].header.bitcoin_hash(); - let total_work_before = self.get_chain_work()?; + let total_work_before = self.update_chain_work()?; if !self.reverse_order { - // fetching headers in ascending order + // fetching headers in ascending order, so verify that the first item in + // `block_headers` connects to a parent in the DB (if it has one) self.insert_block_headers_after(insert_height, block_headers) .map_err(|e| { error!("Failed to insert block headers: {:?}", &e); @@ -864,7 +782,8 @@ impl SpvClient { e })?; } else { - // fetching headers in descending order + // fetching headers in descending order, so verify that the last item in + // `block_headers` connects to a child in the DB (if it has one) self.insert_block_headers_before(insert_height, block_headers) .map_err(|e| { error!("Failed to insert block headers: {:?}", &e); @@ -895,7 +814,7 @@ impl SpvClient { "New headers represent less work than the old headers ({} < {})", total_work_before, total_work_after ); - return Err(btc_error::InvalidDifficulty); + return Err(btc_error::InvalidChainWork); } debug!( @@ -932,6 +851,15 @@ impl SpvClient { Ok(()) } + #[cfg(test)] + pub fn test_write_block_headers( + &mut self, + height: u64, + headers: Vec, + ) -> Result<(), btc_error> { + self.write_block_headers(height, headers) + } + /// Insert block headers into the headers DB. /// Verify that the first header's parent exists and connects with this header chain, and verify that /// the headers are themselves contiguous. @@ -1033,23 +961,6 @@ impl SpvClient { } } - match self.read_block_header(start_height)? { - Some(parent_header) => { - // contiguous? - if block_headers[0].header.prev_blockhash != parent_header.header.bitcoin_hash() { - warn!("Received discontiguous headers at height {}: we have parent {:?} ({}), but were given {:?} ({})", - start_height, &parent_header.header, parent_header.header.bitcoin_hash(), &block_headers[0].header, &block_headers[0].header.bitcoin_hash()); - return Err(btc_error::NoncontiguousHeader); - } - } - None => { - debug!( - "No header for parent block {}, so will not validate continuity", - start_height - 1 - ); - } - } - // store them self.write_block_headers(start_height + 1, block_headers) } @@ -1650,15 +1561,16 @@ mod test { assert_eq!(spv_client.read_block_headers(0, 10).unwrap(), all_headers); - // should fail - if let Err(btc_error::NoncontiguousHeader) = - spv_client.insert_block_headers_before(2, headers.clone()) - { - } else { - assert!(false); - } + // should succeed, since we only check that the last header connects + // to its child, if the child is stored at all + spv_client + .insert_block_headers_before(1, headers.clone()) + .unwrap(); + spv_client + .insert_block_headers_before(2, headers.clone()) + .unwrap(); - // should fail + // should fail now, since there's a child to check if let Err(btc_error::NoncontiguousHeader) = spv_client.insert_block_headers_before(1, headers.clone()) { From 6c9221aa7af4fb7ac5b26192a143128b91d68430 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 13:57:39 -0400 Subject: [PATCH 24/92] fix: update sync_with_indexer() to assume that find_chain_reorg() does the bookkeeping on the SPV DB, so it doesn't have to --- src/burnchains/burnchain.rs | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/burnchains/burnchain.rs b/src/burnchains/burnchain.rs index 0b58d02d9..d29740f36 100644 --- a/src/burnchains/burnchain.rs +++ b/src/burnchains/burnchain.rs @@ -980,27 +980,24 @@ impl Burnchain { if sync_height + 1 < orig_header_height { // a reorg happened warn!( - "Dropping headers higher than {} due to burnchain reorg", + "Dropped headers higher than {} due to burnchain reorg", sync_height ); - indexer.drop_headers(sync_height)?; } // get latest headers. - debug!("Sync headers from {}", sync_height); + let highest_header = indexer.get_highest_header_height()?; - let end_block = indexer.sync_headers(sync_height, None)?; - let mut start_block = match sync_height { - 0 => 0, - _ => sync_height, - }; + debug!("Sync headers from {}", highest_header); + let end_block = indexer.sync_headers(highest_header, None)?; + let mut start_block = sync_height; if db_height < start_block { start_block = db_height; } debug!( "Sync'ed headers from {} to {}. DB at {}", - start_block, end_block, db_height + highest_header, end_block, db_height ); if start_block == db_height && db_height == end_block { // all caught up @@ -1218,22 +1215,22 @@ impl Burnchain { let db_height = burn_chain_tip.block_height; - // handle reorgs + // handle reorgs (which also updates our best-known chain work and headers DB) let (sync_height, did_reorg) = Burnchain::sync_reorg(indexer)?; if did_reorg { // a reorg happened warn!( - "Dropping headers higher than {} due to burnchain reorg", + "Dropped headers higher than {} due to burnchain reorg", sync_height ); - indexer.drop_headers(sync_height)?; } // get latest headers. debug!("Sync headers from {}", sync_height); - // fetch all headers, no matter what - let mut end_block = indexer.sync_headers(sync_height, None)?; + // fetch all new headers + let highest_header = indexer.get_highest_header_height()?; + let mut end_block = indexer.sync_headers(highest_header, None)?; if did_reorg && sync_height > 0 { // a reorg happened, and the last header fetched // is on a smaller fork than the one we just @@ -1258,7 +1255,7 @@ impl Burnchain { debug!( "Sync'ed headers from {} to {}. DB at {}", - sync_height, end_block, db_height + highest_header, end_block, db_height ); if let Some(target_block_height) = target_block_height_opt { From 7670d02acde965a01f43e0eea5f8e23177b11b09 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 13:58:12 -0400 Subject: [PATCH 25/92] chore: add big-endian code for uint256, which is easier to read --- stacks-common/src/util/uint.rs | 46 +++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/stacks-common/src/util/uint.rs b/stacks-common/src/util/uint.rs index bcf69dfe1..a448cc0e1 100644 --- a/stacks-common/src/util/uint.rs +++ b/stacks-common/src/util/uint.rs @@ -131,7 +131,7 @@ macro_rules! construct_uint { $name(ret) } - /// as byte array + /// as litte-endian byte array pub fn to_u8_slice(&self) -> [u8; $n_words * 8] { let mut ret = [0u8; $n_words * 8]; for i in 0..$n_words { @@ -143,6 +143,18 @@ macro_rules! construct_uint { ret } + /// as big-endian byte array + pub fn to_u8_slice_be(&self) -> [u8; $n_words * 8] { + let mut ret = [0u8; $n_words * 8]; + for i in 0..$n_words { + let bytes = self.0[i].to_le_bytes(); + for j in 0..bytes.len() { + ret[$n_words * 8 - 1 - (i * 8 + j)] = bytes[j]; + } + } + ret + } + /// from a little-endian hex string /// padding is expected pub fn from_hex_le(hex: &str) -> Option<$name> { @@ -167,6 +179,31 @@ macro_rules! construct_uint { pub fn to_hex_le(&self) -> String { to_hex(&self.to_u8_slice()) } + + /// from a big-endian hex string + /// padding is expected + pub fn from_hex_be(hex: &str) -> Option<$name> { + let bytes = hex_bytes(hex).ok()?; + if bytes.len() % 8 != 0 { + return None; + } + if bytes.len() / 8 != $n_words { + return None; + } + let mut ret = [0u64; $n_words]; + for i in 0..(bytes.len() / 8) { + let mut next_bytes = [0u8; 8]; + next_bytes.copy_from_slice(&bytes[8 * i..(8 * (i + 1))]); + let next = u64::from_be_bytes(next_bytes); + ret[(bytes.len() / 8) - 1 - i] = next; + } + Some($name(ret)) + } + + /// to a big-endian hex string + pub fn to_hex_be(&self) -> String { + to_hex(&self.to_u8_slice_be()) + } } impl ::std::ops::Add<$name> for $name { @@ -701,10 +738,17 @@ mod tests { #[test] pub fn hex_codec() { let init = Uint256::from_u64(0xDEADBEEFDEADBEEF); + // little-endian representation let hex_init = "efbeaddeefbeadde000000000000000000000000000000000000000000000000"; assert_eq!(Uint256::from_hex_le(&hex_init).unwrap(), init); assert_eq!(&init.to_hex_le(), hex_init); assert_eq!(Uint256::from_hex_le(&init.to_hex_le()).unwrap(), init); + + // big-endian representation + let hex_init = "000000000000000000000000000000000000000000000000deadbeefdeadbeef"; + assert_eq!(Uint256::from_hex_be(&hex_init).unwrap(), init); + assert_eq!(&init.to_hex_be(), hex_init); + assert_eq!(Uint256::from_hex_be(&init.to_hex_be()).unwrap(), init); } } From 437381e339854956aeb0d6fec4374ce32bb1f7cb Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 15:43:56 -0400 Subject: [PATCH 26/92] fix: test that the SPV chain tip is within 2 hours of now --- src/burnchains/bitcoin/indexer.rs | 136 +++++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 2 deletions(-) diff --git a/src/burnchains/bitcoin/indexer.rs b/src/burnchains/bitcoin/indexer.rs index c35d9616b..2eab6170e 100644 --- a/src/burnchains/bitcoin/indexer.rs +++ b/src/burnchains/bitcoin/indexer.rs @@ -46,6 +46,7 @@ use stacks_common::deps_common::bitcoin::blockdata::block::LoneBlockHeader; use stacks_common::deps_common::bitcoin::network::message::NetworkMessage; use stacks_common::deps_common::bitcoin::network::serialize::BitcoinHash; use stacks_common::deps_common::bitcoin::network::serialize::Error as btc_serialization_err; +use stacks_common::util::get_epoch_time_secs; use stacks_common::util::log; use crate::core::{ @@ -762,6 +763,35 @@ impl BitcoinIndexer { Ok(new_tip) } + + /// Verify that the last block header we have is within 2 hours of now. + /// Return burnchain_error::TrySyncAgain if not, and delete the offending header + pub fn check_chain_tip_timestamp(&mut self) -> Result<(), burnchain_error> { + // if there was no target block height, then verify that the highest header fetched is within + // 2 hours of now. Remove headers that don't meet this criterion. + let highest_header_height = self.get_highest_header_height()?; + if highest_header_height == 0 { + return Err(burnchain_error::TrySyncAgain); + } + + let highest_header = self + .read_headers(highest_header_height, highest_header_height + 1)? + .pop() + .expect("FATAL: no header at highest known height"); + let now = get_epoch_time_secs(); + if now - 2 * 60 * 60 <= (highest_header.block_header.header.time as u64) + && (highest_header.block_header.header.time as u64) <= now + 2 * 60 * 60 + { + // we're good + return Ok(()); + } + warn!( + "Header at height {} is not wihtin 2 hours of now (is at {})", + highest_header_height, highest_header.block_header.header.time + ); + self.drop_headers(highest_header_height.saturating_sub(1))?; + return Err(burnchain_error::TrySyncAgain); + } } impl Drop for BitcoinIndexer { @@ -942,11 +972,18 @@ impl BurnchainIndexer for BitcoinIndexer { return Ok(end_height.unwrap()); } - self.sync_last_headers(start_height, end_height) + let new_height = self + .sync_last_headers(start_height, end_height) .map_err(|e| match e { btc_error::TimedOut => burnchain_error::TrySyncAgain, x => burnchain_error::Bitcoin(x), - }) + })?; + + // make sure the headers are up-to-date if we have no target height + if end_height.is_none() { + self.check_chain_tip_timestamp()?; + } + Ok(new_height) } /// Drop headers after a given height -- i.e. to accomodate a reorg @@ -988,6 +1025,7 @@ mod test { deserialize, serialize, BitcoinHash, }; use stacks_common::deps_common::bitcoin::util::hash::Sha256dHash; + use stacks_common::util::get_epoch_time_secs; use stacks_common::util::uint::Uint256; use std::env; @@ -3186,4 +3224,98 @@ mod test { let total_work_after = spv_client.update_chain_work().unwrap(); assert!(total_work_after > total_work_before); } + + #[test] + fn test_check_header_timestamp() { + let db_path = "/tmp/test-indexer-check-header-timestamp.dat"; + + if fs::metadata(db_path).is_ok() { + fs::remove_file(db_path).unwrap(); + } + + let headers = vec![ + LoneBlockHeader { + header: BlockHeader { + bits: 545259519, + merkle_root: Sha256dHash::from_hex( + "20bee96458517fc5082a9720ce6207b5742f2b18e4e0a7e7373342725d80f88c", + ) + .unwrap(), + nonce: 2, + prev_blockhash: Sha256dHash::from_hex( + "0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206", + ) + .unwrap(), + time: (get_epoch_time_secs() - 1) as u32, + version: 0x20000000, + }, + tx_count: VarInt(0), + }, + LoneBlockHeader { + header: BlockHeader { + bits: 545259519, + merkle_root: Sha256dHash::from_hex( + "39d1a6f1ee7a5903797f92ec89e4c58549013f38114186fc2eb6e5218cb2d0ac", + ) + .unwrap(), + nonce: 1, + prev_blockhash: Sha256dHash::from_hex( + "606d31daaaa5919f3720d8440dd99d31f2a4e4189c65879f19ae43268425e74b", + ) + .unwrap(), + time: (get_epoch_time_secs() - 1) as u32, + version: 0x20000000, + }, + tx_count: VarInt(0), + }, + LoneBlockHeader { + header: BlockHeader { + bits: 545259519, + merkle_root: Sha256dHash::from_hex( + "a7e04ed25f589938eb5627abb7b5913dd77b8955bcdf72d7f111d0a71e346e47", + ) + .unwrap(), + nonce: 4, + prev_blockhash: Sha256dHash::from_hex( + "2fa2f451ac27f0e5cd3760ba6cdf34ef46adb76a44d96bc0f3bf3e713dd955f0", + ) + .unwrap(), + time: 1587626882, + version: 0x20000000, + }, + tx_count: VarInt(0), + }, + ]; + + // set up SPV client so we don't have chain work at first + let mut spv_client = SpvClient::new_without_migration( + &db_path, + 0, + None, + BitcoinNetworkType::Regtest, + true, + false, + ) + .unwrap(); + + spv_client + .test_write_block_headers(0, headers.clone()) + .unwrap(); + assert_eq!(spv_client.get_highest_header_height().unwrap(), 2); + + let mut indexer = BitcoinIndexer::new( + BitcoinIndexerConfig::test_default(db_path.to_string()), + BitcoinIndexerRuntime::new(BitcoinNetworkType::Regtest), + ); + + if let Err(burnchain_error::TrySyncAgain) = indexer.check_chain_tip_timestamp() { + } else { + panic!("stale tip not detected"); + } + + // peeled + assert_eq!(spv_client.get_highest_header_height().unwrap(), 1); + assert!(indexer.check_chain_tip_timestamp().is_ok()); + assert_eq!(spv_client.get_highest_header_height().unwrap(), 1); + } } From debf40c35cc27ce14715afe8db968a844125b005 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 15:44:34 -0400 Subject: [PATCH 27/92] refactor: better name --- src/burnchains/burnchain.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/burnchains/burnchain.rs b/src/burnchains/burnchain.rs index d29740f36..20c7500b5 100644 --- a/src/burnchains/burnchain.rs +++ b/src/burnchains/burnchain.rs @@ -1229,8 +1229,8 @@ impl Burnchain { debug!("Sync headers from {}", sync_height); // fetch all new headers - let highest_header = indexer.get_highest_header_height()?; - let mut end_block = indexer.sync_headers(highest_header, None)?; + let highest_header_height = indexer.get_highest_header_height()?; + let mut end_block = indexer.sync_headers(highest_header_height, None)?; if did_reorg && sync_height > 0 { // a reorg happened, and the last header fetched // is on a smaller fork than the one we just @@ -1255,7 +1255,7 @@ impl Burnchain { debug!( "Sync'ed headers from {} to {}. DB at {}", - highest_header, end_block, db_height + highest_header_height, end_block, db_height ); if let Some(target_block_height) = target_block_height_opt { From 1cc98b552f8d2ed00e4e6d36b2fa7271b927edaa Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Thu, 26 May 2022 21:35:12 -0400 Subject: [PATCH 28/92] fix: prime the .reorg DB with the parent of the first header we expect to download (off-by-one error) --- src/burnchains/bitcoin/indexer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/burnchains/bitcoin/indexer.rs b/src/burnchains/bitcoin/indexer.rs index 2eab6170e..b703a66e4 100644 --- a/src/burnchains/bitcoin/indexer.rs +++ b/src/burnchains/bitcoin/indexer.rs @@ -456,9 +456,9 @@ impl BitcoinIndexer { let interval_start_block = start_block / BLOCK_DIFFICULTY_CHUNK_SIZE - 2; let base_block = interval_start_block * BLOCK_DIFFICULTY_CHUNK_SIZE; let interval_headers = - canonical_spv_client.read_block_headers(base_block, start_block)?; + canonical_spv_client.read_block_headers(base_block, start_block + 1)?; assert!( - interval_headers.len() == (start_block - base_block) as usize, + interval_headers.len() >= (start_block - base_block) as usize, "BUG: missing headers for {}-{}", base_block, start_block From 9f510fd59432467aec2c2cef715c02d2231fb61d Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Sat, 28 May 2022 22:21:03 -0400 Subject: [PATCH 29/92] fix: use .saturating_sub() when determining the header range to copy over to the .reorg DB, and use K/V logging. Also, add multi-word test vectors to hex codec for uint256 --- src/burnchains/bitcoin/indexer.rs | 4 ++-- stacks-common/src/util/uint.rs | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/burnchains/bitcoin/indexer.rs b/src/burnchains/bitcoin/indexer.rs index b703a66e4..f6aef4b82 100644 --- a/src/burnchains/bitcoin/indexer.rs +++ b/src/burnchains/bitcoin/indexer.rs @@ -453,7 +453,7 @@ impl BitcoinIndexer { // * needs the last difficulty interval of headers (note that the current // interval is `start_block / BLOCK_DIFFICULTY_CHUNK_SIZE - 1). // * needs the last interval's chain work calculation - let interval_start_block = start_block / BLOCK_DIFFICULTY_CHUNK_SIZE - 2; + let interval_start_block = (start_block / BLOCK_DIFFICULTY_CHUNK_SIZE).saturating_sub(2); let base_block = interval_start_block * BLOCK_DIFFICULTY_CHUNK_SIZE; let interval_headers = canonical_spv_client.read_block_headers(base_block, start_block + 1)?; @@ -703,7 +703,7 @@ impl BitcoinIndexer { let reorg_total_work = reorg_spv_client.update_chain_work()?; let orig_total_work = orig_spv_client.get_chain_work()?; - debug!("Bitcoin headers history is consistent up to {}. Orig chainwork: {}, reorg chainwork: {}", new_tip, orig_total_work, reorg_total_work); + debug!("Bitcoin headers history is consistent up to {}", new_tip; "Orig chainwork" => %origin_total_work, "Reorg chainwork" => %reorg_total_work)i; if orig_total_work < reorg_total_work { let reorg_tip = reorg_spv_client.get_headers_height()?; diff --git a/stacks-common/src/util/uint.rs b/stacks-common/src/util/uint.rs index a448cc0e1..3c5c0d8e8 100644 --- a/stacks-common/src/util/uint.rs +++ b/stacks-common/src/util/uint.rs @@ -147,10 +147,9 @@ macro_rules! construct_uint { pub fn to_u8_slice_be(&self) -> [u8; $n_words * 8] { let mut ret = [0u8; $n_words * 8]; for i in 0..$n_words { - let bytes = self.0[i].to_le_bytes(); - for j in 0..bytes.len() { - ret[$n_words * 8 - 1 - (i * 8 + j)] = bytes[j]; - } + let word_end = $n_words * 8 - (i * 8); + let word_start = word_end - 8; + ret[word_start..word_end].copy_from_slice(&self.0[i].to_be_bytes()); } ret } @@ -737,16 +736,17 @@ mod tests { #[test] pub fn hex_codec() { - let init = Uint256::from_u64(0xDEADBEEFDEADBEEF); + let init = + Uint256::from_u64(0xDEADBEEFDEADBEEF) << 64 | Uint256::from_u64(0x0102030405060708); // little-endian representation - let hex_init = "efbeaddeefbeadde000000000000000000000000000000000000000000000000"; + let hex_init = "0807060504030201efbeaddeefbeadde00000000000000000000000000000000"; assert_eq!(Uint256::from_hex_le(&hex_init).unwrap(), init); assert_eq!(&init.to_hex_le(), hex_init); assert_eq!(Uint256::from_hex_le(&init.to_hex_le()).unwrap(), init); // big-endian representation - let hex_init = "000000000000000000000000000000000000000000000000deadbeefdeadbeef"; + let hex_init = "00000000000000000000000000000000deadbeefdeadbeef0102030405060708"; assert_eq!(Uint256::from_hex_be(&hex_init).unwrap(), init); assert_eq!(&init.to_hex_be(), hex_init); assert_eq!(Uint256::from_hex_be(&init.to_hex_be()).unwrap(), init); From e1319a9e81c5e390cd212789ffe3615ae0ada6a7 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Sat, 28 May 2022 22:45:19 -0400 Subject: [PATCH 30/92] fix: compile-time error --- src/burnchains/bitcoin/indexer.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/burnchains/bitcoin/indexer.rs b/src/burnchains/bitcoin/indexer.rs index f6aef4b82..a0b8a0f25 100644 --- a/src/burnchains/bitcoin/indexer.rs +++ b/src/burnchains/bitcoin/indexer.rs @@ -703,7 +703,9 @@ impl BitcoinIndexer { let reorg_total_work = reorg_spv_client.update_chain_work()?; let orig_total_work = orig_spv_client.get_chain_work()?; - debug!("Bitcoin headers history is consistent up to {}", new_tip; "Orig chainwork" => %origin_total_work, "Reorg chainwork" => %reorg_total_work)i; + debug!("Bitcoin headers history is consistent up to {}", new_tip; + "Orig chainwork" => %orig_total_work, + "Reorg chainwork" => %reorg_total_work); if orig_total_work < reorg_total_work { let reorg_tip = reorg_spv_client.get_headers_height()?; From 5b326f9d35e3fbbdde8649704268115b5f3ce85c Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Sat, 28 May 2022 22:45:44 -0400 Subject: [PATCH 31/92] chore: cargo fmt --- src/burnchains/bitcoin/indexer.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/burnchains/bitcoin/indexer.rs b/src/burnchains/bitcoin/indexer.rs index a0b8a0f25..402ef1436 100644 --- a/src/burnchains/bitcoin/indexer.rs +++ b/src/burnchains/bitcoin/indexer.rs @@ -453,7 +453,8 @@ impl BitcoinIndexer { // * needs the last difficulty interval of headers (note that the current // interval is `start_block / BLOCK_DIFFICULTY_CHUNK_SIZE - 1). // * needs the last interval's chain work calculation - let interval_start_block = (start_block / BLOCK_DIFFICULTY_CHUNK_SIZE).saturating_sub(2); + let interval_start_block = + (start_block / BLOCK_DIFFICULTY_CHUNK_SIZE).saturating_sub(2); let base_block = interval_start_block * BLOCK_DIFFICULTY_CHUNK_SIZE; let interval_headers = canonical_spv_client.read_block_headers(base_block, start_block + 1)?; From 762e007cafda91c1b05539e08d51947c79263885 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 31 May 2022 12:08:22 -0400 Subject: [PATCH 32/92] chore: add changelog for SPV chain work fix --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad7801a17..4c849273e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to the versioning scheme outlined in the [README.md](README.md). +## [2.05.0.2.1] + +### Fixed +- Fixed a security bug in the SPV client whereby the chain work was not being + considered at all when determining the canonical Bitcoin fork. The SPV client +now only accepts a new Bitcoin fork if it has a higher chain work than any other +previously-seen chain (#3152). + ## [2.05.0.2.0] ### IMPORTANT! READ THIS FIRST From 8bf9fcd6f1d2393dee1c077608d536d4e4e8688e Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Tue, 7 Jun 2022 09:06:37 -0700 Subject: [PATCH 33/92] feat: add coverage flags to clarity-cli --- src/clarity_cli.rs | 169 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 153 insertions(+), 16 deletions(-) diff --git a/src/clarity_cli.rs b/src/clarity_cli.rs index 16aed6c84..265a3fc66 100644 --- a/src/clarity_cli.rs +++ b/src/clarity_cli.rs @@ -16,6 +16,7 @@ use std::convert::TryInto; use std::env; +use std::ffi::OsStr; use std::fs; use std::io; use std::io::{Read, Write}; @@ -23,6 +24,8 @@ use std::iter::Iterator; use std::path::PathBuf; use std::process; +use clarity::util::get_epoch_time_ms; +use clarity::vm::coverage::CoverageReporter; use rand::Rng; use rusqlite::types::ToSql; use rusqlite::Row; @@ -421,6 +424,7 @@ pub fn vm_execute(program: &str) -> Result, Error> { LimitedCostTracker::new_free(), DEFAULT_CLI_EPOCH, ); + global_context.coverage_reporting = Some(CoverageReporter::new()); global_context.execute(|g| { let parsed = ast::build_ast(&contract_id, program, &mut ())?.expressions; eval_all(&parsed, &mut contract_context, g) @@ -1237,6 +1241,11 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option (i32, Option { + (Ok(result), cost, coverage) => { + if let Some(ref coverage_folder) = coverage_folder { + let mut coverage_file = PathBuf::from(coverage_folder); + coverage_file.push(&format!("eval_{}", get_epoch_time_ms())); + coverage_file.set_extension("clarcov"); + + coverage + .expect( + "Failed to recover coverage reporter when coverage was requested", + ) + .to_file(&coverage_file) + .expect("Coverage reference file generation failure"); + } let mut result_json = json!({ "output": serde_json::to_value(&result).unwrap(), "success": true, @@ -1268,7 +1298,19 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option { + (Err(error), cost, coverage) => { + if let Some(ref coverage_folder) = coverage_folder { + let mut coverage_file = PathBuf::from(coverage_folder); + coverage_file.push(&format!("eval_{}", get_epoch_time_ms())); + coverage_file.set_extension("clarcov"); + + coverage + .expect( + "Failed to recover coverage reporter when coverage was requested", + ) + .to_file(&coverage_file) + .expect("Coverage reference file generation failure"); + } let mut result_json = json!({ "error": { "runtime": serde_json::to_value(&format!("{}", error)).unwrap() @@ -1357,6 +1399,11 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option { let mut argv: Vec = args.into_iter().map(|x| x.clone()).collect(); + let coverage_folder = if let Ok(covarg) = consume_arg(&mut argv, &["--c"], true) { + covarg + } else { + None + }; let costs = if let Ok(Some(_)) = consume_arg(&mut argv, &["--costs"], false) { true } else { @@ -1382,14 +1429,15 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option (i32, Option (i32, Option { let result_and_cost = with_env_costs(mainnet, &header_db, &mut marf, |vm_env| { - vm_env - .initialize_contract(contract_identifier, &contract_content) + if coverage_folder.is_some() { + vm_env.set_coverage_reporter(CoverageReporter::new()); + } + ( + vm_env.initialize_contract( + contract_identifier, + &contract_content, + ), + vm_env.take_coverage_reporter(), + ) }); - (header_db, marf, Ok((analysis, result_and_cost))) + // let (result) + let ((result, coverage), cost) = result_and_cost; + (header_db, marf, Ok((analysis, (result, cost, coverage)))) } } }); match analysis_result_and_cost { - Ok((contract_analysis, (Ok((_x, asset_map, events)), cost))) => { + Ok((contract_analysis, (Ok((_x, asset_map, events)), cost, coverage))) => { let mut result = json!({ "message": "Contract initialized!" }); @@ -1431,6 +1505,19 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option (i32, Option { let mut argv: Vec = args.into_iter().map(|x| x.clone()).collect(); + let coverage_folder = if let Ok(covarg) = consume_arg(&mut argv, &["--c"], true) { + covarg + } else { + None + }; let costs = if let Ok(Some(_)) = consume_arg(&mut argv, &["--costs"], false) { true @@ -1526,14 +1618,36 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option { + (Ok((x, asset_map, events)), cost, coverage) => { if let Value::Response(data) = x { + if let Some(ref coverage_folder) = coverage_folder { + let mut coverage_file = PathBuf::from(coverage_folder); + coverage_file.push(&format!("execute_{}", get_epoch_time_ms())); + coverage_file.set_extension("clarcov"); + + coverage + .expect("Failed to recover coverage reporter when coverage was requested") + .to_file(&coverage_file) + .expect("Coverage reference file generation failure"); + } if data.committed { let mut result = json!({ "message": "Transaction executed and committed.", @@ -1576,7 +1690,7 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option { + (Err(error), ..) => { let result = json!({ "error": { "runtime": "Transaction execution error.", @@ -1588,6 +1702,29 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option { + let mut register_files = vec![]; + let mut coverage_files = vec![]; + let coverage_folder = &args[1]; + let lcov_output_file = &args[2]; + for folder_entry in + fs::read_dir(coverage_folder).expect("Failed to read the coverage folder") + { + let folder_entry = + folder_entry.expect("Failed to read entry in the coverage folder"); + let entry_path = folder_entry.path(); + if entry_path.is_file() { + if entry_path.extension() == Some(OsStr::new("clarcovref")) { + register_files.push(entry_path) + } else if entry_path.extension() == Some(OsStr::new("clarcov")) { + coverage_files.push(entry_path) + } + } + } + CoverageReporter::produce_lcov(lcov_output_file, ®ister_files, &coverage_files) + .expect("Failed to produce an lcov output"); + (0, None) + } _ => { print_usage(invoked_by); (1, None) From beedf34775a6f0516e1f68caa1c654326d93990d Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Wed, 8 Jun 2022 07:42:33 -0700 Subject: [PATCH 34/92] fix: cleanup saving coverage into function --- src/clarity_cli.rs | 67 +++++++++++++++------------------------------- 1 file changed, 21 insertions(+), 46 deletions(-) diff --git a/src/clarity_cli.rs b/src/clarity_cli.rs index 265a3fc66..c7878e6ed 100644 --- a/src/clarity_cli.rs +++ b/src/clarity_cli.rs @@ -431,6 +431,23 @@ pub fn vm_execute(program: &str) -> Result, Error> { }) } +fn save_coverage(coverage_folder: Option, coverage: Option, prefix: &str) { + match (coverage_folder, coverage) { + (Some(coverage_folder), Some(coverage)) => { + let mut coverage_file = PathBuf::from(coverage_folder); + coverage_file.push(&format!("{}_{}", prefix, get_epoch_time_ms())); + coverage_file.set_extension("clarcov"); + + coverage + .to_file(&coverage_file) + .expect("Coverage reference file generation failure"); + } + (None, None) => (), + (None, Some(_)) => (), + (Some(_), None) => (), + } +} + struct CLIHeadersDB { db_path: String, conn: Connection, @@ -1276,18 +1293,7 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option { - if let Some(ref coverage_folder) = coverage_folder { - let mut coverage_file = PathBuf::from(coverage_folder); - coverage_file.push(&format!("eval_{}", get_epoch_time_ms())); - coverage_file.set_extension("clarcov"); - - coverage - .expect( - "Failed to recover coverage reporter when coverage was requested", - ) - .to_file(&coverage_file) - .expect("Coverage reference file generation failure"); - } + save_coverage(coverage_folder, coverage, "eval"); let mut result_json = json!({ "output": serde_json::to_value(&result).unwrap(), "success": true, @@ -1299,18 +1305,7 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option { - if let Some(ref coverage_folder) = coverage_folder { - let mut coverage_file = PathBuf::from(coverage_folder); - coverage_file.push(&format!("eval_{}", get_epoch_time_ms())); - coverage_file.set_extension("clarcov"); - - coverage - .expect( - "Failed to recover coverage reporter when coverage was requested", - ) - .to_file(&coverage_file) - .expect("Coverage reference file generation failure"); - } + save_coverage(coverage_folder, coverage, "eval"); let mut result_json = json!({ "error": { "runtime": serde_json::to_value(&format!("{}", error)).unwrap() @@ -1505,18 +1500,7 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option (i32, Option { if let Value::Response(data) = x { - if let Some(ref coverage_folder) = coverage_folder { - let mut coverage_file = PathBuf::from(coverage_folder); - coverage_file.push(&format!("execute_{}", get_epoch_time_ms())); - coverage_file.set_extension("clarcov"); - - coverage - .expect("Failed to recover coverage reporter when coverage was requested") - .to_file(&coverage_file) - .expect("Coverage reference file generation failure"); - } + save_coverage(coverage_folder, coverage, "execute"); if data.committed { let mut result = json!({ "message": "Transaction executed and committed.", From 4bb2a4745020da7e16c34b199614f8f6b6a31b5c Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Wed, 8 Jun 2022 07:43:06 -0700 Subject: [PATCH 35/92] chore: comment cleanup --- src/clarity_cli.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clarity_cli.rs b/src/clarity_cli.rs index c7878e6ed..fb15aedae 100644 --- a/src/clarity_cli.rs +++ b/src/clarity_cli.rs @@ -1484,7 +1484,6 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option Date: Fri, 10 Jun 2022 09:31:25 -0700 Subject: [PATCH 36/92] fix: cargo fmt --- src/clarity_cli.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/clarity_cli.rs b/src/clarity_cli.rs index fb15aedae..4ad3c906b 100644 --- a/src/clarity_cli.rs +++ b/src/clarity_cli.rs @@ -431,7 +431,11 @@ pub fn vm_execute(program: &str) -> Result, Error> { }) } -fn save_coverage(coverage_folder: Option, coverage: Option, prefix: &str) { +fn save_coverage( + coverage_folder: Option, + coverage: Option, + prefix: &str, +) { match (coverage_folder, coverage) { (Some(coverage_folder), Some(coverage)) => { let mut coverage_file = PathBuf::from(coverage_folder); From db3c7a233e8d7c492ba24c7143b5ac065f58ea9b Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Sun, 12 Jun 2022 10:05:14 -0500 Subject: [PATCH 37/92] replace unwrap with error --- clarity/src/vm/analysis/errors.rs | 2 ++ clarity/src/vm/types/mod.rs | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/clarity/src/vm/analysis/errors.rs b/clarity/src/vm/analysis/errors.rs index 4cd89c971..61a59583a 100644 --- a/clarity/src/vm/analysis/errors.rs +++ b/clarity/src/vm/analysis/errors.rs @@ -170,6 +170,7 @@ pub enum CheckErrors { // strings InvalidCharactersDetected, + InvalidUTF8Encoding, // secp256k1 signature InvalidSecp65k1Signature, @@ -405,6 +406,7 @@ impl DiagnosableError for CheckErrors { CheckErrors::TraitReferenceNotAllowed => format!("trait references can not be stored"), CheckErrors::ContractOfExpectsTrait => format!("trait reference expected"), CheckErrors::InvalidCharactersDetected => format!("invalid characters detected"), + CheckErrors::InvalidUTF8Encoding => format!("invalid UTF8 encoding"), CheckErrors::InvalidSecp65k1Signature => format!("invalid seckp256k1 signature"), CheckErrors::TypeAlreadyAnnotatedFailure | CheckErrors::CheckerImplementationFailure => { format!("internal error - please file an issue on github.com/blockstack/blockstack-core") diff --git a/clarity/src/vm/types/mod.rs b/clarity/src/vm/types/mod.rs index b8b48ff72..8ec216db9 100644 --- a/clarity/src/vm/types/mod.rs +++ b/clarity/src/vm/types/mod.rs @@ -786,8 +786,9 @@ impl Value { let matched = captures.name("value").unwrap(); let scalar_value = window[matched.start()..matched.end()].to_string(); let unicode_char = { - let u = u32::from_str_radix(&scalar_value, 16).unwrap(); - let c = char::from_u32(u).unwrap(); + let u = u32::from_str_radix(&scalar_value, 16) + .map_err(|_| CheckErrors::InvalidUTF8Encoding)?; + let c = char::from_u32(u).ok_or_else(|| CheckErrors::InvalidUTF8Encoding)?; let mut encoded_char: Vec = vec![0; c.len_utf8()]; c.encode_utf8(&mut encoded_char[..]); encoded_char From 55719e0202776c0ba60fe84f76966b4f51f41e43 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Mon, 27 Jun 2022 12:59:34 -0500 Subject: [PATCH 38/92] fixed comment --- src/core/mempool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 3bc4e2c5e..f604e293b 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -1009,8 +1009,8 @@ impl MemPoolDB { /// highest-fee-first order. This method is interruptable -- in the `settings` struct, the /// caller may choose how long to spend iterating before this method stops. /// - /// `todo` returns an option to a `TransactionEvent` representing the outcome, or None if we - /// hit an error that wasn't transaction specific. + /// `todo` returns an option to a `TransactionEvent` representing the outcome, or None to indicate + /// that iteration through the mempool should be halted. /// /// `output_events` is modified in place, adding all substantive transaction events output by `todo`. pub fn iterate_candidates( From eebcaeb24198b5468ca511e71824a9f88a90ee5d Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Mon, 27 Jun 2022 15:07:32 -0500 Subject: [PATCH 39/92] don't push `Skipped` events to the observer --- src/core/mempool.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index f604e293b..19596fae9 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -1012,7 +1012,8 @@ impl MemPoolDB { /// `todo` returns an option to a `TransactionEvent` representing the outcome, or None to indicate /// that iteration through the mempool should be halted. /// - /// `output_events` is modified in place, adding all substantive transaction events output by `todo`. + /// `output_events` is modified in place, adding all substantive transaction events (success and error + /// events, but not skipped) output by `todo`. pub fn iterate_candidates( &mut self, clarity_tx: &mut C, @@ -1090,7 +1091,14 @@ impl MemPoolDB { // Run `todo` on the transaction. match todo(clarity_tx, &consider, self.cost_estimator.as_mut())? { Some(tx_event) => { - output_events.push(tx_event); + match tx_event { + TransactionEvent::Skipped(_) => { + // don't push `Skipped` events to the observer + } + _ => { + output_events.push(tx_event); + } + } } None => { debug!("Mempool iteration early exit from iterator"); From d4dd2e6c2fe4c7d79c89ce45ee90291b89490d59 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Mon, 27 Jun 2022 17:13:19 -0500 Subject: [PATCH 40/92] add to the changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad7801a17..183b0831d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE ## [2.05.0.2.0] +- Updates to the logging of transaction events (#3139). + ### IMPORTANT! READ THIS FIRST Please read the following **WARNINGs** in their entirety before upgrading. From 08b0516ae8012393a1806517777e26b95e7f25b3 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Fri, 1 Jul 2022 10:58:11 -0500 Subject: [PATCH 41/92] feat: add tx/block to prometheus outputs --- src/chainstate/stacks/db/blocks.rs | 3 ++- src/monitoring/mod.rs | 7 +++++++ src/monitoring/prometheus.rs | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/chainstate/stacks/db/blocks.rs b/src/chainstate/stacks/db/blocks.rs index 79f9838c0..4f4e63d13 100644 --- a/src/chainstate/stacks/db/blocks.rs +++ b/src/chainstate/stacks/db/blocks.rs @@ -82,7 +82,7 @@ use crate::chainstate::coordinator::BlockEventDispatcher; use crate::chainstate::stacks::address::StacksAddressExtensions; use crate::chainstate::stacks::StacksBlockHeader; use crate::chainstate::stacks::StacksMicroblockHeader; -use crate::monitoring::set_last_execution_cost_observed; +use crate::monitoring::{set_last_block_transaction_count, set_last_execution_cost_observed}; use crate::util_lib::boot::boot_code_id; use crate::{types, util}; use stacks_common::types::chainstate::BurnchainHeaderHash; @@ -5463,6 +5463,7 @@ impl StacksChainState { chainstate_tx.log_transactions_processed(&new_tip.index_block_hash(), &tx_receipts); + set_last_block_transaction_count(block.txs.len() as u64); set_last_execution_cost_observed(&block_execution_cost, &block_limit); let epoch_receipt = StacksEpochReceipt { diff --git a/src/monitoring/mod.rs b/src/monitoring/mod.rs index 82b49d65f..03cd5fc5d 100644 --- a/src/monitoring/mod.rs +++ b/src/monitoring/mod.rs @@ -125,6 +125,13 @@ pub fn set_last_execution_cost_observed( } } +/// Log the number of transactions in the latest block. +#[allow(unused_variables)] +pub fn set_last_block_transaction_count(transactions_in_block: u64) { + #[cfg(feature = "monitoring_prom")] + prometheus::LAST_BLOCK_TRANSACTION_COUNT.set(transactions_in_block as f64); +} + pub fn increment_btc_ops_sent_counter() { #[cfg(feature = "monitoring_prom")] prometheus::BTC_OPS_SENT_COUNTER.inc(); diff --git a/src/monitoring/prometheus.rs b/src/monitoring/prometheus.rs index 9a9d7f8f4..d583d301c 100644 --- a/src/monitoring/prometheus.rs +++ b/src/monitoring/prometheus.rs @@ -116,6 +116,11 @@ lazy_static! { "`execution_cost_runtime` for the last block observed." )).unwrap(); + pub static ref LAST_BLOCK_TRANSACTION_COUNT: Gauge = register_gauge!(opts!( + "stacks_node_last_block_transaction_count", + "Number of transactions in the last block." + )).unwrap(); + pub static ref ACTIVE_MINERS_COUNT_GAUGE: IntGauge = register_int_gauge!(opts!( "stacks_node_active_miners_total", "Total number of active miners" From 1b778106e65b988530c79851f952888f2d36155c Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Fri, 1 Jul 2022 11:09:21 -0500 Subject: [PATCH 42/92] fixes to changelog --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 183b0831d..f1d4a108b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to the versioning scheme outlined in the [README.md](README.md). -## [2.05.0.2.0] +## Upcoming - Updates to the logging of transaction events (#3139). +- Added prometheus output for "transactions in last block" (#3138). + +## [2.05.0.2.0] ### IMPORTANT! READ THIS FIRST From f0b4e71402988aade24436a86b23a110694737f0 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Tue, 5 Jul 2022 20:59:26 -0500 Subject: [PATCH 43/92] review: use IntGauge for LAST_BLOCK_TRANSACTION_COUNT --- src/monitoring/prometheus.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monitoring/prometheus.rs b/src/monitoring/prometheus.rs index d583d301c..b5642e53f 100644 --- a/src/monitoring/prometheus.rs +++ b/src/monitoring/prometheus.rs @@ -116,7 +116,7 @@ lazy_static! { "`execution_cost_runtime` for the last block observed." )).unwrap(); - pub static ref LAST_BLOCK_TRANSACTION_COUNT: Gauge = register_gauge!(opts!( + pub static ref LAST_BLOCK_TRANSACTION_COUNT: IntGauge = register_gauge!(opts!( "stacks_node_last_block_transaction_count", "Number of transactions in the last block." )).unwrap(); From 4d2b296ad3739fa356fdedc6f2ea2a61630cc005 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Wed, 6 Jul 2022 09:53:23 -0500 Subject: [PATCH 44/92] fix: build --- src/monitoring/prometheus.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monitoring/prometheus.rs b/src/monitoring/prometheus.rs index b5642e53f..b532863ca 100644 --- a/src/monitoring/prometheus.rs +++ b/src/monitoring/prometheus.rs @@ -116,7 +116,7 @@ lazy_static! { "`execution_cost_runtime` for the last block observed." )).unwrap(); - pub static ref LAST_BLOCK_TRANSACTION_COUNT: IntGauge = register_gauge!(opts!( + pub static ref LAST_BLOCK_TRANSACTION_COUNT: IntGauge = register_int_gauge!(opts!( "stacks_node_last_block_transaction_count", "Number of transactions in the last block." )).unwrap(); From e2f164e2a1d897f077329b8182ac521990892022 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Fri, 10 Jun 2022 19:10:48 -0500 Subject: [PATCH 45/92] chore: TOML config parser improvements --- testnet/stacks-node/src/config.rs | 120 ++++++++++++------ testnet/stacks-node/src/main.rs | 16 ++- .../src/tests/neon_integrations.rs | 2 +- 3 files changed, 97 insertions(+), 41 deletions(-) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index 6e402e1dc..003886f02 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -39,7 +39,7 @@ const LEADER_KEY_TX_ESTIM_SIZE: u64 = 290; const BLOCK_COMMIT_TX_ESTIM_SIZE: u64 = 350; const INV_REWARD_CYCLES_TESTNET: u64 = 6; -#[derive(Clone, Deserialize, Default)] +#[derive(Clone, Deserialize, Default, Debug)] pub struct ConfigFile { pub burnchain: Option, pub node: Option, @@ -59,6 +59,36 @@ pub struct LegacyMstxConfigFile { mod tests { use super::*; + #[test] + fn check_invalid_config_files() { + ConfigFile::from_path("some_path").expect_err("path does not exist"); + ConfigFile::from_str("//[node]").expect_err("invalid toml comment"); + } + + #[test] + fn check_invalid_configs() { + Config::from_config_file(ConfigFile::from_str( + r#" + [node] + seed = "invalid-hex-value" + "# + ).unwrap()).expect_err("invalid [node]seed hex encoding"); + + Config::from_config_file(ConfigFile::from_str( + r#" + [node] + local_peer_seed = "invalid-hex-value" + "# + ).unwrap()).expect_err("invalid [node]local_peer_seed hex encoding"); + + Config::from_config_file(ConfigFile::from_str( + r#" + [burnchain] + peer_host = "bitcoin2.blockstack.com" + "# + ).unwrap()).expect_err("invalid [burnchain].peer_host"); + } + #[test] fn should_load_legacy_mstx_balances_toml() { let config = ConfigFile::from_str( @@ -80,6 +110,7 @@ mod tests { amount = 10000000000000000 "#, ); + let config = config.unwrap(); assert!(config.ustx_balance.is_some()); let balances = config .ustx_balance @@ -105,13 +136,20 @@ mod tests { } impl ConfigFile { - pub fn from_path(path: &str) -> ConfigFile { - let content_str = fs::read_to_string(path).unwrap(); - Self::from_str(&content_str) + pub fn from_path(path: &str) -> Result { + let content_str = fs::read_to_string(path); + if content_str.is_err() { + return Err(format!("Invalid path")); + } + Self::from_str(&content_str.unwrap()) } - pub fn from_str(content: &str) -> ConfigFile { - let mut config: ConfigFile = toml::from_str(content).unwrap(); + pub fn from_str(content: &str) -> Result { + let toml_result = toml::from_str(content); + if toml_result.is_err() { + return Err(format!("Invalid toml: {}", toml_result.err().unwrap())); + } + let mut config: ConfigFile = toml_result.unwrap(); let legacy_config: LegacyMstxConfigFile = toml::from_str(content).unwrap(); if let Some(mstx_balance) = legacy_config.mstx_balance { warn!("'mstx_balance' inside toml config is deprecated, replace with 'ustx_balance'"); @@ -120,7 +158,7 @@ impl ConfigFile { None => Some(mstx_balance), }; } - config + Ok(config) } pub fn xenon() -> ConfigFile { @@ -284,7 +322,7 @@ impl ConfigFile { } } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Config { pub burnchain: BurnchainConfig, pub node: NodeConfig, @@ -327,7 +365,7 @@ lazy_static! { } impl Config { - pub fn from_config_file(config_file: ConfigFile) -> Config { + pub fn from_config_file(config_file: ConfigFile) -> Result { let default_node_config = NodeConfig::default(); let (mut node, bootstrap_node, deny_nodes) = match config_file.node { Some(node) => { @@ -336,7 +374,12 @@ impl Config { name: node.name.unwrap_or(default_node_config.name), seed: match node.seed { Some(seed) => { - hex_bytes(&seed).expect("Seed should be a hex encoded string") + match hex_bytes(&seed) { + Err(_error) => { + return Err(format!("[node]seed should be a hex encoded string")); + } + Ok(seed) => seed, + } } None => default_node_config.seed, }, @@ -352,7 +395,12 @@ impl Config { }, local_peer_seed: match node.local_peer_seed { Some(seed) => { - hex_bytes(&seed).expect("Seed should be a hex encoded string") + match hex_bytes(&seed) { + Err(_error) => { + return Err(format!("[node]local_peer_seed should be a hex encoded string")); + } + Ok(seed) => seed, + } } None => default_node_config.local_peer_seed, }, @@ -404,19 +452,17 @@ impl Config { burnchain.magic_bytes = mainnet_magic.clone(); } if burnchain.magic_bytes != mainnet_magic { - panic!( - "Attempted to run mainnet node with bad magic bytes '{}'", - burnchain.magic_bytes.as_ref().unwrap() - ); + return Err(format!("Attempted to run mainnet node with bad magic bytes '{}'", + burnchain.magic_bytes.as_ref().unwrap())); } if node.use_test_genesis_chainstate == Some(true) { - panic!("Attempted to run mainnet node with `use_test_genesis_chainstate`"); + return Err(format!("Attempted to run mainnet node with `use_test_genesis_chainstate`")); } if let Some(ref balances) = config_file.ustx_balance { if balances.len() > 0 { - panic!( + return Err(format!( "Attempted to run mainnet node with specified `initial_balances`" - ); + )); } } } @@ -445,9 +491,11 @@ impl Config { // Using std::net::LookupHost would be preferable, but it's // unfortunately unstable at this point. // https://doc.rust-lang.org/1.6.0/std/net/struct.LookupHost.html - let mut addrs_iter = - format!("{}:1", peer_host).to_socket_addrs().unwrap(); - let sock_addr = addrs_iter.next().unwrap(); + let addrs_iter = format!("{}:1", peer_host).to_socket_addrs(); + if addrs_iter.is_err() { + return Err(format!("Invalid [burnchain].peer_host")); + } + let sock_addr = addrs_iter.unwrap().next().unwrap(); format!("{}", sock_addr.ip()) } None => default_burnchain_config.peer_host, @@ -528,14 +576,12 @@ impl Config { ]; if !supported_modes.contains(&burnchain.mode.as_str()) { - panic!( - "Setting burnchain.network not supported (should be: {})", - supported_modes.join(", ") - ) + return Err(format!("Setting burnchain.network not supported (should be: {})", + supported_modes.join(", "))); } if burnchain.mode == "helium" && burnchain.local_mining_public_key.is_none() { - panic!("Config is missing the setting `burnchain.local_mining_public_key` (mandatory for helium)") + return Err(format!("Config is missing the setting `burnchain.local_mining_public_key` (mandatory for helium)")); } if let Some(bootstrap_node) = bootstrap_node { @@ -757,7 +803,7 @@ impl Config { None => FeeEstimationConfig::default(), }; - Config { + Ok(Config { node, burnchain, initial_balances, @@ -765,7 +811,7 @@ impl Config { connection_options, estimation, miner, - } + }) } fn get_burnchain_path(&self) -> PathBuf { @@ -1011,7 +1057,7 @@ impl BurnchainConfig { } } -#[derive(Clone, Deserialize, Default)] +#[derive(Clone, Deserialize, Default, Debug)] pub struct BurnchainConfigFile { pub chain: Option, pub burn_fee_cap: Option, @@ -1449,7 +1495,7 @@ impl MinerConfig { } } -#[derive(Clone, Default, Deserialize)] +#[derive(Clone, Default, Deserialize, Debug)] pub struct ConnectionOptionsFile { pub inbox_maxlen: Option, pub outbox_maxlen: Option, @@ -1492,7 +1538,7 @@ pub struct ConnectionOptionsFile { pub antientropy_public: Option, } -#[derive(Clone, Deserialize, Default)] +#[derive(Clone, Deserialize, Default, Debug)] pub struct NodeConfigFile { pub name: Option, pub seed: Option, @@ -1517,7 +1563,7 @@ pub struct NodeConfigFile { pub use_test_genesis_chainstate: Option, } -#[derive(Clone, Deserialize)] +#[derive(Clone, Deserialize, Debug)] pub struct FeeEstimationConfigFile { pub cost_estimator: Option, pub fee_estimator: Option, @@ -1542,7 +1588,7 @@ impl Default for FeeEstimationConfigFile { } } -#[derive(Clone, Deserialize, Default)] +#[derive(Clone, Deserialize, Default, Debug)] pub struct MinerConfigFile { pub min_tx_fee: Option, pub first_attempt_time_ms: Option, @@ -1551,19 +1597,19 @@ pub struct MinerConfigFile { pub probability_pick_no_estimate_tx: Option, } -#[derive(Clone, Deserialize, Default)] +#[derive(Clone, Deserialize, Default, Debug)] pub struct EventObserverConfigFile { pub endpoint: String, pub events_keys: Vec, } -#[derive(Clone, Default)] +#[derive(Clone, Default, Debug)] pub struct EventObserverConfig { pub endpoint: String, pub events_keys: Vec, } -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum EventKeyType { SmartContractEvent((QualifiedContractIdentifier, String)), AssetEvent(AssetIdentifier), @@ -1641,7 +1687,7 @@ pub struct InitialBalance { pub amount: u64, } -#[derive(Clone, Deserialize, Default)] +#[derive(Clone, Deserialize, Default, Debug)] pub struct InitialBalanceFile { pub address: String, pub amount: u64, diff --git a/testnet/stacks-node/src/main.rs b/testnet/stacks-node/src/main.rs index 06eef337d..d5a1cc54f 100644 --- a/testnet/stacks-node/src/main.rs +++ b/testnet/stacks-node/src/main.rs @@ -113,7 +113,12 @@ fn main() { let config_path: String = args.value_from_str("--config").unwrap(); args.finish().unwrap(); info!("Loading config at path {}", config_path); - ConfigFile::from_path(&config_path) + let config_file_result = ConfigFile::from_path(&config_path); + if config_file_result.is_err() { + warn!("Invalid config file: {}", config_file_result.err().unwrap()); + process::exit(1); + } + config_file_result.unwrap() } "version" => { println!("{}", &version()); @@ -123,7 +128,7 @@ fn main() { let seed = { let config_path: Option = args.opt_value_from_str("--config").unwrap(); if let Some(config_path) = config_path { - let conf = Config::from_config_file(ConfigFile::from_path(&config_path)); + let conf = Config::from_config_file(ConfigFile::from_path(&config_path).unwrap()).unwrap(); args.finish().unwrap(); conf.node.seed } else { @@ -151,7 +156,12 @@ fn main() { } }; - let conf = Config::from_config_file(config_file); + let conf_result = Config::from_config_file(config_file); + if conf_result.is_err() { + warn!("Invalid config: {}", conf_result.err().unwrap()); + process::exit(1); + } + let conf = conf_result.unwrap(); debug!("node configuration {:?}", &conf.node); debug!("burnchain configuration {:?}", &conf.burnchain); debug!("connection configuration {:?}", &conf.connection_options); diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 18066e815..b6f717be6 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -100,7 +100,7 @@ pub fn neon_integration_test_conf() -> (Config, StacksAddress) { conf.burnchain.commit_anchor_block_within = 0; // test to make sure config file parsing is correct - let magic_bytes = Config::from_config_file(ConfigFile::xenon()) + let magic_bytes = Config::from_config_file(ConfigFile::xenon()).unwrap() .burnchain .magic_bytes; assert_eq!(magic_bytes.as_bytes(), &['T' as u8, '2' as u8]); From 04bcfe23fb0640198808f9e73ef67f16bc8085da Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 21 Jun 2022 12:43:54 -0500 Subject: [PATCH 46/92] clean up --- testnet/stacks-node/src/config.rs | 128 ++++++++++++++++-------------- testnet/stacks-node/src/main.rs | 39 ++++++--- 2 files changed, 98 insertions(+), 69 deletions(-) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index 003886f02..fc4b749c3 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -60,33 +60,57 @@ mod tests { use super::*; #[test] - fn check_invalid_config_files() { - ConfigFile::from_path("some_path").expect_err("path does not exist"); - ConfigFile::from_str("//[node]").expect_err("invalid toml comment"); + fn test_config_file() { + assert_eq!( + format!("Invalid path: No such file or directory (os error 2)"), + ConfigFile::from_path("some_path").unwrap_err() + ); + assert_eq!( + format!("Invalid toml: unexpected character found: `/` at line 1 column 1"), + ConfigFile::from_str("//[node]").unwrap_err() + ); + assert!(ConfigFile::from_str("").is_ok()); } #[test] - fn check_invalid_configs() { - Config::from_config_file(ConfigFile::from_str( - r#" - [node] - seed = "invalid-hex-value" - "# - ).unwrap()).expect_err("invalid [node]seed hex encoding"); + fn test_config() { + assert_eq!( + format!("[node]seed should be a hex encoded string"), + Config::from_config_file( + ConfigFile::from_str( + r#" + [node] + seed = "invalid-hex-value" + "#, + ).unwrap() + ).unwrap_err() + ); - Config::from_config_file(ConfigFile::from_str( - r#" - [node] - local_peer_seed = "invalid-hex-value" - "# - ).unwrap()).expect_err("invalid [node]local_peer_seed hex encoding"); + assert_eq!( + format!("[node]local_peer_seed should be a hex encoded string"), + Config::from_config_file( + ConfigFile::from_str( + r#" + [node] + local_peer_seed = "invalid-hex-value" + "#, + ).unwrap() + ).unwrap_err() + ); - Config::from_config_file(ConfigFile::from_str( - r#" - [burnchain] - peer_host = "bitcoin2.blockstack.com" - "# - ).unwrap()).expect_err("invalid [burnchain].peer_host"); + assert_eq!( + format!("Invalid [burnchain]peer_host: failed to lookup address information: nodename nor servname provided, or not known"), + Config::from_config_file( + ConfigFile::from_str( + r#" + [burnchain] + peer_host = "bitcoin2.blockstack.com" + "#, + ).unwrap() + ).unwrap_err() + ); + + assert!(Config::from_config_file(ConfigFile::from_str("").unwrap()).is_ok()); } #[test] @@ -137,19 +161,13 @@ mod tests { impl ConfigFile { pub fn from_path(path: &str) -> Result { - let content_str = fs::read_to_string(path); - if content_str.is_err() { - return Err(format!("Invalid path")); - } - Self::from_str(&content_str.unwrap()) + let content = fs::read_to_string(path).map_err(|e| format!("Invalid path: {}", &e))?; + Self::from_str(&content) } pub fn from_str(content: &str) -> Result { - let toml_result = toml::from_str(content); - if toml_result.is_err() { - return Err(format!("Invalid toml: {}", toml_result.err().unwrap())); - } - let mut config: ConfigFile = toml_result.unwrap(); + let mut config: ConfigFile = + toml::from_str(content).map_err(|e| format!("Invalid toml: {}", e))?; let legacy_config: LegacyMstxConfigFile = toml::from_str(content).unwrap(); if let Some(mstx_balance) = legacy_config.mstx_balance { warn!("'mstx_balance' inside toml config is deprecated, replace with 'ustx_balance'"); @@ -373,14 +391,7 @@ impl Config { let node_config = NodeConfig { name: node.name.unwrap_or(default_node_config.name), seed: match node.seed { - Some(seed) => { - match hex_bytes(&seed) { - Err(_error) => { - return Err(format!("[node]seed should be a hex encoded string")); - } - Ok(seed) => seed, - } - } + Some(seed) => hex_bytes(&seed).map_err(|e| format!("[node]seed should be a hex encoded string"))?, None => default_node_config.seed, }, working_dir: node.working_dir.unwrap_or(default_node_config.working_dir), @@ -394,14 +405,7 @@ impl Config { None => format!("http://{}", rpc_bind), }, local_peer_seed: match node.local_peer_seed { - Some(seed) => { - match hex_bytes(&seed) { - Err(_error) => { - return Err(format!("[node]local_peer_seed should be a hex encoded string")); - } - Ok(seed) => seed, - } - } + Some(seed) => hex_bytes(&seed).map_err(|e| format!("[node]local_peer_seed should be a hex encoded string"))?, None => default_node_config.local_peer_seed, }, miner: node.miner.unwrap_or(default_node_config.miner), @@ -452,11 +456,15 @@ impl Config { burnchain.magic_bytes = mainnet_magic.clone(); } if burnchain.magic_bytes != mainnet_magic { - return Err(format!("Attempted to run mainnet node with bad magic bytes '{}'", - burnchain.magic_bytes.as_ref().unwrap())); + return Err(format!( + "Attempted to run mainnet node with bad magic bytes '{}'", + burnchain.magic_bytes.as_ref().unwrap() + )); } if node.use_test_genesis_chainstate == Some(true) { - return Err(format!("Attempted to run mainnet node with `use_test_genesis_chainstate`")); + return Err(format!( + "Attempted to run mainnet node with `use_test_genesis_chainstate`" + )); } if let Some(ref balances) = config_file.ustx_balance { if balances.len() > 0 { @@ -491,11 +499,13 @@ impl Config { // Using std::net::LookupHost would be preferable, but it's // unfortunately unstable at this point. // https://doc.rust-lang.org/1.6.0/std/net/struct.LookupHost.html - let addrs_iter = format!("{}:1", peer_host).to_socket_addrs(); - if addrs_iter.is_err() { - return Err(format!("Invalid [burnchain].peer_host")); - } - let sock_addr = addrs_iter.unwrap().next().unwrap(); + let mut sock_addrs = format!("{}::1", &peer_host).to_socket_addrs().map_err(|e| format!("Invalid [burnchain]peer_host: {}", &e))?; + let sock_addr = match sock_addrs.next() { + Some(addr) => addr, + None => { + return Err(format!("No IP address could be queried for '{}'", &peer_host)); + } + }; format!("{}", sock_addr.ip()) } None => default_burnchain_config.peer_host, @@ -576,8 +586,10 @@ impl Config { ]; if !supported_modes.contains(&burnchain.mode.as_str()) { - return Err(format!("Setting burnchain.network not supported (should be: {})", - supported_modes.join(", "))); + return Err(format!( + "Setting burnchain.network not supported (should be: {})", + supported_modes.join(", ") + )); } if burnchain.mode == "helium" && burnchain.local_mining_public_key.is_none() { diff --git a/testnet/stacks-node/src/main.rs b/testnet/stacks-node/src/main.rs index d5a1cc54f..9ab7ae2bf 100644 --- a/testnet/stacks-node/src/main.rs +++ b/testnet/stacks-node/src/main.rs @@ -109,16 +109,32 @@ fn main() { args.finish().unwrap(); ConfigFile::mainnet() } + "check-config" => { + let config_path: String = args.value_from_str("--config").unwrap(); + args.finish().unwrap(); + info!("Loading config at path {}", config_path); + match ConfigFile::from_path(&config_path) { + Ok(config_file) => { + info!("Loaded config!"); + process::exit(0); + }, + Err(e) => { + warn!("Invalid config file: {}", e); + process::exit(1); + } + } + } "start" => { let config_path: String = args.value_from_str("--config").unwrap(); args.finish().unwrap(); info!("Loading config at path {}", config_path); - let config_file_result = ConfigFile::from_path(&config_path); - if config_file_result.is_err() { - warn!("Invalid config file: {}", config_file_result.err().unwrap()); - process::exit(1); + match ConfigFile::from_path(&config_path) { + Ok(config_file) => config_file, + Err(e) => { + warn!("Invalid config file: {}", e); + process::exit(1); + } } - config_file_result.unwrap() } "version" => { println!("{}", &version()); @@ -156,12 +172,13 @@ fn main() { } }; - let conf_result = Config::from_config_file(config_file); - if conf_result.is_err() { - warn!("Invalid config: {}", conf_result.err().unwrap()); - process::exit(1); - } - let conf = conf_result.unwrap(); + let conf = match Config::from_config_file(config_file) { + Ok(conf) => conf, + Err(e) => { + warn!("Invalid config: {}", e); + process::exit(1); + } + }; debug!("node configuration {:?}", &conf.node); debug!("burnchain configuration {:?}", &conf.burnchain); debug!("connection configuration {:?}", &conf.connection_options); From d0c48f9ef926f7441865d6eb41392697bb84d6b8 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 21 Jun 2022 12:59:23 -0500 Subject: [PATCH 47/92] clean up --- testnet/stacks-node/src/main.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/testnet/stacks-node/src/main.rs b/testnet/stacks-node/src/main.rs index 9ab7ae2bf..b147b85b3 100644 --- a/testnet/stacks-node/src/main.rs +++ b/testnet/stacks-node/src/main.rs @@ -113,16 +113,23 @@ fn main() { let config_path: String = args.value_from_str("--config").unwrap(); args.finish().unwrap(); info!("Loading config at path {}", config_path); - match ConfigFile::from_path(&config_path) { - Ok(config_file) => { - info!("Loaded config!"); - process::exit(0); - }, + let config_file = match ConfigFile::from_path(&config_path) { + Ok(config_file) => config_file, Err(e) => { warn!("Invalid config file: {}", e); process::exit(1); } - } + }; + let conf = match Config::from_config_file(config_file) { + Ok(conf) => { + info!("Loaded config!"); + process::exit(0); + }, + Err(e) => { + warn!("Invalid config: {}", e); + process::exit(1); + } + }; } "start" => { let config_path: String = args.value_from_str("--config").unwrap(); From b200b85cdd4aaf82ae75229fdc16166980ab5cf3 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 21 Jun 2022 13:02:46 -0500 Subject: [PATCH 48/92] clean up --- testnet/stacks-node/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testnet/stacks-node/src/main.rs b/testnet/stacks-node/src/main.rs index b147b85b3..889d2d6bc 100644 --- a/testnet/stacks-node/src/main.rs +++ b/testnet/stacks-node/src/main.rs @@ -253,6 +253,8 @@ start\t\tStart a node with a config of your own. Can be used for joining a netwo \t\tExample: \t\t stacks-node start --config=/path/to/config.toml +check-config\t\tValidates the config file without starting up the node. Uses same arguments as start subcommand. + version\t\tDisplay information about the current version and our release cycle. key-for-seed\tOutput the associated secret key for a burnchain signer created with a given seed. From 0eea7e439a891cd995f94e8c5dcf392901a90c42 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 21 Jun 2022 13:16:10 -0500 Subject: [PATCH 49/92] rustfmt --- testnet/stacks-node/src/config.rs | 28 ++++++++++++++++++++-------- testnet/stacks-node/src/main.rs | 6 ++++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index fc4b749c3..394a0489c 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -82,8 +82,10 @@ mod tests { [node] seed = "invalid-hex-value" "#, - ).unwrap() - ).unwrap_err() + ) + .unwrap() + ) + .unwrap_err() ); assert_eq!( @@ -94,8 +96,10 @@ mod tests { [node] local_peer_seed = "invalid-hex-value" "#, - ).unwrap() - ).unwrap_err() + ) + .unwrap() + ) + .unwrap_err() ); assert_eq!( @@ -391,7 +395,8 @@ impl Config { let node_config = NodeConfig { name: node.name.unwrap_or(default_node_config.name), seed: match node.seed { - Some(seed) => hex_bytes(&seed).map_err(|e| format!("[node]seed should be a hex encoded string"))?, + Some(seed) => hex_bytes(&seed) + .map_err(|e| format!("[node]seed should be a hex encoded string"))?, None => default_node_config.seed, }, working_dir: node.working_dir.unwrap_or(default_node_config.working_dir), @@ -405,7 +410,9 @@ impl Config { None => format!("http://{}", rpc_bind), }, local_peer_seed: match node.local_peer_seed { - Some(seed) => hex_bytes(&seed).map_err(|e| format!("[node]local_peer_seed should be a hex encoded string"))?, + Some(seed) => hex_bytes(&seed).map_err(|e| { + format!("[node]local_peer_seed should be a hex encoded string") + })?, None => default_node_config.local_peer_seed, }, miner: node.miner.unwrap_or(default_node_config.miner), @@ -499,11 +506,16 @@ impl Config { // Using std::net::LookupHost would be preferable, but it's // unfortunately unstable at this point. // https://doc.rust-lang.org/1.6.0/std/net/struct.LookupHost.html - let mut sock_addrs = format!("{}::1", &peer_host).to_socket_addrs().map_err(|e| format!("Invalid [burnchain]peer_host: {}", &e))?; + let mut sock_addrs = format!("{}::1", &peer_host) + .to_socket_addrs() + .map_err(|e| format!("Invalid [burnchain]peer_host: {}", &e))?; let sock_addr = match sock_addrs.next() { Some(addr) => addr, None => { - return Err(format!("No IP address could be queried for '{}'", &peer_host)); + return Err(format!( + "No IP address could be queried for '{}'", + &peer_host + )); } }; format!("{}", sock_addr.ip()) diff --git a/testnet/stacks-node/src/main.rs b/testnet/stacks-node/src/main.rs index 889d2d6bc..220972425 100644 --- a/testnet/stacks-node/src/main.rs +++ b/testnet/stacks-node/src/main.rs @@ -124,7 +124,7 @@ fn main() { Ok(conf) => { info!("Loaded config!"); process::exit(0); - }, + } Err(e) => { warn!("Invalid config: {}", e); process::exit(1); @@ -151,7 +151,9 @@ fn main() { let seed = { let config_path: Option = args.opt_value_from_str("--config").unwrap(); if let Some(config_path) = config_path { - let conf = Config::from_config_file(ConfigFile::from_path(&config_path).unwrap()).unwrap(); + let conf = + Config::from_config_file(ConfigFile::from_path(&config_path).unwrap()) + .unwrap(); args.finish().unwrap(); conf.node.seed } else { From 396efd02f2d00d68eafb11cc63ad111b25646c72 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 21 Jun 2022 13:54:01 -0500 Subject: [PATCH 50/92] add neon_integrations.rs --- testnet/stacks-node/src/tests/neon_integrations.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index b6f717be6..9acfb3917 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -100,7 +100,8 @@ pub fn neon_integration_test_conf() -> (Config, StacksAddress) { conf.burnchain.commit_anchor_block_within = 0; // test to make sure config file parsing is correct - let magic_bytes = Config::from_config_file(ConfigFile::xenon()).unwrap() + let magic_bytes = Config::from_config_file(ConfigFile::xenon()) + .unwrap() .burnchain .magic_bytes; assert_eq!(magic_bytes.as_bytes(), &['T' as u8, '2' as u8]); From 20a3bb68de8ad92ee4db6faa0246e5c10e90edde Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Wed, 22 Jun 2022 14:16:53 -0500 Subject: [PATCH 51/92] use consistent toml naming --- testnet/stacks-node/src/config.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index 394a0489c..b5a3b1ce9 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -75,7 +75,7 @@ mod tests { #[test] fn test_config() { assert_eq!( - format!("[node]seed should be a hex encoded string"), + format!("node.seed should be a hex encoded string"), Config::from_config_file( ConfigFile::from_str( r#" @@ -89,7 +89,7 @@ mod tests { ); assert_eq!( - format!("[node]local_peer_seed should be a hex encoded string"), + format!("node.local_peer_seed should be a hex encoded string"), Config::from_config_file( ConfigFile::from_str( r#" @@ -103,7 +103,7 @@ mod tests { ); assert_eq!( - format!("Invalid [burnchain]peer_host: failed to lookup address information: nodename nor servname provided, or not known"), + format!("Invalid burnchain.peer_host: failed to lookup address information: nodename nor servname provided, or not known"), Config::from_config_file( ConfigFile::from_str( r#" @@ -396,7 +396,7 @@ impl Config { name: node.name.unwrap_or(default_node_config.name), seed: match node.seed { Some(seed) => hex_bytes(&seed) - .map_err(|e| format!("[node]seed should be a hex encoded string"))?, + .map_err(|e| format!("node.seed should be a hex encoded string"))?, None => default_node_config.seed, }, working_dir: node.working_dir.unwrap_or(default_node_config.working_dir), @@ -411,7 +411,7 @@ impl Config { }, local_peer_seed: match node.local_peer_seed { Some(seed) => hex_bytes(&seed).map_err(|e| { - format!("[node]local_peer_seed should be a hex encoded string") + format!("node.local_peer_seed should be a hex encoded string") })?, None => default_node_config.local_peer_seed, }, @@ -508,7 +508,7 @@ impl Config { // https://doc.rust-lang.org/1.6.0/std/net/struct.LookupHost.html let mut sock_addrs = format!("{}::1", &peer_host) .to_socket_addrs() - .map_err(|e| format!("Invalid [burnchain]peer_host: {}", &e))?; + .map_err(|e| format!("Invalid burnchain.peer_host: {}", &e))?; let sock_addr = match sock_addrs.next() { Some(addr) => addr, None => { From f2ffb40a6eb77d37bbdff022dfad955f619e15f7 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 5 Jul 2022 16:30:44 -0500 Subject: [PATCH 52/92] fix test case --- testnet/stacks-node/src/config.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index b5a3b1ce9..49420a901 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -102,16 +102,18 @@ mod tests { .unwrap_err() ); + let expected_err_prefix = "Invalid burnchain.peer_host: failed to lookup address information:"; + let actual_err_msg = Config::from_config_file( + ConfigFile::from_str( + r#" + [burnchain] + peer_host = "bitcoin2.blockstack.com" + "#, + ).unwrap() + ).unwrap_err(); assert_eq!( - format!("Invalid burnchain.peer_host: failed to lookup address information: nodename nor servname provided, or not known"), - Config::from_config_file( - ConfigFile::from_str( - r#" - [burnchain] - peer_host = "bitcoin2.blockstack.com" - "#, - ).unwrap() - ).unwrap_err() + expected_err_prefix, + &actual_err_msg[..expected_err_prefix.len()] ); assert!(Config::from_config_file(ConfigFile::from_str("").unwrap()).is_ok()); From e93c89db29f40e2bd8d9cb6396068a01253f4601 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 5 Jul 2022 16:34:01 -0500 Subject: [PATCH 53/92] fmt --- testnet/stacks-node/src/config.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index 49420a901..de9f80fa1 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -102,15 +102,18 @@ mod tests { .unwrap_err() ); - let expected_err_prefix = "Invalid burnchain.peer_host: failed to lookup address information:"; + let expected_err_prefix = + "Invalid burnchain.peer_host: failed to lookup address information:"; let actual_err_msg = Config::from_config_file( ConfigFile::from_str( r#" [burnchain] peer_host = "bitcoin2.blockstack.com" "#, - ).unwrap() - ).unwrap_err(); + ) + .unwrap(), + ) + .unwrap_err(); assert_eq!( expected_err_prefix, &actual_err_msg[..expected_err_prefix.len()] From 3be25029c8854b4035d579f0714e3a29da92191d Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 5 Jul 2022 19:25:32 -0500 Subject: [PATCH 54/92] fix --- testnet/stacks-node/src/config.rs | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index de9f80fa1..b205d273d 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -102,21 +102,16 @@ mod tests { .unwrap_err() ); - let expected_err_prefix = - "Invalid burnchain.peer_host: failed to lookup address information:"; - let actual_err_msg = Config::from_config_file( - ConfigFile::from_str( - r#" - [burnchain] - peer_host = "bitcoin2.blockstack.com" - "#, - ) - .unwrap(), - ) - .unwrap_err(); assert_eq!( - expected_err_prefix, - &actual_err_msg[..expected_err_prefix.len()] + format!("Invalid burnchain.peer_host: failed to lookup address information: nodename nor servname provided, or not known"), + Config::from_config_file( + ConfigFile::from_str( + r#" + [burnchain] + peer_host = "bitcoin2.blockstack.com" + "#, + ).unwrap() + ).unwrap_err() ); assert!(Config::from_config_file(ConfigFile::from_str("").unwrap()).is_ok()); @@ -511,7 +506,7 @@ impl Config { // Using std::net::LookupHost would be preferable, but it's // unfortunately unstable at this point. // https://doc.rust-lang.org/1.6.0/std/net/struct.LookupHost.html - let mut sock_addrs = format!("{}::1", &peer_host) + let mut sock_addrs = format!("{}:1", &peer_host) .to_socket_addrs() .map_err(|e| format!("Invalid burnchain.peer_host: {}", &e))?; let sock_addr = match sock_addrs.next() { From 41432f26b3133ff67ee1c98a13c44625902ca47c Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 5 Jul 2022 21:42:26 -0500 Subject: [PATCH 55/92] fix --- testnet/stacks-node/src/config.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index b205d273d..6eac291ed 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -102,16 +102,21 @@ mod tests { .unwrap_err() ); + let expected_err_prefix = + "Invalid burnchain.peer_host: failed to lookup address information:"; + let actual_err_msg = Config::from_config_file( + ConfigFile::from_str( + r#" + [burnchain] + peer_host = "bitcoin2.blockstack.com" + "#, + ) + .unwrap(), + ) + .unwrap_err(); assert_eq!( - format!("Invalid burnchain.peer_host: failed to lookup address information: nodename nor servname provided, or not known"), - Config::from_config_file( - ConfigFile::from_str( - r#" - [burnchain] - peer_host = "bitcoin2.blockstack.com" - "#, - ).unwrap() - ).unwrap_err() + expected_err_prefix, + &actual_err_msg[..expected_err_prefix.len()] ); assert!(Config::from_config_file(ConfigFile::from_str("").unwrap()).is_ok()); From 0323e799074b6fe0ca647985f66cefe8f23e9e54 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 1 Jul 2022 12:43:55 -0500 Subject: [PATCH 56/92] fix: #3184 alter miner re-attempt logic to always allow a "second attempt" --- testnet/stacks-node/src/neon_node.rs | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index 9cf3bd2cb..def38885a 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -1702,7 +1702,15 @@ impl StacksNode { }; // has the tip changed from our previously-mined block for this epoch? - let attempt = { + let attempt = if last_mined_blocks.len() <= 1 { + // always mine if we've not mined a block for this epoch yet, or + // if we've mined just one attempt, unconditionally try again (so we + // can use `subsequent_miner_time_ms` in this attempt) + if last_mined_blocks.len() == 1 { + debug!("Have only attempted one block; unconditionally trying again"); + } + last_mined_blocks.len() as u64 + 1 + } else { let mut best_attempt = 0; debug!( "Consider {} in-flight Stacks tip(s)", @@ -1717,20 +1725,12 @@ impl StacksNode { &prev_block.my_burn_hash, &prev_block.anchored_block.txs.len() ); - if prev_block.anchored_block.txs.len() == 1 { - if last_mined_blocks.len() == 1 { - // this is an empty block, and we've only tried once before. We should always - // try again, with the `subsequent_miner_time_ms` allotment, in order to see if - // we can make a bigger block - debug!("Have only mined one empty block off of {}/{} height {}; unconditionally trying again", &prev_block.parent_consensus_hash, &prev_block.anchored_block.block_hash(), prev_block.anchored_block.header.total_work.work); - best_attempt = 1; - break; - } else if prev_block.attempt == 1 { - // Don't let the fact that we've built an empty block during this sortition - // prevent us from trying again. - best_attempt = 1; - continue; - } + + if prev_block.anchored_block.txs.len() == 1 && prev_block.attempt == 1 { + // Don't let the fact that we've built an empty block during this sortition + // prevent us from trying again. + best_attempt = 1; + continue; } if prev_block.parent_consensus_hash == parent_consensus_hash && prev_block.my_burn_hash == burn_block.burn_header_hash From 11950ef305bcc91d2516235e89acd3894d610374 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 1 Jul 2022 14:49:37 -0500 Subject: [PATCH 57/92] add CHANGELOG.md entry --- CHANGELOG.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cab2be9ae..cc5818cbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,21 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE - Updates to the logging of transaction events (#3139). - Added prometheus output for "transactions in last block" (#3138). +### Fixed + +- Fixed default miner behavior regarding block assembly + attempts. Previously, the miner would only attempt to assemble a + larger block after their first attempt (by Bitcoin RBF) if new + microblock or block data arrived. This changes the miner to always + attempt a second block assembly (#3184). + ## [2.05.0.2.1] ### Fixed - Fixed a security bug in the SPV client whereby the chain work was not being considered at all when determining the canonical Bitcoin fork. The SPV client -now only accepts a new Bitcoin fork if it has a higher chain work than any other -previously-seen chain (#3152). + now only accepts a new Bitcoin fork if it has a higher chain work than any other + previously-seen chain (#3152). ## [2.05.0.2.0] From 1468e6404db3eb44b3c4be775b92d3a866757a87 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Thu, 21 Jul 2022 18:04:09 -0500 Subject: [PATCH 58/92] fix type error in monitoring_prom build --- src/monitoring/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/monitoring/mod.rs b/src/monitoring/mod.rs index 03cd5fc5d..166364937 100644 --- a/src/monitoring/mod.rs +++ b/src/monitoring/mod.rs @@ -128,8 +128,10 @@ pub fn set_last_execution_cost_observed( /// Log the number of transactions in the latest block. #[allow(unused_variables)] pub fn set_last_block_transaction_count(transactions_in_block: u64) { + // Saturating cast from u64 to i64 #[cfg(feature = "monitoring_prom")] - prometheus::LAST_BLOCK_TRANSACTION_COUNT.set(transactions_in_block as f64); + prometheus::LAST_BLOCK_TRANSACTION_COUNT + .set(i64::try_from(transactions_in_block).unwrap_or_else(|_| i64::MAX)); } pub fn increment_btc_ops_sent_counter() { From 219f1d867eaa0538cdf434fe6d92fced15953c2f Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Fri, 22 Jul 2022 09:57:01 -0400 Subject: [PATCH 59/92] docs: fix output type of `as-max-len?` --- clarity/src/vm/docs/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity/src/vm/docs/mod.rs b/clarity/src/vm/docs/mod.rs index 1bb96b234..10c2b92dd 100644 --- a/clarity/src/vm/docs/mod.rs +++ b/clarity/src/vm/docs/mod.rs @@ -560,7 +560,7 @@ and outputs a list of the same type with max_len += 1.", const ASSERTS_MAX_LEN_API: SpecialAPI = SpecialAPI { input_type: "sequence_A, uint", - output_type: "sequence_A", + output_type: "(optional sequence_A)", signature: "(as-max-len? sequence max_length)", description: "The `as-max-len?` function takes a sequence argument and a uint-valued, literal length argument. From d11e77b7f27b2664fb0e147e576a933181376ac5 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 22 Jul 2022 12:40:26 -0400 Subject: [PATCH 60/92] chore: remove compile-time test warning --- src/chainstate/burn/db/sortdb.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chainstate/burn/db/sortdb.rs b/src/chainstate/burn/db/sortdb.rs index 09f3e0ebb..cdfafa945 100644 --- a/src/chainstate/burn/db/sortdb.rs +++ b/src/chainstate/burn/db/sortdb.rs @@ -7891,7 +7891,7 @@ pub mod tests { // drop descendancy information { - let mut db_tx = db.tx_begin().unwrap(); + let db_tx = db.tx_begin().unwrap(); db_tx .execute("DELETE FROM block_commit_parents", NO_PARAMS) .unwrap(); From c3611ec3361e679cf98d7f0a916c33e9a47df59d Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 22 Jul 2022 12:40:46 -0400 Subject: [PATCH 61/92] feat: TemporarilyBlacklisted error variant for mempool rejection --- src/chainstate/stacks/db/blocks.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/chainstate/stacks/db/blocks.rs b/src/chainstate/stacks/db/blocks.rs index 4f4e63d13..14f8c34ea 100644 --- a/src/chainstate/stacks/db/blocks.rs +++ b/src/chainstate/stacks/db/blocks.rs @@ -157,6 +157,7 @@ pub enum MemPoolRejection { TransferAmountMustBePositive, DBError(db_error), EstimatorError(EstimatorError), + TemporarilyBlacklisted, Other(String), } @@ -304,6 +305,7 @@ impl MemPoolRejection { "ServerFailureDatabase", Some(json!({"message": e.to_string()})), ), + TemporarilyBlacklisted => ("TemporarilyBlacklisted", None), Other(s) => ("ServerFailureOther", Some(json!({ "message": s }))), }; let mut result = json!({ From 3a9d10d85b0110c59b0e322e76dbf40a0f23d7e0 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 22 Jul 2022 12:41:05 -0400 Subject: [PATCH 62/92] refactor: expose handle_clarity_runtime_error() --- src/chainstate/stacks/db/transactions.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/chainstate/stacks/db/transactions.rs b/src/chainstate/stacks/db/transactions.rs index 319ccb241..bcc4250e4 100644 --- a/src/chainstate/stacks/db/transactions.rs +++ b/src/chainstate/stacks/db/transactions.rs @@ -256,7 +256,7 @@ impl From for MemPoolRejection { } } -enum ClarityRuntimeTxError { +pub enum ClarityRuntimeTxError { Acceptable { error: clarity_error, err_type: &'static str, @@ -266,7 +266,7 @@ enum ClarityRuntimeTxError { Rejectable(clarity_error), } -fn handle_clarity_runtime_error(error: clarity_error) -> ClarityRuntimeTxError { +pub fn handle_clarity_runtime_error(error: clarity_error) -> ClarityRuntimeTxError { match error { // runtime errors are okay clarity_error::Interpreter(InterpreterError::Runtime(_, _)) => { @@ -925,7 +925,7 @@ impl StacksChainState { return Err(Error::CostOverflowError(cost_before, cost_after, budget)); } ClarityRuntimeTxError::Rejectable(e) => { - error!("Unexpected error invalidating transaction: if included, this will invalidate a block"; + error!("Unexpected error in validating transaction: if included, this will invalidate a block"; "contract_name" => %contract_id, "function_name" => %contract_call.function_name, "function_args" => %VecDisplay(&contract_call.function_args), From 441e1d0e6b1acb2c6f9ae38484c26e53b7946200 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 22 Jul 2022 12:41:27 -0400 Subject: [PATCH 63/92] feat: identify transactions that fail at *runtime* due to an analysis-time error. This ideally should never happen, but there are a few cases where it does (and, it means that miners spend non-trivial compute time on transactions they can't claim tx fees for). Identify these transactions as "problematic" transactions, and drop them from the mempool and blacklist them so they can't keep circulating in the network. --- src/chainstate/stacks/miner.rs | 846 +++++++++++++++++++++++++++++---- 1 file changed, 764 insertions(+), 82 deletions(-) diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index 2ba60c735..a94e696fc 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -25,6 +25,9 @@ use crate::burnchains::PublicKey; use crate::chainstate::burn::db::sortdb::{SortitionDB, SortitionDBConn, SortitionHandleTx}; use crate::chainstate::burn::operations::*; use crate::chainstate::burn::*; +use crate::chainstate::stacks::db::transactions::{ + handle_clarity_runtime_error, ClarityRuntimeTxError, +}; use crate::chainstate::stacks::db::unconfirmed::UnconfirmedState; use crate::chainstate::stacks::db::{ blocks::MemPoolRejection, ChainstateTx, ClarityTx, MinerRewardInfo, StacksChainState, @@ -57,7 +60,10 @@ use crate::types::chainstate::BurnchainHeaderHash; use crate::types::chainstate::StacksBlockId; use crate::types::chainstate::TrieHash; use crate::types::chainstate::{BlockHeaderHash, StacksAddress, StacksWorkScore}; +use clarity::vm::analysis::{CheckError, CheckErrors}; use clarity::vm::clarity::TransactionConnection; +use clarity::vm::errors::Error as InterpreterError; +use clarity::vm::types::TypeSignature; #[derive(Debug, Clone)] pub struct BlockBuilderSettings { @@ -161,6 +167,13 @@ pub struct TransactionSkipped { pub error: Error, } +/// Represents a transaction that is problematic and should be dropped. +#[derive(Debug)] +pub struct TransactionProblematic { + pub tx: StacksTransaction, + pub error: Error, +} + /// Represents an event for a successful transaction. This transaction should be added to the block. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct TransactionSuccessEvent { @@ -187,6 +200,14 @@ pub struct TransactionSkippedEvent { pub error: String, } +/// Represents an event for a transaction that needs to be dropped from the mempool for some reason +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct TransactionProblematicEvent { + #[serde(deserialize_with = "hex_deserialize", serialize_with = "hex_serialize")] + pub txid: Txid, + pub error: String, +} + fn hex_serialize(txid: &Txid, s: S) -> Result { let inst = txid.to_hex(); s.serialize_str(inst.as_str()) @@ -213,6 +234,10 @@ pub enum TransactionResult { ProcessingError(TransactionError), /// Transaction wasn't ready to be be processed, but might succeed later. Skipped(TransactionSkipped), + /// Transaction is problematic (e.g. a DDoS vector) and should be dropped. + /// This error variant is a placeholder for fixing Clarity VM quirks in the next network + /// upgrade. + Problematic(TransactionProblematic), } /// This struct is used to transmit data about transaction results through either the `mined_block` @@ -226,6 +251,8 @@ pub enum TransactionEvent { /// Transaction wasn't ready to be be processed, but might succeed later. /// The bool represents whether mempool propagation should halt or continue Skipped(TransactionSkippedEvent), + /// Transaction is problematic and will be dropped + Problematic(TransactionProblematicEvent), } impl TransactionResult { @@ -261,6 +288,17 @@ impl TransactionResult { ); } + /// Logs a queryable message for the case where `tx` is problematic and needs to be dropped. + pub fn log_transaction_problematic(tx: &StacksTransaction, err: &Error) { + info!( + "Tx processing problematic"; + "event_name" => "transaction_result", + "tx_id" => %tx.txid(), + "event_type" => "problematic", + "reason" => %err, + ) + } + /// Creates a `TransactionResult` backed by `TransactionSuccess`. /// This method logs "transaction success" as a side effect. pub fn success( @@ -312,6 +350,16 @@ impl TransactionResult { }) } + /// Creates a `TransactionResult` backed by `TransactionProblematic`. + /// This method logs "transaction problematic" as a side effect. + pub fn problematic(transaction: &StacksTransaction, error: Error) -> TransactionResult { + Self::log_transaction_problematic(transaction, &error); + TransactionResult::Problematic(TransactionProblematic { + tx: transaction.clone(), + error: error, + }) + } + pub fn convert_to_event(&self) -> TransactionEvent { match &self { TransactionResult::Success(TransactionSuccess { tx, fee, receipt }) => { @@ -334,6 +382,12 @@ impl TransactionResult { error: error.to_string(), }) } + TransactionResult::Problematic(TransactionProblematic { tx, error }) => { + TransactionEvent::Problematic(TransactionProblematicEvent { + txid: tx.txid(), + error: error.to_string(), + }) + } } } @@ -374,6 +428,41 @@ impl TransactionResult { _ => panic!("Tried to `unwrap_error` a non-error result."), } } + + /// Is a given transaction-processing error evidence of a problematic transaction? + /// We can't clone() the error, nor use a reference, so we have to return it. + /// Returns (true, error) if so + /// Returns (false, error) if none + pub fn is_problematic(tx: &StacksTransaction, error: Error) -> (bool, Error) { + let error = match error { + Error::ClarityError(e) => match handle_clarity_runtime_error(e) { + ClarityRuntimeTxError::Rejectable(e) => { + // this transaction would invalidate the whole block, so don't re-consider it + info!("Problematic transaction would invalidate the block, so dropping from mempool"; "txid" => %tx.txid(), "error" => %e); + return (true, Error::ClarityError(e)); + } + // recover original ClarityError + ClarityRuntimeTxError::Acceptable { error, .. } => Error::ClarityError(error), + ClarityRuntimeTxError::CostError(cost, budget) => { + Error::ClarityError(clarity_error::CostError(cost, budget)) + } + ClarityRuntimeTxError::AbortedByCallback(val, assets, events) => { + Error::ClarityError(clarity_error::AbortedByCallback(val, assets, events)) + } + }, + Error::InvalidFee => { + // The transaction didn't have enough STX left over after it was run. + // While such a transaction *could* be mineable in the future, e.g. depending on + // which code paths were hit, the user should really have attached an appropriate + // tx fee in the first place. In Stacks 2.1, the code will debit the fee first, so + // this will no longer be an issue. + info!("Problematic transaction caused InvalidFee"; "txid" => %tx.txid()); + return (true, Error::InvalidFee); + } + e => e, + }; + (false, error) + } } /// @@ -613,6 +702,7 @@ impl<'a> StacksMicroblockBuilder<'a> { /// Returns Ok(TransactionResult::Success) if the transaction was mined into this microblock. /// Returns Ok(TransactionResult::Skipped) if the transaction was not mined, but can be mined later. /// Returns Ok(TransactionResult::Error) if the transaction was not mined due to an error. + /// Returns Ok(TransactionResult::Problematic) if the transaction should be dropped from the mempool. /// Returns Err(e) if an error occurs during the function. /// /// This calls `StacksChainState::process_transaction` and also checks certain pre-conditions @@ -689,38 +779,43 @@ impl<'a> StacksMicroblockBuilder<'a> { match StacksChainState::process_transaction(clarity_tx, &tx, quiet) { Ok((fee, receipt)) => Ok(TransactionResult::success(&tx, fee, receipt)), Err(e) => { - match &e { - Error::CostOverflowError(cost_before, cost_after, total_budget) => { - // note: this path _does_ not perform the tx block budget % heuristic, - // because this code path is not directly called with a mempool handle. - clarity_tx.reset_cost(cost_before.clone()); - if total_budget.proportion_largest_dimension(&cost_before) - < TX_BLOCK_LIMIT_PROPORTION_HEURISTIC - { - warn!( - "Transaction {} consumed over {}% of block budget, marking as invalid; budget was {}", - tx.txid(), - 100 - TX_BLOCK_LIMIT_PROPORTION_HEURISTIC, - &total_budget - ); - return Ok(TransactionResult::error( - &tx, - Error::TransactionTooBigError, - )); - } else { - warn!( - "Transaction {} reached block cost {}; budget was {}", - tx.txid(), - &cost_after, - &total_budget - ); - return Ok(TransactionResult::skipped_due_to_error( - &tx, - Error::BlockTooBigError, - )); + let (is_problematic, e) = TransactionResult::is_problematic(&tx, e); + if is_problematic { + Ok(TransactionResult::problematic(&tx, e)) + } else { + match &e { + Error::CostOverflowError(cost_before, cost_after, total_budget) => { + // note: this path _does_ not perform the tx block budget % heuristic, + // because this code path is not directly called with a mempool handle. + clarity_tx.reset_cost(cost_before.clone()); + if total_budget.proportion_largest_dimension(&cost_before) + < TX_BLOCK_LIMIT_PROPORTION_HEURISTIC + { + warn!( + "Transaction {} consumed over {}% of block budget, marking as invalid; budget was {}", + tx.txid(), + 100 - TX_BLOCK_LIMIT_PROPORTION_HEURISTIC, + &total_budget + ); + return Ok(TransactionResult::error( + &tx, + Error::TransactionTooBigError, + )); + } else { + warn!( + "Transaction {} reached block cost {}; budget was {}", + tx.txid(), + &cost_after, + &total_budget + ); + return Ok(TransactionResult::skipped_due_to_error( + &tx, + Error::BlockTooBigError, + )); + } } + _ => Ok(TransactionResult::error(&tx, e)), } - _ => Ok(TransactionResult::error(&tx, e)), } } } @@ -798,6 +893,10 @@ impl<'a> StacksMicroblockBuilder<'a> { } continue; } + TransactionResult::Problematic(TransactionProblematic { tx, .. }) => { + test_debug!("Exclude problematic tx {} from microblock", tx.txid()); + continue; + } } } Err(e) => { @@ -854,6 +953,7 @@ impl<'a> StacksMicroblockBuilder<'a> { .expect("Microblock already open and processing"); let mut invalidated_txs = vec![]; + let mut to_drop_and_blacklist = vec![]; let mut bytes_so_far = self.runtime.bytes_so_far; let mut num_txs = self.runtime.num_mined; @@ -975,6 +1075,11 @@ impl<'a> StacksMicroblockBuilder<'a> { } return Ok(Some(result_event)) } + TransactionResult::Problematic(TransactionProblematic { tx, .. }) => { + debug!("Drop problematic transaction {}", &tx.txid()); + to_drop_and_blacklist.push(tx.txid()); + Ok(Some(result_event)) + } } } Err(e) => Err(e), @@ -982,6 +1087,14 @@ impl<'a> StacksMicroblockBuilder<'a> { }, ); + if to_drop_and_blacklist.len() > 0 { + debug!( + "Dropping and blacklisting {} problematic transaction(s)", + &to_drop_and_blacklist.len() + ); + let _ = mem_pool.drop_and_blacklist_txs(&to_drop_and_blacklist); + } + if intermediate_result.is_err() { break; } @@ -1014,6 +1127,7 @@ impl<'a> StacksMicroblockBuilder<'a> { mem_pool.drop_txs(&invalidated_txs)?; event_dispatcher.mempool_txs_dropped(invalidated_txs, MemPoolDropReason::TOO_EXPENSIVE); + event_dispatcher.mempool_txs_dropped(to_drop_and_blacklist, MemPoolDropReason::PROBLEMATIC); match result { Ok(_) => {} @@ -1230,6 +1344,9 @@ impl StacksBlockBuilder { TransactionResult::Success(s) => Ok(TransactionResult::Success(s)), TransactionResult::Skipped(TransactionSkipped { error, .. }) | TransactionResult::ProcessingError(TransactionError { error, .. }) => Err(error), + TransactionResult::Problematic(TransactionProblematic { tx, .. }) => { + Err(Error::ProblematicTransaction(tx.txid())) + } } } @@ -1295,34 +1412,44 @@ impl StacksBlockBuilder { let (fee, receipt) = match StacksChainState::process_transaction(clarity_tx, tx, quiet) { Ok((fee, receipt)) => (fee, receipt), - Err(e) => match e { - Error::CostOverflowError(cost_before, cost_after, total_budget) => { - clarity_tx.reset_cost(cost_before.clone()); - if total_budget.proportion_largest_dimension(&cost_before) - < TX_BLOCK_LIMIT_PROPORTION_HEURISTIC - { - warn!( - "Transaction {} consumed over {}% of block budget, marking as invalid; budget was {}", - tx.txid(), - 100 - TX_BLOCK_LIMIT_PROPORTION_HEURISTIC, - &total_budget - ); - return TransactionResult::error(&tx, Error::TransactionTooBigError); - } else { - warn!( - "Transaction {} reached block cost {}; budget was {}", - tx.txid(), - &cost_after, - &total_budget - ); - return TransactionResult::skipped_due_to_error( - &tx, - Error::BlockTooBigError, - ); + Err(e) => { + let (is_problematic, e) = TransactionResult::is_problematic(&tx, e); + if is_problematic { + return TransactionResult::problematic(&tx, e); + } else { + match e { + Error::CostOverflowError(cost_before, cost_after, total_budget) => { + clarity_tx.reset_cost(cost_before.clone()); + if total_budget.proportion_largest_dimension(&cost_before) + < TX_BLOCK_LIMIT_PROPORTION_HEURISTIC + { + warn!( + "Transaction {} consumed over {}% of block budget, marking as invalid; budget was {}", + tx.txid(), + 100 - TX_BLOCK_LIMIT_PROPORTION_HEURISTIC, + &total_budget + ); + return TransactionResult::error( + &tx, + Error::TransactionTooBigError, + ); + } else { + warn!( + "Transaction {} reached block cost {}; budget was {}", + tx.txid(), + &cost_after, + &total_budget + ); + return TransactionResult::skipped_due_to_error( + &tx, + Error::BlockTooBigError, + ); + } + } + _ => return TransactionResult::error(&tx, e), } } - _ => return TransactionResult::error(&tx, e), - }, + } }; info!("Include tx"; "tx" => %tx.txid(), @@ -1351,34 +1478,44 @@ impl StacksBlockBuilder { let (fee, receipt) = match StacksChainState::process_transaction(clarity_tx, tx, quiet) { Ok((fee, receipt)) => (fee, receipt), - Err(e) => match e { - Error::CostOverflowError(cost_before, cost_after, total_budget) => { - clarity_tx.reset_cost(cost_before.clone()); - if total_budget.proportion_largest_dimension(&cost_before) - < TX_BLOCK_LIMIT_PROPORTION_HEURISTIC - { - warn!( - "Transaction {} consumed over {}% of block budget, marking as invalid; budget was {}", - tx.txid(), - 100 - TX_BLOCK_LIMIT_PROPORTION_HEURISTIC, - &total_budget - ); - return TransactionResult::error(&tx, Error::TransactionTooBigError); - } else { - warn!( - "Transaction {} reached block cost {}; budget was {}", - tx.txid(), - &cost_after, - &total_budget - ); - return TransactionResult::skipped_due_to_error( - &tx, - Error::BlockTooBigError, - ); + Err(e) => { + let (is_problematic, e) = TransactionResult::is_problematic(&tx, e); + if is_problematic { + return TransactionResult::problematic(&tx, e); + } else { + match e { + Error::CostOverflowError(cost_before, cost_after, total_budget) => { + clarity_tx.reset_cost(cost_before.clone()); + if total_budget.proportion_largest_dimension(&cost_before) + < TX_BLOCK_LIMIT_PROPORTION_HEURISTIC + { + warn!( + "Transaction {} consumed over {}% of block budget, marking as invalid; budget was {}", + tx.txid(), + 100 - TX_BLOCK_LIMIT_PROPORTION_HEURISTIC, + &total_budget + ); + return TransactionResult::error( + &tx, + Error::TransactionTooBigError, + ); + } else { + warn!( + "Transaction {} reached block cost {}; budget was {}", + tx.txid(), + &cost_after, + &total_budget + ); + return TransactionResult::skipped_due_to_error( + &tx, + Error::BlockTooBigError, + ); + } + } + _ => return TransactionResult::error(&tx, e), } } - _ => return TransactionResult::error(&tx, e), - }, + } }; debug!( "Include tx {} ({}) in microblock", @@ -1798,6 +1935,11 @@ impl StacksBlockBuilder { ); continue; } + Err(Error::ProblematicTransaction(txid)) => { + test_debug!("Encountered problematic transaction. Aborting"); + return Err(Error::ProblematicTransaction(txid)); + } + Err(e) => { warn!("Failed to apply tx {}: {:?}", &tx.txid(), &e); continue; @@ -1975,6 +2117,7 @@ impl StacksBlockBuilder { let mut mined_sponsor_nonces: HashMap = HashMap::new(); // map addrs of mined transaction sponsors to the nonces we used let mut invalidated_txs = vec![]; + let mut to_drop_and_blacklist = vec![]; let mut block_limit_hit = BlockLimitFunction::NO_LIMIT_HIT; let deadline = ts_start + (max_miner_time_ms as u128); @@ -2122,12 +2265,23 @@ impl StacksBlockBuilder { } } } + TransactionResult::Problematic(TransactionProblematic { + tx, .. + }) => { + // drop from the mempool + debug!("Drop and blacklist problematic transaction {}", &tx.txid()); + to_drop_and_blacklist.push(tx.txid()); + } } Ok(Some(result_event)) }, ); + if to_drop_and_blacklist.len() > 0 { + let _ = mempool.drop_and_blacklist_txs(&to_drop_and_blacklist); + } + if intermediate_result.is_err() { break; } @@ -2144,6 +2298,7 @@ impl StacksBlockBuilder { if let Some(observer) = event_observer { observer.mempool_txs_dropped(invalidated_txs, MemPoolDropReason::TOO_EXPENSIVE); + observer.mempool_txs_dropped(to_drop_and_blacklist, MemPoolDropReason::PROBLEMATIC); } match result { @@ -6931,6 +7086,38 @@ pub mod test { sign_standard_singlesig_tx(payload.into(), sender, nonce, tx_fee) } + pub fn make_user_contract_call( + sender: &StacksPrivateKey, + nonce: u64, + tx_fee: u64, + contract_addr: &StacksAddress, + contract_name: &str, + contract_function: &str, + args: Vec, + ) -> StacksTransaction { + let mut tx_contract_call = StacksTransaction::new( + TransactionVersion::Testnet, + TransactionAuth::from_p2pkh(sender).unwrap(), + TransactionPayload::new_contract_call( + contract_addr.clone(), + contract_name, + contract_function, + args, + ) + .unwrap(), + ); + + tx_contract_call.chain_id = 0x80000000; + tx_contract_call.auth.set_origin_nonce(nonce); + tx_contract_call.auth.set_tx_fee(tx_fee); + tx_contract_call.post_condition_mode = TransactionPostConditionMode::Allow; + + let mut tx_signer = StacksTransactionSigner::new(&tx_contract_call); + tx_signer.sign_origin(sender).unwrap(); + let tx_contract_call_signed = tx_signer.get_tx().unwrap(); + tx_contract_call_signed + } + pub fn make_user_stacks_transfer( sender: &StacksPrivateKey, nonce: u64, @@ -10212,6 +10399,501 @@ pub mod test { .0 } + // verify that the problematic checker works + #[test] + fn test_is_tx_problematic() { + let privk = StacksPrivateKey::from_hex( + "42faca653724860da7a41bfcef7e6ba78db55146f6900de8cb2a9f760ffac70c01", + ) + .unwrap(); + let privk_extra = StacksPrivateKey::from_hex( + "f67c7437f948ca1834602b28595c12ac744f287a4efaf70d437042a6afed81bc01", + ) + .unwrap(); + let mut privks_expensive = vec![]; + let mut addrs_expensive = vec![]; + let mut initial_balances = vec![]; + let num_blocks = 10; + for i in 0..num_blocks { + let pk = StacksPrivateKey::new(); + let addr = StacksAddress::from_public_keys( + C32_ADDRESS_VERSION_TESTNET_SINGLESIG, + &AddressHashMode::SerializeP2PKH, + 1, + &vec![StacksPublicKey::from_private(&pk)], + ) + .unwrap(); + + privks_expensive.push(pk); + addrs_expensive.push(addr.clone()); + initial_balances.push((addr.to_account_principal(), 10000000000)); + } + + let addr = StacksAddress::from_public_keys( + C32_ADDRESS_VERSION_TESTNET_SINGLESIG, + &AddressHashMode::SerializeP2PKH, + 1, + &vec![StacksPublicKey::from_private(&privk)], + ) + .unwrap(); + let addr_extra = StacksAddress::from_public_keys( + C32_ADDRESS_VERSION_TESTNET_SINGLESIG, + &AddressHashMode::SerializeP2PKH, + 1, + &vec![StacksPublicKey::from_private(&privk_extra)], + ) + .unwrap(); + + initial_balances.push((addr.to_account_principal(), 100000000000)); + initial_balances.push((addr_extra.to_account_principal(), 200000000000)); + + let mut peer_config = TestPeerConfig::new("test_is_tx_problematic", 2018, 2019); + peer_config.initial_balances = initial_balances; + peer_config.epochs = Some(vec![ + StacksEpoch { + epoch_id: StacksEpochId::Epoch20, + start_height: 0, + end_height: 1, + block_limit: ExecutionCost::max_value(), + network_epoch: PEER_VERSION_EPOCH_2_0, + }, + StacksEpoch { + epoch_id: StacksEpochId::Epoch2_05, + start_height: 1, + end_height: i64::MAX as u64, + block_limit: ExecutionCost::max_value(), + network_epoch: PEER_VERSION_EPOCH_2_05, + }, + ]); + + let mut peer = TestPeer::new(peer_config); + + let chainstate_path = peer.chainstate_path.clone(); + + let first_stacks_block_height = { + let sn = + SortitionDB::get_canonical_burn_chain_tip(&peer.sortdb.as_ref().unwrap().conn()) + .unwrap(); + sn.block_height + }; + + let recipient_addr_str = "ST1RFD5Q2QPK3E0F08HG9XDX7SSC7CNRS0QR0SGEV"; + let recipient = StacksAddress::from_string(recipient_addr_str).unwrap(); + + let mut last_block = None; + for tenure_id in 0..num_blocks { + // send transactions to the mempool + let tip = + SortitionDB::get_canonical_burn_chain_tip(&peer.sortdb.as_ref().unwrap().conn()) + .unwrap(); + + let (burn_ops, stacks_block, microblocks) = peer.make_tenure( + |ref mut miner, + ref mut sortdb, + ref mut chainstate, + vrf_proof, + ref parent_opt, + ref parent_microblock_header_opt| { + let parent_tip = match parent_opt { + None => StacksChainState::get_genesis_header_info(chainstate.db()).unwrap(), + Some(block) => { + let ic = sortdb.index_conn(); + let snapshot = + SortitionDB::get_block_snapshot_for_winning_stacks_block( + &ic, + &tip.sortition_id, + &block.block_hash(), + ) + .unwrap() + .unwrap(); // succeeds because we don't fork + StacksChainState::get_anchored_block_header_info( + chainstate.db(), + &snapshot.consensus_hash, + &snapshot.winning_stacks_block_hash, + ) + .unwrap() + .unwrap() + } + }; + + let parent_header_hash = parent_tip.anchored_header.block_hash(); + let parent_consensus_hash = parent_tip.consensus_hash.clone(); + let coinbase_tx = make_coinbase(miner, tenure_id); + + let mut mempool = + MemPoolDB::open_test(false, 0x80000000, &chainstate_path).unwrap(); + + let mut expected_txids = vec![]; + expected_txids.push(coinbase_tx.txid()); + + let mut problematic_txids = vec![]; + + if tenure_id == 2 { + // make a contract that, when instantiated, spends way too much STX. + // Should result in an Error::InvalidFee, causing the tx to get evicted + // from the mempool. + let contract_spends_too_much = + "(begin + (stx-transfer? (stx-get-balance tx-sender) tx-sender 'ST1RFD5Q2QPK3E0F08HG9XDX7SSC7CNRS0QR0SGEV) + )".to_string(); + + let contract_spends_too_much_tx = make_user_contract_publish( + &privks_expensive[tenure_id], + 0, + (2 * contract_spends_too_much.len()) as u64, + &format!("hello-world-{}", &tenure_id), + &contract_spends_too_much + ); + let contract_spends_too_much_txid = contract_spends_too_much_tx.txid(); + + // attempting to build an anchored block with this tx should cause this tx + // to get flagged as problematic + let block_builder = StacksBlockBuilder::make_regtest_block_builder( + &parent_tip, + vrf_proof.clone(), + tip.total_burn, + Hash160::from_node_public_key(&StacksPublicKey::from_private(&miner.next_microblock_privkey())) + ) + .unwrap(); + + if let Err(ChainstateError::ProblematicTransaction(txid)) = StacksBlockBuilder::make_anchored_block_from_txs( + block_builder, + chainstate, + &sortdb.index_conn(), + vec![coinbase_tx.clone(), contract_spends_too_much_tx.clone()] + ) { + assert_eq!(txid, contract_spends_too_much_txid); + } + else { + panic!("Did not get Error::ProblematicTransaction"); + } + + // for tenure_id == 3: + // make a contract that, when called, will cause the caller to spend too + // much stx + let contract_call_spends_too_much = + "(define-public (spend-too-much) + (begin + (print { balance: (stx-get-balance tx-sender) }) + (stx-transfer? (stx-get-balance tx-sender) tx-sender 'ST1RFD5Q2QPK3E0F08HG9XDX7SSC7CNRS0QR0SGEV) + ) + )".to_string(); + + let contract_call_spends_too_much_tx = make_user_contract_publish( + &privks_expensive[tenure_id], + 0, + (2 * contract_call_spends_too_much.len()) as u64, + "spend-too-much", + &contract_call_spends_too_much + ); + + expected_txids.push(contract_call_spends_too_much_tx.txid()); + + // for tenure_id == 4: + // make a contract that, when called, will result in a CheckError at + // runtime + let runtime_checkerror_trait = + " + (define-trait foo + ( + (lolwut () (response bool uint)) + ) + ) + ".to_string(); + + let runtime_checkerror_impl = + " + (impl-trait .foo.foo) + + (define-public (lolwut) + (ok true) + ) + ".to_string(); + + let runtime_checkerror = format!( + " + (use-trait trait .foo.foo) + + (define-data-var mutex bool true) + + (define-public (flip) + (ok (var-set mutex (not (var-get mutex)))) + ) + + ;; triggers checkerror at runtime because gets coerced + ;; into a principal when `internal` is called. + (define-public (test (ref )) + (ok (internal (if (var-get mutex) + (some ref) + none + ))) + ) + + ;; triggers a checkerror at runtime because the code in + ;; `at-block` is buggy + (define-public (test-past (ref )) + (at-block 0x{} (test ref)) + ) + + (define-private (internal (ref (optional ))) true) + ", + &last_block.clone().unwrap() + ); + + let runtime_checkerror_trait_tx = make_user_contract_publish( + &privks_expensive[tenure_id], + 1, + (2 * runtime_checkerror_trait.len()) as u64, + "foo", + &runtime_checkerror_trait + ); + + let runtime_checkerror_impl_tx = make_user_contract_publish( + &privks_expensive[tenure_id], + 2, + (2 * runtime_checkerror_impl.len()) as u64, + "foo-impl", + &runtime_checkerror_impl + ); + + let runtime_checkerror_tx = make_user_contract_publish( + &privks_expensive[tenure_id], + 3, + (2 * runtime_checkerror.len()) as u64, + "trait-checkerror", + &runtime_checkerror + ); + + expected_txids.push(runtime_checkerror_trait_tx.txid()); + expected_txids.push(runtime_checkerror_impl_tx.txid()); + expected_txids.push(runtime_checkerror_tx.txid()); + + for tx in &[&contract_call_spends_too_much_tx, &runtime_checkerror_trait_tx, &runtime_checkerror_impl_tx, &runtime_checkerror_tx] { + mempool + .submit( + chainstate, + &parent_consensus_hash, + &parent_header_hash, + tx, + None, + &ExecutionCost::max_value(), + &StacksEpochId::Epoch2_05, + ) + .unwrap(); + } + + // the same tx, but with nonce 4 (since we expect the `spends-too-much` contract to get + // mined, as well as the other problem setup txs) + let contract_spends_too_much_tx = make_user_contract_publish( + &privks_expensive[tenure_id], + 4, + (2 * contract_spends_too_much.len()) as u64, + &format!("hello-world-{}", &tenure_id), + &contract_spends_too_much + ); + let contract_spends_too_much_txid = contract_spends_too_much_tx.txid(); + problematic_txids.push(contract_spends_too_much_txid); + + // put this into the mempool anyway, so we can verify it gets rejected + mempool + .submit( + chainstate, + &parent_consensus_hash, + &parent_header_hash, + &contract_spends_too_much_tx, + None, + &ExecutionCost::max_value(), + &StacksEpochId::Epoch2_05, + ) + .unwrap(); + } + + if tenure_id == 3 { + // call spend-too-much and verify that it's flagged as problematic + let spend_too_much = make_user_contract_call( + &privks_expensive[tenure_id], + 0, + 2000, + &addrs_expensive[2], + "spend-too-much", + "spend-too-much", + vec![] + ); + + // attempting to build an anchored block with this tx should cause this tx + // to get flagged as problematic + let block_builder = StacksBlockBuilder::make_regtest_block_builder( + &parent_tip, + vrf_proof.clone(), + tip.total_burn, + Hash160::from_node_public_key(&StacksPublicKey::from_private(&miner.next_microblock_privkey())) + ) + .unwrap(); + + if let Err(ChainstateError::ProblematicTransaction(txid)) = StacksBlockBuilder::make_anchored_block_from_txs( + block_builder, + chainstate, + &sortdb.index_conn(), + vec![coinbase_tx.clone(), spend_too_much.clone()] + ) { + assert_eq!(txid, spend_too_much.txid()); + } + else { + panic!("Did not get Error::ProblematicTransaction"); + } + + problematic_txids.push(spend_too_much.txid()); + mempool + .submit( + chainstate, + &parent_consensus_hash, + &parent_header_hash, + &spend_too_much, + None, + &ExecutionCost::max_value(), + &StacksEpochId::Epoch2_05, + ) + .unwrap(); + } + + if tenure_id == 4 { + // call trait-checkerror.test and verify that it's flagged as problematic + let runtime_checkerror_problematic = make_user_contract_call( + &privks_expensive[tenure_id], + 0, + 2000, + &addrs_expensive[2], + "trait-checkerror", + "test", + vec![Value::Principal(PrincipalData::Contract(QualifiedContractIdentifier::parse(&format!("{}.foo-impl", &addrs_expensive[2])).unwrap()))], + ); + + // attempting to build an anchored block with this tx should cause this tx + // to get flagged as problematic + let block_builder = StacksBlockBuilder::make_regtest_block_builder( + &parent_tip, + vrf_proof.clone(), + tip.total_burn, + Hash160::from_node_public_key(&StacksPublicKey::from_private(&miner.next_microblock_privkey())) + ) + .unwrap(); + + if let Err(ChainstateError::ProblematicTransaction(txid)) = StacksBlockBuilder::make_anchored_block_from_txs( + block_builder, + chainstate, + &sortdb.index_conn(), + vec![coinbase_tx.clone(), runtime_checkerror_problematic.clone()] + ) { + assert_eq!(txid, runtime_checkerror_problematic.txid()); + } + else { + panic!("Did not get Error::ProblematicTransaction"); + } + + problematic_txids.push(runtime_checkerror_problematic.txid()); + mempool + .submit( + chainstate, + &parent_consensus_hash, + &parent_header_hash, + &runtime_checkerror_problematic, + None, + &ExecutionCost::max_value(), + &StacksEpochId::Epoch2_05, + ) + .unwrap(); + } + + if tenure_id == 5 { + // call trait-checkerror.test-past and verify that it's flagged as problematic + let runtime_checkerror_problematic = make_user_contract_call( + &privks_expensive[tenure_id], + 0, + 2000, + &addrs_expensive[2], + "trait-checkerror", + "test-past", + vec![Value::Principal(PrincipalData::Contract(QualifiedContractIdentifier::parse(&format!("{}.foo-impl", &addrs_expensive[2])).unwrap()))], + ); + + // attempting to build an anchored block with this tx should cause this tx + // to get flagged as problematic + let block_builder = StacksBlockBuilder::make_regtest_block_builder( + &parent_tip, + vrf_proof.clone(), + tip.total_burn, + Hash160::from_node_public_key(&StacksPublicKey::from_private(&miner.next_microblock_privkey())) + ) + .unwrap(); + + if let Err(ChainstateError::ProblematicTransaction(txid)) = StacksBlockBuilder::make_anchored_block_from_txs( + block_builder, + chainstate, + &sortdb.index_conn(), + vec![coinbase_tx.clone(), runtime_checkerror_problematic.clone()] + ) { + assert_eq!(txid, runtime_checkerror_problematic.txid()); + } + else { + panic!("Did not get Error::ProblematicTransaction"); + } + + problematic_txids.push(runtime_checkerror_problematic.txid()); + mempool + .submit( + chainstate, + &parent_consensus_hash, + &parent_header_hash, + &runtime_checkerror_problematic, + None, + &ExecutionCost::max_value(), + &StacksEpochId::Epoch2_05, + ) + .unwrap(); + } + + // all problematic txids are present + for problematic_txid in problematic_txids.iter() { + assert!(mempool.has_tx(problematic_txid)); + } + + let anchored_block = StacksBlockBuilder::build_anchored_block( + chainstate, + &sortdb.index_conn(), + &mut mempool, + &parent_tip, + tip.total_burn, + vrf_proof, + Hash160([tenure_id as u8; 20]), + &coinbase_tx, + BlockBuilderSettings::limited(), + None, + ) + .unwrap(); + + // all problematic txids are absent + for problematic_txid in problematic_txids.iter() { + assert!(!mempool.has_tx(problematic_txid)); + } + + // make sure the right txs get included + let txids : Vec<_> = anchored_block.0.txs.iter().map(|tx| tx.txid()).collect(); + assert_eq!(txids, expected_txids); + + (anchored_block.0, vec![]) + }, + ); + + let (_, _, consensus_hash) = peer.next_burnchain_block(burn_ops.clone()); + peer.process_stacks_epoch_at_tip(&stacks_block, µblocks); + + last_block = Some(StacksBlockHeader::make_index_block_hash( + &consensus_hash, + &stacks_block.block_hash(), + )); + } + } + static CONTRACT: &'static str = " (define-map my-map int int) (define-private (do (input bool)) From 2a965385dbf625c8689b5038d75958903254417c Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 22 Jul 2022 12:42:30 -0400 Subject: [PATCH 64/92] feat: ProblematicTransaction error variant --- src/chainstate/stacks/mod.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/chainstate/stacks/mod.rs b/src/chainstate/stacks/mod.rs index 5dcacd2aa..bd5420884 100644 --- a/src/chainstate/stacks/mod.rs +++ b/src/chainstate/stacks/mod.rs @@ -118,6 +118,7 @@ pub enum Error { PoxAlreadyLocked, PoxInsufficientBalance, PoxNoRewardCycle, + ProblematicTransaction(Txid), } impl From for Error { @@ -188,6 +189,11 @@ impl fmt::Display for Error { r ) } + Error::ProblematicTransaction(ref txid) => write!( + f, + "Transaction {} is problematic and will not be mined again", + txid + ), } } } @@ -221,6 +227,7 @@ impl error::Error for Error { Error::PoxInsufficientBalance => None, Error::PoxNoRewardCycle => None, Error::StacksTransactionSkipped(ref _r) => None, + Error::ProblematicTransaction(ref _txid) => None, } } } @@ -254,6 +261,7 @@ impl Error { Error::PoxInsufficientBalance => "PoxInsufficientBalance", Error::PoxNoRewardCycle => "PoxNoRewardCycle", Error::StacksTransactionSkipped(ref _r) => "StacksTransactionSkipped", + Error::ProblematicTransaction(ref _txid) => "ProblematicTransaction", } } @@ -543,6 +551,14 @@ pub struct TransactionContractCall { pub function_args: Vec, } +impl TransactionContractCall { + pub fn contract_identifier(&self) -> QualifiedContractIdentifier { + let standard_principal = + StandardPrincipalData(self.address.version, self.address.bytes.0.clone()); + QualifiedContractIdentifier::new(standard_principal, self.contract_name.clone()) + } +} + /// A transaction that instantiates a smart contract #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct TransactionSmartContract { From 0181134ff7e02430b601f18749b0ff7b76a9a6c5 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 22 Jul 2022 12:42:46 -0400 Subject: [PATCH 65/92] feat: add a blacklisted txid table (bumping the schema version), which contains all txids of transactions the miner found to be problematic (i.e. can't be mined, and shouldn't ever be relayed again). Add an API to drop and blacklist transactions. --- src/core/mempool.rs | 143 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 139 insertions(+), 4 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 19596fae9..a7b164055 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -103,6 +103,10 @@ pub const MAX_BLOOM_COUNTER_TXS: u32 = 8192; // how far back in time (in Stacks blocks) does the bloom counter maintain tx records? pub const BLOOM_COUNTER_DEPTH: usize = 2; +// how long will a transaction be blacklisted? +// about as long as it takes for it to be garbage-collected +pub const DEFAULT_BLACKLIST_TIMEOUT: u64 = 24 * 60 * 60 * 2; + // maximum many tx tags we'll send before sending a bloom filter instead. // The parameter choice here is due to performance -- calculating a tag set can be slower than just // loading the bloom filter, even though the bloom filter is larger. @@ -180,6 +184,7 @@ pub enum MemPoolDropReason { REPLACE_BY_FEE, STALE_COLLECT, TOO_EXPENSIVE, + PROBLEMATIC, } pub struct ConsiderTransaction { @@ -204,6 +209,7 @@ impl std::fmt::Display for MemPoolDropReason { MemPoolDropReason::TOO_EXPENSIVE => write!(f, "TooExpensive"), MemPoolDropReason::REPLACE_ACROSS_FORK => write!(f, "ReplaceAcrossFork"), MemPoolDropReason::REPLACE_BY_FEE => write!(f, "ReplaceByFee"), + MemPoolDropReason::PROBLEMATIC => write!(f, "Problematic"), } } } @@ -414,6 +420,21 @@ const MEMPOOL_SCHEMA_3_BLOOM_STATE: &'static [&'static str] = &[ "#, ]; +const MEMPOOL_SCHEMA_4_BLACKLIST: &'static [&'static str] = &[ + r#" + -- List of transactions that will never be stored to the mempool again, for as long as the rows exist. + -- `arrival_time` indicates when the entry was created. This is used to garbage-collect the list. + -- A transaction that is blacklisted may still be served from the mempool, but it will never be (re)submitted. + CREATE TABLE IF NOT EXISTS tx_blacklist( + txid TEXT PRIMARY KEY NOT NULL, + arrival_time INTEGER NOT NULL + ); + "#, + r#" + INSERT INTO schema_version (version) VALUES (4) + "#, +]; + const MEMPOOL_INDEXES: &'static [&'static str] = &[ "CREATE INDEX IF NOT EXISTS by_txid ON mempool(txid);", "CREATE INDEX IF NOT EXISTS by_height ON mempool(height);", @@ -425,6 +446,7 @@ const MEMPOOL_INDEXES: &'static [&'static str] = &[ "CREATE INDEX IF NOT EXISTS fee_by_txid ON fee_estimates(txid);", "CREATE INDEX IF NOT EXISTS by_ordered_hashed_txid ON randomized_txids(hashed_txid ASC);", "CREATE INDEX IF NOT EXISTS by_hashed_txid ON randomized_txids(txid,hashed_txid);", + "CREATE INDEX IF NOT EXISTS by_arrival_time_desc ON tx_blacklist(arrival_time DESC);", ]; pub struct MemPoolDB { @@ -435,6 +457,7 @@ pub struct MemPoolDB { max_tx_tags: u32, cost_estimator: Box, metric: Box, + pub blacklist_timeout: u64, } pub struct MemPoolTx<'a> { @@ -692,6 +715,9 @@ impl MemPoolDB { MemPoolDB::instantiate_bloom_state(tx)?; } 3 => { + MemPoolDB::instantiate_tx_blacklist(tx)?; + } + 4 => { break; } _ => { @@ -736,6 +762,15 @@ impl MemPoolDB { Ok(()) } + /// Instantiate the tx blacklist schema + fn instantiate_tx_blacklist(tx: &DBTx) -> Result<(), db_error> { + for sql_exec in MEMPOOL_SCHEMA_4_BLACKLIST { + tx.execute_batch(sql_exec)?; + } + + Ok(()) + } + pub fn db_path(chainstate_root_path: &str) -> Result { let mut path = PathBuf::from(chainstate_root_path); @@ -815,6 +850,7 @@ impl MemPoolDB { max_tx_tags: DEFAULT_MAX_TX_TAGS, cost_estimator, metric, + blacklist_timeout: DEFAULT_BLACKLIST_TIMEOUT, }) } @@ -1563,6 +1599,12 @@ impl MemPoolDB { block_limit: &ExecutionCost, stacks_epoch_id: &StacksEpochId, ) -> Result<(), MemPoolRejection> { + if self.is_tx_blacklisted(&tx.txid())? { + // don't re-store this transaction + test_debug!("Transaction {} is temporarily blacklisted", &tx.txid()); + return Err(MemPoolRejection::TemporarilyBlacklisted); + } + let estimator_result = cost_estimates::estimate_fee_rate( tx, self.cost_estimator.as_ref(), @@ -1613,6 +1655,12 @@ impl MemPoolDB { let tx = StacksTransaction::consensus_deserialize(&mut &tx_bytes[..]) .map_err(MemPoolRejection::DeserializationFailure)?; + if self.is_tx_blacklisted(&tx.txid())? { + // don't re-store this transaction + test_debug!("Transaction {} is temporarily blacklisted", &tx.txid()); + return Err(MemPoolRejection::TemporarilyBlacklisted); + } + let estimator_result = cost_estimates::estimate_fee_rate( &tx, self.cost_estimator.as_ref(), @@ -1650,17 +1698,104 @@ impl MemPoolDB { Ok(()) } - /// Drop transactions from the mempool - pub fn drop_txs(&mut self, txids: &[Txid]) -> Result<(), db_error> { - let mempool_tx = self.tx_begin()?; + /// Blacklist transactions from the mempool + /// Do not call directly; it's `pub` only for testing + pub fn inner_blacklist_txs<'a>( + tx: &DBTx<'a>, + txids: &[Txid], + now: u64, + ) -> Result<(), db_error> { + for txid in txids { + let sql = "INSERT OR REPLACE INTO tx_blacklist (txid, arrival_time) VALUES (?1, ?2)"; + let args: &[&dyn ToSql] = &[&txid, &u64_to_sql(now)?]; + tx.execute(sql, args)?; + } + Ok(()) + } + + /// garbage-collect the tx blacklist -- delete any transactions whose blacklist timeout has + /// been exceeded + pub fn garbage_collect_tx_blacklist<'a>( + tx: &DBTx<'a>, + now: u64, + timeout: u64, + ) -> Result<(), db_error> { + let sql = "DELETE FROM tx_blacklist WHERE arrival_time + ?1 < ?2"; + let args: &[&dyn ToSql] = &[&u64_to_sql(timeout)?, &u64_to_sql(now)?]; + tx.execute(sql, args)?; + Ok(()) + } + + /// when was a tx blacklisted? + fn get_blacklisted_tx_arrival_time( + conn: &DBConn, + txid: &Txid, + ) -> Result, db_error> { + let sql = "SELECT arrival_time FROM tx_blacklist WHERE txid = ?1"; + let args: &[&dyn ToSql] = &[&txid]; + query_row(conn, sql, args) + } + + /// is a tx blacklisted as of the given timestamp? + fn inner_is_tx_blacklisted( + conn: &DBConn, + txid: &Txid, + now: u64, + timeout: u64, + ) -> Result { + match MemPoolDB::get_blacklisted_tx_arrival_time(conn, txid)? { + None => Ok(false), + Some(arrival_time) => Ok(now < arrival_time + timeout), + } + } + + /// is a tx blacklisted? + pub fn is_tx_blacklisted(&self, txid: &Txid) -> Result { + MemPoolDB::inner_is_tx_blacklisted( + self.conn(), + txid, + get_epoch_time_secs(), + self.blacklist_timeout, + ) + } + + /// Inner code body for dropping transactions. + /// Note that the bloom filter will *NOT* be updated. That's the caller's job, if desired. + fn inner_drop_txs<'a>(tx: &DBTx<'a>, txids: &[Txid]) -> Result<(), db_error> { let sql = "DELETE FROM mempool WHERE txid = ?"; for txid in txids.iter() { - mempool_tx.execute(sql, &[txid])?; + tx.execute(sql, &[txid])?; } + Ok(()) + } + + /// Drop transactions from the mempool. Does not update the bloom filter, thereby ensuring that + /// these transactions will still show up as present to the mempool sync logic. + pub fn drop_txs(&mut self, txids: &[Txid]) -> Result<(), db_error> { + let mempool_tx = self.tx_begin()?; + MemPoolDB::inner_drop_txs(&mempool_tx, txids)?; mempool_tx.commit()?; Ok(()) } + /// Drop and blacklist transactions, so we don't re-broadcast them or re-fetch them. + /// Do *NOT* remove them from the bloom filter. This will cause them to continue to be + /// reported as present, which is exactly what we want because we don't want these transactions + /// to be seen again (so we don't want anyone accidentally "helpfully" pushing them to us, nor + /// do we want the mempool sync logic to "helpfully" re-discover and re-download them). + pub fn drop_and_blacklist_txs(&mut self, txids: &[Txid]) -> Result<(), db_error> { + let now = get_epoch_time_secs(); + let blacklist_timeout = self.blacklist_timeout; + + let mempool_tx = self.tx_begin()?; + MemPoolDB::inner_drop_txs(&mempool_tx, txids)?; + MemPoolDB::inner_blacklist_txs(&mempool_tx, txids, now)?; + MemPoolDB::garbage_collect_tx_blacklist(&mempool_tx, now, blacklist_timeout)?; + mempool_tx.commit()?; + + Ok(()) + } + #[cfg(test)] pub fn dump_txs(&self) { let sql = "SELECT * FROM mempool"; From 95f0fbc4ca6386382a563fd8efe2aef667c1a58e Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 22 Jul 2022 12:43:35 -0400 Subject: [PATCH 66/92] feat: unit test for dropping and blacklisting transactions --- src/core/tests/mod.rs | 106 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/src/core/tests/mod.rs b/src/core/tests/mod.rs index 805e05601..2f9091b35 100644 --- a/src/core/tests/mod.rs +++ b/src/core/tests/mod.rs @@ -60,9 +60,9 @@ use clarity::vm::{ }; use stacks_common::address::AddressHashMode; use stacks_common::types::chainstate::TrieHash; -use stacks_common::util::get_epoch_time_ms; use stacks_common::util::hash::Hash160; use stacks_common::util::secp256k1::MessageSignature; +use stacks_common::util::{get_epoch_time_ms, get_epoch_time_secs}; use stacks_common::util::{hash::hex_bytes, hash::to_hex, hash::*, log, secp256k1::*}; use crate::chainstate::stacks::index::TrieHashExtension; @@ -2061,3 +2061,107 @@ fn test_decode_tx_stream() { } } } + +#[test] +fn test_drop_and_blacklist_txs() { + let mut chainstate = instantiate_chainstate(false, 0x80000000, "test_drop_and_blacklist_txs"); + let chainstate_path = chainstate_path("test_drop_and_blacklist_txs"); + let mut mempool = MemPoolDB::open_test(false, 0x80000000, &chainstate_path).unwrap(); + + let addr = StacksAddress { + version: 1, + bytes: Hash160([0xff; 20]), + }; + let mut txs = vec![]; + let block_height = 10; + + let mut mempool_tx = mempool.tx_begin().unwrap(); + for i in 0..10 { + let pk = StacksPrivateKey::new(); + let mut tx = StacksTransaction { + version: TransactionVersion::Testnet, + chain_id: 0x80000000, + auth: TransactionAuth::from_p2pkh(&pk).unwrap(), + anchor_mode: TransactionAnchorMode::Any, + post_condition_mode: TransactionPostConditionMode::Allow, + post_conditions: vec![], + payload: TransactionPayload::TokenTransfer( + addr.to_account_principal(), + 123, + TokenTransferMemo([0u8; 34]), + ), + }; + tx.set_tx_fee(1000); + tx.set_origin_nonce(0); + + let txid = tx.txid(); + let tx_bytes = tx.serialize_to_vec(); + let origin_addr = tx.origin_address(); + let origin_nonce = tx.get_origin_nonce(); + let sponsor_addr = tx.sponsor_address().unwrap_or(origin_addr.clone()); + let sponsor_nonce = tx.get_sponsor_nonce().unwrap_or(origin_nonce); + let tx_fee = tx.get_tx_fee(); + + // should succeed + MemPoolDB::try_add_tx( + &mut mempool_tx, + &mut chainstate, + &ConsensusHash([0x1 + (block_height as u8); 20]), + &BlockHeaderHash([0x2 + (block_height as u8); 32]), + txid.clone(), + tx_bytes, + tx_fee, + block_height as u64, + &origin_addr, + origin_nonce, + &sponsor_addr, + sponsor_nonce, + None, + ) + .unwrap(); + + eprintln!("Added {} {}", i, &txid); + txs.push(tx); + } + mempool_tx.commit().unwrap(); + let txids: Vec<_> = txs.iter().map(|tx| tx.txid()).collect(); + + for tx in txs.iter() { + assert!(!mempool.is_tx_blacklisted(&tx.txid()).unwrap()); + assert!(mempool.has_tx(&tx.txid())); + } + + let mempool_tx = mempool.tx_begin().unwrap(); + MemPoolDB::inner_blacklist_txs(&mempool_tx, &txids, get_epoch_time_secs()).unwrap(); + mempool_tx.commit().unwrap(); + + for tx in txs.iter() { + assert!(mempool.is_tx_blacklisted(&tx.txid()).unwrap()); + assert!(mempool.has_tx(&tx.txid())); + } + + let mempool_tx = mempool.tx_begin().unwrap(); + MemPoolDB::garbage_collect_tx_blacklist(&mempool_tx, get_epoch_time_secs() + 1, 0).unwrap(); + mempool_tx.commit().unwrap(); + + for tx in txs.iter() { + assert!(!mempool.is_tx_blacklisted(&tx.txid()).unwrap()); + assert!(mempool.has_tx(&tx.txid())); + } + + mempool.drop_and_blacklist_txs(&txids).unwrap(); + + for tx in txs.iter() { + assert!(mempool.is_tx_blacklisted(&tx.txid()).unwrap()); + assert!(!mempool.has_tx(&tx.txid())); + } + + let mempool_tx = mempool.tx_begin().unwrap(); + MemPoolDB::garbage_collect_tx_blacklist(&mempool_tx, get_epoch_time_secs() + 2, 0).unwrap(); + mempool_tx.commit().unwrap(); + + for tx in txs.iter() { + assert!(!mempool.is_tx_blacklisted(&tx.txid()).unwrap()); + assert!(!mempool.has_tx(&tx.txid())); + } +} From 606640f80b5b80e3bc4877f9ad9b3a641cebe016 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 22 Jul 2022 12:43:50 -0400 Subject: [PATCH 67/92] feat: unit test to verify that blacklisted transactions never get relayed --- src/net/p2p.rs | 214 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 213 insertions(+), 1 deletion(-) diff --git a/src/net/p2p.rs b/src/net/p2p.rs index 772a89f3b..5182cf895 100644 --- a/src/net/p2p.rs +++ b/src/net/p2p.rs @@ -5116,7 +5116,7 @@ impl PeerNetwork { } /// Store a single transaction - /// Return true if stored; false if it was a dup. + /// Return true if stored; false if it was a dup or if it's temporarily blacklisted. /// Has to be done here, since only the p2p network has the unconfirmed state. fn store_transaction( mempool: &mut MemPoolDB, @@ -6154,4 +6154,216 @@ mod test { } }); } + + #[test] + #[ignore] + fn test_mempool_sync_2_peers_blacklisted() { + with_timeout(600, || { + // peer 1 gets some transactions; peer 2 blacklists some of them; + // verify peer 2 gets only the non-blacklisted ones. + let mut peer_1_config = + TestPeerConfig::new("test_mempool_sync_2_peers_paginated", 2218, 2219); + let mut peer_2_config = + TestPeerConfig::new("test_mempool_sync_2_peers_paginated", 2220, 2221); + + peer_1_config.add_neighbor(&peer_2_config.to_neighbor()); + peer_2_config.add_neighbor(&peer_1_config.to_neighbor()); + + peer_1_config.connection_opts.mempool_sync_interval = 1; + peer_2_config.connection_opts.mempool_sync_interval = 1; + + let num_txs = 1024; + let pks: Vec<_> = (0..num_txs).map(|_| StacksPrivateKey::new()).collect(); + let addrs: Vec<_> = pks.iter().map(|pk| to_addr(pk)).collect(); + let initial_balances: Vec<_> = addrs + .iter() + .map(|a| (a.to_account_principal(), 1000000000)) + .collect(); + + peer_1_config.initial_balances = initial_balances.clone(); + peer_2_config.initial_balances = initial_balances.clone(); + + let mut peer_1 = TestPeer::new(peer_1_config); + let mut peer_2 = TestPeer::new(peer_2_config); + + let num_blocks = 10; + let first_stacks_block_height = { + let sn = SortitionDB::get_canonical_burn_chain_tip( + &peer_1.sortdb.as_ref().unwrap().conn(), + ) + .unwrap(); + sn.block_height + 1 + }; + + for i in 0..num_blocks { + let (burn_ops, stacks_block, microblocks) = peer_2.make_default_tenure(); + + peer_1.next_burnchain_block(burn_ops.clone()); + peer_2.next_burnchain_block(burn_ops.clone()); + + peer_1.process_stacks_epoch_at_tip(&stacks_block, µblocks); + peer_2.process_stacks_epoch_at_tip(&stacks_block, µblocks); + } + + let addr = StacksAddress { + version: C32_ADDRESS_VERSION_TESTNET_SINGLESIG, + bytes: Hash160([0xff; 20]), + }; + + // fill peer 1 with lots of transactions + let mut txs = HashMap::new(); + let mut peer_1_mempool = peer_1.mempool.take().unwrap(); + let mut mempool_tx = peer_1_mempool.tx_begin().unwrap(); + let mut peer_2_blacklist = vec![]; + for i in 0..num_txs { + let pk = &pks[i]; + let mut tx = StacksTransaction { + version: TransactionVersion::Testnet, + chain_id: 0x80000000, + auth: TransactionAuth::from_p2pkh(&pk).unwrap(), + anchor_mode: TransactionAnchorMode::Any, + post_condition_mode: TransactionPostConditionMode::Allow, + post_conditions: vec![], + payload: TransactionPayload::TokenTransfer( + addr.to_account_principal(), + 123, + TokenTransferMemo([0u8; 34]), + ), + }; + tx.set_tx_fee(1000); + tx.set_origin_nonce(0); + + let mut tx_signer = StacksTransactionSigner::new(&tx); + tx_signer.sign_origin(&pk).unwrap(); + + let tx = tx_signer.get_tx().unwrap(); + + let txid = tx.txid(); + let tx_bytes = tx.serialize_to_vec(); + let origin_addr = tx.origin_address(); + let origin_nonce = tx.get_origin_nonce(); + let sponsor_addr = tx.sponsor_address().unwrap_or(origin_addr.clone()); + let sponsor_nonce = tx.get_sponsor_nonce().unwrap_or(origin_nonce); + let tx_fee = tx.get_tx_fee(); + + txs.insert(tx.txid(), tx.clone()); + + // should succeed + MemPoolDB::try_add_tx( + &mut mempool_tx, + peer_1.chainstate(), + &ConsensusHash([0x1 + (num_blocks as u8); 20]), + &BlockHeaderHash([0x2 + (num_blocks as u8); 32]), + txid.clone(), + tx_bytes, + tx_fee, + num_blocks, + &origin_addr, + origin_nonce, + &sponsor_addr, + sponsor_nonce, + None, + ) + .unwrap(); + + eprintln!("Added {} {}", i, &txid); + + if i % 2 == 0 { + // peer 2 blacklists even-numbered txs + peer_2_blacklist.push(txid); + } + } + mempool_tx.commit().unwrap(); + peer_1.mempool = Some(peer_1_mempool); + + // peer 2 blacklists them all + let mut peer_2_mempool = peer_2.mempool.take().unwrap(); + + // blacklisted txs never time out + peer_2_mempool.blacklist_timeout = u64::MAX / 2; + + let mempool_tx = peer_2_mempool.tx_begin().unwrap(); + MemPoolDB::inner_blacklist_txs(&mempool_tx, &peer_2_blacklist, get_epoch_time_secs()) + .unwrap(); + mempool_tx.commit().unwrap(); + + peer_2.mempool = Some(peer_2_mempool); + + let num_burn_blocks = { + let sn = SortitionDB::get_canonical_burn_chain_tip( + peer_1.sortdb.as_ref().unwrap().conn(), + ) + .unwrap(); + sn.block_height + 1 + }; + + let mut round = 0; + let mut peer_1_mempool_txs = 0; + let mut peer_2_mempool_txs = 0; + + while peer_1_mempool_txs < num_txs || peer_2_mempool_txs < num_txs / 2 { + if let Ok(mut result) = peer_1.step() { + let lp = peer_1.network.local_peer.clone(); + peer_1 + .with_db_state(|sortdb, chainstate, relayer, mempool| { + relayer.process_network_result( + &lp, + &mut result, + sortdb, + chainstate, + mempool, + false, + None, + None, + ) + }) + .unwrap(); + } + + if let Ok(mut result) = peer_2.step() { + let lp = peer_2.network.local_peer.clone(); + peer_2 + .with_db_state(|sortdb, chainstate, relayer, mempool| { + relayer.process_network_result( + &lp, + &mut result, + sortdb, + chainstate, + mempool, + false, + None, + None, + ) + }) + .unwrap(); + } + + round += 1; + + let mp = peer_1.mempool.take().unwrap(); + peer_1_mempool_txs = MemPoolDB::get_all_txs(mp.conn()).unwrap().len(); + peer_1.mempool.replace(mp); + + let mp = peer_2.mempool.take().unwrap(); + peer_2_mempool_txs = MemPoolDB::get_all_txs(mp.conn()).unwrap().len(); + peer_2.mempool.replace(mp); + + info!( + "Peer 1: {}, Peer 2: {}", + peer_1_mempool_txs, peer_2_mempool_txs + ); + } + + info!("Completed mempool sync in {} step(s)", round); + + let mp = peer_2.mempool.take().unwrap(); + let peer_2_mempool_txs = MemPoolDB::get_all_txs(mp.conn()).unwrap(); + peer_2.mempool.replace(mp); + + for tx in peer_2_mempool_txs { + assert_eq!(&tx.tx, txs.get(&tx.tx.txid()).unwrap()); + assert!(!peer_2_blacklist.contains(&tx.tx.txid())); + } + }); + } } From b364c213b5590bba8ea7d9c0b4806552b7e5329f Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 25 Jul 2022 14:28:07 -0400 Subject: [PATCH 68/92] fix: store the private key to local_peer if given to PeerDB::connect() --- CHANGELOG.md | 9 ++++++- src/net/db.rs | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1d4a108b..a30c55db9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,16 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE ## Upcoming -- Updates to the logging of transaction events (#3139). +### Added - Added prometheus output for "transactions in last block" (#3138). +### Changed +- Updates to the logging of transaction events (#3139). + +### Fixed +- Make it so that a new peer private key in the config file will propagate to + the peer database (#3165). + ## [2.05.0.2.0] ### IMPORTANT! READ THIS FIRST diff --git a/src/net/db.rs b/src/net/db.rs index b0e177d2b..e8487ae16 100644 --- a/src/net/db.rs +++ b/src/net/db.rs @@ -587,6 +587,9 @@ impl PeerDB { PeerDB::refresh_allows(&mut tx)?; PeerDB::refresh_denies(&mut tx)?; PeerDB::clear_initial_peers(&mut tx)?; + if let Some(privkey) = privkey_opt { + PeerDB::set_local_private_key(&mut tx, &privkey, key_expires)?; + } if let Some(neighbors) = initial_neighbors { for neighbor in neighbors { @@ -2325,4 +2328,68 @@ mod test { assert_eq!(n1.allowed, 0); assert_eq!(n2.allowed, 0); } + + #[test] + fn test_connect_new_key() { + let key1 = Secp256k1PrivateKey::new(); + let key2 = Secp256k1PrivateKey::new(); + + let path = "/tmp/test-connect-new-key.db".to_string(); + if fs::metadata(&path).is_ok() { + fs::remove_file(&path).unwrap(); + } + + let db = PeerDB::connect( + &path, + true, + 0x80000000, + 0, + Some(key1.clone()), + i64::MAX as u64, + PeerAddress::from_ipv4(127, 0, 0, 1), + 12345, + UrlString::try_from("http://foo.com").unwrap(), + &vec![], + None, + ) + .unwrap(); + let local_peer = PeerDB::get_local_peer(db.conn()).unwrap(); + assert_eq!(local_peer.private_key, key1); + + assert!(fs::metadata(&path).is_ok()); + + let db = PeerDB::connect( + &path, + true, + 0x80000000, + 0, + None, + i64::MAX as u64, + PeerAddress::from_ipv4(127, 0, 0, 1), + 12345, + UrlString::try_from("http://foo.com").unwrap(), + &vec![], + None, + ) + .unwrap(); + let local_peer = PeerDB::get_local_peer(db.conn()).unwrap(); + assert_eq!(local_peer.private_key, key1); + + let db = PeerDB::connect( + &path, + true, + 0x80000000, + 0, + Some(key2.clone()), + i64::MAX as u64, + PeerAddress::from_ipv4(127, 0, 0, 1), + 12345, + UrlString::try_from("http://foo.com").unwrap(), + &vec![], + None, + ) + .unwrap(); + let local_peer = PeerDB::get_local_peer(db.conn()).unwrap(); + assert_eq!(local_peer.private_key, key2); + } } From 0e241ab32e740633298b3129bee2ea8982272a57 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 25 Jul 2022 14:57:03 -0400 Subject: [PATCH 69/92] fix: limit the size of the blacklist by deleting random txids if we go over. --- src/core/mempool.rs | 54 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index a7b164055..55588a713 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -106,6 +106,7 @@ pub const BLOOM_COUNTER_DEPTH: usize = 2; // how long will a transaction be blacklisted? // about as long as it takes for it to be garbage-collected pub const DEFAULT_BLACKLIST_TIMEOUT: u64 = 24 * 60 * 60 * 2; +pub const DEFAULT_BLACKLIST_MAX_SIZE: u64 = 134217728; // 2**27 -- the blacklist table can reach at most 4GB at 128 bytes per record // maximum many tx tags we'll send before sending a bloom filter instead. // The parameter choice here is due to performance -- calculating a tag set can be slower than just @@ -431,6 +432,30 @@ const MEMPOOL_SCHEMA_4_BLACKLIST: &'static [&'static str] = &[ ); "#, r#" + -- Count the number of entries in the blacklist + CREATE TABLE IF NOT EXISTS tx_blacklist_size( + size INTEGER NOT NULL + ); + "#, + r#" + -- Maintain a count of the size of the blacklist + CREATE TRIGGER IF NOT EXISTS tx_blacklist_size_inc + AFTER INSERT ON tx_blacklist + BEGIN + UPDATE tx_blacklist_size SET size = size + 1; + END + "#, + r#" + CREATE TRIGGER IF NOT EXISTS tx_blacklist_size_dec + AFTER DELETE ON tx_blacklist + BEGIN + UPDATE tx_blacklist_size SET size = size - 1; + END + "#, + r#" + INSERT INTO tx_blacklist_size (size) VALUES (0) + "#, + r#" INSERT INTO schema_version (version) VALUES (4) "#, ]; @@ -458,6 +483,7 @@ pub struct MemPoolDB { cost_estimator: Box, metric: Box, pub blacklist_timeout: u64, + pub blacklist_max_size: u64, } pub struct MemPoolTx<'a> { @@ -851,6 +877,7 @@ impl MemPoolDB { cost_estimator, metric, blacklist_timeout: DEFAULT_BLACKLIST_TIMEOUT, + blacklist_max_size: DEFAULT_BLACKLIST_MAX_SIZE, }) } @@ -1719,10 +1746,29 @@ impl MemPoolDB { tx: &DBTx<'a>, now: u64, timeout: u64, + max_size: u64, ) -> Result<(), db_error> { let sql = "DELETE FROM tx_blacklist WHERE arrival_time + ?1 < ?2"; let args: &[&dyn ToSql] = &[&u64_to_sql(timeout)?, &u64_to_sql(now)?]; tx.execute(sql, args)?; + + // if we get too big, then drop some txs at random + let sql = "SELECT size FROM tx_blacklist_size"; + let sz = query_int(tx, sql, NO_PARAMS)? as u64; + if sz > max_size { + let to_delete = sz - max_size; + let txids: Vec = query_rows( + tx, + "SELECT txid FROM tx_blacklist ORDER BY RANDOM() LIMIT ?1", + &[&u64_to_sql(to_delete)? as &dyn ToSql], + )?; + for txid in txids.into_iter() { + tx.execute( + "DELETE FROM tx_blacklist WHERE txid = ?1", + &[&txid as &dyn ToSql], + )?; + } + } Ok(()) } @@ -1786,11 +1832,17 @@ impl MemPoolDB { pub fn drop_and_blacklist_txs(&mut self, txids: &[Txid]) -> Result<(), db_error> { let now = get_epoch_time_secs(); let blacklist_timeout = self.blacklist_timeout; + let blacklist_max_size = self.blacklist_max_size; let mempool_tx = self.tx_begin()?; MemPoolDB::inner_drop_txs(&mempool_tx, txids)?; MemPoolDB::inner_blacklist_txs(&mempool_tx, txids, now)?; - MemPoolDB::garbage_collect_tx_blacklist(&mempool_tx, now, blacklist_timeout)?; + MemPoolDB::garbage_collect_tx_blacklist( + &mempool_tx, + now, + blacklist_timeout, + blacklist_max_size, + )?; mempool_tx.commit()?; Ok(()) From 913b0f9dde51f726c1638b89afe1d6a77bf5d49d Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 25 Jul 2022 14:57:41 -0400 Subject: [PATCH 70/92] fix: test coverage for blacklist size shrinkage due to being too big --- src/core/tests/mod.rs | 129 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 5 deletions(-) diff --git a/src/core/tests/mod.rs b/src/core/tests/mod.rs index 2f9091b35..da5e7c929 100644 --- a/src/core/tests/mod.rs +++ b/src/core/tests/mod.rs @@ -2063,9 +2063,10 @@ fn test_decode_tx_stream() { } #[test] -fn test_drop_and_blacklist_txs() { - let mut chainstate = instantiate_chainstate(false, 0x80000000, "test_drop_and_blacklist_txs"); - let chainstate_path = chainstate_path("test_drop_and_blacklist_txs"); +fn test_drop_and_blacklist_txs_by_time() { + let mut chainstate = + instantiate_chainstate(false, 0x80000000, "test_drop_and_blacklist_txs_by_time"); + let chainstate_path = chainstate_path("test_drop_and_blacklist_txs_by_time"); let mut mempool = MemPoolDB::open_test(false, 0x80000000, &chainstate_path).unwrap(); let addr = StacksAddress { @@ -2131,6 +2132,7 @@ fn test_drop_and_blacklist_txs() { assert!(mempool.has_tx(&tx.txid())); } + // blacklist some txs let mempool_tx = mempool.tx_begin().unwrap(); MemPoolDB::inner_blacklist_txs(&mempool_tx, &txids, get_epoch_time_secs()).unwrap(); mempool_tx.commit().unwrap(); @@ -2140,8 +2142,15 @@ fn test_drop_and_blacklist_txs() { assert!(mempool.has_tx(&tx.txid())); } + // purge blacklisted txs by time let mempool_tx = mempool.tx_begin().unwrap(); - MemPoolDB::garbage_collect_tx_blacklist(&mempool_tx, get_epoch_time_secs() + 1, 0).unwrap(); + MemPoolDB::garbage_collect_tx_blacklist( + &mempool_tx, + get_epoch_time_secs() + 1, + 0, + i64::MAX as u64, + ) + .unwrap(); mempool_tx.commit().unwrap(); for tx in txs.iter() { @@ -2156,8 +2165,15 @@ fn test_drop_and_blacklist_txs() { assert!(!mempool.has_tx(&tx.txid())); } + // purge blacklisted txs by time let mempool_tx = mempool.tx_begin().unwrap(); - MemPoolDB::garbage_collect_tx_blacklist(&mempool_tx, get_epoch_time_secs() + 2, 0).unwrap(); + MemPoolDB::garbage_collect_tx_blacklist( + &mempool_tx, + get_epoch_time_secs() + 2, + 0, + i64::MAX as u64, + ) + .unwrap(); mempool_tx.commit().unwrap(); for tx in txs.iter() { @@ -2165,3 +2181,106 @@ fn test_drop_and_blacklist_txs() { assert!(!mempool.has_tx(&tx.txid())); } } + +#[test] +fn test_drop_and_blacklist_txs_by_size() { + let mut chainstate = + instantiate_chainstate(false, 0x80000000, "test_drop_and_blacklist_txs_by_size"); + let chainstate_path = chainstate_path("test_drop_and_blacklist_txs_by_size"); + let mut mempool = MemPoolDB::open_test(false, 0x80000000, &chainstate_path).unwrap(); + + let addr = StacksAddress { + version: 1, + bytes: Hash160([0xff; 20]), + }; + let mut txs = vec![]; + let block_height = 10; + + let mut mempool_tx = mempool.tx_begin().unwrap(); + for i in 0..10 { + let pk = StacksPrivateKey::new(); + let mut tx = StacksTransaction { + version: TransactionVersion::Testnet, + chain_id: 0x80000000, + auth: TransactionAuth::from_p2pkh(&pk).unwrap(), + anchor_mode: TransactionAnchorMode::Any, + post_condition_mode: TransactionPostConditionMode::Allow, + post_conditions: vec![], + payload: TransactionPayload::TokenTransfer( + addr.to_account_principal(), + 123, + TokenTransferMemo([0u8; 34]), + ), + }; + tx.set_tx_fee(1000); + tx.set_origin_nonce(0); + + let txid = tx.txid(); + let tx_bytes = tx.serialize_to_vec(); + let origin_addr = tx.origin_address(); + let origin_nonce = tx.get_origin_nonce(); + let sponsor_addr = tx.sponsor_address().unwrap_or(origin_addr.clone()); + let sponsor_nonce = tx.get_sponsor_nonce().unwrap_or(origin_nonce); + let tx_fee = tx.get_tx_fee(); + + // should succeed + MemPoolDB::try_add_tx( + &mut mempool_tx, + &mut chainstate, + &ConsensusHash([0x1 + (block_height as u8); 20]), + &BlockHeaderHash([0x2 + (block_height as u8); 32]), + txid.clone(), + tx_bytes, + tx_fee, + block_height as u64, + &origin_addr, + origin_nonce, + &sponsor_addr, + sponsor_nonce, + None, + ) + .unwrap(); + + eprintln!("Added {} {}", i, &txid); + txs.push(tx); + } + mempool_tx.commit().unwrap(); + let txids: Vec<_> = txs.iter().map(|tx| tx.txid()).collect(); + + for tx in txs.iter() { + assert!(!mempool.is_tx_blacklisted(&tx.txid()).unwrap()); + assert!(mempool.has_tx(&tx.txid())); + } + + // blacklist some txs + let mempool_tx = mempool.tx_begin().unwrap(); + MemPoolDB::inner_blacklist_txs(&mempool_tx, &txids, get_epoch_time_secs()).unwrap(); + mempool_tx.commit().unwrap(); + + for tx in txs.iter() { + assert!(mempool.is_tx_blacklisted(&tx.txid()).unwrap()); + assert!(mempool.has_tx(&tx.txid())); + } + + // purge blacklisted txs by size + let mempool_tx = mempool.tx_begin().unwrap(); + MemPoolDB::garbage_collect_tx_blacklist( + &mempool_tx, + get_epoch_time_secs() + 1, + i64::MAX as u64, + 5, + ) + .unwrap(); + mempool_tx.commit().unwrap(); + + // 5 txs remain blacklisted + let mut num_blacklisted = 0; + for tx in txs.iter() { + if mempool.is_tx_blacklisted(&tx.txid()).unwrap() { + num_blacklisted += 1; + } + assert!(mempool.has_tx(&tx.txid())); + } + + assert_eq!(num_blacklisted, 5); +} From 91f10b008ad4bc034037aa699bee06d931623572 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Mon, 25 Jul 2022 14:10:42 -0500 Subject: [PATCH 71/92] Add BLOCKSTACK_FORMAT_TIME env var --- stacks-common/src/util/log.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/stacks-common/src/util/log.rs b/stacks-common/src/util/log.rs index d203bbe7b..15d427a66 100644 --- a/stacks-common/src/util/log.rs +++ b/stacks-common/src/util/log.rs @@ -41,15 +41,21 @@ fn print_msg_header(mut rd: &mut dyn RecordDecorator, record: &Record) -> io::Re write!(rd, " ")?; rd.start_timestamp()?; - let elapsed = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)); - write!( - rd, - "[{:5}.{:06}]", - elapsed.as_secs(), - elapsed.subsec_nanos() / 1000 - )?; + let system_time = SystemTime::now(); + if env::var("BLOCKSTACK_FORMAT_TIME") != Ok("1".into()) { + let elapsed = system_time + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)); + write!( + rd, + "[{:5}.{:06}]", + elapsed.as_secs(), + elapsed.subsec_nanos() / 1000 + )?; + } else { + let datetime: DateTime = system_time.into(); + write!(rd, "[{}]", datetime.format("%Y-%m-%d %H:%M:%S"))?; + } write!(rd, " ")?; write!(rd, "[{}:{}]", record.file(), record.line())?; write!(rd, " ")?; From 011c340eec6e947b71b0a20c73875993e29ae43d Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Mon, 25 Jul 2022 14:29:35 -0500 Subject: [PATCH 72/92] Annotate flaky tests --- .github/workflows/bitcoin-tests.yml | 4 ++-- testnet/stacks-node/src/tests/neon_integrations.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 3eea180f0..120c51143 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -48,7 +48,7 @@ jobs: - tests::neon_integrations::bitcoind_forking_test - tests::neon_integrations::should_fix_2771 - tests::neon_integrations::pox_integration_test - - tests::neon_integrations::mining_events_integration_test + - tests::neon_integrations::mining_events_integration_test_FLAKY_TEST_FIX_ME - tests::bitcoin_regtest::bitcoind_integration_test - tests::should_succeed_handling_malformed_and_valid_txs - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test @@ -59,7 +59,7 @@ jobs: - tests::neon_integrations::filter_low_fee_tx_integration_test - tests::neon_integrations::filter_long_runtime_tx_integration_test - tests::neon_integrations::mining_transactions_is_fair - - tests::neon_integrations::microblock_large_tx_integration_test + - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY_TEST_FIX_ME - tests::neon_integrations::block_large_tx_integration_test - tests::neon_integrations::microblock_limit_hit_integration_test - tests::neon_integrations::block_limit_hit_integration_test diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 9acfb3917..bd1efeb13 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -3866,7 +3866,7 @@ fn cost_voting_integration() { #[test] #[ignore] -fn mining_events_integration_test() { +fn mining_events_integration_test_FLAKY_TEST_FIX_ME() { if env::var("BITCOIND_TEST") != Ok("1".into()) { return; } @@ -4603,7 +4603,7 @@ fn block_large_tx_integration_test() { #[test] #[ignore] -fn microblock_large_tx_integration_test() { +fn microblock_large_tx_integration_test_FLAKY_TEST_FIX_ME() { if env::var("BITCOIND_TEST") != Ok("1".into()) { return; } From e2c69befa074f8625806c031127f2f912504a02d Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 26 Jul 2022 11:18:28 -0400 Subject: [PATCH 73/92] fix: ::Skipped() events are no longer reported by the miner since there are so many of them --- testnet/stacks-node/src/tests/neon_integrations.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 9acfb3917..7159b4173 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -3968,7 +3968,7 @@ fn mining_events_integration_test() { // check tx events in the first microblock // 1 success: 1 contract publish, 2 error (on chain transactions) let microblock_tx_events = &mined_microblock_events[0].tx_events; - assert_eq!(microblock_tx_events.len(), 3); + assert_eq!(microblock_tx_events.len(), 1); // contract publish match µblock_tx_events[0] { @@ -3993,15 +3993,6 @@ fn mining_events_integration_test() { } _ => panic!("unexpected event type"), } - for i in 1..3 { - // on chain only transactions will be skipped in a microblock - match µblock_tx_events[i] { - TransactionEvent::Skipped(TransactionSkippedEvent { error, .. }) => { - assert_eq!(error, "Invalid transaction anchor mode for streamed data"); - } - _ => panic!("unexpected event type"), - } - } // check mined block events let mined_block_events = test_observer::get_mined_blocks(); From 0bffeb106bccef5082da9c0bbeace7f51457f215 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 26 Jul 2022 15:52:16 -0500 Subject: [PATCH 74/92] update --- .github/workflows/bitcoin-tests.yml | 4 ++-- testnet/stacks-node/src/tests/neon_integrations.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 120c51143..1874fd76c 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -48,7 +48,7 @@ jobs: - tests::neon_integrations::bitcoind_forking_test - tests::neon_integrations::should_fix_2771 - tests::neon_integrations::pox_integration_test - - tests::neon_integrations::mining_events_integration_test_FLAKY_TEST_FIX_ME + - tests::neon_integrations::mining_events_integration_test_BROKEN - tests::bitcoin_regtest::bitcoind_integration_test - tests::should_succeed_handling_malformed_and_valid_txs - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test @@ -59,7 +59,7 @@ jobs: - tests::neon_integrations::filter_low_fee_tx_integration_test - tests::neon_integrations::filter_long_runtime_tx_integration_test - tests::neon_integrations::mining_transactions_is_fair - - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY_TEST_FIX_ME + - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY - tests::neon_integrations::block_large_tx_integration_test - tests::neon_integrations::microblock_limit_hit_integration_test - tests::neon_integrations::block_limit_hit_integration_test diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index bd1efeb13..5a74a98ed 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -3866,7 +3866,7 @@ fn cost_voting_integration() { #[test] #[ignore] -fn mining_events_integration_test_FLAKY_TEST_FIX_ME() { +fn mining_events_integration_test_BROKEN() { if env::var("BITCOIND_TEST") != Ok("1".into()) { return; } @@ -4603,7 +4603,7 @@ fn block_large_tx_integration_test() { #[test] #[ignore] -fn microblock_large_tx_integration_test_FLAKY_TEST_FIX_ME() { +fn microblock_large_tx_integration_test_FLAKY() { if env::var("BITCOIND_TEST") != Ok("1".into()) { return; } From 432425804e9d6526bb6eceadc8a196d33ac73193 Mon Sep 17 00:00:00 2001 From: Igor Sylvester Date: Tue, 26 Jul 2022 17:09:55 -0500 Subject: [PATCH 75/92] update --- stacks-common/src/util/log.rs | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/stacks-common/src/util/log.rs b/stacks-common/src/util/log.rs index 15d427a66..b3697baee 100644 --- a/stacks-common/src/util/log.rs +++ b/stacks-common/src/util/log.rs @@ -26,6 +26,8 @@ use std::time::{Duration, SystemTime}; lazy_static! { pub static ref LOGGER: Logger = make_logger(); + pub static ref STACKS_LOG_FORMAT_TIME: Result = + env::var("STACKS_LOG_FORMAT_TIME"); } struct TermFormat { decorator: D, @@ -42,19 +44,22 @@ fn print_msg_header(mut rd: &mut dyn RecordDecorator, record: &Record) -> io::Re rd.start_timestamp()?; let system_time = SystemTime::now(); - if env::var("BLOCKSTACK_FORMAT_TIME") != Ok("1".into()) { - let elapsed = system_time - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)); - write!( - rd, - "[{:5}.{:06}]", - elapsed.as_secs(), - elapsed.subsec_nanos() / 1000 - )?; - } else { - let datetime: DateTime = system_time.into(); - write!(rd, "[{}]", datetime.format("%Y-%m-%d %H:%M:%S"))?; + match &*STACKS_LOG_FORMAT_TIME { + Err(e) => { + let elapsed = system_time + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)); + write!( + rd, + "[{:5}.{:06}]", + elapsed.as_secs(), + elapsed.subsec_nanos() / 1000 + )?; + } + Ok(ref format) => { + let datetime: DateTime = system_time.into(); + write!(rd, "[{}]", datetime.format(format))?; + } } write!(rd, " ")?; write!(rd, "[{}:{}]", record.file(), record.line())?; From 694aba7bdf1199b9ea68cb1ed87308c792c1fc1c Mon Sep 17 00:00:00 2001 From: Igor Date: Wed, 27 Jul 2022 12:44:51 -0500 Subject: [PATCH 76/92] update --- .github/workflows/bitcoin-tests.yml | 2 +- testnet/stacks-node/src/tests/neon_integrations.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 1874fd76c..e4a19e1c5 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -48,7 +48,7 @@ jobs: - tests::neon_integrations::bitcoind_forking_test - tests::neon_integrations::should_fix_2771 - tests::neon_integrations::pox_integration_test - - tests::neon_integrations::mining_events_integration_test_BROKEN + - tests::neon_integrations::mining_events_integration_test - tests::bitcoin_regtest::bitcoind_integration_test - tests::should_succeed_handling_malformed_and_valid_txs - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 3353b10b7..a769835ea 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -3866,7 +3866,7 @@ fn cost_voting_integration() { #[test] #[ignore] -fn mining_events_integration_test_BROKEN() { +fn mining_events_integration_test() { if env::var("BITCOIND_TEST") != Ok("1".into()) { return; } From bc15aad528d8dfec56056eb3ed557dadfcc14771 Mon Sep 17 00:00:00 2001 From: Igor Date: Thu, 28 Jul 2022 11:23:50 -0500 Subject: [PATCH 77/92] update --- stacks-common/src/util/log.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/stacks-common/src/util/log.rs b/stacks-common/src/util/log.rs index b3697baee..26de14e67 100644 --- a/stacks-common/src/util/log.rs +++ b/stacks-common/src/util/log.rs @@ -26,8 +26,7 @@ use std::time::{Duration, SystemTime}; lazy_static! { pub static ref LOGGER: Logger = make_logger(); - pub static ref STACKS_LOG_FORMAT_TIME: Result = - env::var("STACKS_LOG_FORMAT_TIME"); + pub static ref STACKS_LOG_FORMAT_TIME: Option = env::var("STACKS_LOG_FORMAT_TIME").ok(); } struct TermFormat { decorator: D, @@ -45,7 +44,7 @@ fn print_msg_header(mut rd: &mut dyn RecordDecorator, record: &Record) -> io::Re rd.start_timestamp()?; let system_time = SystemTime::now(); match &*STACKS_LOG_FORMAT_TIME { - Err(e) => { + None => { let elapsed = system_time .duration_since(SystemTime::UNIX_EPOCH) .unwrap_or(Duration::from_secs(0)); @@ -56,7 +55,7 @@ fn print_msg_header(mut rd: &mut dyn RecordDecorator, record: &Record) -> io::Re elapsed.subsec_nanos() / 1000 )?; } - Ok(ref format) => { + Some(ref format) => { let datetime: DateTime = system_time.into(); write!(rd, "[{}]", datetime.format(format))?; } From c56e6adfd2ac7bbd3ba7ecb1ac1eca54fb04e83d Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 29 Jul 2022 09:51:15 -0500 Subject: [PATCH 78/92] test: add integration test for #3185 --- .github/workflows/bitcoin-tests.yml | 1 + .../src/tests/neon_integrations.rs | 91 +++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 3eea180f0..0937a6b82 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -38,6 +38,7 @@ jobs: fail-fast: false matrix: test-name: + - tests::neon_integrations::miner_submit_twice - tests::neon_integrations::microblock_integration_test - tests::neon_integrations::size_check_integration_test - tests::neon_integrations::cost_voting_integration diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 7159b4173..f20582aaf 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -2265,6 +2265,97 @@ fn filter_long_runtime_tx_integration_test() { channel.stop_chains_coordinator(); } +#[test] +#[ignore] +fn miner_submit_twice() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let spender_sk = StacksPrivateKey::new(); + let spender_addr: PrincipalData = to_addr(&spender_sk).into(); + let contract_content = " + (define-public (foo (a int)) + (ok (* 2 (+ a 1)))) + (define-private (bar) + (foo 56)) + "; + let tx_1 = make_contract_publish(&spender_sk, 0, 50_000, "first-contract", contract_content); + let tx_2 = make_contract_publish(&spender_sk, 1, 50_000, "second-contract", contract_content); + + let (mut conf, _) = neon_integration_test_conf(); + conf.initial_balances.push(InitialBalance { + address: spender_addr.clone(), + amount: 1049230, + }); + + // all transactions have high-enough fees... + conf.miner.min_tx_fee = 1; + conf.node.mine_microblocks = false; + // one should be mined in first attempt, and two should be in second attempt + conf.miner.first_attempt_time_ms = 20; + conf.miner.subsequent_attempt_time_ms = 30_000; + + // note: this test depends on timing of how long it takes to assemble a block, + // but it won't flake if the miner behaves correctly: a correct miner should + // always be able to mine both transactions by the end of this test. an incorrect + // miner may sometimes pass this test though, if they can successfully mine a + // 2-transaction block in 20 ms *OR* if they are slow enough that they mine a + // 0-transaction block in that time (because this would trigger a re-attempt, which + // is exactly what this test is measuring). + // + // The "fixed" behavior is the corner case where a miner did a "first attempt", which + // included 1 or more transaction, but they could have made a second attempt with + // more transactions. + + let mut btcd_controller = BitcoinCoreController::new(conf.clone()); + btcd_controller + .start_bitcoind() + .map_err(|_e| ()) + .expect("Failed starting bitcoind"); + + let mut btc_regtest_controller = BitcoinRegtestController::new(conf.clone(), None); + let http_origin = format!("http://{}", &conf.node.rpc_bind); + + btc_regtest_controller.bootstrap_chain(201); + + eprintln!("Chain bootstrapped..."); + + let mut run_loop = neon::RunLoop::new(conf); + let blocks_processed = run_loop.get_blocks_processed_arc(); + + let channel = run_loop.get_coordinator_channel().unwrap(); + + thread::spawn(move || run_loop.start(None, 0)); + + // give the run loop some time to start up! + wait_for_runloop(&blocks_processed); + + // first block wakes up the run loop + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // first block will hold our VRF registration + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // second block will be the first mined Stacks block + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + submit_tx(&http_origin, &tx_1); + submit_tx(&http_origin, &tx_2); + + // mine a couple more blocks + // waiting enough time between them that a second attempt could be made. + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + thread::sleep(Duration::from_secs(15)); + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // 1 transaction mined + let account = get_account(&http_origin, &spender_addr); + assert_eq!(account.nonce, 2); + + channel.stop_chains_coordinator(); +} + #[test] #[ignore] fn mining_transactions_is_fair() { From c6e56cf3e2213610569965c2ca4ac640d5ee900a Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 29 Jul 2022 13:20:43 -0400 Subject: [PATCH 79/92] fix: explicitly set CARGO_MANIFEST_DIR in a bid to fix CI --- .github/actions/bitcoin-int-tests/Dockerfile.code-cov | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov index ed90c1d71..733f879b7 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov +++ b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov @@ -2,6 +2,8 @@ FROM rust:bullseye AS test WORKDIR /build +ENV CARGO_MANIFEST_DIR="$(pwd)" + RUN rustup override set nightly-2022-01-14 && \ rustup component add llvm-tools-preview && \ cargo install grcov From 60114d149491b507d1f7c58f52a79f7f3fb373d3 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 29 Jul 2022 13:20:43 -0400 Subject: [PATCH 80/92] fix: explicitly set CARGO_MANIFEST_DIR in a bid to fix CI --- .github/actions/bitcoin-int-tests/Dockerfile.code-cov | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov index ed90c1d71..733f879b7 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov +++ b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov @@ -2,6 +2,8 @@ FROM rust:bullseye AS test WORKDIR /build +ENV CARGO_MANIFEST_DIR="$(pwd)" + RUN rustup override set nightly-2022-01-14 && \ rustup component add llvm-tools-preview && \ cargo install grcov From dc2d1e06a1925ac6bbd900ee90f7ac8929c1f4f6 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Sun, 31 Jul 2022 09:27:55 -0400 Subject: [PATCH 81/92] fix: make attachment enqueuing fail specifically due to DB errors, since that's the only way it can fail --- src/net/atlas/download.rs | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/net/atlas/download.rs b/src/net/atlas/download.rs index a2ab7e1d8..273fff3d5 100644 --- a/src/net/atlas/download.rs +++ b/src/net/atlas/download.rs @@ -33,6 +33,7 @@ use crate::net::NeighborKey; use crate::net::{GetAttachmentResponse, GetAttachmentsInvResponse}; use crate::net::{HttpRequestMetadata, HttpRequestType, HttpResponseType, PeerHost, Requestable}; use crate::types::chainstate::StacksBlockId; +use crate::util_lib::db::Error as DBError; use crate::util_lib::strings; use crate::util_lib::strings::UrlString; use clarity::vm::types::QualifiedContractIdentifier; @@ -240,7 +241,7 @@ impl AttachmentsDownloader { new_attachments: &mut HashSet, atlasdb: &mut AtlasDB, initial_batch: bool, - ) -> Result, net_error> { + ) -> Result, DBError> { if new_attachments.is_empty() { return Ok(vec![]); } @@ -251,9 +252,7 @@ impl AttachmentsDownloader { // Are we dealing with an empty hash - allowed for undoing onchain binding if attachment_instance.content_hash == Hash160::empty() { // todo(ludo) insert or update ? - atlasdb - .insert_uninstantiated_attachment_instance(&attachment_instance, true) - .map_err(|e| net_error::DBError(e))?; + atlasdb.insert_uninstantiated_attachment_instance(&attachment_instance, true)?; debug!("Atlas: inserting and pairing new attachment instance with empty hash"); resolved_attachments.push((attachment_instance, Attachment::empty())); continue; @@ -261,9 +260,7 @@ impl AttachmentsDownloader { // Do we already have a matching validated attachment if let Ok(Some(entry)) = atlasdb.find_attachment(&attachment_instance.content_hash) { - atlasdb - .insert_uninstantiated_attachment_instance(&attachment_instance, true) - .map_err(|e| net_error::DBError(e))?; + atlasdb.insert_uninstantiated_attachment_instance(&attachment_instance, true)?; debug!( "Atlas: inserting and pairing new attachment instance to existing attachment" ); @@ -275,12 +272,8 @@ impl AttachmentsDownloader { if let Ok(Some(attachment)) = atlasdb.find_uninstantiated_attachment(&attachment_instance.content_hash) { - atlasdb - .insert_instantiated_attachment(&attachment) - .map_err(|e| net_error::DBError(e))?; - atlasdb - .insert_uninstantiated_attachment_instance(&attachment_instance, true) - .map_err(|e| net_error::DBError(e))?; + atlasdb.insert_instantiated_attachment(&attachment)?; + atlasdb.insert_uninstantiated_attachment_instance(&attachment_instance, true)?; debug!("Atlas: inserting and pairing new attachment instance to inboxed attachment, now validated"); resolved_attachments.push((attachment_instance, attachment)); continue; @@ -300,9 +293,7 @@ impl AttachmentsDownloader { }; if !initial_batch { - atlasdb - .insert_uninstantiated_attachment_instance(&attachment_instance, false) - .map_err(|e| net_error::DBError(e))?; + atlasdb.insert_uninstantiated_attachment_instance(&attachment_instance, false)?; } } From 960cdf82e4d42789878e7243f631958c3030453b Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Sun, 31 Jul 2022 09:28:23 -0400 Subject: [PATCH 82/92] fix: make top-level inv sync state machine infallible -- it absorbs all errors that could happen --- src/net/inv.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/net/inv.rs b/src/net/inv.rs index 06e1c3471..e0774a1f4 100644 --- a/src/net/inv.rs +++ b/src/net/inv.rs @@ -2249,7 +2249,7 @@ impl PeerNetwork { &mut self, sortdb: &SortitionDB, ibd: bool, - ) -> Result<(bool, bool, Vec, Vec), net_error> { + ) -> (bool, bool, Vec, Vec) { PeerNetwork::with_inv_state(self, |network, inv_state| { debug!( "{:?}: Inventory state has {} block stats tracked", @@ -2481,6 +2481,7 @@ impl PeerNetwork { Ok((false, false, vec![], vec![])) } }) + .expect("FATAL: network not connected") } pub fn with_inv_state(network: &mut PeerNetwork, handler: F) -> Result From 2545a6443750f27400bbbae7c788b0947459c0ca Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Sun, 31 Jul 2022 09:28:53 -0400 Subject: [PATCH 83/92] fix: make process_new_sockets() infallible -- it doesn't need to return Err(..) if the http server isn't initialized --- src/net/server.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/net/server.rs b/src/net/server.rs index 0f302295e..0aaf1a2ab 100644 --- a/src/net/server.rs +++ b/src/net/server.rs @@ -375,7 +375,7 @@ impl HttpPeer { mempool: &MemPoolDB, chainstate: &mut StacksChainState, poll_state: &mut NetworkPollState, - ) -> Result, net_error> { + ) -> Vec { let mut registered = vec![]; for (hint_event_id, client_sock) in poll_state.new.drain() { @@ -420,7 +420,7 @@ impl HttpPeer { registered.push(event_id); } - Ok(registered) + registered } /// Process network traffic on a HTTP conversation. @@ -680,9 +680,9 @@ impl HttpPeer { mempool: &mut MemPoolDB, mut poll_state: NetworkPollState, handler_args: &RPCHandlerArgs, - ) -> Result, net_error> { + ) -> Vec { // set up new inbound conversations - self.process_new_sockets(network_state, mempool, chainstate, &mut poll_state)?; + self.process_new_sockets(network_state, mempool, chainstate, &mut poll_state); // set up connected sockets self.process_connecting_sockets(network_state, mempool, chainstate, &mut poll_state); @@ -716,7 +716,7 @@ impl HttpPeer { // clear out slow or non-responsive peers self.disconnect_unresponsive(network_state); - Ok(stacks_msgs) + stacks_msgs } } From 155e0a580931f9dd4d6bc9e5360690d3db53f0b6 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Sun, 31 Jul 2022 09:29:19 -0400 Subject: [PATCH 84/92] fix: make the network state-machine processing steps (in particular, mempool sync) infallible -- make them absorb all errors and simply retry their execution instead of propagating them to the caller of PeerNetwork::run() --- src/net/p2p.rs | 435 +++++++++++++++++++++++++------------------------ 1 file changed, 222 insertions(+), 213 deletions(-) diff --git a/src/net/p2p.rs b/src/net/p2p.rs index 772a89f3b..48d3fe3a2 100644 --- a/src/net/p2p.rs +++ b/src/net/p2p.rs @@ -1601,14 +1601,11 @@ impl PeerNetwork { } /// Process new inbound TCP connections we just accepted. - /// Returns the event IDs of sockets we need to register - fn process_new_sockets( - &mut self, - poll_state: &mut NetworkPollState, - ) -> Result, net_error> { + /// Returns the event IDs of sockets we need to register. + fn process_new_sockets(&mut self, poll_state: &mut NetworkPollState) -> Vec { if self.network.is_none() { - test_debug!("{:?}: network not connected", &self.local_peer); - return Err(net_error::NotConnected); + warn!("{:?}: network not connected", &self.local_peer); + return vec![]; } let mut registered = vec![]; @@ -1644,7 +1641,7 @@ impl PeerNetwork { } None => { debug!("{:?}: network not connected", &self.local_peer); - return Err(net_error::NotConnected); + return vec![]; } }; @@ -1656,7 +1653,7 @@ impl PeerNetwork { registered.push(event_id); } - Ok(registered) + registered } /// Process network traffic on a p2p conversation. @@ -2024,7 +2021,7 @@ impl PeerNetwork { } /// Regenerate our session private key and re-handshake with everyone. - fn rekey(&mut self, old_local_peer_opt: Option<&LocalPeer>) -> () { + fn rekey(&mut self, old_local_peer_opt: Option<&LocalPeer>) { assert!(old_local_peer_opt.is_some()); let _old_local_peer = old_local_peer_opt.unwrap(); @@ -2132,10 +2129,10 @@ impl PeerNetwork { /// Update the state of our neighbor walk. /// Return true if we finish, and true if we're throttled - fn do_network_neighbor_walk(&mut self, ibd: bool) -> Result { + fn do_network_neighbor_walk(&mut self, ibd: bool) -> bool { if cfg!(test) && self.connection_opts.disable_neighbor_walk { test_debug!("neighbor walk is disabled"); - return Ok(true); + return true; } debug!("{:?}: walk peer graph", &self.local_peer); @@ -2149,7 +2146,7 @@ impl PeerNetwork { self.process_neighbor_walk(walk_result); } } - Ok(done) + done } /// Do a mempool sync. Return any transactions we might receive. @@ -2159,12 +2156,12 @@ impl PeerNetwork { mempool: &MemPoolDB, chainstate: &mut StacksChainState, ibd: bool, - ) -> Result>, net_error> { + ) -> Option> { if ibd { - return Ok(None); + return None; } - match self.do_mempool_sync(dns_client_opt, mempool, chainstate)? { + match self.do_mempool_sync(dns_client_opt, mempool, chainstate) { (true, txs_opt) => { // did we run to completion? if let Some(txs) = txs_opt { @@ -2176,9 +2173,9 @@ impl PeerNetwork { self.mempool_sync_deadline = get_epoch_time_secs() + self.connection_opts.mempool_sync_interval; - return Ok(Some(txs)); + return Some(txs); } else { - return Ok(None); + return None; } } (false, txs_opt) => { @@ -2190,9 +2187,9 @@ impl PeerNetwork { txs.len() ); - return Ok(Some(txs)); + return Some(txs); } else { - return Ok(None); + return None; } } } @@ -2427,9 +2424,10 @@ impl PeerNetwork { } /// Learn our publicly-routable IP address - fn do_get_public_ip(&mut self) -> Result { + /// return true if we're done with this state machine + fn do_get_public_ip(&mut self) -> bool { if !self.need_public_ip() { - return Ok(true); + return true; } if self.local_peer.public_ip_address.is_some() && self.public_ip_requested_at + self.connection_opts.public_ip_request_timeout @@ -2441,48 +2439,34 @@ impl PeerNetwork { &self.local_peer, self.public_ip_requested_at + self.connection_opts.public_ip_request_timeout ); - return Ok(true); + return true; } match self.do_learn_public_ip() { Ok(b) => { if !b { test_debug!("{:?}: try do_learn_public_ip again", &self.local_peer); - return Ok(false); + return false; } } Err(e) => { - test_debug!( + warn!( "{:?}: failed to learn public IP: {:?}", - &self.local_peer, - &e + &self.local_peer, &e ); self.public_ip_reset(); - - match e { - net_error::NoSuchNeighbor => { - // haven't connected to anyone yet - return Ok(true); - } - _ => { - return Err(e); - } - }; + return true; } } - Ok(true) + true } /// Update the state of our neighbors' block inventories. /// Return true if we finish - fn do_network_inv_sync( - &mut self, - sortdb: &SortitionDB, - ibd: bool, - ) -> Result<(bool, bool), net_error> { + fn do_network_inv_sync(&mut self, sortdb: &SortitionDB, ibd: bool) -> (bool, bool) { if cfg!(test) && self.connection_opts.disable_inv_sync { test_debug!("{:?}: inv sync is disabled", &self.local_peer); - return Ok((true, false)); + return (true, false); } debug!("{:?}: network inventory sync", &self.local_peer); @@ -2493,7 +2477,7 @@ impl PeerNetwork { // synchronize peer block inventories let (done, throttled, broken_neighbors, dead_neighbors) = - self.sync_inventories(sortdb, ibd)?; + self.sync_inventories(sortdb, ibd); // disconnect and ban broken peers for broken in broken_neighbors.into_iter() { @@ -2505,7 +2489,7 @@ impl PeerNetwork { self.deregister_neighbor(&dead); } - Ok((done, throttled)) + (done, throttled) } /// Download blocks, and add them to our network result. @@ -2517,10 +2501,10 @@ impl PeerNetwork { dns_client: &mut DNSClient, ibd: bool, network_result: &mut NetworkResult, - ) -> Result { + ) -> bool { if self.connection_opts.disable_block_download { debug!("{:?}: block download is disabled", &self.local_peer); - return Ok(true); + return true; } if self.block_downloader.is_none() { @@ -2543,19 +2527,20 @@ impl PeerNetwork { "{:?}: no progress can be made on the block downloader -- not connected", &self.local_peer ); - return Ok(true); + return true; } Err(net_error::Transient(s)) => { // not fatal, but just skip and try again info!("Transient network error while downloading blocks: {}", &s); - return Ok(true); + return true; } Err(e) => { warn!( "{:?}: Failed to download blocks: {:?}", &self.local_peer, &e ); - return Err(e); + // done + return true; } }; @@ -2616,10 +2601,11 @@ impl PeerNetwork { self.num_downloader_passes += 1; } - Ok(done && at_chain_tip) + done && at_chain_tip } - /// Find the next block to push + /// Find the next block to push. + /// Mask database errors if they occur fn find_next_push_block( &mut self, nk: &NeighborKey, @@ -2629,10 +2615,10 @@ impl PeerNetwork { chainstate: &StacksChainState, local_blocks_inv: &BlocksInvData, block_stats: &NeighborBlockStats, - ) -> Result, net_error> { + ) -> Option<(ConsensusHash, StacksBlock)> { let start_block_height = self.burnchain.reward_cycle_to_block_height(reward_cycle); if !local_blocks_inv.has_ith_block((height - start_block_height) as u16) { - return Ok(None); + return None; } if block_stats.inv.get_block_height() >= height && !block_stats.inv.has_ith_block(height) { let ancestor_sn = match self.get_ancestor_sortition_snapshot(sortdb, height) { @@ -2642,7 +2628,7 @@ impl PeerNetwork { "{:?}: AntiEntropy: Failed to query ancestor block height {}: {:?}", &self.local_peer, height, &e ); - return Ok(None); + return None; } }; @@ -2654,14 +2640,21 @@ impl PeerNetwork { &chainstate.blocks_path, &ancestor_sn.consensus_hash, &ancestor_sn.winning_stacks_block_hash, - )? { - Some(block) => block, - None => { + ) { + Ok(Some(block)) => block, + Ok(None) => { debug!( "{:?}: AntiEntropy: No such block {}", &self.local_peer, &index_block_hash ); - return Ok(None); + return None; + } + Err(e) => { + warn!( + "{:?}: AntiEntropy: failed to load block {}: {:?}", + &self.local_peer, &index_block_hash, &e + ); + return None; } }; @@ -2669,13 +2662,14 @@ impl PeerNetwork { "{:?}: AntiEntropy: Peer {:?} is missing Stacks block {} from height {}, which we have", &self.local_peer, nk, &index_block_hash, height ); - return Ok(Some((ancestor_sn.consensus_hash, block))); + return Some((ancestor_sn.consensus_hash, block)); } else { - return Ok(None); + return None; } } /// Find the next confirmed microblock stream to push. + /// Mask database errors fn find_next_push_microblocks( &mut self, nk: &NeighborKey, @@ -2685,10 +2679,10 @@ impl PeerNetwork { chainstate: &StacksChainState, local_blocks_inv: &BlocksInvData, block_stats: &NeighborBlockStats, - ) -> Result)>, net_error> { + ) -> Option<(ConsensusHash, BlockHeaderHash, Vec)> { let start_block_height = self.burnchain.reward_cycle_to_block_height(reward_cycle); if !local_blocks_inv.has_ith_microblock_stream((height - start_block_height) as u16) { - return Ok(None); + return None; } if block_stats.inv.get_block_height() >= height && !block_stats.inv.has_ith_microblock_stream(height) @@ -2700,7 +2694,7 @@ impl PeerNetwork { "{:?}: AntiEntropy: Failed to query ancestor block height {}: {:?}", &self.local_peer, height, &e ); - return Ok(None); + return None; } }; @@ -2719,7 +2713,7 @@ impl PeerNetwork { &ancestor_sn.consensus_hash, &ancestor_sn.winning_stacks_block_hash, ); - return Ok(None); + return None; } Err(e) => { debug!( @@ -2729,7 +2723,7 @@ impl PeerNetwork { &ancestor_sn.winning_stacks_block_hash, &e ); - return Ok(None); + return None; } }; @@ -2749,7 +2743,7 @@ impl PeerNetwork { &block_info.consensus_hash, &block_info.anchored_block_hash, ); - return Ok(None); + return None; } Err(e) => { debug!("{:?}: AntiEntropy: Failed to load processed microblocks in-between {}/{} and {}/{}: {:?}", @@ -2760,7 +2754,7 @@ impl PeerNetwork { &block_info.anchored_block_hash, &e ); - return Ok(None); + return None; } }; @@ -2772,28 +2766,24 @@ impl PeerNetwork { "{:?}: AntiEntropy: Peer {:?} is missing Stacks microblocks {} from height {}, which we have", &self.local_peer, nk, &index_block_hash, height ); - return Ok(Some(( + return Some(( block_info.parent_consensus_hash, block_info.parent_anchored_block_hash, microblocks, - ))); + )); } else { - return Ok(None); + return None; } } /// Push any blocks and microblock streams that we're holding onto out to our neighbors. /// Start with the most-recently-arrived data, since this node is likely to have already /// fetched older data via the block-downloader. - fn try_push_local_data( - &mut self, - sortdb: &SortitionDB, - chainstate: &StacksChainState, - ) -> Result<(), net_error> { + fn try_push_local_data(&mut self, sortdb: &SortitionDB, chainstate: &StacksChainState) { if self.antientropy_last_push_ts + self.connection_opts.antientropy_retry >= get_epoch_time_secs() { - return Ok(()); + return; } self.antientropy_last_push_ts = get_epoch_time_secs(); @@ -2806,7 +2796,7 @@ impl PeerNetwork { if num_public_inbound > 0 && !self.connection_opts.antientropy_public { // we're likely not NAT'ed, and we're not supposed to push blocks to the public. - return Ok(()); + return; } if self.relay_handles.len() as u64 @@ -2818,12 +2808,12 @@ impl PeerNetwork { &self.local_peer, self.relay_handles.len() ); - return Ok(()); + return; } if self.inv_state.is_none() { // nothing to do - return Ok(()); + return; } let mut total_blocks_to_broadcast = 0; @@ -2852,7 +2842,7 @@ impl PeerNetwork { self.antientropy_start_reward_cycle = reward_cycle_finish; if neighbor_keys.len() == 0 { - return Ok(()); + return; } debug!( @@ -2916,7 +2906,7 @@ impl PeerNetwork { chainstate, &local_blocks_inv, block_stats, - )? + ) { let index_block_hash = StacksBlockHeader::make_index_block_hash( &consensus_hash, @@ -2975,7 +2965,7 @@ impl PeerNetwork { chainstate, &local_blocks_inv, block_stats, - )? + ) { let index_block_hash = StacksBlockHeader::make_index_block_hash( &parent_consensus_hash, @@ -3033,11 +3023,12 @@ impl PeerNetwork { continue; } Err(e) => { + // should be unreachable, but why tempt fate? debug!( "{:?}: AntiEntropy: Failed to push blocks to {:?}: {:?}", &self.local_peer, &nk, &e ); - return Err(e); + break; } }; @@ -3119,9 +3110,9 @@ impl PeerNetwork { .truncate_pox_inventory(&network.burnchain, reward_cycle); } Ok(()) - })?; + }) + .expect("FATAL: with_inv_state() should be infallible"); } - Ok(()) } /// Extract an IP address from a UrlString if it exists @@ -3416,13 +3407,13 @@ impl PeerNetwork { dns_client_opt: &mut Option<&mut DNSClient>, mempool: &MemPoolDB, chainstate: &mut StacksChainState, - ) -> Result<(bool, Option>), net_error> { + ) -> (bool, Option>) { if get_epoch_time_secs() <= self.mempool_sync_deadline { debug!( "{:?}: Wait until {} to do a mempool sync", &self.local_peer, self.mempool_sync_deadline ); - return Ok((true, None)); + return (true, None); } if self.mempool_sync_timeout == 0 { @@ -3436,7 +3427,7 @@ impl PeerNetwork { &self.local_peer ); self.mempool_sync_reset(); - return Ok((true, None)); + return (true, None); } } @@ -3451,37 +3442,49 @@ impl PeerNetwork { match cur_state { MempoolSyncState::PickOutboundPeer => { // 1. pick a random outbound conversation. - if let Some(next_state) = - self.mempool_sync_pick_outbound_peer(dns_client_opt, &Txid([0u8; 32]))? - { - // success! can advance to either resolve a URL or to send a query - self.mempool_state = next_state; - } else { - // done - self.mempool_sync_reset(); - return Ok((true, None)); + match self.mempool_sync_pick_outbound_peer(dns_client_opt, &Txid([0u8; 32])) { + Ok(Some(next_state)) => { + // success! can advance to either resolve a URL or to send a query + self.mempool_state = next_state; + } + Ok(None) => { + // done + self.mempool_sync_reset(); + return (true, None); + } + Err(e) => { + // done; need reset + warn!("mempool_sync_pick_outbound_peer returned {:?}", &e); + self.mempool_sync_reset(); + return (true, None); + } } } MempoolSyncState::ResolveURL(ref url_str, ref dns_request, ref page_id) => { // 2. resolve its data URL - match self.mempool_sync_resolve_data_url( - url_str, - dns_request, - dns_client_opt, - )? { - (false, Some(addr)) => { + match self.mempool_sync_resolve_data_url(url_str, dns_request, dns_client_opt) { + Ok((false, Some(addr))) => { // success! advance self.mempool_state = MempoolSyncState::SendQuery(url_str.clone(), addr, page_id.clone()); } - (false, None) => { + Ok((false, None)) => { // try again later - return Ok((false, None)); + return (false, None); } - (true, _) => { + Ok((true, _)) => { // done self.mempool_sync_reset(); - return Ok((true, None)); + return (true, None); + } + Err(e) => { + // failed + warn!( + "mempool_sync_resolve_data_url({}) failed: {:?}", + url_str, &e + ); + self.mempool_sync_reset(); + return (true, None); } } } @@ -3497,27 +3500,33 @@ impl PeerNetwork { mempool, chainstate, page_id.clone(), - )? { - (false, Some(event_id)) => { + ) { + Ok((false, Some(event_id))) => { // success! advance debug!("{:?}: Mempool sync query {} for mempool transactions at {} on event {}", &self.local_peer, url, page_id, event_id); self.mempool_state = MempoolSyncState::RecvResponse(url.clone(), addr.clone(), event_id); } - (false, None) => { + Ok((false, None)) => { // try again later - return Ok((false, None)); + return (false, None); } - (true, _) => { + Ok((true, _)) => { // done self.mempool_sync_reset(); - return Ok((true, None)); + return (true, None); + } + Err(e) => { + // done + warn!("mempool_sync_send_query({}) returned {:?}", url, &e); + self.mempool_sync_reset(); + return (true, None); } } } MempoolSyncState::RecvResponse(ref url, ref addr, ref event_id) => { - match self.mempool_sync_recv_response(*event_id)? { - (true, next_page_id_opt, Some(txs)) => { + match self.mempool_sync_recv_response(*event_id) { + Ok((true, next_page_id_opt, Some(txs))) => { debug!( "{:?}: Mempool sync received {} transactions; next page is {:?}", &self.local_peer, @@ -3542,25 +3551,31 @@ impl PeerNetwork { true } }; - return Ok((ret, Some(txs))); + return (ret, Some(txs)); } - (true, _, None) => { + Ok((true, _, None)) => { // done! did not get data self.mempool_sync_reset(); - return Ok((true, None)); + return (true, None); } - (false, _, None) => { + Ok((false, _, None)) => { // still receiving; try again later - return Ok((false, None)); + return (false, None); } - (false, _, Some(_)) => { + Ok((false, _, Some(_))) => { // should never happen if cfg!(test) { panic!("Reached invalid state in {:?}, aborting...", &cur_state); } warn!("Reached invalid state in {:?}, resetting...", &cur_state); self.mempool_sync_reset(); - return Ok((true, None)); + return (true, None); + } + Err(e) => { + // likely a network error + warn!("mempool_sync_recv_response returned {:?}", &e); + self.mempool_sync_reset(); + return (true, None); } } } @@ -3579,7 +3594,7 @@ impl PeerNetwork { download_backpressure: bool, ibd: bool, network_result: &mut NetworkResult, - ) -> Result { + ) -> bool { // do some Actual Work(tm) let mut do_prune = false; let mut did_cycle = false; @@ -3605,22 +3620,15 @@ impl PeerNetwork { self.work_state = PeerNetworkWorkState::BlockInvSync; } else { // (re)determine our public IP address - match self.do_get_public_ip() { - Ok(b) => { - if b { - self.work_state = PeerNetworkWorkState::BlockInvSync; - } - } - Err(e) => { - info!("Failed to query public IP ({:?}) skipping", &e); - self.work_state = PeerNetworkWorkState::BlockInvSync; - } + let done = self.do_get_public_ip(); + if done { + self.work_state = PeerNetworkWorkState::BlockInvSync; } } } PeerNetworkWorkState::BlockInvSync => { // synchronize peer block inventories - let (inv_done, inv_throttled) = self.do_network_inv_sync(sortdb, ibd)?; + let (inv_done, inv_throttled) = self.do_network_inv_sync(sortdb, ibd); if inv_done { if !download_backpressure { // proceed to get blocks, if we're not backpressured @@ -3721,13 +3729,15 @@ impl PeerNetwork { let (consensus_hash, _) = SortitionDB::get_canonical_stacks_chain_tip_hash( sortdb.conn(), - )?; + ) + .expect("FATAL: failed to load canonical stacks chain tip hash from sortition DB"); let stacks_tip_sortition_height = SortitionDB::get_block_snapshot_consensus( sortdb.conn(), &consensus_hash, - )? + ) + .expect("FATAL: failed to query sortition DB") .map(|sn| sn.block_height) .unwrap_or(self.burnchain.first_block_height) .saturating_sub(self.burnchain.first_block_height); @@ -3783,14 +3793,15 @@ impl PeerNetwork { // go fetch blocks match dns_client_opt { Some(ref mut dns_client) => { - if self.do_network_block_download( + let done = self.do_network_block_download( sortdb, mempool, chainstate, *dns_client, ibd, network_result, - )? { + ); + if done { // advance work state self.work_state = PeerNetworkWorkState::AntiEntropy; } @@ -3812,15 +3823,7 @@ impl PeerNetwork { &self.local_peer ); } else { - match self.try_push_local_data(sortdb, chainstate) { - Ok(_) => {} - Err(e) => { - debug!( - "{:?}: Failed to push local data: {:?}", - &self.local_peer, &e - ); - } - }; + self.try_push_local_data(sortdb, chainstate); } self.work_state = PeerNetworkWorkState::Prune; } @@ -3848,7 +3851,7 @@ impl PeerNetwork { ); } - Ok(do_prune) + do_prune } fn do_attachment_downloads( @@ -3857,12 +3860,18 @@ impl PeerNetwork { chainstate: &mut StacksChainState, mut dns_client_opt: Option<&mut DNSClient>, network_result: &mut NetworkResult, - ) -> Result<(), net_error> { + ) { if self.attachments_downloader.is_none() { - self.atlasdb.evict_expired_uninstantiated_attachments()?; self.atlasdb - .evict_expired_unresolved_attachment_instances()?; - let initial_batch = self.atlasdb.find_unresolved_attachment_instances()?; + .evict_expired_uninstantiated_attachments() + .expect("FATAL: atlasdb error: evict_expired_uninstantiated_attachments"); + self.atlasdb + .evict_expired_unresolved_attachment_instances() + .expect("FATAL: atlasdb error: evict_expired_unresolved_attachment_instances"); + let initial_batch = self + .atlasdb + .find_unresolved_attachment_instances() + .expect("FATAL: atlasdb error: find_unresolved_attachment_instances"); self.init_attachments_downloader(initial_batch); } @@ -3887,7 +3896,7 @@ impl PeerNetwork { } Ok(dead_events) }, - )?; + ).expect("FATAL: with_attachments_downloader() should be infallible (and it is not initialized)"); let _ = PeerNetwork::with_network_state( self, @@ -3913,7 +3922,6 @@ impl PeerNetwork { ); } } - Ok(()) } /// Given an event ID, find the other event ID corresponding @@ -4680,7 +4688,7 @@ impl PeerNetwork { unsolicited: HashMap>, ibd: bool, buffer: bool, - ) -> Result>, net_error> { + ) -> HashMap> { let mut unhandled: HashMap> = HashMap::new(); for (event_id, messages) in unsolicited.into_iter() { let neighbor_key = match self.peers.get(&event_id) { @@ -4737,7 +4745,7 @@ impl PeerNetwork { } } } - Ok(unhandled) + unhandled } /// Find unauthenticated inbound conversations @@ -4753,10 +4761,10 @@ impl PeerNetwork { /// Find inbound conversations that have authenticated, given a list of event ids to search /// for. Add them to our network pingbacks - fn schedule_network_pingbacks(&mut self, event_ids: Vec) -> Result<(), net_error> { + fn schedule_network_pingbacks(&mut self, event_ids: Vec) { if cfg!(test) && self.connection_opts.disable_pingbacks { test_debug!("{:?}: pingbacks are disabled for testing", &self.local_peer); - return Ok(()); + return; } // clear timed-out pingbacks @@ -4796,7 +4804,7 @@ impl PeerNetwork { &addr.addrbytes, addr.port, ) - .map_err(net_error::DBError)?; + .expect("FATAL: failed to read from peer database"); if neighbor_opt.is_some() { debug!( @@ -4842,7 +4850,6 @@ impl PeerNetwork { &self.local_peer, self.walk_pingbacks.len() ); - Ok(()) } /// Count up the number of inbound neighbors that have public IP addresses (i.e. that we have @@ -4950,13 +4957,8 @@ impl PeerNetwork { if sn.burn_header_hash != self.burnchain_tip.burn_header_hash { // try processing previously-buffered messages (best-effort) let buffered_messages = mem::replace(&mut self.pending_messages, HashMap::new()); - ret = self.handle_unsolicited_messages( - sortdb, - chainstate, - buffered_messages, - ibd, - false, - )?; + ret = + self.handle_unsolicited_messages(sortdb, chainstate, buffered_messages, ibd, false); } // update cached stacks chain view for /v2/info @@ -4980,14 +4982,14 @@ impl PeerNetwork { download_backpressure: bool, ibd: bool, mut poll_state: NetworkPollState, - ) -> Result<(), net_error> { + ) { if self.network.is_none() { - test_debug!("{:?}: network not connected", &self.local_peer); - return Err(net_error::NotConnected); + warn!("{:?}: network not connected", &self.local_peer); + return; } // set up new inbound conversations - self.process_new_sockets(&mut poll_state)?; + self.process_new_sockets(&mut poll_state); // set up sockets that have finished connecting self.process_connecting_sockets(&mut poll_state); @@ -5006,11 +5008,11 @@ impl PeerNetwork { self.deregister_peer(error_event); } let unhandled_messages = - self.handle_unsolicited_messages(sortdb, chainstate, unsolicited_messages, ibd, true)?; + self.handle_unsolicited_messages(sortdb, chainstate, unsolicited_messages, ibd, true); network_result.consume_unsolicited(unhandled_messages); // schedule now-authenticated inbound convos for pingback - self.schedule_network_pingbacks(unauthenticated_inbounds)?; + self.schedule_network_pingbacks(unauthenticated_inbounds); // do some Actual Work(tm) // do this _after_ processing new sockets, so the act of opening a socket doesn't trample @@ -5023,35 +5025,36 @@ impl PeerNetwork { download_backpressure, ibd, network_result, - )?; + ); if do_prune { // prune back our connections if it's been a while // (only do this if we're done with all other tasks). // Also, process banned peers. - let mut dead_events = self.process_bans()?; - for dead in dead_events.drain(..) { - debug!( - "{:?}: Banned connection on event {}", - &self.local_peer, dead - ); - self.deregister_peer(dead); + if let Ok(mut dead_events) = self.process_bans() { + for dead in dead_events.drain(..) { + debug!( + "{:?}: Banned connection on event {}", + &self.local_peer, dead + ); + self.deregister_peer(dead); + } } self.prune_connections(); } // In parallel, do a neighbor walk - self.do_network_neighbor_walk(ibd)?; + self.do_network_neighbor_walk(ibd); // In parallel, do a mempool sync. // Remember any txs we get, so we can feed them to the relayer thread. if let Some(mut txs) = - self.do_network_mempool_sync(&mut dns_client_opt, mempool, chainstate, ibd)? + self.do_network_mempool_sync(&mut dns_client_opt, mempool, chainstate, ibd) { network_result.synced_transactions.append(&mut txs); } // download attachments - self.do_attachment_downloads(mempool, chainstate, dns_client_opt, network_result)?; + self.do_attachment_downloads(mempool, chainstate, dns_client_opt, network_result); // remove timed-out requests from other threads for (_, convo) in self.peers.iter_mut() { @@ -5077,10 +5080,15 @@ impl PeerNetwork { // is our key about to expire? do we need to re-key? // NOTE: must come last since it invalidates local_peer if self.local_peer.private_key_expire < self.chain_view.burn_block_height + 1 { - self.peerdb.rekey( - self.local_peer.private_key_expire + self.connection_opts.private_key_lifetime, - )?; - let new_local_peer = self.load_local_peer()?; + self.peerdb + .rekey( + self.local_peer.private_key_expire + self.connection_opts.private_key_lifetime, + ) + .expect("FATAL: failed to rekey peer DB"); + + let new_local_peer = self + .load_local_peer() + .expect("FATAL: failed to load local peer from peer DB"); let old_local_peer = self.local_peer.clone(); self.local_peer = new_local_peer; self.rekey(Some(&old_local_peer)); @@ -5111,8 +5119,6 @@ impl PeerNetwork { } } } - - Ok(()) } /// Store a single transaction @@ -5229,6 +5235,9 @@ impl PeerNetwork { /// -- runs the p2p and http peer main loop /// Returns the table of unhandled network messages to be acted upon, keyed by the neighbors /// that sent them (i.e. keyed by their event IDs) + /// + /// This method can only fail if the internal network object (self.network) is not + /// instantiated. pub fn run( &mut self, sortdb: &SortitionDB, @@ -5267,32 +5276,31 @@ impl PeerNetwork { ); // update local-peer state - self.refresh_local_peer()?; + self.refresh_local_peer() + .expect("FATAL: failed to read local peer from the peer DB"); // update burnchain view, before handling any HTTP connections - let unsolicited_buffered_messages = self.refresh_burnchain_view(sortdb, chainstate, ibd)?; + let unsolicited_buffered_messages = self + .refresh_burnchain_view(sortdb, chainstate, ibd) + .expect("FATAL: failed to refresh burnchain view"); + network_result.consume_unsolicited(unsolicited_buffered_messages); // update PoX view, before handling any HTTP connections - self.refresh_sortition_view(sortdb)?; + self.refresh_sortition_view(sortdb) + .expect("FATAL: failed to refresh sortition view from sortition DB"); // This operation needs to be performed before any early return: // Events are being parsed and dispatched here once and we want to // enqueue them. - match PeerNetwork::with_attachments_downloader(self, |network, attachments_downloader| { - let mut known_attachments = attachments_downloader.enqueue_new_attachments( - attachment_requests, - &mut network.atlasdb, - false, - )?; + PeerNetwork::with_attachments_downloader(self, |network, attachments_downloader| { + let mut known_attachments = attachments_downloader + .enqueue_new_attachments(attachment_requests, &mut network.atlasdb, false) + .expect("FATAL: failed to store new attachments to the atlas DB"); network_result.attachments.append(&mut known_attachments); Ok(()) - }) { - Ok(_) => {} - Err(e) => { - warn!("Atlas: updating attachment inventory failed: {}", e); - } - } + }) + .expect("FATAL: with_attachments_downloader should be infallable (not connected)"); PeerNetwork::with_network_state(self, |ref mut network, ref mut network_state| { let http_stacks_msgs = PeerNetwork::with_http(network, |ref mut net, ref mut http| { @@ -5305,10 +5313,11 @@ impl PeerNetwork { http_poll_state, handler_args, ) - })?; + }); network_result.consume_http_uploads(http_stacks_msgs); Ok(()) - })?; + }) + .expect("FATAL: with_network_state should be infallable (not connected)"); self.dispatch_network( &mut network_result, @@ -5319,7 +5328,7 @@ impl PeerNetwork { download_backpressure, ibd, p2p_poll_state, - )?; + ); debug!("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< End Network Dispatch <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"); Ok(network_result) @@ -5485,7 +5494,7 @@ mod test { let mut p2p_poll_state = poll_states.remove(&p2p.p2p_network_handle).unwrap(); - p2p.process_new_sockets(&mut p2p_poll_state).unwrap(); + p2p.process_new_sockets(&mut p2p_poll_state); p2p.process_connecting_sockets(&mut p2p_poll_state); total_disconnected += p2p.disconnect_unresponsive(); @@ -5562,7 +5571,7 @@ mod test { let mut p2p_poll_state = poll_states.remove(&p2p.p2p_network_handle).unwrap(); - p2p.process_new_sockets(&mut p2p_poll_state).unwrap(); + p2p.process_new_sockets(&mut p2p_poll_state); p2p.process_connecting_sockets(&mut p2p_poll_state); thread::sleep(time::Duration::from_millis(1000)); @@ -5656,7 +5665,7 @@ mod test { let mut p2p_poll_state = poll_state.remove(&p2p.p2p_network_handle).unwrap(); - p2p.process_new_sockets(&mut p2p_poll_state).unwrap(); + p2p.process_new_sockets(&mut p2p_poll_state); p2p.process_connecting_sockets(&mut p2p_poll_state); let mut banned = p2p.process_bans().unwrap(); From 438589ce82d9d9613a45ce81351d852ff256b5ff Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Sun, 31 Jul 2022 09:33:27 -0400 Subject: [PATCH 85/92] fix: the network state machine can only fail if it was not properly instantiated (which should always result in a panic) --- testnet/stacks-node/src/neon_node.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index def38885a..ce58cf9df 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -774,10 +774,9 @@ fn spawn_peer( } } Err(e) => { - error!("P2P: Failed to process network dispatch: {:?}", &e); - if config.is_node_event_driven() { - panic!(); - } + // this is only reachable if the network is not instantiated correctly -- + // i.e. you didn't connect it + panic!("P2P: Failed to process network dispatch: {:?}", &e); } }; From bc22e348a9e35c7fc9c1b326ba82f93d5f4e0a3e Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 1 Aug 2022 14:25:39 -0500 Subject: [PATCH 86/92] Added changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb790eb7..052d078cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE ### Added - Added prometheus output for "transactions in last block" (#3138). +- Added envrionement variable STACKS_LOG_FORMAT_TIME to set the time format + stacks-node uses for logging. + Example: STACKS_LOG_FORMAT_TIME="%Y-%m-%d %H:%M:%S" cargo stacks-node ### Changed - Updates to the logging of transaction events (#3139). From c21870f33a8128785e9a256090a95f2d7ec8be5c Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 2 Aug 2022 11:22:15 -0400 Subject: [PATCH 87/92] refactor: make PeerNetwork::with_inv_state() take a closure that returns R instead of Result --- src/net/download.rs | 2 +- src/net/inv.rs | 22 +++++++++++++--------- src/net/p2p.rs | 5 ++--- src/net/relay.rs | 4 +--- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/net/download.rs b/src/net/download.rs index 5f1ec0c50..2defd5215 100644 --- a/src/net/download.rs +++ b/src/net/download.rs @@ -1223,7 +1223,7 @@ impl PeerNetwork { start_sortition_height, start_sortition_height + scan_batch_size, ) - })?; + })??; debug!( "{:?}: {} availability calculated over {} sortitions ({}-{})", diff --git a/src/net/inv.rs b/src/net/inv.rs index e0774a1f4..cbef9d147 100644 --- a/src/net/inv.rs +++ b/src/net/inv.rs @@ -2280,7 +2280,7 @@ impl PeerNetwork { &network.local_peer, inv_state.last_rescanned_at + inv_state.sync_interval ); - return Ok((true, true, vec![], vec![])); + return (true, true, vec![], vec![]); } for (nk, stats) in inv_state.block_stats.iter_mut() { @@ -2476,9 +2476,9 @@ impl PeerNetwork { network.connection_opts.num_neighbors as usize, ); - Ok((true, false, broken_peers, dead_peers)) + (true, false, broken_peers, dead_peers) } else { - Ok((false, false, vec![], vec![])) + (false, false, vec![], vec![]) } }) .expect("FATAL: network not connected") @@ -2486,7 +2486,7 @@ impl PeerNetwork { pub fn with_inv_state(network: &mut PeerNetwork, handler: F) -> Result where - F: FnOnce(&mut PeerNetwork, &mut InvState) -> Result, + F: FnOnce(&mut PeerNetwork, &mut InvState) -> R, { let mut inv_state = network.inv_state.take(); let res = match inv_state { @@ -2494,7 +2494,7 @@ impl PeerNetwork { test_debug!("{:?}: inv state not connected", &network.local_peer); Err(net_error::NotConnected) } - Some(ref mut invs) => handler(network, invs), + Some(ref mut invs) => Ok(handler(network, invs)), }; network.inv_state = inv_state; res @@ -2560,15 +2560,19 @@ impl PeerNetwork { func: F, ) -> Result where - F: FnOnce(&mut PeerNetwork, &mut NeighborBlockStats) -> Result, + F: FnOnce(&mut PeerNetwork, &mut NeighborBlockStats) -> R, { - PeerNetwork::with_inv_state(self, |network, inv_state| { + match PeerNetwork::with_inv_state(self, |network, inv_state| { if let Some(nstats) = inv_state.block_stats.get_mut(nk) { - func(network, nstats) + Ok(func(network, nstats)) } else { Err(net_error::PeerNotConnected) } - }) + }) { + Ok(Ok(x)) => Ok(x), + Ok(Err(x)) => Err(x), + Err(x) => Err(x), + } } /// Get the local block inventory for a reward cycle diff --git a/src/net/p2p.rs b/src/net/p2p.rs index 48d3fe3a2..13a2ed987 100644 --- a/src/net/p2p.rs +++ b/src/net/p2p.rs @@ -3014,7 +3014,7 @@ impl PeerNetwork { } } } - Ok((local_blocks, local_microblocks)) + (local_blocks, local_microblocks) }, ) { Ok(x) => x, @@ -3109,9 +3109,8 @@ impl PeerNetwork { .inv .truncate_pox_inventory(&network.burnchain, reward_cycle); } - Ok(()) }) - .expect("FATAL: with_inv_state() should be infallible"); + .expect("FATAL: with_inv_state() should be infallible (not connected)"); } } diff --git a/src/net/relay.rs b/src/net/relay.rs index afe339682..6d3acfe84 100644 --- a/src/net/relay.rs +++ b/src/net/relay.rs @@ -1359,7 +1359,7 @@ impl PeerNetwork { } } } - Ok(recipients) + recipients })?; // make a normalized random sample of inbound recipients, but don't send to an inbound peer @@ -1561,7 +1561,6 @@ impl PeerNetwork { } } } - Ok(()) }) } @@ -1604,7 +1603,6 @@ impl PeerNetwork { } } } - Ok(()) }) } From d588625382bffc83745b465109176fc3e8fddfff Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 2 Aug 2022 13:26:48 -0400 Subject: [PATCH 88/92] refactor: take a BlockSnapshot, not a BurnchainTip, when spawning a Neon node because we only really need the snapshot --- testnet/stacks-node/src/neon_node.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index def38885a..44825dbf4 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -1229,7 +1229,7 @@ enum LeaderKeyRegistrationState { impl StacksNode { pub fn spawn( runloop: &RunLoop, - last_burn_block: Option, + last_burn_block: Option, coord_comms: CoordinatorChannels, attachments_rx: Receiver>, ) -> StacksNode { @@ -1403,7 +1403,6 @@ impl StacksNode { // setup the relayer channel let (relay_send, relay_recv) = sync_channel(RELAYER_MAX_BUFFER); - let last_burn_block = last_burn_block.map(|x| x.block_snapshot); let last_sortition = Arc::new(Mutex::new(last_burn_block)); let burnchain_signer = keychain.get_burnchain_signer(); From ba1a64aab5efcda672dad9f7c76f87b659601c36 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 2 Aug 2022 13:27:23 -0400 Subject: [PATCH 89/92] fix: start the p2p thread immediately if there are available sortitions in the DB. If there aren't, then wait for one sortition to complete before spawning it. --- testnet/stacks-node/src/run_loop/neon.rs | 78 +++++++++++++++--------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/testnet/stacks-node/src/run_loop/neon.rs b/testnet/stacks-node/src/run_loop/neon.rs index df2101828..7587c5699 100644 --- a/testnet/stacks-node/src/run_loop/neon.rs +++ b/testnet/stacks-node/src/run_loop/neon.rs @@ -19,6 +19,7 @@ use stacks::burnchains::bitcoin::address::BitcoinAddress; use stacks::burnchains::bitcoin::address::BitcoinAddressType; use stacks::burnchains::{Address, Burnchain}; use stacks::chainstate::burn::db::sortdb::SortitionDB; +use stacks::chainstate::burn::BlockSnapshot; use stacks::chainstate::coordinator::comm::{CoordinatorChannels, CoordinatorReceivers}; use stacks::chainstate::coordinator::{ migrate_chainstate_dbs, BlockEventDispatcher, ChainsCoordinator, CoordinatorCommunication, @@ -491,29 +492,36 @@ impl RunLoop { } } - /// Get the sortition DB's highest block height - fn get_sortition_db_height(sortdb: &SortitionDB, burnchain_config: &Burnchain) -> u64 { - let sortition_db_height = { - let (stacks_ch, _) = SortitionDB::get_canonical_stacks_chain_tip_hash(sortdb.conn()) - .expect("BUG: failed to load canonical stacks chain tip hash"); + /// Get the sortition DB's highest block height, aligned to a reward cycle boundary, and the + /// highest sortition. + /// Returns (height at rc start, sortition) + fn get_reward_cycle_sortition_db_height( + sortdb: &SortitionDB, + burnchain_config: &Burnchain, + ) -> (u64, BlockSnapshot) { + let (stacks_ch, _) = SortitionDB::get_canonical_stacks_chain_tip_hash(sortdb.conn()) + .expect("BUG: failed to load canonical stacks chain tip hash"); - match SortitionDB::get_block_snapshot_consensus(sortdb.conn(), &stacks_ch) - .expect("BUG: failed to query sortition DB") - { - Some(sn) => burnchain_config.reward_cycle_to_block_height( - burnchain_config - .block_height_to_reward_cycle(sn.block_height) - .expect("BUG: snapshot preceeds first reward cycle"), - ), - None => { - let sn = SortitionDB::get_first_block_snapshot(&sortdb.conn()) - .expect("BUG: failed to get first-ever block snapshot"); - - sn.block_height - } + let sn = match SortitionDB::get_block_snapshot_consensus(sortdb.conn(), &stacks_ch) + .expect("BUG: failed to query sortition DB") + { + Some(sn) => sn, + None => { + debug!("No canonical stacks chain tip hash present"); + let sn = SortitionDB::get_first_block_snapshot(&sortdb.conn()) + .expect("BUG: failed to get first-ever block snapshot"); + sn } }; - sortition_db_height + + ( + burnchain_config.reward_cycle_to_block_height( + burnchain_config + .block_height_to_reward_cycle(sn.block_height) + .expect("BUG: snapshot preceeds first reward cycle"), + ), + sn, + ) } /// Starts the node runloop. @@ -542,33 +550,47 @@ impl RunLoop { let (coordinator_thread_handle, attachments_rx) = self.spawn_chains_coordinator(&burnchain_config, coordinator_receivers); self.instantiate_pox_watchdog(); + self.start_prometheus(); // We announce a new burn block so that the chains coordinator // can resume prior work and handle eventual unprocessed sortitions // stored during a previous session. coordinator_senders.announce_new_burn_block(); - // Wait for some sortitions! - let mut burnchain_tip = burnchain - .wait_for_sortitions(None) - .expect("Unable to get burnchain tip"); + // Make sure at least one sortition has happened + let sortdb = burnchain.sortdb_mut(); + let (rc_aligned_height, sn) = + RunLoop::get_reward_cycle_sortition_db_height(&sortdb, &burnchain_config); + + let burnchain_tip_snapshot = if sn.block_height == burnchain_config.first_block_height { + // need at least one sortition to happen. + burnchain + .wait_for_sortitions(Some(1)) + .expect("Unable to get burnchain tip") + .block_snapshot + } else { + sn + }; // Boot up the p2p network and relayer, and figure out how many sortitions we have so far // (it could be non-zero if the node is resuming from chainstate) let mut node = StacksNode::spawn( self, - Some(burnchain_tip.clone()), + Some(burnchain_tip_snapshot), coordinator_senders.clone(), attachments_rx, ); - let sortdb = burnchain.sortdb_mut(); - let mut sortition_db_height = RunLoop::get_sortition_db_height(&sortdb, &burnchain_config); + + // Wait for all pending sortitions to process + let mut burnchain_tip = burnchain + .wait_for_sortitions(None) + .expect("Unable to get burnchain tip"); // Start the runloop debug!("Begin run loop"); - self.start_prometheus(); self.counters.bump_blocks_processed(); + let mut sortition_db_height = rc_aligned_height; let mut burnchain_height = sortition_db_height; let mut num_sortitions_in_last_cycle = 1; From b3c13a861f4c1d3eb9ce45b09a8fe41d966b40f6 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 2 Aug 2022 13:41:31 -0400 Subject: [PATCH 90/92] fix: wait_for_sortitions() takes a block height --- testnet/stacks-node/src/run_loop/neon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testnet/stacks-node/src/run_loop/neon.rs b/testnet/stacks-node/src/run_loop/neon.rs index 7587c5699..8882ae1ac 100644 --- a/testnet/stacks-node/src/run_loop/neon.rs +++ b/testnet/stacks-node/src/run_loop/neon.rs @@ -565,7 +565,7 @@ impl RunLoop { let burnchain_tip_snapshot = if sn.block_height == burnchain_config.first_block_height { // need at least one sortition to happen. burnchain - .wait_for_sortitions(Some(1)) + .wait_for_sortitions(Some(sn.block_height + 1)) .expect("Unable to get burnchain tip") .block_snapshot } else { From 7b56663b06b1ddcaee9f0e48bb95b3b5179cb22a Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 2 Aug 2022 14:17:31 -0400 Subject: [PATCH 91/92] chore: add CHANGELOG.md entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb790eb7..d569e2a2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE larger block after their first attempt (by Bitcoin RBF) if new microblock or block data arrived. This changes the miner to always attempt a second block assembly (#3184). +- Fixed a deadlock condition that can arise when the node boots from existing + chainstate and feeds the P2P thread too many Atlas attachment records (#3216). ## [2.05.0.2.1] From 4e9a86d036f63f2ac553ec05405fc258f5b1f872 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 2 Aug 2022 20:56:07 -0400 Subject: [PATCH 92/92] chore: add changelog for this PR --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb790eb7..1636a2ea6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE larger block after their first attempt (by Bitcoin RBF) if new microblock or block data arrived. This changes the miner to always attempt a second block assembly (#3184). +- Fixed a bug in the node whereby the node would encounter a deadlock when + processing attachment requests before the P2P thread had started (#3236). ## [2.05.0.2.1]