From d38b7ac87f94cb3dbca733bfacd2690768ee7069 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Thu, 14 Jul 2022 21:49:17 -0700 Subject: [PATCH 001/178] Move puppet-chain to contrib/tools - Move puppet-chain src to contrib/tools - Update base Dockerfiles to not build/copy puppet-chain --- .cargo/config | 3 +- .vscode/launch.json | 468 +++--- Cargo.lock | 15 - Cargo.toml | 3 +- Dockerfile | 12 +- Dockerfile.stretch => Dockerfile.debian | 15 +- .../tools}/puppet-chain/.dockerignore | 0 contrib/tools/puppet-chain/Cargo.lock | 1329 +++++++++++++++++ .../tools}/puppet-chain/Cargo.toml | 4 + .../tools}/puppet-chain/Dockerfile | 0 .../tools}/puppet-chain/README.md | 0 .../tools}/puppet-chain/config.toml.default | 0 .../puppet-chain/local-leader.toml.default | 0 .../tools}/puppet-chain/src/main.rs | 0 14 files changed, 1545 insertions(+), 304 deletions(-) rename Dockerfile.stretch => Dockerfile.debian (57%) rename {testnet => contrib/tools}/puppet-chain/.dockerignore (100%) create mode 100644 contrib/tools/puppet-chain/Cargo.lock rename {testnet => contrib/tools}/puppet-chain/Cargo.toml (91%) rename {testnet => contrib/tools}/puppet-chain/Dockerfile (100%) rename {testnet => contrib/tools}/puppet-chain/README.md (100%) rename {testnet => contrib/tools}/puppet-chain/config.toml.default (100%) rename {testnet => contrib/tools}/puppet-chain/local-leader.toml.default (100%) rename {testnet => contrib/tools}/puppet-chain/src/main.rs (100%) diff --git a/.cargo/config b/.cargo/config index dda826266..c1dd5d640 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,3 +1,2 @@ [alias] -stacks-node = "run --package stacks-node --" -puppet-chain = "run --package puppet-chain --" +stacks-node = "run --package stacks-node --" \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 225e311de..ea804a69a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,266 +1,208 @@ { - "version": "0.2.0", - "configurations": [ - { - "type": "lldb", - "request": "launch", - "name": "executable 'blockstack-core'", - "cargo": { - "args": [ - "build", - "--bin=stacks-inspect", - "--package=blockstack-core" - ], - "filter": { - "name": "stacks-inspect", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "executable 'clarity-cli'", - "cargo": { - "args": [ - "build", - "--bin=clarity-cli", - "--package=blockstack-core" - ], - "filter": { - "name": "clarity-cli", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "executable 'blockstack-cli'", - "cargo": { - "args": [ - "build", - "--bin=blockstack-cli", - "--package=blockstack-core" - ], - "filter": { - "name": "blockstack-cli", - "kind": "bin" - } - }, - "args": ["generate-sk"], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "executable 'puppet-chain'", - "cargo": { - "args": [ - "build", - "--bin=puppet-chain", - "--package=puppet-chain" - ], - "filter": { - "name": "puppet-chain", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "executable 'stacks-node' -- mocknet", - "cargo": { - "args": [ - "build", - "--bin=stacks-node", - "--package=stacks-node" - ], - "filter": { - "name": "stacks-node", - "kind": "bin" - } - }, - "args": ["mocknet"], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "unit tests in library 'blockstack_lib'", - "cargo": { - "args": [ - "test", - "--no-run", - "--lib", - "--package=blockstack-core" - ], - "filter": { - "name": "blockstack_lib", - "kind": "lib" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "unit tests in executable 'blockstack-core'", - "cargo": { - "args": [ - "test", - "--no-run", - "--bin=stacks-inspect", - "--package=blockstack-core" - ], - "filter": { - "name": "stacks-inspect", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "unit tests in executable 'clarity-cli'", - "cargo": { - "args": [ - "test", - "--no-run", - "--bin=clarity-cli", - "--package=blockstack-core" - ], - "filter": { - "name": "clarity-cli", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "unit tests in executable 'blockstack-cli'", - "cargo": { - "args": [ - "test", - "--no-run", - "--bin=blockstack-cli", - "--package=blockstack-core" - ], - "filter": { - "name": "blockstack-cli", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "unit tests in executable 'stacks-node'", - "cargo": { - "args": [ - "test", - "--no-run", - "--bin=stacks-node", - "--package=stacks-node" - ], - "filter": { - "name": "stacks-node", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "unit tests in executable 'puppet-chain'", - "cargo": { - "args": [ - "test", - "--no-run", - "--bin=puppet-chain", - "--package=puppet-chain" - ], - "filter": { - "name": "puppet-chain", - "kind": "bin" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "benchmark 'marf_bench'", - "cargo": { - "args": [ - "test", - "--no-run", - "--bench=marf_bench", - "--package=blockstack-core" - ], - "filter": { - "name": "marf_bench", - "kind": "bench" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "benchmark 'large_contract_bench'", - "cargo": { - "args": [ - "test", - "--no-run", - "--bench=large_contract_bench", - "--package=blockstack-core" - ], - "filter": { - "name": "large_contract_bench", - "kind": "bench" - } - }, - "args": [], - "cwd": "${workspaceFolder}" - }, - { - "type": "lldb", - "request": "launch", - "name": "benchmark 'block_limits'", - "cargo": { - "args": [ - "test", - "--no-run", - "--bench=block_limits", - "--package=blockstack-core" - ], - "filter": { - "name": "block_limits", - "kind": "bench" - } - }, - "args": [], - "cwd": "${workspaceFolder}" + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "executable 'blockstack-core'", + "cargo": { + "args": ["build", "--bin=stacks-inspect", "--package=blockstack-core"], + "filter": { + "name": "stacks-inspect", + "kind": "bin" } - ] + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "executable 'clarity-cli'", + "cargo": { + "args": ["build", "--bin=clarity-cli", "--package=blockstack-core"], + "filter": { + "name": "clarity-cli", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "executable 'blockstack-cli'", + "cargo": { + "args": ["build", "--bin=blockstack-cli", "--package=blockstack-core"], + "filter": { + "name": "blockstack-cli", + "kind": "bin" + } + }, + "args": ["generate-sk"], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "executable 'stacks-node' -- mocknet", + "cargo": { + "args": ["build", "--bin=stacks-node", "--package=stacks-node"], + "filter": { + "name": "stacks-node", + "kind": "bin" + } + }, + "args": ["mocknet"], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "unit tests in library 'blockstack_lib'", + "cargo": { + "args": ["test", "--no-run", "--lib", "--package=blockstack-core"], + "filter": { + "name": "blockstack_lib", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "unit tests in executable 'blockstack-core'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=stacks-inspect", + "--package=blockstack-core" + ], + "filter": { + "name": "stacks-inspect", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "unit tests in executable 'clarity-cli'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=clarity-cli", + "--package=blockstack-core" + ], + "filter": { + "name": "clarity-cli", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "unit tests in executable 'blockstack-cli'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=blockstack-cli", + "--package=blockstack-core" + ], + "filter": { + "name": "blockstack-cli", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "unit tests in executable 'stacks-node'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=stacks-node", + "--package=stacks-node" + ], + "filter": { + "name": "stacks-node", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "benchmark 'marf_bench'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bench=marf_bench", + "--package=blockstack-core" + ], + "filter": { + "name": "marf_bench", + "kind": "bench" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "benchmark 'large_contract_bench'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bench=large_contract_bench", + "--package=blockstack-core" + ], + "filter": { + "name": "large_contract_bench", + "kind": "bench" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "benchmark 'block_limits'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bench=block_limits", + "--package=blockstack-core" + ], + "filter": { + "name": "block_limits", + "kind": "bench" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] } diff --git a/Cargo.lock b/Cargo.lock index 65a2666f9..cd12019b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1908,21 +1908,6 @@ dependencies = [ "cc", ] -[[package]] -name = "puppet-chain" -version = "0.1.0" -dependencies = [ - "async-h1", - "async-std", - "base64 0.12.3", - "http-types", - "rand 0.7.3", - "serde", - "serde_derive", - "serde_json", - "toml", -] - [[package]] name = "quick-error" version = "1.2.3" diff --git a/Cargo.toml b/Cargo.toml index b4ecf0fc5..58eccaccf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -133,5 +133,4 @@ members = [ ".", "clarity", "stx-genesis", - "testnet/stacks-node", - "testnet/puppet-chain"] + "testnet/stacks-node"] diff --git a/Dockerfile b/Dockerfile index 2b16c4155..09493c983 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,19 +7,11 @@ ARG GIT_COMMIT='No Commit Info' WORKDIR /src COPY . . - RUN apk add --no-cache musl-dev - RUN mkdir /out - RUN cd testnet/stacks-node && cargo build --features monitoring_prom,slog_json --release -RUN cd testnet/puppet-chain && cargo build --release - RUN cp target/release/stacks-node /out -RUN cp target/release/puppet-chain /out - -FROM alpine - -COPY --from=build /out/ /bin/ +FROM --platform=${TARGETPLATFORM} alpine +COPY --from=build /out/stacks-node /bin/ CMD ["stacks-node", "mainnet"] diff --git a/Dockerfile.stretch b/Dockerfile.debian similarity index 57% rename from Dockerfile.stretch rename to Dockerfile.debian index 8cfc0a9db..82a4e2385 100644 --- a/Dockerfile.stretch +++ b/Dockerfile.debian @@ -1,24 +1,15 @@ -FROM rust:stretch as build +FROM rust:bullseye as build ARG STACKS_NODE_VERSION="No Version Info" ARG GIT_BRANCH='No Branch Info' ARG GIT_COMMIT='No Commit Info' WORKDIR /src - COPY . . - RUN mkdir /out - RUN cd testnet/stacks-node && cargo build --features monitoring_prom,slog_json --release -RUN cd testnet/puppet-chain && cargo build --release - RUN cp target/release/stacks-node /out -RUN cp target/release/puppet-chain /out - -FROM debian:stretch-slim - -RUN apt update && apt install -y netcat -COPY --from=build /out/ /bin/ +FROM --platform=${TARGETPLATFORM} debian:bullseye +COPY --from=build /out/stacks-node /bin/ CMD ["stacks-node", "mainnet"] diff --git a/testnet/puppet-chain/.dockerignore b/contrib/tools/puppet-chain/.dockerignore similarity index 100% rename from testnet/puppet-chain/.dockerignore rename to contrib/tools/puppet-chain/.dockerignore diff --git a/contrib/tools/puppet-chain/Cargo.lock b/contrib/tools/puppet-chain/Cargo.lock new file mode 100644 index 000000000..e3d8d1088 --- /dev/null +++ b/contrib/tools/puppet-chain/Cargo.lock @@ -0,0 +1,1329 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aead" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" +dependencies = [ + "aes-soft", + "aesni", + "cipher", +] + +[[package]] +name = "aes-gcm" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aes-soft" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" +dependencies = [ + "cipher", + "opaque-debug", +] + +[[package]] +name = "aesni" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" +dependencies = [ + "cipher", + "opaque-debug", +] + +[[package]] +name = "anyhow" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" + +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-dup" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7427a12b8dc09291528cfb1da2447059adb4a257388c2acd6497a79d55cf6f7c" +dependencies = [ + "futures-io", + "simple-mutex", +] + +[[package]] +name = "async-executor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5262ed948da60dd8956c6c5aca4d4163593dddb7b32d73267c93dab7b2e98940" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "num_cpus", + "once_cell", +] + +[[package]] +name = "async-h1" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8101020758a4fc3a7c326cb42aa99e9fa77cbfb76987c128ad956406fe1f70a7" +dependencies = [ + "async-channel", + "async-dup", + "async-std", + "futures-core", + "http-types", + "httparse", + "log", + "pin-project", +] + +[[package]] +name = "async-io" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e18f61464ae81cde0a23e713ae8fd299580c54d697a35820cfd0625b8b0e07" +dependencies = [ + "concurrent-queue", + "futures-lite", + "libc", + "log", + "once_cell", + "parking", + "polling", + "slab", + "socket2", + "waker-fn", + "winapi", +] + +[[package]] +name = "async-lock" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-attributes", + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" + +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + +[[package]] +name = "bumpalo" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" + +[[package]] +name = "cache-padded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "const_fn" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" + +[[package]] +name = "cookie" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" +dependencies = [ + "aes-gcm", + "base64 0.13.0", + "hkdf", + "hmac", + "percent-encoding", + "rand 0.8.5", + "sha2", + "time", + "version_check", +] + +[[package]] +name = "cpufeatures" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +dependencies = [ + "libc", +] + +[[package]] +name = "cpuid-bool" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" + +[[package]] +name = "crossbeam-utils" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "crypto-mac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "ctor" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "ctr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" +dependencies = [ + "cipher", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "event-listener" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" + +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "generic-array" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "ghash" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "gloo-timers" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fb7d06c1c8cc2a29bee7ec961009a0b2caa0793ee4900c2ffb348734ba1c8f9" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hkdf" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "hmac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "http-types" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" +dependencies = [ + "anyhow", + "async-channel", + "async-std", + "base64 0.13.0", + "cookie", + "futures-lite", + "infer", + "pin-project-lite", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs", + "serde_urlencoded", + "url", +] + +[[package]] +name = "httparse" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + +[[package]] +name = "js-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", + "value-bag", +] + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "polling" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "685404d509889fade3e86fe3a5803bca2ec09b0c0778d5ada6ec8bf7a8de5259" +dependencies = [ + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "winapi", +] + +[[package]] +name = "polyval" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" +dependencies = [ + "cpuid-bool", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro2" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "puppet-chain" +version = "0.1.0" +dependencies = [ + "async-h1", + "async-std", + "base64 0.12.3", + "http-types", + "rand 0.7.3", + "serde", + "serde_derive", + "serde_json", + "toml", +] + +[[package]] +name = "quote" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom 0.2.7", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0171ebb889e45aa68b44aee0859b3eede84c6f5f5c228e6f140c0b2a0a46cad6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1d3230c1de7932af58ad8ffbe1d784bd55efd5a9d84ac24f69c72d83543dfb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_qs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +dependencies = [ + "percent-encoding", + "serde", + "thiserror", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer", + "cfg-if", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "simple-mutex" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38aabbeafa6f6dead8cebf246fe9fae1f9215c8d29b3a69f93bd62a9e4a3dcd6" +dependencies = [ + "event-listener", +] + +[[package]] +name = "slab" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" + +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" + +[[package]] +name = "unicode-normalization" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "universal-hash" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", + "serde", +] + +[[package]] +name = "value-bag" +version = "1.0.0-alpha.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" +dependencies = [ + "ctor", + "version_check", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de9a9cec1733468a8c657e57fa2413d2ae2c0129b95e87c5b72b8ace4d13f31f" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" + +[[package]] +name = "web-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/testnet/puppet-chain/Cargo.toml b/contrib/tools/puppet-chain/Cargo.toml similarity index 91% rename from testnet/puppet-chain/Cargo.toml rename to contrib/tools/puppet-chain/Cargo.toml index d14a2c835..68bee027c 100644 --- a/testnet/puppet-chain/Cargo.toml +++ b/contrib/tools/puppet-chain/Cargo.toml @@ -14,3 +14,7 @@ serde_derive = "1" serde_json = { version = "1.0", features = ["arbitrary_precision"] } toml = "0.5" rand = "0.7.2" + +[workspace] +members = [ + "."] \ No newline at end of file diff --git a/testnet/puppet-chain/Dockerfile b/contrib/tools/puppet-chain/Dockerfile similarity index 100% rename from testnet/puppet-chain/Dockerfile rename to contrib/tools/puppet-chain/Dockerfile diff --git a/testnet/puppet-chain/README.md b/contrib/tools/puppet-chain/README.md similarity index 100% rename from testnet/puppet-chain/README.md rename to contrib/tools/puppet-chain/README.md diff --git a/testnet/puppet-chain/config.toml.default b/contrib/tools/puppet-chain/config.toml.default similarity index 100% rename from testnet/puppet-chain/config.toml.default rename to contrib/tools/puppet-chain/config.toml.default diff --git a/testnet/puppet-chain/local-leader.toml.default b/contrib/tools/puppet-chain/local-leader.toml.default similarity index 100% rename from testnet/puppet-chain/local-leader.toml.default rename to contrib/tools/puppet-chain/local-leader.toml.default diff --git a/testnet/puppet-chain/src/main.rs b/contrib/tools/puppet-chain/src/main.rs similarity index 100% rename from testnet/puppet-chain/src/main.rs rename to contrib/tools/puppet-chain/src/main.rs From 98b5b7fe7698cdf848c6022efec54839e28625a6 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Fri, 15 Jul 2022 15:46:26 -0700 Subject: [PATCH 002/178] ignore puppet-chain target --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 49aea8d27..472d70fe4 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ testnet/index.html # will have compiled files and executables /target/ /testnet/helium/target/ +/contrib/tools/puppet-chain/target/ # These are backup files generated by rustfmt **/*.rs.bk From 8ff40c604517b86cf128d9512f5441af0691d002 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Wed, 20 Jul 2022 16:04:12 -0700 Subject: [PATCH 003/178] Reverting Dockerfiles to develop - incorrectly using some logic from a different PR from same repo - removed 'puppet-chain' from dockerfiles --- Dockerfile | 10 ++++++++-- Dockerfile.debian => Dockerfile.stretch | 13 ++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) rename Dockerfile.debian => Dockerfile.stretch (70%) diff --git a/Dockerfile b/Dockerfile index 09493c983..055cc3df7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,11 +7,17 @@ ARG GIT_COMMIT='No Commit Info' WORKDIR /src COPY . . + RUN apk add --no-cache musl-dev + RUN mkdir /out + RUN cd testnet/stacks-node && cargo build --features monitoring_prom,slog_json --release + RUN cp target/release/stacks-node /out -FROM --platform=${TARGETPLATFORM} alpine -COPY --from=build /out/stacks-node /bin/ +FROM alpine + +COPY --from=build /out/ /bin/ + CMD ["stacks-node", "mainnet"] diff --git a/Dockerfile.debian b/Dockerfile.stretch similarity index 70% rename from Dockerfile.debian rename to Dockerfile.stretch index 82a4e2385..7f5148dfe 100644 --- a/Dockerfile.debian +++ b/Dockerfile.stretch @@ -1,15 +1,22 @@ -FROM rust:bullseye as build +FROM rust:stretch as build ARG STACKS_NODE_VERSION="No Version Info" ARG GIT_BRANCH='No Branch Info' ARG GIT_COMMIT='No Commit Info' WORKDIR /src + COPY . . + RUN mkdir /out + RUN cd testnet/stacks-node && cargo build --features monitoring_prom,slog_json --release + RUN cp target/release/stacks-node /out -FROM --platform=${TARGETPLATFORM} debian:bullseye -COPY --from=build /out/stacks-node /bin/ +FROM debian:stretch-slim + +RUN apt update && apt install -y netcat +COPY --from=build /out/ /bin/ + CMD ["stacks-node", "mainnet"] From 8a3b176f86931e8112e511d2bb9dbe2d79cf0044 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 1 Aug 2022 16:07:34 -0500 Subject: [PATCH 004/178] Add config file --- README.md | 2 +- .../conf/mainnet-mockminer-conf.toml | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 testnet/stacks-node/conf/mainnet-mockminer-conf.toml diff --git a/README.md b/README.md index 0b736ff68..99a746886 100644 --- a/README.md +++ b/README.md @@ -332,7 +332,7 @@ seed = "YOUR PRIVATE KEY" wait_time_for_microblocks = 10000 # Run as a mock-miner, to test mining without spending BTC. # Mutually exclusive with `miner`. -#mock_miner = True +#mock_mining = True [miner] # Smallest allowed tx fee, in microSTX diff --git a/testnet/stacks-node/conf/mainnet-mockminer-conf.toml b/testnet/stacks-node/conf/mainnet-mockminer-conf.toml new file mode 100644 index 000000000..6d91c4ba5 --- /dev/null +++ b/testnet/stacks-node/conf/mainnet-mockminer-conf.toml @@ -0,0 +1,21 @@ +[node] +# working_dir = "/dir/to/save/chainstate" +rpc_bind = "0.0.0.0:20443" +p2p_bind = "0.0.0.0:20444" +mock_mining = true +bootstrap_node = "02da7a464ac770ae8337a343670778b93410f2f3fef6bea98dd1c3e9224459d36b@seed-0.mainnet.stacks.co:20444,02afeae522aab5f8c99a00ddf75fbcb4a641e052dd48836408d9cf437344b63516@seed-1.mainnet.stacks.co:20444,03652212ea76be0ed4cd83a25c06e57819993029a7b9999f7d63c36340b34a4e62@seed-2.mainnet.stacks.co:20444" + +[burnchain] +chain = "bitcoin" +mode = "mainnet" +peer_host = "bitcoind.stacks.co" +username = "blockstack" +password = "blockstacksystem" +rpc_port = 8332 +peer_port = 8333 + +# Used for sending events to a local stacks-blockchain-api service +# [[events_observer]] +# endpoint = "localhost:3700" +# retry_count = 255 +# events_keys = ["*"] \ No newline at end of file From 8491730ea9f2d40fa1d0b5fb212809aa7c9df1a4 Mon Sep 17 00:00:00 2001 From: Igor Date: Mon, 1 Aug 2022 16:42:20 -0500 Subject: [PATCH 005/178] fix --- README.md | 3 +-- testnet/stacks-node/conf/mainnet-mockminer-conf.toml | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 99a746886..33e71e855 100644 --- a/README.md +++ b/README.md @@ -330,8 +330,7 @@ miner = True seed = "YOUR PRIVATE KEY" # How long to wait for microblocks to arrive before mining a block to confirm them (in milliseconds) wait_time_for_microblocks = 10000 -# Run as a mock-miner, to test mining without spending BTC. -# Mutually exclusive with `miner`. +# Run as a mock-miner, to test mining without spending BTC. Needs miner=True. #mock_mining = True [miner] diff --git a/testnet/stacks-node/conf/mainnet-mockminer-conf.toml b/testnet/stacks-node/conf/mainnet-mockminer-conf.toml index 6d91c4ba5..cff7c4f46 100644 --- a/testnet/stacks-node/conf/mainnet-mockminer-conf.toml +++ b/testnet/stacks-node/conf/mainnet-mockminer-conf.toml @@ -1,7 +1,8 @@ [node] -# working_dir = "/dir/to/save/chainstate" +working_dir = "/dir/to/save/chainstate" rpc_bind = "0.0.0.0:20443" p2p_bind = "0.0.0.0:20444" +miner = true mock_mining = true bootstrap_node = "02da7a464ac770ae8337a343670778b93410f2f3fef6bea98dd1c3e9224459d36b@seed-0.mainnet.stacks.co:20444,02afeae522aab5f8c99a00ddf75fbcb4a641e052dd48836408d9cf437344b63516@seed-1.mainnet.stacks.co:20444,03652212ea76be0ed4cd83a25c06e57819993029a7b9999f7d63c36340b34a4e62@seed-2.mainnet.stacks.co:20444" From 7402238d9d4ca01ac7e6c0bd8261b363b06e0602 Mon Sep 17 00:00:00 2001 From: Igor Date: Tue, 16 Aug 2022 14:28:00 -0500 Subject: [PATCH 006/178] cache lex_matchers --- clarity/src/vm/ast/parser/mod.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/clarity/src/vm/ast/parser/mod.rs b/clarity/src/vm/ast/parser/mod.rs index 6782d69a5..cbf014312 100644 --- a/clarity/src/vm/ast/parser/mod.rs +++ b/clarity/src/vm/ast/parser/mod.rs @@ -124,14 +124,8 @@ lazy_static! { ); pub static ref CLARITY_NAME_REGEX: String = format!(r#"([[:word:]]|[-!?+<>=/*]){{1,{}}}"#, MAX_STRING_LEN); -} -pub fn lex(input: &str) -> ParseResult> { - // Aaron: I'd like these to be static, but that'd require using - // lazy_static (or just hand implementing that), and I'm not convinced - // it's worth either (1) an extern macro, or (2) the complexity of hand implementing. - - let lex_matchers: &[LexMatcher] = &[ + static ref lex_matchers: Vec = vec![ LexMatcher::new( r##"u"(?P((\\")|([[ -~]&&[^"]]))*)""##, TokenType::StringUTF8Literal, @@ -187,7 +181,9 @@ pub fn lex(input: &str) -> ParseResult> { TokenType::Variable, ), ]; +} +pub fn lex(input: &str) -> ParseResult> { let mut context = LexContext::ExpectNothing; let mut line_indices = get_lines_at(input); From 4c76b401a7d8ac528005f0689709c80f6bca2da1 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Tue, 16 Aug 2022 15:47:21 -0700 Subject: [PATCH 007/178] Update CHANGELOG for 2.05.0.3.0 --- CHANGELOG.md | 104 ++++++++++++++++++++++++++++----------------------- 1 file changed, 57 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b658c5090..d41ac5f8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,18 +5,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to the versioning scheme outlined in the [README.md](README.md). -## Upcoming +## [2.05.0.3.0] ### Added + - Added prometheus output for "transactions in last block" (#3138). - Added envrionement variable STACKS_LOG_FORMAT_TIME to set the time format - stacks-node uses for logging. + stacks-node uses for logging. (#3219) Example: STACKS_LOG_FORMAT_TIME="%Y-%m-%d %H:%M:%S" cargo stacks-node +- Added mock-miner sample config (#3225) ### Changed + - Updates to the logging of transaction events (#3139). +- Moved puppet-chain to `./contrib/tools` directory and disabled compiling by default (#3200) +- Optimized mempool walk to speed up miner loop 6x (#3229) ### Fixed + - Make it so that a new peer private key in the config file will propagate to the peer database (#3165). - Fixed default miner behavior regarding block assembly @@ -30,12 +36,15 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE from sockets, but instead propagate them to the outer caller. This would lead to a node crash in nodes connected to event observers, which expect the P2P state machine to only report fatal errors (#3228) +- Spawn the p2p thread before processing number of sortitions. Fixes issue (#3216) where sync from genesis paused (#3236) +- Drop well-formed "problematic" transactions that result in miner performance degradation (#3212) ## [2.05.0.2.1] ### Fixed + - Fixed a security bug in the SPV client whereby the chain work was not being - considered at all when determining the canonical Bitcoin fork. The SPV client + considered at all when determining the canonical Bitcoin fork. The SPV client now only accepts a new Bitcoin fork if it has a higher chain work than any other previously-seen chain (#3152). @@ -60,8 +69,9 @@ It is highly recommended that you **back up your chainstate** before running this version of the software on it. ### Changed + - The MARF implementation will now defer calculating the root hash of a new trie - until the moment the trie is committed to disk. This avoids gratuitous hash + until the moment the trie is committed to disk. This avoids gratuitous hash calculations, and yields a performance improvement of anywhere between 10x and 200x (#3041). - The MARF implementation will now store tries to an external file for instances @@ -72,8 +82,8 @@ this version of the software on it. by an environment variable (#3042). - Sortition processing performance has been improved by about an order of magnitude, by avoiding a slew of expensive database reads (#3045). -- Updated chains coordinator so that before a Stacks block or a burn block is processed, - an event is sent through the event dispatcher. This fixes #3015. +- Updated chains coordinator so that before a Stacks block or a burn block is processed, + an event is sent through the event dispatcher. This fixes #3015. - Expose a node's public key and public key hash160 (i.e. what appears in /v2/neighbors) via the /v2/info API endpoint (#3046) - Reduced the default subsequent block attempt timeout from 180 seconds to 30 @@ -84,7 +94,8 @@ this version of the software on it. ## [2.05.0.1.0] -### Added +### Added + - A new fee estimator intended to produce fewer over-estimates, by having less sensitivity to outliers. Its characteristic features are: 1) use a window to forget past estimates instead of exponential averaging, 2) use weighted @@ -92,46 +103,46 @@ this version of the software on it. assess empty space in blocks as having paid the "minimum fee", so that empty space is accounted for, 4) use random "fuzz" so that in busy times the fees can change dynamically. (#2972) -- Implements anti-entropy protocol for querying transactions from other +- Implements anti-entropy protocol for querying transactions from other nodes' mempools. Before, nodes wouldn't sync mempool contents with one another. (#2884) -- Structured logging in the mining code paths. This will shine light +- Structured logging in the mining code paths. This will shine light on what happens to transactions (successfully added, skipped or errored) that the miner considers while buildings blocks. (#2975) - Added the mined microblock event, which includes information on transaction events that occurred in the course of mining (will provide insight on whether a transaction was successfully added to the block, skipped, or had a processing error). (#2975) -- For v2 endpoints, can now specify the `tip` parameter to `latest`. If +- For v2 endpoints, can now specify the `tip` parameter to `latest`. If `tip=latest`, the node will try to run the query off of the latest tip. (#2778) -- Adds the /v2/headers endpoint, which returns a sequence of SIP-003-encoded - block headers and consensus hashes (see the ExtendedStacksHeader struct that +- Adds the /v2/headers endpoint, which returns a sequence of SIP-003-encoded + block headers and consensus hashes (see the ExtendedStacksHeader struct that this PR adds to represent this data). (#2862) -- Adds the /v2/data_var endpoint, which returns a contract's data variable +- Adds the /v2/data_var endpoint, which returns a contract's data variable value and a MARF proof of its existence. (#2862) - Fixed a bug in the unconfirmed state processing logic that could lead to a denial of service (node crash) for nodes that mine microblocks (#2970) - Added prometheus metric that tracks block fullness by logging the percentage of each - cost dimension that is consumed in a given block (#3025). - + cost dimension that is consumed in a given block (#3025). ### Changed -- Updated the mined block event. It now includes information on transaction + +- Updated the mined block event. It now includes information on transaction events that occurred in the course of mining (will provide insight - on whether a transaction was successfully added to the block, + on whether a transaction was successfully added to the block, skipped, or had a processing error). (#2975) - Updated some of the logic in the block assembly for the miner and the follower to consolidate similar logic. Added functions `setup_block` and `finish_block`. (#2946) -- Makes the p2p state machine more reactive to newly-arrived - `BlocksAvailable` and `MicroblocksAvailable` messages for block and microblock - streams that this node does not have. If such messages arrive during an inventory - sync, the p2p state machine will immediately transition from the inventory sync - work state to the block downloader work state, and immediately proceed to fetch +- Makes the p2p state machine more reactive to newly-arrived + `BlocksAvailable` and `MicroblocksAvailable` messages for block and microblock + streams that this node does not have. If such messages arrive during an inventory + sync, the p2p state machine will immediately transition from the inventory sync + work state to the block downloader work state, and immediately proceed to fetch the available block or microblock stream. (#2862) - Nodes will push recently-obtained blocks and microblock streams to outbound neighbors if their cached inventories indicate that they do not yet have them -(#2986). + (#2986). - Nodes will no longer perform full inventory scans on their peers, except during boot-up, in a bid to minimize block-download stalls (#2986). - Nodes will process sortitions in parallel to downloading the Stacks blocks for @@ -140,19 +151,20 @@ this version of the software on it. top of parent blocks that are no longer the chain tip (#2969). - Several database indexes have been updated to avoid table scans, which significantly improves most RPC endpoint speed and cuts node spin-up time in -half (#2989, #3005). + half (#2989, #3005). - Fixed a rare denial-of-service bug whereby a node that processes a very deep burnchain reorg can get stuck, and be rendered unable to process further -sortitions. This has never happened in production, but it can be replicated in -tests (#2989). -- Updated what indices are created, and ensures that indices are created even + sortitions. This has never happened in production, but it can be replicated in + tests (#2989). +- Updated what indices are created, and ensures that indices are created even after the database is initialized (#3029). -### Fixed +### Fixed + - Updates the lookup key for contracts in the pessimistic cost estimator. Before, contracts - published by different principals with the same name would have had the same + published by different principals with the same name would have had the same key in the cost estimator. (#2984) -- Fixed a few prometheus metrics to be more accurate compared to `/v2` endpoints +- Fixed a few prometheus metrics to be more accurate compared to `/v2` endpoints when polling data (#2987) - Fixed an error message from the type-checker that shows up when the type of a parameter refers to a trait defined in the same contract (#3064). @@ -212,7 +224,7 @@ compatible with chainstate directories from 2.0.11.3.0. ## [2.0.11.3.0] This software update is a point-release to change the transaction selection -logic in the default miner to prioritize by fee instead of nonce sequence. This +logic in the default miner to prioritize by fee instead of nonce sequence. This release's chainstate directory is compatible with chainstate directories from 2.0.11.2.0. @@ -220,8 +232,8 @@ release's chainstate directory is compatible with chainstate directories from - The node will enforce a soft deadline for mining a block, so that a node operator can control how frequently their node attempts to mine a block -regardless of how congested the mempool is. The timeout parameters are -controlled in the `[miner]` section of the node's config file (#2823). + regardless of how congested the mempool is. The timeout parameters are + controlled in the `[miner]` section of the node's config file (#2823). ## Changed @@ -266,11 +278,11 @@ to reset their chain states. - Two bugs that caused problems syncing with the bitcoin chain during a bitcoin reorg have been fixed (#2771, #2780). - Documentation is fixed in cases where string and buffer types are allowed - but not covered in the documentation. (#2676) + but not covered in the documentation. (#2676) ## [2.0.11.1.0] -This software update is our monthly release. It introduces fixes and features for both developers and miners. +This software update is our monthly release. It introduces fixes and features for both developers and miners. This release's chainstate directory is compatible with chainstate directories from 2.0.11.0.0. ## Added @@ -289,14 +301,14 @@ This release's chainstate directory is compatible with chainstate directories fr - Improved mempool walk order (#2514). - Renamed database `tx_tracking.db` to `tx_tracking.sqlite` (#2666). - -## Fixed -- Alter the miner to prioritize spending the most recent UTXO when building a transaction, +## Fixed + +- Alter the miner to prioritize spending the most recent UTXO when building a transaction, instead of the largest UTXO. In the event of a tie, it uses the smallest UTXO first (#2661). - Fix trait rpc lookups for implicitly implemented traits (#2602). - Fix `v2/pox` endpoint, broken on Mocknet (#2634). -- Align cost limits on mocknet, testnet and mainnet (#2660). +- Align cost limits on mocknet, testnet and mainnet (#2660). - Log peer addresses in the HTTP server (#2667) - Mine microblocks if there are no recent unprocessed Stacks blocks @@ -314,8 +326,7 @@ compatible with prior chainstate directories. - Log transactions in local db table via setting env `STACKS_TRANSACTION_LOG=1` - New prometheus metrics for mempool transaction processing times and outstanding mempool transactions -- New RPC endpoint with path `/v2/traits/contractAddr/contractName/traitContractName - /traitContractAddr/traitName` to determine whether a given trait is implemented +- New RPC endpoint with path `/v2/traits/contractAddr/contractName/traitContractName /traitContractAddr/traitName` to determine whether a given trait is implemented within the specified contract (either explicitly or implicitly). - Re-activate the Atlas network for propagating and storing transaction attachments. This re-enables off-chain BNS name storage. @@ -329,8 +340,8 @@ compatible with prior chainstate directories. - The `/v2/pox` RPC endpoint was updated to include more useful information about the current and next PoX cycles. For details, see `docs/rpc-endpoints.md` - -## Fixed + +## Fixed - Fixed faulty logic in the mempool that was still treating the transaction fee as a fee rate, which prevented replace-by-fee from working as expected. @@ -359,7 +370,7 @@ node. ## [2.0.9] This is a hotfix release for improved handling of arriving Stacks blocks through -both the RPC interface and the P2P ineterface. The chainstate directory of +both the RPC interface and the P2P ineterface. The chainstate directory of 2.0.9 is compatible with the 2.0.8 chainstate. ## Fixed @@ -388,7 +399,6 @@ valid block data if its descendant microblock stream is invalid for some reason. - Do not delete a valid parent Stacks block. - ## [2.0.6] - 2021-02-15 The database schema has not changed since 2.0.5, so when spinning up a @@ -457,9 +467,9 @@ node from an earlier chainstate, you must use a fresh working directory. - Enabled WAL mode for the chainstate databases. This allows much more concurrency in the `stacks-node`, and improves network performance - across the board. **NOTE:** *This changed the database schema, any + across the board. **NOTE:** _This changed the database schema, any running node would need to re-initialize their nodes from a new chain - state when upgrading*. + state when upgrading_. - Default value `wait_time_for_microblocks`: from 60s to 30s - The mempool now performs more transfer semantics checks before admitting a transaction (e.g., reject if origin = recipient): see issue #2354 From f8628cd7389298951302a9c0fdd1c43a591e350f Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Tue, 16 Aug 2022 15:51:09 -0700 Subject: [PATCH 008/178] revert formatting changes --- CHANGELOG.md | 94 ++++++++++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d41ac5f8a..f3dda274a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,12 +39,12 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE - Spawn the p2p thread before processing number of sortitions. Fixes issue (#3216) where sync from genesis paused (#3236) - Drop well-formed "problematic" transactions that result in miner performance degradation (#3212) + ## [2.05.0.2.1] ### Fixed - - Fixed a security bug in the SPV client whereby the chain work was not being - considered at all when determining the canonical Bitcoin fork. The SPV client + considered at all when determining the canonical Bitcoin fork. The SPV client now only accepts a new Bitcoin fork if it has a higher chain work than any other previously-seen chain (#3152). @@ -69,9 +69,8 @@ It is highly recommended that you **back up your chainstate** before running this version of the software on it. ### Changed - - The MARF implementation will now defer calculating the root hash of a new trie - until the moment the trie is committed to disk. This avoids gratuitous hash + until the moment the trie is committed to disk. This avoids gratuitous hash calculations, and yields a performance improvement of anywhere between 10x and 200x (#3041). - The MARF implementation will now store tries to an external file for instances @@ -82,8 +81,8 @@ this version of the software on it. by an environment variable (#3042). - Sortition processing performance has been improved by about an order of magnitude, by avoiding a slew of expensive database reads (#3045). -- Updated chains coordinator so that before a Stacks block or a burn block is processed, - an event is sent through the event dispatcher. This fixes #3015. +- Updated chains coordinator so that before a Stacks block or a burn block is processed, + an event is sent through the event dispatcher. This fixes #3015. - Expose a node's public key and public key hash160 (i.e. what appears in /v2/neighbors) via the /v2/info API endpoint (#3046) - Reduced the default subsequent block attempt timeout from 180 seconds to 30 @@ -94,8 +93,7 @@ this version of the software on it. ## [2.05.0.1.0] -### Added - +### Added - A new fee estimator intended to produce fewer over-estimates, by having less sensitivity to outliers. Its characteristic features are: 1) use a window to forget past estimates instead of exponential averaging, 2) use weighted @@ -103,46 +101,46 @@ this version of the software on it. assess empty space in blocks as having paid the "minimum fee", so that empty space is accounted for, 4) use random "fuzz" so that in busy times the fees can change dynamically. (#2972) -- Implements anti-entropy protocol for querying transactions from other +- Implements anti-entropy protocol for querying transactions from other nodes' mempools. Before, nodes wouldn't sync mempool contents with one another. (#2884) -- Structured logging in the mining code paths. This will shine light +- Structured logging in the mining code paths. This will shine light on what happens to transactions (successfully added, skipped or errored) that the miner considers while buildings blocks. (#2975) - Added the mined microblock event, which includes information on transaction events that occurred in the course of mining (will provide insight on whether a transaction was successfully added to the block, skipped, or had a processing error). (#2975) -- For v2 endpoints, can now specify the `tip` parameter to `latest`. If +- For v2 endpoints, can now specify the `tip` parameter to `latest`. If `tip=latest`, the node will try to run the query off of the latest tip. (#2778) -- Adds the /v2/headers endpoint, which returns a sequence of SIP-003-encoded - block headers and consensus hashes (see the ExtendedStacksHeader struct that +- Adds the /v2/headers endpoint, which returns a sequence of SIP-003-encoded + block headers and consensus hashes (see the ExtendedStacksHeader struct that this PR adds to represent this data). (#2862) -- Adds the /v2/data_var endpoint, which returns a contract's data variable +- Adds the /v2/data_var endpoint, which returns a contract's data variable value and a MARF proof of its existence. (#2862) - Fixed a bug in the unconfirmed state processing logic that could lead to a denial of service (node crash) for nodes that mine microblocks (#2970) - Added prometheus metric that tracks block fullness by logging the percentage of each - cost dimension that is consumed in a given block (#3025). + cost dimension that is consumed in a given block (#3025). + ### Changed - -- Updated the mined block event. It now includes information on transaction +- Updated the mined block event. It now includes information on transaction events that occurred in the course of mining (will provide insight - on whether a transaction was successfully added to the block, + on whether a transaction was successfully added to the block, skipped, or had a processing error). (#2975) - Updated some of the logic in the block assembly for the miner and the follower to consolidate similar logic. Added functions `setup_block` and `finish_block`. (#2946) -- Makes the p2p state machine more reactive to newly-arrived - `BlocksAvailable` and `MicroblocksAvailable` messages for block and microblock - streams that this node does not have. If such messages arrive during an inventory - sync, the p2p state machine will immediately transition from the inventory sync - work state to the block downloader work state, and immediately proceed to fetch +- Makes the p2p state machine more reactive to newly-arrived + `BlocksAvailable` and `MicroblocksAvailable` messages for block and microblock + streams that this node does not have. If such messages arrive during an inventory + sync, the p2p state machine will immediately transition from the inventory sync + work state to the block downloader work state, and immediately proceed to fetch the available block or microblock stream. (#2862) - Nodes will push recently-obtained blocks and microblock streams to outbound neighbors if their cached inventories indicate that they do not yet have them - (#2986). +(#2986). - Nodes will no longer perform full inventory scans on their peers, except during boot-up, in a bid to minimize block-download stalls (#2986). - Nodes will process sortitions in parallel to downloading the Stacks blocks for @@ -151,20 +149,19 @@ this version of the software on it. top of parent blocks that are no longer the chain tip (#2969). - Several database indexes have been updated to avoid table scans, which significantly improves most RPC endpoint speed and cuts node spin-up time in - half (#2989, #3005). +half (#2989, #3005). - Fixed a rare denial-of-service bug whereby a node that processes a very deep burnchain reorg can get stuck, and be rendered unable to process further - sortitions. This has never happened in production, but it can be replicated in - tests (#2989). -- Updated what indices are created, and ensures that indices are created even +sortitions. This has never happened in production, but it can be replicated in +tests (#2989). +- Updated what indices are created, and ensures that indices are created even after the database is initialized (#3029). -### Fixed - +### Fixed - Updates the lookup key for contracts in the pessimistic cost estimator. Before, contracts - published by different principals with the same name would have had the same + published by different principals with the same name would have had the same key in the cost estimator. (#2984) -- Fixed a few prometheus metrics to be more accurate compared to `/v2` endpoints +- Fixed a few prometheus metrics to be more accurate compared to `/v2` endpoints when polling data (#2987) - Fixed an error message from the type-checker that shows up when the type of a parameter refers to a trait defined in the same contract (#3064). @@ -224,7 +221,7 @@ compatible with chainstate directories from 2.0.11.3.0. ## [2.0.11.3.0] This software update is a point-release to change the transaction selection -logic in the default miner to prioritize by fee instead of nonce sequence. This +logic in the default miner to prioritize by fee instead of nonce sequence. This release's chainstate directory is compatible with chainstate directories from 2.0.11.2.0. @@ -232,8 +229,8 @@ release's chainstate directory is compatible with chainstate directories from - The node will enforce a soft deadline for mining a block, so that a node operator can control how frequently their node attempts to mine a block - regardless of how congested the mempool is. The timeout parameters are - controlled in the `[miner]` section of the node's config file (#2823). +regardless of how congested the mempool is. The timeout parameters are +controlled in the `[miner]` section of the node's config file (#2823). ## Changed @@ -278,11 +275,11 @@ to reset their chain states. - Two bugs that caused problems syncing with the bitcoin chain during a bitcoin reorg have been fixed (#2771, #2780). - Documentation is fixed in cases where string and buffer types are allowed - but not covered in the documentation. (#2676) + but not covered in the documentation. (#2676) ## [2.0.11.1.0] -This software update is our monthly release. It introduces fixes and features for both developers and miners. +This software update is our monthly release. It introduces fixes and features for both developers and miners. This release's chainstate directory is compatible with chainstate directories from 2.0.11.0.0. ## Added @@ -301,14 +298,14 @@ This release's chainstate directory is compatible with chainstate directories fr - Improved mempool walk order (#2514). - Renamed database `tx_tracking.db` to `tx_tracking.sqlite` (#2666). + +## Fixed -## Fixed - -- Alter the miner to prioritize spending the most recent UTXO when building a transaction, +- Alter the miner to prioritize spending the most recent UTXO when building a transaction, instead of the largest UTXO. In the event of a tie, it uses the smallest UTXO first (#2661). - Fix trait rpc lookups for implicitly implemented traits (#2602). - Fix `v2/pox` endpoint, broken on Mocknet (#2634). -- Align cost limits on mocknet, testnet and mainnet (#2660). +- Align cost limits on mocknet, testnet and mainnet (#2660). - Log peer addresses in the HTTP server (#2667) - Mine microblocks if there are no recent unprocessed Stacks blocks @@ -326,7 +323,8 @@ compatible with prior chainstate directories. - Log transactions in local db table via setting env `STACKS_TRANSACTION_LOG=1` - New prometheus metrics for mempool transaction processing times and outstanding mempool transactions -- New RPC endpoint with path `/v2/traits/contractAddr/contractName/traitContractName /traitContractAddr/traitName` to determine whether a given trait is implemented +- New RPC endpoint with path `/v2/traits/contractAddr/contractName/traitContractName + /traitContractAddr/traitName` to determine whether a given trait is implemented within the specified contract (either explicitly or implicitly). - Re-activate the Atlas network for propagating and storing transaction attachments. This re-enables off-chain BNS name storage. @@ -340,8 +338,8 @@ compatible with prior chainstate directories. - The `/v2/pox` RPC endpoint was updated to include more useful information about the current and next PoX cycles. For details, see `docs/rpc-endpoints.md` - -## Fixed + +## Fixed - Fixed faulty logic in the mempool that was still treating the transaction fee as a fee rate, which prevented replace-by-fee from working as expected. @@ -370,7 +368,7 @@ node. ## [2.0.9] This is a hotfix release for improved handling of arriving Stacks blocks through -both the RPC interface and the P2P ineterface. The chainstate directory of +both the RPC interface and the P2P ineterface. The chainstate directory of 2.0.9 is compatible with the 2.0.8 chainstate. ## Fixed @@ -399,6 +397,7 @@ valid block data if its descendant microblock stream is invalid for some reason. - Do not delete a valid parent Stacks block. + ## [2.0.6] - 2021-02-15 The database schema has not changed since 2.0.5, so when spinning up a @@ -467,9 +466,9 @@ node from an earlier chainstate, you must use a fresh working directory. - Enabled WAL mode for the chainstate databases. This allows much more concurrency in the `stacks-node`, and improves network performance - across the board. **NOTE:** _This changed the database schema, any + across the board. **NOTE:** *This changed the database schema, any running node would need to re-initialize their nodes from a new chain - state when upgrading_. + state when upgrading*. - Default value `wait_time_for_microblocks`: from 60s to 30s - The mempool now performs more transfer semantics checks before admitting a transaction (e.g., reject if origin = recipient): see issue #2354 @@ -492,3 +491,4 @@ node from an earlier chainstate, you must use a fresh working directory. next `LeaderBlockCommit` operations using the UTXOs of the previous transaction with a replacement by fee. The fee increase increments can be configured with the setting `rbf_fee_increment`. +SAVE TO CACHER From ce21a95d3a82e28fe4a33cf94854fde79d38a5b6 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Tue, 16 Aug 2022 15:51:43 -0700 Subject: [PATCH 009/178] Fix typo --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3dda274a..a84db6add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -491,4 +491,3 @@ node from an earlier chainstate, you must use a fresh working directory. next `LeaderBlockCommit` operations using the UTXOs of the previous transaction with a replacement by fee. The fee increase increments can be configured with the setting `rbf_fee_increment`. -SAVE TO CACHER From c835cf99b347c907319f6a99873ef43c4ceeeefe Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Thu, 18 Aug 2022 08:07:08 -0700 Subject: [PATCH 010/178] Remove line about non-existent PR 3229 --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a84db6add..1b47512e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,6 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE - Updates to the logging of transaction events (#3139). - Moved puppet-chain to `./contrib/tools` directory and disabled compiling by default (#3200) -- Optimized mempool walk to speed up miner loop 6x (#3229) ### Fixed From cb22fdbb0fe9b833cb7780bf976d1615af5528e2 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Fri, 19 Aug 2022 07:25:26 -0700 Subject: [PATCH 011/178] disable docker builds --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b20a74ac2..859491802 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: workflow_dispatch: inputs: tag: - description: 'The tag to create (optional)' + description: "The tag to create (optional)" required: false concurrency: @@ -96,7 +96,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - platform: [ windows-x64, macos-x64, macos-arm64, linux-x64, linux-musl-x64, linux-armv7, linux-arm64 ] + platform: + [ + windows-x64, + macos-x64, + macos-arm64, + linux-x64, + linux-musl-x64, + linux-armv7, + linux-arm64, + ] steps: - uses: actions/checkout@v2 @@ -117,7 +126,7 @@ jobs: STACKS_NODE_VERSION=${{ github.event.inputs.tag || env.GITHUB_SHA_SHORT }} GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} - + - name: Compress artifact run: zip --junk-paths ${{ matrix.platform }} ./dist/${{ matrix.platform }}/* @@ -128,7 +137,7 @@ jobs: path: ${{ matrix.platform }}.zip call-docker-platforms-workflow: - if: ${{ github.event.inputs.tag != '' }} + if: ${{ false }} uses: stacks-network/stacks-blockchain/.github/workflows/docker-platforms.yml@master with: tag: ${{ github.event.inputs.tag }} @@ -142,6 +151,7 @@ jobs: # - a tag was pushed up # - this workflow was invoked against a non-master branch (a Docker image tag with the name of the branch will be published) build-publish: + if: ${{ false }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -187,6 +197,7 @@ jobs: # - a tag was pushed up # - this workflow was invoked against a non-master branch (a Docker image tag with the name of the branch will be published) build-publish-stretch: + if: ${{ false }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -276,7 +287,16 @@ jobs: - create-release strategy: matrix: - platform: [ windows-x64, macos-x64, macos-arm64, linux-x64, linux-musl-x64, linux-armv7, linux-arm64 ] + platform: + [ + windows-x64, + macos-x64, + macos-arm64, + linux-x64, + linux-musl-x64, + linux-armv7, + linux-arm64, + ] steps: - uses: actions/checkout@v2 From ca1f5d675cfab9957bea0da5e32b8cfd1709d175 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Fri, 19 Aug 2022 08:10:20 -0700 Subject: [PATCH 012/178] build docker images for quay --- .github/workflows/ci.yml | 21 ++++++++++----------- .github/workflows/docker-platforms.yml | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 859491802..8acaa91db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,13 +137,12 @@ jobs: path: ${{ matrix.platform }}.zip call-docker-platforms-workflow: - if: ${{ false }} uses: stacks-network/stacks-blockchain/.github/workflows/docker-platforms.yml@master with: tag: ${{ github.event.inputs.tag }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} # Build docker image, tag it with the git tag and `latest` if running on master branch, and publish under the following conditions # Will publish if: @@ -151,7 +150,6 @@ jobs: # - a tag was pushed up # - this workflow was invoked against a non-master branch (a Docker image tag with the name of the branch will be published) build-publish: - if: ${{ false }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -172,11 +170,12 @@ jobs: type=ref,event=pr ${{ github.event.inputs.tag }} - - name: Login to DockerHub + - name: Login to Quay uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} - name: Build/Tag/Push Image uses: docker/build-push-action@v2 @@ -197,7 +196,6 @@ jobs: # - a tag was pushed up # - this workflow was invoked against a non-master branch (a Docker image tag with the name of the branch will be published) build-publish-stretch: - if: ${{ false }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -234,11 +232,12 @@ jobs: type=ref,event=pr ${{ env.STRETCH_TAG }} - - name: Login to DockerHub + - name: Login to Quay uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} - name: Build/Tag/Push Image uses: docker/build-push-action@v2 diff --git a/.github/workflows/docker-platforms.yml b/.github/workflows/docker-platforms.yml index 7ee44b3ed..fb62a785e 100644 --- a/.github/workflows/docker-platforms.yml +++ b/.github/workflows/docker-platforms.yml @@ -10,9 +10,9 @@ on: required: true type: string secrets: - DOCKERHUB_USERNAME: + QUAY_USERNAME: required: true - DOCKERHUB_PASSWORD: + QUAY_PASSWORD: required: true env: @@ -46,11 +46,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub + - name: Login to QUAY uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} - name: Build/Tag/Push Image uses: docker/build-push-action@v2 @@ -105,11 +106,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub + - name: Login to quay uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} - name: Build/Tag/Push Image uses: docker/build-push-action@v2 From b28b81fea25178e71cfb9d62384ee176e7ff1926 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Fri, 19 Aug 2022 08:22:29 -0700 Subject: [PATCH 013/178] disabling these PR workflows --- .github/workflows/clarity-js-sdk-pr.yml | 46 ++++----- .github/workflows/docs-pr.yml | 130 ++++++++++++------------ 2 files changed, 88 insertions(+), 88 deletions(-) diff --git a/.github/workflows/clarity-js-sdk-pr.yml b/.github/workflows/clarity-js-sdk-pr.yml index fd28738cf..e64a7f61b 100644 --- a/.github/workflows/clarity-js-sdk-pr.yml +++ b/.github/workflows/clarity-js-sdk-pr.yml @@ -28,29 +28,29 @@ jobs: repository: ${{ env.CLARITY_JS_SDK_REPOSITORY }} ref: master - - name: Determine Release Version - run: | - RELEASE_VERSION=$(echo ${GITHUB_REF#refs/*/} | tr / -) - echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV + # - name: Determine Release Version + # run: | + # RELEASE_VERSION=$(echo ${GITHUB_REF#refs/*/} | tr / -) + # echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV - - name: Update SDK Tag - run: sed -i "s@CORE_SDK_TAG = \".*\"@CORE_SDK_TAG = \"$RELEASE_VERSION\"@g" packages/clarity-native-bin/src/index.ts + # - name: Update SDK Tag + # run: sed -i "s@CORE_SDK_TAG = \".*\"@CORE_SDK_TAG = \"$RELEASE_VERSION\"@g" packages/clarity-native-bin/src/index.ts - - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.GH_TOKEN }} - commit-message: "chore: update clarity-native-bin tag" - committer: ${{ env.COMMIT_USER }} <${{ env.COMMIT_EMAIL }}> - author: ${{ env.COMMIT_USER }} <${{ env.COMMIT_EMAIL }}> - branch: auto/update-bin-tag - delete-branch: true - title: "clarity-native-bin tag update: ${{ env.RELEASE_VERSION }}" - labels: | - dependencies - body: | - :robot: This is an automated pull request created from a new release in [stacks-blockchain](https://github.com/blockstack/stacks-blockchain/releases). + # - name: Create Pull Request + # uses: peter-evans/create-pull-request@v3 + # with: + # token: ${{ secrets.GH_TOKEN }} + # commit-message: "chore: update clarity-native-bin tag" + # committer: ${{ env.COMMIT_USER }} <${{ env.COMMIT_EMAIL }}> + # author: ${{ env.COMMIT_USER }} <${{ env.COMMIT_EMAIL }}> + # branch: auto/update-bin-tag + # delete-branch: true + # title: "clarity-native-bin tag update: ${{ env.RELEASE_VERSION }}" + # labels: | + # dependencies + # body: | + # :robot: This is an automated pull request created from a new release in [stacks-blockchain](https://github.com/blockstack/stacks-blockchain/releases). - Updates the clarity-native-bin tag. - assignees: zone117x - reviewers: zone117x + # Updates the clarity-native-bin tag. + # assignees: zone117x + # reviewers: zone117x diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml index 6f43ecb9c..b143447f1 100644 --- a/.github/workflows/docs-pr.yml +++ b/.github/workflows/docs-pr.yml @@ -19,7 +19,7 @@ on: push: branches: [master] -jobs: +jobs: dist: runs-on: ubuntu-latest env: @@ -27,73 +27,73 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Build docs - env: - DOCKER_BUILDKIT: 1 - run: rm -rf docs-output && docker build -o docs-output -f ./.github/actions/docsgen/Dockerfile.docsgen . + # - name: Build docs + # env: + # DOCKER_BUILDKIT: 1 + # run: rm -rf docs-output && docker build -o docs-output -f ./.github/actions/docsgen/Dockerfile.docsgen . - - name: Checkout latest docs - uses: actions/checkout@v2 - with: - token: ${{ secrets.DOCS_GITHUB_TOKEN }} - repository: ${{ env.TARGET_REPOSITORY }} - path: docs.blockstack + # - name: Checkout latest docs + # uses: actions/checkout@v2 + # with: + # token: ${{ secrets.DOCS_GITHUB_TOKEN }} + # repository: ${{ env.TARGET_REPOSITORY }} + # path: docs.blockstack - - name: Branch and commit - id: push - run: | - cd docs.blockstack - git config user.email "kantai+robot@gmail.com" - git config user.name "PR Robot" - git fetch --unshallow - git checkout -b $ROBOT_BRANCH - cp ../docs-output/clarity-reference.json ./src/_data/clarity-reference.json - cp ../docs-output/boot-contracts-reference.json ./src/_data/boot-contracts-reference.json - git add src/_data/clarity-reference.json - git add src/_data/boot-contracts-reference.json - if $(git diff --staged --quiet --exit-code); then - echo "No reference.json changes, stopping" - echo "::set-output name=open_pr::0" - else - git remote add robot https://github.com/$ROBOT_OWNER/$ROBOT_REPO - git commit -m "auto: update Clarity references JSONs from stacks-blockchain@${GITHUB_SHA}" - git push robot $ROBOT_BRANCH - echo "::set-output name=open_pr::1" - fi - - name: Open PR - if: ${{ steps.push.outputs.open_pr == '1' }} - uses: actions/github-script@v2 - with: - github-token: ${{ secrets.DOCS_GITHUB_TOKEN }} - script: | - // get env vars - const process = require("process"); - const robot_owner = process.env.ROBOT_OWNER; - const robot_branch = process.env.ROBOT_BRANCH; - const head = `${robot_owner}:${robot_branch}`; - const owner = process.env.TARGET_OWNER; - const repo = process.env.TARGET_REPO; + # - name: Branch and commit + # id: push + # run: | + # cd docs.blockstack + # git config user.email "kantai+robot@gmail.com" + # git config user.name "PR Robot" + # git fetch --unshallow + # git checkout -b $ROBOT_BRANCH + # cp ../docs-output/clarity-reference.json ./src/_data/clarity-reference.json + # cp ../docs-output/boot-contracts-reference.json ./src/_data/boot-contracts-reference.json + # git add src/_data/clarity-reference.json + # git add src/_data/boot-contracts-reference.json + # if $(git diff --staged --quiet --exit-code); then + # echo "No reference.json changes, stopping" + # echo "::set-output name=open_pr::0" + # else + # git remote add robot https://github.com/$ROBOT_OWNER/$ROBOT_REPO + # git commit -m "auto: update Clarity references JSONs from stacks-blockchain@${GITHUB_SHA}" + # git push robot $ROBOT_BRANCH + # echo "::set-output name=open_pr::1" + # fi + # - name: Open PR + # if: ${{ steps.push.outputs.open_pr == '1' }} + # uses: actions/github-script@v2 + # with: + # github-token: ${{ secrets.DOCS_GITHUB_TOKEN }} + # script: | + # // get env vars + # const process = require("process"); + # const robot_owner = process.env.ROBOT_OWNER; + # const robot_branch = process.env.ROBOT_BRANCH; + # const head = `${robot_owner}:${robot_branch}`; + # const owner = process.env.TARGET_OWNER; + # const repo = process.env.TARGET_REPO; - console.log(`Checking PR with params: head= ${head} owner= ${owner} repo= ${repo}`); + # console.log(`Checking PR with params: head= ${head} owner= ${owner} repo= ${repo}`); - // check if a pull exists - const existingPulls = await github.pulls.list({ - owner, repo, state: "open" }); - const myPulls = existingPulls.data.filter( pull => pull.user.login == robot_owner ); - console.log(myPulls); + # // check if a pull exists + # const existingPulls = await github.pulls.list({ + # owner, repo, state: "open" }); + # const myPulls = existingPulls.data.filter( pull => pull.user.login == robot_owner ); + # console.log(myPulls); - for (myPull of myPulls) { - // close any open PRs - const pull_number = myPull.number; - console.log(`Closing PR: ${ pull_number }`); - await github.pulls.update({ owner, repo, pull_number, state: "closed" }); - } + # for (myPull of myPulls) { + # // close any open PRs + # const pull_number = myPull.number; + # console.log(`Closing PR: ${ pull_number }`); + # await github.pulls.update({ owner, repo, pull_number, state: "closed" }); + # } - // Open PR if one doesn't exist - console.log("Opening the new PR."); - let result = await github.pulls.create({ - owner, repo, head, - base: "master", - title: "Auto: Update API documentation from stacks-blockchain", - body: "Update API documentation from the latest in `stacks-blockchain`", - }); + # // Open PR if one doesn't exist + # console.log("Opening the new PR."); + # let result = await github.pulls.create({ + # owner, repo, head, + # base: "master", + # title: "Auto: Update API documentation from stacks-blockchain", + # body: "Update API documentation from the latest in `stacks-blockchain`", + # }); From e4ab78326efde2e65ee05c87916a00f702ccd07d Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Fri, 19 Aug 2022 08:24:51 -0700 Subject: [PATCH 014/178] inherit secret --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8acaa91db..6633eb3f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,6 +138,7 @@ jobs: call-docker-platforms-workflow: uses: stacks-network/stacks-blockchain/.github/workflows/docker-platforms.yml@master + secrets: inherit with: tag: ${{ github.event.inputs.tag }} secrets: From 1a34c115089434e3f23abf198122cd5d58b4c921 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 31 Aug 2022 13:41:12 -0400 Subject: [PATCH 015/178] feat: release 2.05.0.3.0 --- CHANGELOG.md | 1 + Cargo.toml | 2 +- clarity/src/vm/analysis/mod.rs | 12 +- .../vm/analysis/type_checker/natives/mod.rs | 2 +- clarity/src/vm/ast/errors.rs | 5 + clarity/src/vm/ast/mod.rs | 83 ++- clarity/src/vm/ast/parser/mod.rs | 60 +- clarity/src/vm/ast/stack_depth_checker.rs | 27 + clarity/src/vm/clarity.rs | 5 +- clarity/src/vm/contexts.rs | 51 +- clarity/src/vm/database/clarity_db.rs | 14 +- clarity/src/vm/docs/contracts.rs | 4 +- clarity/src/vm/docs/mod.rs | 21 +- clarity/src/vm/mod.rs | 20 +- clarity/src/vm/test_util/mod.rs | 7 + clarity/src/vm/tests/assets.rs | 51 +- clarity/src/vm/tests/contracts.rs | 42 +- clarity/src/vm/tests/events.rs | 3 +- clarity/src/vm/tests/traits.rs | 63 +- src/chainstate/burn/db/sortdb.rs | 121 +++- src/chainstate/coordinator/tests.rs | 10 +- src/chainstate/stacks/boot/contract_tests.rs | 49 +- src/chainstate/stacks/boot/mod.rs | 2 + src/chainstate/stacks/db/blocks.rs | 22 +- src/chainstate/stacks/db/contracts.rs | 1 - src/chainstate/stacks/db/mod.rs | 22 +- src/chainstate/stacks/db/transactions.rs | 361 ++++++++--- src/chainstate/stacks/db/unconfirmed.rs | 3 + src/chainstate/stacks/miner.rs | 192 ++++-- src/chainstate/stacks/mod.rs | 1 + src/chainstate/stacks/transaction.rs | 1 - src/clarity_cli.rs | 71 ++- src/clarity_vm/clarity.rs | 84 ++- src/clarity_vm/database/mod.rs | 9 + src/clarity_vm/tests/analysis_costs.rs | 7 +- src/clarity_vm/tests/costs.rs | 25 +- src/clarity_vm/tests/forking.rs | 15 +- src/clarity_vm/tests/large_contract.rs | 33 +- src/clarity_vm/tests/simple_tests.rs | 1 + src/core/mod.rs | 3 + src/net/http.rs | 6 +- src/net/mod.rs | 3 + src/net/p2p.rs | 21 +- src/net/relay.rs | 584 +++++++++++++++--- src/net/rpc.rs | 131 ++-- stacks-common/src/util/macros.rs | 2 +- testnet/stacks-node/Cargo.toml | 3 + testnet/stacks-node/src/config.rs | 4 + testnet/stacks-node/src/neon_node.rs | 183 +++++- testnet/stacks-node/src/run_loop/neon.rs | 10 +- testnet/stacks-node/src/syncctl.rs | 4 +- .../src/tests/neon_integrations.rs | 22 +- 52 files changed, 2017 insertions(+), 462 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b47512e2..b14cc2aac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE state machine to only report fatal errors (#3228) - Spawn the p2p thread before processing number of sortitions. Fixes issue (#3216) where sync from genesis paused (#3236) - Drop well-formed "problematic" transactions that result in miner performance degradation (#3212) +- Ignore blocks that include problematic transactions ## [2.05.0.2.1] diff --git a/Cargo.toml b/Cargo.toml index 58eccaccf..e5e7ca7fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,7 +117,7 @@ default = ["developer-mode"] developer-mode = [] monitoring_prom = ["prometheus"] slog_json = ["slog-json", "stacks_common/slog_json", "clarity/slog_json"] - +testing = [] [profile.dev.package.regex] opt-level = 2 diff --git a/clarity/src/vm/analysis/mod.rs b/clarity/src/vm/analysis/mod.rs index b6b46d8b6..95833fa21 100644 --- a/clarity/src/vm/analysis/mod.rs +++ b/clarity/src/vm/analysis/mod.rs @@ -40,12 +40,20 @@ use self::contract_interface_builder::build_contract_interface; use self::read_only_checker::ReadOnlyChecker; use self::trait_checker::TraitChecker; use self::type_checker::TypeChecker; +use crate::vm::ast::build_ast_with_rules; +use crate::vm::ast::ASTRules; /// Used by CLI tools like the docs generator. Not used in production pub fn mem_type_check(snippet: &str) -> CheckResult<(Option, ContractAnalysis)> { - use crate::vm::ast::parse; let contract_identifier = QualifiedContractIdentifier::transient(); - let mut contract = parse(&contract_identifier, snippet).unwrap(); + let mut contract = build_ast_with_rules( + &contract_identifier, + snippet, + &mut (), + ASTRules::PrecheckSize, + ) + .unwrap() + .expressions; let mut marf = MemoryBackingStore::new(); let mut analysis_db = marf.as_analysis_db(); let cost_tracker = LimitedCostTracker::new_free(); diff --git a/clarity/src/vm/analysis/type_checker/natives/mod.rs b/clarity/src/vm/analysis/type_checker/natives/mod.rs index befca5542..3a7899c64 100644 --- a/clarity/src/vm/analysis/type_checker/natives/mod.rs +++ b/clarity/src/vm/analysis/type_checker/natives/mod.rs @@ -207,7 +207,7 @@ pub fn check_special_tuple_cons( })?; let tuple_signature = TupleTypeSignature::try_from(tuple_type_data) - .map_err(|_| CheckErrors::BadTupleConstruction)?; + .map_err(|_e| CheckErrors::BadTupleConstruction)?; Ok(TypeSignature::TupleType(tuple_signature)) } diff --git a/clarity/src/vm/ast/errors.rs b/clarity/src/vm/ast/errors.rs index c7cb11e20..3ae20a6d4 100644 --- a/clarity/src/vm/ast/errors.rs +++ b/clarity/src/vm/ast/errors.rs @@ -31,6 +31,7 @@ pub enum ParseErrors { MemoryBalanceExceeded(u64, u64), TooManyExpressions, ExpressionStackDepthTooDeep, + VaryExpressionStackDepthTooDeep, FailedCapturingInput, SeparatorExpected(String), SeparatorExpectedAfterColon(String), @@ -235,6 +236,10 @@ impl DiagnosableError for ParseErrors { "AST has too deep of an expression nesting. The maximum stack depth is {}", MAX_CALL_STACK_DEPTH ), + ParseErrors::VaryExpressionStackDepthTooDeep => format!( + "AST has too deep of an expression nesting. The maximum stack depth is {}", + MAX_CALL_STACK_DEPTH + ), ParseErrors::InvalidCharactersDetected => format!("invalid characters detected"), ParseErrors::InvalidEscaping => format!("invalid escaping detected in string"), ParseErrors::CostComputationFailed(s) => format!("Cost computation failed: {}", s), diff --git a/clarity/src/vm/ast/mod.rs b/clarity/src/vm/ast/mod.rs index 9e0d27cbf..4688a5b96 100644 --- a/clarity/src/vm/ast/mod.rs +++ b/clarity/src/vm/ast/mod.rs @@ -33,6 +33,7 @@ use self::definition_sorter::DefinitionSorter; use self::errors::ParseResult; use self::expression_identifier::ExpressionIdentifier; use self::stack_depth_checker::StackDepthChecker; +use self::stack_depth_checker::VaryStackDepthChecker; use self::sugar_expander::SugarExpander; use self::traits_resolver::TraitsResolver; use self::types::BuildASTPass; @@ -40,6 +41,7 @@ pub use self::types::ContractAST; use crate::vm::costs::cost_functions::ClarityCostFunction; /// Legacy function +#[cfg(test)] pub fn parse( contract_identifier: &QualifiedContractIdentifier, source_code: &str, @@ -48,7 +50,43 @@ pub fn parse( Ok(ast.expressions) } -pub fn build_ast( +// AST parser rulesets to apply. +define_u8_enum!(ASTRules { + Typical = 0, + PrecheckSize = 1 +}); + +/// This is the part of the AST parser that runs without respect to cost analysis, specifically +/// pertaining to verifying that the AST is reasonably-sized. +/// Used mainly to filter transactions that might be too costly, as an optimization heuristic. +pub fn ast_check_size( + contract_identifier: &QualifiedContractIdentifier, + source_code: &str, +) -> ParseResult { + let pre_expressions = parser::parse(source_code)?; + let mut contract_ast = ContractAST::new(contract_identifier.clone(), pre_expressions); + StackDepthChecker::run_pass(&mut contract_ast)?; + VaryStackDepthChecker::run_pass(&mut contract_ast)?; + Ok(contract_ast) +} + +/// Build an AST according to a ruleset +pub fn build_ast_with_rules( + contract_identifier: &QualifiedContractIdentifier, + source_code: &str, + cost_track: &mut T, + ruleset: ASTRules, +) -> ParseResult { + match ruleset { + ASTRules::Typical => build_ast_typical(contract_identifier, source_code, cost_track), + ASTRules::PrecheckSize => { + build_ast_precheck_size(contract_identifier, source_code, cost_track) + } + } +} + +/// Build an AST with the typical rules +fn build_ast_typical( contract_identifier: &QualifiedContractIdentifier, source_code: &str, cost_track: &mut T, @@ -58,7 +96,7 @@ pub fn build_ast( cost_track, source_code.len() as u64, )?; - let pre_expressions = parser::parse(source_code)?; + let pre_expressions = parser::parse_no_stack_limit(source_code)?; let mut contract_ast = ContractAST::new(contract_identifier.clone(), pre_expressions); StackDepthChecker::run_pass(&mut contract_ast)?; ExpressionIdentifier::run_pre_expression_pass(&mut contract_ast)?; @@ -69,14 +107,49 @@ pub fn build_ast( Ok(contract_ast) } +/// Built an AST, but pre-check the size of the AST before doing more work +fn build_ast_precheck_size( + contract_identifier: &QualifiedContractIdentifier, + source_code: &str, + cost_track: &mut T, +) -> ParseResult { + runtime_cost( + ClarityCostFunction::AstParse, + cost_track, + source_code.len() as u64, + )?; + let mut contract_ast = ast_check_size(contract_identifier, source_code)?; + ExpressionIdentifier::run_pre_expression_pass(&mut contract_ast)?; + DefinitionSorter::run_pass(&mut contract_ast, cost_track)?; + TraitsResolver::run_pass(&mut contract_ast)?; + SugarExpander::run_pass(&mut contract_ast)?; + ExpressionIdentifier::run_expression_pass(&mut contract_ast)?; + Ok(contract_ast) +} + +/// Test compatibility +#[cfg(any(test, feature = "testing"))] +pub fn build_ast( + contract_identifier: &QualifiedContractIdentifier, + source_code: &str, + cost_track: &mut T, +) -> ParseResult { + build_ast_typical(contract_identifier, source_code, cost_track) +} + #[cfg(test)] mod test { - use std::collections::HashMap; - - use crate::vm::ast::build_ast; + use crate::vm::ast::errors::ParseErrors; + use crate::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER; + use crate::vm::ast::{build_ast, build_ast_with_rules, ASTRules}; use crate::vm::costs::LimitedCostTracker; + use crate::vm::costs::*; use crate::vm::representations::depth_traverse; use crate::vm::types::QualifiedContractIdentifier; + use crate::vm::ClarityCostFunction; + use crate::vm::ClarityName; + use crate::vm::MAX_CALL_STACK_DEPTH; + use std::collections::HashMap; #[test] fn test_expression_identification_tuples() { diff --git a/clarity/src/vm/ast/parser/mod.rs b/clarity/src/vm/ast/parser/mod.rs index 6782d69a5..e0c0605f5 100644 --- a/clarity/src/vm/ast/parser/mod.rs +++ b/clarity/src/vm/ast/parser/mod.rs @@ -26,6 +26,9 @@ use stacks_common::util::hash::hex_bytes; use std::cmp; use std::convert::TryInto; +use crate::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER; +use crate::vm::MAX_CALL_STACK_DEPTH; + pub const CONTRACT_MIN_NAME_LENGTH: usize = 1; pub const CONTRACT_MAX_NAME_LENGTH: usize = 40; @@ -124,14 +127,8 @@ lazy_static! { ); pub static ref CLARITY_NAME_REGEX: String = format!(r#"([[:word:]]|[-!?+<>=/*]){{1,{}}}"#, MAX_STRING_LEN); -} -pub fn lex(input: &str) -> ParseResult> { - // Aaron: I'd like these to be static, but that'd require using - // lazy_static (or just hand implementing that), and I'm not convinced - // it's worth either (1) an extern macro, or (2) the complexity of hand implementing. - - let lex_matchers: &[LexMatcher] = &[ + static ref lex_matchers: Vec = vec![ LexMatcher::new( r##"u"(?P((\\")|([[ -~]&&[^"]]))*)""##, TokenType::StringUTF8Literal, @@ -187,7 +184,10 @@ pub fn lex(input: &str) -> ParseResult> { TokenType::Variable, ), ]; +} +/// Lex the contract, permitting nesting of lists and tuples up to `max_nesting`. +fn inner_lex(input: &str, max_nesting: u64) -> ParseResult> { let mut context = LexContext::ExpectNothing; let mut line_indices = get_lines_at(input); @@ -198,6 +198,9 @@ pub fn lex(input: &str) -> ParseResult> { let mut munch_index = 0; let mut column_pos: u32 = 1; let mut did_match = true; + + let mut nesting_depth = 0; + while did_match && munch_index < input.len() { if let Some(next_line_ix) = next_line_break { if munch_index > next_line_ix { @@ -255,9 +258,19 @@ pub fn lex(input: &str) -> ParseResult> { let token = match matcher.handler { TokenType::LParens => { context = LexContext::ExpectNothing; + nesting_depth += 1; + if nesting_depth > max_nesting { + return Err(ParseError::new( + ParseErrors::VaryExpressionStackDepthTooDeep, + )); + } Ok(LexItem::LeftParen) } - TokenType::RParens => Ok(LexItem::RightParen), + TokenType::RParens => { + // if this underflows, the contract is invalid anyway + nesting_depth = nesting_depth.saturating_sub(1); + Ok(LexItem::RightParen) + } TokenType::Whitespace => { context = LexContext::ExpectNothing; Ok(LexItem::Whitespace) @@ -274,9 +287,19 @@ pub fn lex(input: &str) -> ParseResult> { } TokenType::LCurly => { context = LexContext::ExpectNothing; + nesting_depth += 1; + if nesting_depth > max_nesting { + return Err(ParseError::new( + ParseErrors::VaryExpressionStackDepthTooDeep, + )); + } Ok(LexItem::LeftCurly) } - TokenType::RCurly => Ok(LexItem::RightCurly), + TokenType::RCurly => { + // if this underflows, the contract is invalid anyway + nesting_depth = nesting_depth.saturating_sub(1); + Ok(LexItem::RightCurly) + } TokenType::Variable => { let value = get_value_or_err(current_slice, captures)?; if value.contains("#") { @@ -427,6 +450,13 @@ pub fn lex(input: &str) -> ParseResult> { } } +pub fn lex(input: &str) -> ParseResult> { + inner_lex( + input, + AST_CALL_STACK_DEPTH_BUFFER + (MAX_CALL_STACK_DEPTH as u64) + 1, + ) +} + fn unescape_ascii_chars(escaped_str: String, allow_unicode_escape: bool) -> ParseResult { let mut unescaped_str = String::new(); let mut chars = escaped_str.chars().into_iter(); @@ -689,7 +719,15 @@ pub fn parse_lexed(mut input: Vec<(LexItem, u32, u32)>) -> ParseResult ParseResult> { - let lexed = lex(input)?; + let lexed = inner_lex( + input, + AST_CALL_STACK_DEPTH_BUFFER + (MAX_CALL_STACK_DEPTH as u64) + 1, + )?; + parse_lexed(lexed) +} + +pub fn parse_no_stack_limit(input: &str) -> ParseResult> { + let lexed = inner_lex(input, u64::MAX)?; parse_lexed(lexed) } @@ -697,11 +735,13 @@ pub fn parse(input: &str) -> ParseResult> { mod test { use crate::vm::ast; use crate::vm::ast::errors::{ParseError, ParseErrors}; + use crate::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER; use crate::vm::representations::{PreSymbolicExpression, PreSymbolicExpressionType}; use crate::vm::types::TraitIdentifier; use crate::vm::types::{ CharType, PrincipalData, QualifiedContractIdentifier, SequenceData, Value, }; + use crate::vm::MAX_CALL_STACK_DEPTH; fn make_atom( x: &str, diff --git a/clarity/src/vm/ast/stack_depth_checker.rs b/clarity/src/vm/ast/stack_depth_checker.rs index ff42ff2bb..345cab138 100644 --- a/clarity/src/vm/ast/stack_depth_checker.rs +++ b/clarity/src/vm/ast/stack_depth_checker.rs @@ -18,6 +18,7 @@ use crate::vm::ast::errors::{ParseError, ParseErrors, ParseResult}; use crate::vm::ast::types::{BuildASTPass, ContractAST}; use crate::vm::representations::PreSymbolicExpression; use crate::vm::representations::PreSymbolicExpressionType::List; +use crate::vm::representations::PreSymbolicExpressionType::Tuple; use crate::vm::MAX_CALL_STACK_DEPTH; @@ -50,3 +51,29 @@ impl BuildASTPass for StackDepthChecker { check(&contract_ast.pre_expressions, 0) } } + +fn check_vary(args: &[PreSymbolicExpression], depth: u64) -> ParseResult<()> { + if depth >= (AST_CALL_STACK_DEPTH_BUFFER + MAX_CALL_STACK_DEPTH as u64) { + return Err(ParseErrors::VaryExpressionStackDepthTooDeep.into()); + } + for expression in args.iter() { + match expression.pre_expr { + List(ref exprs) => check_vary(exprs, depth + 1), + Tuple(ref exprs) => check_vary(exprs, depth + 1), + _ => { + // Other symbolic expressions don't have depth + // impacts. + Ok(()) + } + }?; + } + Ok(()) +} + +pub struct VaryStackDepthChecker; + +impl BuildASTPass for VaryStackDepthChecker { + fn run_pass(contract_ast: &mut ContractAST) -> ParseResult<()> { + check_vary(&contract_ast.pre_expressions, 0) + } +} diff --git a/clarity/src/vm/clarity.rs b/clarity/src/vm/clarity.rs index 7bc80371d..fbf377ddb 100644 --- a/clarity/src/vm/clarity.rs +++ b/clarity/src/vm/clarity.rs @@ -2,6 +2,7 @@ use crate::vm::analysis; use crate::vm::analysis::ContractAnalysis; use crate::vm::analysis::{AnalysisDatabase, CheckError, CheckErrors}; use crate::vm::ast::errors::{ParseError, ParseErrors}; +use crate::vm::ast::ASTRules; use crate::vm::ast::ContractAST; use crate::vm::contexts::Environment; use crate::vm::contexts::{AssetMap, OwnedEnvironment}; @@ -164,9 +165,11 @@ pub trait TransactionConnection: ClarityConnection { &mut self, identifier: &QualifiedContractIdentifier, contract_content: &str, + ast_rules: ASTRules, ) -> Result<(ContractAST, ContractAnalysis), Error> { self.with_analysis_db(|db, mut cost_track| { - let ast_result = ast::build_ast(identifier, contract_content, &mut cost_track); + let ast_result = + ast::build_ast_with_rules(identifier, contract_content, &mut cost_track, ast_rules); let mut contract_ast = match ast_result { Ok(x) => x, diff --git a/clarity/src/vm/contexts.rs b/clarity/src/vm/contexts.rs index b458f3bc6..67275dabe 100644 --- a/clarity/src/vm/contexts.rs +++ b/clarity/src/vm/contexts.rs @@ -20,6 +20,7 @@ use std::fmt; use std::mem::replace; use crate::vm::ast; +use crate::vm::ast::ASTRules; use crate::vm::ast::ContractAST; use crate::vm::callables::{DefinedFunction, FunctionIdentifier}; use crate::vm::contracts::Contract; @@ -43,7 +44,6 @@ use crate::vm::types::{ Value, }; use crate::vm::{eval, is_reserved}; - use crate::{types::chainstate::StacksBlockId, types::StacksEpochId}; use crate::vm::costs::cost_functions::ClarityCostFunction; @@ -635,9 +635,10 @@ impl<'a> OwnedEnvironment<'a> { &mut self, contract_identifier: QualifiedContractIdentifier, contract_content: &str, + ast_rules: ASTRules, ) -> Result<((), AssetMap, Vec)> { self.execute_in_env(contract_identifier.issuer.clone().into(), |exec_env| { - exec_env.initialize_contract(contract_identifier, contract_content) + exec_env.initialize_contract(contract_identifier, contract_content, ast_rules) }) } @@ -712,15 +713,25 @@ impl<'a> OwnedEnvironment<'a> { ) } + pub fn eval_read_only_with_rules( + &mut self, + contract: &QualifiedContractIdentifier, + program: &str, + ast_rules: ast::ASTRules, + ) -> Result<(Value, AssetMap, Vec)> { + self.execute_in_env( + QualifiedContractIdentifier::transient().issuer.into(), + |exec_env| exec_env.eval_read_only_with_rules(contract, program, ast_rules), + ) + } + + #[cfg(any(test, feature = "testing"))] pub fn eval_read_only( &mut self, contract: &QualifiedContractIdentifier, program: &str, ) -> Result<(Value, AssetMap, Vec)> { - self.execute_in_env( - QualifiedContractIdentifier::transient().issuer.into(), - |exec_env| exec_env.eval_read_only(contract, program), - ) + self.eval_read_only_with_rules(contract, program, ast::ASTRules::Typical) } pub fn begin(&mut self) { @@ -857,12 +868,14 @@ impl<'a, 'b> Environment<'a, 'b> { ) } - pub fn eval_read_only( + pub fn eval_read_only_with_rules( &mut self, contract_identifier: &QualifiedContractIdentifier, program: &str, + rules: ast::ASTRules, ) -> Result { - let parsed = ast::build_ast(contract_identifier, program, self)?.expressions; + let parsed = + ast::build_ast_with_rules(contract_identifier, program, self, rules)?.expressions; if parsed.len() < 1 { return Err(RuntimeErrorType::ParseError( @@ -895,10 +908,19 @@ impl<'a, 'b> Environment<'a, 'b> { result } - pub fn eval_raw(&mut self, program: &str) -> Result { + #[cfg(any(test, feature = "testing"))] + pub fn eval_read_only( + &mut self, + contract_identifier: &QualifiedContractIdentifier, + program: &str, + ) -> Result { + self.eval_read_only_with_rules(contract_identifier, program, ast::ASTRules::Typical) + } + + pub fn eval_raw_with_rules(&mut self, program: &str, rules: ast::ASTRules) -> Result { let contract_id = QualifiedContractIdentifier::transient(); - let parsed = ast::build_ast(&contract_id, program, self)?.expressions; + let parsed = ast::build_ast_with_rules(&contract_id, program, self, rules)?.expressions; if parsed.len() < 1 { return Err(RuntimeErrorType::ParseError( "Expected a program of at least length 1".to_string(), @@ -910,6 +932,11 @@ impl<'a, 'b> Environment<'a, 'b> { result } + #[cfg(any(test, feature = "testing"))] + pub fn eval_raw(&mut self, program: &str) -> Result { + self.eval_raw_with_rules(program, ast::ASTRules::Typical) + } + /// Used only for contract-call! cost short-circuiting. Once the short-circuited cost /// has been evaluated and assessed, the contract-call! itself is executed "for free". pub fn run_free(&mut self, to_run: F) -> A @@ -1061,8 +1088,10 @@ impl<'a, 'b> Environment<'a, 'b> { &mut self, contract_identifier: QualifiedContractIdentifier, contract_content: &str, + ast_rules: ASTRules, ) -> Result<()> { - let contract_ast = ast::build_ast(&contract_identifier, contract_content, self)?; + let contract_ast = + ast::build_ast_with_rules(&contract_identifier, contract_content, self, ast_rules)?; self.initialize_contract_from_ast(contract_identifier, &contract_ast, &contract_content) } diff --git a/clarity/src/vm/database/clarity_db.rs b/clarity/src/vm/database/clarity_db.rs index 0f60f9a3d..abefb834e 100644 --- a/clarity/src/vm/database/clarity_db.rs +++ b/clarity/src/vm/database/clarity_db.rs @@ -18,6 +18,7 @@ use std::collections::{HashMap, VecDeque}; use std::convert::{TryFrom, TryInto}; use crate::vm::analysis::{AnalysisDatabase, ContractAnalysis}; +use crate::vm::ast::ASTRules; use crate::vm::contracts::Contract; use crate::vm::costs::CostOverflowingMath; use crate::vm::costs::ExecutionCost; @@ -107,6 +108,7 @@ pub trait BurnStateDB { ) -> Option; fn get_stacks_epoch(&self, height: u32) -> Option; fn get_stacks_epoch_by_epoch_id(&self, epoch_id: &StacksEpochId) -> Option; + fn get_ast_rules(&self, height: u32) -> ASTRules; } impl HeadersDB for &dyn HeadersDB { @@ -153,6 +155,10 @@ impl BurnStateDB for &dyn BurnStateDB { fn get_stacks_epoch_by_epoch_id(&self, epoch_id: &StacksEpochId) -> Option { (*self).get_stacks_epoch_by_epoch_id(epoch_id) } + + fn get_ast_rules(&self, height: u32) -> ASTRules { + (*self).get_ast_rules(height) + } } pub struct NullHeadersDB {} @@ -236,6 +242,10 @@ impl BurnStateDB for NullBurnStateDB { fn get_stacks_epoch_by_epoch_id(&self, _epoch_id: &StacksEpochId) -> Option { self.get_stacks_epoch(0) } + + fn get_ast_rules(&self, _height: u32) -> ASTRules { + ASTRules::Typical + } } impl<'a> ClarityDatabase<'a> { @@ -590,7 +600,9 @@ impl<'a> ClarityDatabase<'a> { /// Get the last-known burnchain block height. /// Note that this is _not_ the burnchain height in which this block was mined! - /// This is the burnchain block height of its parent. + /// This is the burnchain block height of the parent of the Stacks block at the current Stacks + /// block height (i.e. that returned by `get_index_block_header_hash` for + /// `get_current_block_height`). pub fn get_current_burnchain_block_height(&mut self) -> u32 { let cur_stacks_height = self.store.get_current_block_height(); let last_mined_bhh = if cur_stacks_height == 0 { diff --git a/clarity/src/vm/docs/contracts.rs b/clarity/src/vm/docs/contracts.rs index c826b9195..704fc0eb6 100644 --- a/clarity/src/vm/docs/contracts.rs +++ b/clarity/src/vm/docs/contracts.rs @@ -6,6 +6,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::iter::FromIterator; use crate::types::StacksEpochId; +use crate::vm::ast::{build_ast_with_rules, ASTRules}; use crate::vm::contexts::GlobalContext; use crate::vm::costs::LimitedCostTracker; use crate::vm::database::MemoryBackingStore; @@ -76,7 +77,8 @@ fn doc_execute(program: &str) -> Result, vm::Error> { DOCS_GENERATION_EPOCH, ); global_context.execute(|g| { - let parsed = vm::ast::build_ast(&contract_id, program, &mut ())?.expressions; + let parsed = build_ast_with_rules(&contract_id, program, &mut (), ASTRules::PrecheckSize)? + .expressions; vm::eval_all(&parsed, &mut contract_context, g) }) } diff --git a/clarity/src/vm/docs/mod.rs b/clarity/src/vm/docs/mod.rs index 10c2b92dd..7523b04c1 100644 --- a/clarity/src/vm/docs/mod.rs +++ b/clarity/src/vm/docs/mod.rs @@ -1816,6 +1816,7 @@ mod test { vm::database::{ClarityDatabase, MemoryBackingStore}, }; + use crate::vm::ast::ASTRules; use crate::vm::costs::ExecutionCost; struct DocHeadersDB {} @@ -1895,6 +1896,10 @@ mod test { fn get_stacks_epoch_by_epoch_id(&self, epoch_id: &StacksEpochId) -> Option { self.get_stacks_epoch(0) } + + fn get_ast_rules(&self, height: u32) -> ASTRules { + ASTRules::PrecheckSize + } } fn docs_execute(store: &mut MemoryBackingStore, program: &str) { @@ -2042,11 +2047,19 @@ mod test { ) .unwrap(); - env.initialize_contract(contract_id, &token_contract_content) - .unwrap(); + env.initialize_contract( + contract_id, + &token_contract_content, + ASTRules::PrecheckSize, + ) + .unwrap(); - env.initialize_contract(trait_def_id, super::DEFINE_TRAIT_API.example) - .unwrap(); + env.initialize_contract( + trait_def_id, + super::DEFINE_TRAIT_API.example, + ASTRules::PrecheckSize, + ) + .unwrap(); } let example = &func_api.example; diff --git a/clarity/src/vm/mod.rs b/clarity/src/vm/mod.rs index 4cf1d5868..ed0fc089d 100644 --- a/clarity/src/vm/mod.rs +++ b/clarity/src/vm/mod.rs @@ -79,7 +79,7 @@ use crate::vm::costs::cost_functions::ClarityCostFunction; pub use crate::vm::functions::stx_transfer_consolidated; use std::convert::{TryFrom, TryInto}; -const MAX_CALL_STACK_DEPTH: usize = 64; +pub const MAX_CALL_STACK_DEPTH: usize = 64; fn lookup_variable(name: &str, context: &LocalContext, env: &mut Environment) -> Result { if name.starts_with(char::is_numeric) || name.starts_with('\'') { @@ -389,8 +389,18 @@ pub fn eval_all( /// that the result is the same before returning the result #[cfg(any(test, feature = "testing"))] pub fn execute_on_network(program: &str, use_mainnet: bool) -> Result> { - let epoch_200_result = execute_in_epoch(program, StacksEpochId::Epoch20, use_mainnet); - let epoch_205_result = execute_in_epoch(program, StacksEpochId::Epoch2_05, use_mainnet); + let epoch_200_result = execute_in_epoch( + program, + StacksEpochId::Epoch20, + ast::ASTRules::PrecheckSize, + use_mainnet, + ); + let epoch_205_result = execute_in_epoch( + program, + StacksEpochId::Epoch2_05, + ast::ASTRules::PrecheckSize, + use_mainnet, + ); assert_eq!( epoch_200_result, epoch_205_result, "Epoch 2.0 and 2.05 should have same execution result, but did not for program `{}`", @@ -409,6 +419,7 @@ pub fn execute(program: &str) -> Result> { pub fn execute_in_epoch( program: &str, epoch: StacksEpochId, + ast_rules: ast::ASTRules, use_mainnet: bool, ) -> Result> { use crate::vm::database::MemoryBackingStore; @@ -420,7 +431,8 @@ pub fn execute_in_epoch( let mut global_context = GlobalContext::new(use_mainnet, conn, LimitedCostTracker::new_free(), epoch); global_context.execute(|g| { - let parsed = ast::build_ast(&contract_id, program, &mut ())?.expressions; + let parsed = + ast::build_ast_with_rules(&contract_id, program, &mut (), ast_rules)?.expressions; eval_all(&parsed, &mut contract_context, g) }) } diff --git a/clarity/src/vm/test_util/mod.rs b/clarity/src/vm/test_util/mod.rs index b83b3174c..90797828a 100644 --- a/clarity/src/vm/test_util/mod.rs +++ b/clarity/src/vm/test_util/mod.rs @@ -1,3 +1,4 @@ +use crate::vm::ast::ASTRules; use crate::vm::costs::ExecutionCost; use crate::vm::database::{BurnStateDB, HeadersDB}; use crate::vm::execute as vm_execute; @@ -19,12 +20,14 @@ use stacks_common::types::{StacksEpochId, PEER_VERSION_EPOCH_2_0}; pub struct UnitTestBurnStateDB { pub epoch_id: StacksEpochId, + pub ast_rules: ASTRules, } pub struct UnitTestHeaderDB {} pub const TEST_HEADER_DB: UnitTestHeaderDB = UnitTestHeaderDB {}; pub const TEST_BURN_STATE_DB: UnitTestBurnStateDB = UnitTestBurnStateDB { epoch_id: StacksEpochId::Epoch20, + ast_rules: ASTRules::Typical, }; pub fn execute(s: &str) -> Value { @@ -169,4 +172,8 @@ impl BurnStateDB for UnitTestBurnStateDB { fn get_stacks_epoch_by_epoch_id(&self, _epoch_id: &StacksEpochId) -> Option { self.get_stacks_epoch(0) } + + fn get_ast_rules(&self, _height: u32) -> ASTRules { + self.ast_rules + } } diff --git a/clarity/src/vm/tests/assets.rs b/clarity/src/vm/tests/assets.rs index 0fdb8ba82..b060c341e 100644 --- a/clarity/src/vm/tests/assets.rs +++ b/clarity/src/vm/tests/assets.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +use crate::vm::ast::ASTRules; use crate::vm::contexts::{AssetMap, AssetMapEntry, GlobalContext, OwnedEnvironment}; use crate::vm::contracts::Contract; use crate::vm::errors::{CheckErrors, Error, RuntimeErrorType}; @@ -178,10 +179,14 @@ fn test_native_stx_ops(owned_env: &mut OwnedEnvironment) { QualifiedContractIdentifier::new(p1_std_principal_data.clone(), "second".into()); owned_env - .initialize_contract(token_contract_id.clone(), contract) + .initialize_contract(token_contract_id.clone(), contract, ASTRules::PrecheckSize) .unwrap(); owned_env - .initialize_contract(second_contract_id.clone(), contract_second) + .initialize_contract( + second_contract_id.clone(), + contract_second, + ASTRules::PrecheckSize, + ) .unwrap(); owned_env.stx_faucet(&(p1_principal), u128::MAX - 1500); @@ -533,7 +538,11 @@ fn test_simple_token_system(owned_env: &mut OwnedEnvironment) { let contract_principal = PrincipalData::Contract(token_contract_id.clone()); owned_env - .initialize_contract(token_contract_id.clone(), tokens_contract) + .initialize_contract( + token_contract_id.clone(), + tokens_contract, + ASTRules::PrecheckSize, + ) .unwrap(); let (result, asset_map, _events) = execute_transaction( @@ -824,7 +833,7 @@ fn test_total_supply(owned_env: &mut OwnedEnvironment) { let token_contract_id = QualifiedContractIdentifier::new(p1_std_principal_data.clone(), "tokens".into()); let err = owned_env - .initialize_contract(token_contract_id.clone(), bad_0) + .initialize_contract(token_contract_id.clone(), bad_0, ASTRules::PrecheckSize) .unwrap_err(); assert!(match err { Error::Unchecked(CheckErrors::TypeValueError(_, _)) => true, @@ -832,7 +841,7 @@ fn test_total_supply(owned_env: &mut OwnedEnvironment) { }); let err = owned_env - .initialize_contract(token_contract_id.clone(), bad_1) + .initialize_contract(token_contract_id.clone(), bad_1, ASTRules::PrecheckSize) .unwrap_err(); assert!(match err { Error::Unchecked(CheckErrors::TypeValueError(_, _)) => true, @@ -840,7 +849,7 @@ fn test_total_supply(owned_env: &mut OwnedEnvironment) { }); owned_env - .initialize_contract(token_contract_id.clone(), contract) + .initialize_contract(token_contract_id.clone(), contract, ASTRules::PrecheckSize) .unwrap(); let (result, _asset_map, _events) = execute_transaction( @@ -907,13 +916,25 @@ fn test_overlapping_nfts(owned_env: &mut OwnedEnvironment) { QualifiedContractIdentifier::new(p1_std_principal_data.clone(), "names-2".into()); owned_env - .initialize_contract(tokens_contract_id.clone(), tokens_contract) + .initialize_contract( + tokens_contract_id.clone(), + tokens_contract, + ASTRules::PrecheckSize, + ) .unwrap(); owned_env - .initialize_contract(names_contract_id.clone(), names_contract) + .initialize_contract( + names_contract_id.clone(), + names_contract, + ASTRules::PrecheckSize, + ) .unwrap(); owned_env - .initialize_contract(names_2_contract_id.clone(), names_contract) + .initialize_contract( + names_2_contract_id.clone(), + names_contract, + ASTRules::PrecheckSize, + ) .unwrap(); } @@ -960,13 +981,21 @@ fn test_simple_naming_system(owned_env: &mut OwnedEnvironment) { let name_hash_cheap_0 = execute("(hash160 100001)"); owned_env - .initialize_contract(tokens_contract_id.clone(), tokens_contract) + .initialize_contract( + tokens_contract_id.clone(), + tokens_contract, + ASTRules::PrecheckSize, + ) .unwrap(); let names_contract_id = QualifiedContractIdentifier::new(p1_std_principal_data.clone(), "names".into()); owned_env - .initialize_contract(names_contract_id.clone(), names_contract) + .initialize_contract( + names_contract_id.clone(), + names_contract, + ASTRules::PrecheckSize, + ) .unwrap(); let (result, _asset_map, _events) = execute_transaction( diff --git a/clarity/src/vm/tests/contracts.rs b/clarity/src/vm/tests/contracts.rs index b0508030a..def0a5180 100644 --- a/clarity/src/vm/tests/contracts.rs +++ b/clarity/src/vm/tests/contracts.rs @@ -18,6 +18,7 @@ use crate::types::chainstate::BlockHeaderHash; use crate::types::chainstate::StacksBlockId; use crate::vm::ast; use crate::vm::ast::errors::ParseErrors; +use crate::vm::ast::ASTRules; use crate::vm::contexts::{Environment, GlobalContext, OwnedEnvironment}; use crate::vm::contracts::Contract; use crate::vm::costs::ExecutionCost; @@ -137,7 +138,11 @@ fn test_get_block_info_eval() { let mut owned_env = OwnedEnvironment::new(marf.as_clarity_db()); let contract_identifier = QualifiedContractIdentifier::local("test-contract").unwrap(); owned_env - .initialize_contract(contract_identifier.clone(), contracts[i]) + .initialize_contract( + contract_identifier.clone(), + contracts[i], + ASTRules::PrecheckSize, + ) .unwrap(); let mut env = owned_env.get_exec_environment(None); @@ -182,11 +187,13 @@ fn test_contract_caller(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("contract-a").unwrap(), contract_a, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("contract-b").unwrap(), contract_b, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -258,11 +265,13 @@ fn test_fully_qualified_contract_call(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("contract-a").unwrap(), contract_a, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("contract-b").unwrap(), contract_b, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -383,11 +392,11 @@ fn test_simple_naming_system(owned_env: &mut OwnedEnvironment) { let mut env = owned_env.get_exec_environment(None); let contract_identifier = QualifiedContractIdentifier::local("tokens").unwrap(); - env.initialize_contract(contract_identifier, tokens_contract) + env.initialize_contract(contract_identifier, tokens_contract, ASTRules::PrecheckSize) .unwrap(); let contract_identifier = QualifiedContractIdentifier::local("names").unwrap(); - env.initialize_contract(contract_identifier, names_contract) + env.initialize_contract(contract_identifier, names_contract, ASTRules::PrecheckSize) .unwrap(); } @@ -524,11 +533,11 @@ fn test_simple_contract_call(owned_env: &mut OwnedEnvironment) { let mut env = owned_env.get_exec_environment(Some(get_principal().expect_principal())); let contract_identifier = QualifiedContractIdentifier::local("factorial-contract").unwrap(); - env.initialize_contract(contract_identifier, contract_1) + env.initialize_contract(contract_identifier, contract_1, ASTRules::PrecheckSize) .unwrap(); let contract_identifier = QualifiedContractIdentifier::local("proxy-compute").unwrap(); - env.initialize_contract(contract_identifier, contract_2) + env.initialize_contract(contract_identifier, contract_2, ASTRules::PrecheckSize) .unwrap(); let args = symbols_from_values(vec![]); @@ -598,11 +607,11 @@ fn test_aborts(owned_env: &mut OwnedEnvironment) { let mut env = owned_env.get_exec_environment(None); let contract_identifier = QualifiedContractIdentifier::local("contract-1").unwrap(); - env.initialize_contract(contract_identifier, contract_1) + env.initialize_contract(contract_identifier, contract_1, ASTRules::PrecheckSize) .unwrap(); let contract_identifier = QualifiedContractIdentifier::local("contract-2").unwrap(); - env.initialize_contract(contract_identifier, contract_2) + env.initialize_contract(contract_identifier, contract_2, ASTRules::PrecheckSize) .unwrap(); env.sender = Some(get_principal_as_principal_data()); @@ -704,8 +713,12 @@ fn test_factorial_contract(owned_env: &mut OwnedEnvironment) { let mut env = owned_env.get_exec_environment(None); let contract_identifier = QualifiedContractIdentifier::local("factorial").unwrap(); - env.initialize_contract(contract_identifier, FACTORIAL_CONTRACT) - .unwrap(); + env.initialize_contract( + contract_identifier, + FACTORIAL_CONTRACT, + ASTRules::PrecheckSize, + ) + .unwrap(); let tx_name = "compute"; let arguments_to_test = [ @@ -804,6 +817,7 @@ fn test_as_max_len() { .initialize_contract( QualifiedContractIdentifier::local("contract").unwrap(), &contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -827,7 +841,7 @@ fn test_ast_stack_depth() { "; assert_eq!( vm_execute(program).unwrap_err(), - RuntimeErrorType::ASTError(ParseErrors::ExpressionStackDepthTooDeep.into()).into() + RuntimeErrorType::ASTError(ParseErrors::VaryExpressionStackDepthTooDeep.into()).into() ); } @@ -873,12 +887,12 @@ fn test_cc_stack_depth() { let mut env = owned_env.get_exec_environment(None); let contract_identifier = QualifiedContractIdentifier::local("c-foo").unwrap(); - env.initialize_contract(contract_identifier, contract_one) + env.initialize_contract(contract_identifier, contract_one, ASTRules::PrecheckSize) .unwrap(); let contract_identifier = QualifiedContractIdentifier::local("c-bar").unwrap(); assert_eq!( - env.initialize_contract(contract_identifier, contract_two) + env.initialize_contract(contract_identifier, contract_two, ASTRules::PrecheckSize) .unwrap_err(), RuntimeErrorType::MaxStackDepthReached.into() ); @@ -911,12 +925,12 @@ fn test_cc_trait_stack_depth() { let mut env = owned_env.get_exec_environment(None); let contract_identifier = QualifiedContractIdentifier::local("c-foo").unwrap(); - env.initialize_contract(contract_identifier, contract_one) + env.initialize_contract(contract_identifier, contract_one, ASTRules::PrecheckSize) .unwrap(); let contract_identifier = QualifiedContractIdentifier::local("c-bar").unwrap(); assert_eq!( - env.initialize_contract(contract_identifier, contract_two) + env.initialize_contract(contract_identifier, contract_two, ASTRules::PrecheckSize) .unwrap_err(), RuntimeErrorType::MaxStackDepthReached.into() ); diff --git a/clarity/src/vm/tests/events.rs b/clarity/src/vm/tests/events.rs index 08a3b7c1c..67719057c 100644 --- a/clarity/src/vm/tests/events.rs +++ b/clarity/src/vm/tests/events.rs @@ -22,6 +22,7 @@ use crate::vm::types::{AssetIdentifier, BuffData, QualifiedContractIdentifier, V use stacks_common::types::StacksEpochId; +use crate::vm::ast::ASTRules; use crate::vm::database::MemoryBackingStore; use crate::vm::tests::{TEST_BURN_STATE_DB, TEST_HEADER_DB}; @@ -35,7 +36,7 @@ fn helper_execute(contract: &str, method: &str) -> (Value, Vec. use crate::vm::analysis::errors::CheckError; +use crate::vm::ast::ASTRules; use crate::vm::contexts::{Environment, GlobalContext, OwnedEnvironment}; use crate::vm::errors::{CheckErrors, Error, RuntimeErrorType}; use crate::vm::execute as vm_execute; +use crate::vm::tests::{execute, symbols_from_values, with_memory_environment}; use crate::vm::types::{ PrincipalData, QualifiedContractIdentifier, ResponseData, TypeSignature, Value, }; use std::convert::TryInto; -use crate::vm::tests::{execute, symbols_from_values, with_memory_environment}; - #[test] fn test_all() { let to_test = [ @@ -70,11 +70,13 @@ fn test_dynamic_dispatch_by_defining_trait(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -114,11 +116,13 @@ fn test_dynamic_dispatch_pass_trait_nested_in_let(owned_env: &mut OwnedEnvironme env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -157,11 +161,13 @@ fn test_dynamic_dispatch_pass_trait(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -199,11 +205,13 @@ fn test_dynamic_dispatch_intra_contract_call(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("contract-defining-trait").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -244,16 +252,19 @@ fn test_dynamic_dispatch_by_implementing_imported_trait(owned_env: &mut OwnedEnv env.initialize_contract( QualifiedContractIdentifier::local("contract-defining-trait").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -296,16 +307,19 @@ fn test_dynamic_dispatch_by_implementing_imported_trait_mul_funcs( env.initialize_contract( QualifiedContractIdentifier::local("contract-defining-trait").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -343,16 +357,19 @@ fn test_dynamic_dispatch_by_importing_trait(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("contract-defining-trait").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -397,26 +414,31 @@ fn test_dynamic_dispatch_including_nested_trait(owned_env: &mut OwnedEnvironment env.initialize_contract( QualifiedContractIdentifier::local("contract-defining-nested-trait").unwrap(), contract_defining_nested_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("contract-defining-trait").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-nested-contract").unwrap(), target_nested_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -456,11 +478,13 @@ fn test_dynamic_dispatch_mismatched_args(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -499,11 +523,13 @@ fn test_dynamic_dispatch_mismatched_returned(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -545,11 +571,13 @@ fn test_reentrant_dynamic_dispatch(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -588,11 +616,13 @@ fn test_readwrite_dynamic_dispatch(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -631,11 +661,13 @@ fn test_readwrite_violation_dynamic_dispatch(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -681,21 +713,25 @@ fn test_bad_call_with_trait(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("defun").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatch").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("implem").unwrap(), impl_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("call").unwrap(), caller_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -733,21 +769,25 @@ fn test_good_call_with_trait(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("defun").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatch").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("implem").unwrap(), impl_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("call").unwrap(), caller_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -786,21 +826,25 @@ fn test_good_call_2_with_trait(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("defun").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatch").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("implem").unwrap(), impl_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("call").unwrap(), caller_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -843,16 +887,19 @@ fn test_dynamic_dispatch_pass_literal_principal_as_trait_in_user_defined_functio env.initialize_contract( QualifiedContractIdentifier::local("contract-defining-trait").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -891,16 +938,19 @@ fn test_contract_of_value(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("defun").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatch").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("implem").unwrap(), impl_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -943,16 +993,19 @@ fn test_contract_of_no_impl(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("defun").unwrap(), contract_defining_trait, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("dispatch").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("implem").unwrap(), impl_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -993,11 +1046,13 @@ fn test_return_trait_with_contract_of_wrapped_in_begin(owned_env: &mut OwnedEnvi env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -1036,11 +1091,13 @@ fn test_return_trait_with_contract_of_wrapped_in_let(owned_env: &mut OwnedEnviro env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } @@ -1077,11 +1134,13 @@ fn test_return_trait_with_contract_of(owned_env: &mut OwnedEnvironment) { env.initialize_contract( QualifiedContractIdentifier::local("dispatching-contract").unwrap(), dispatching_contract, + ASTRules::PrecheckSize, ) .unwrap(); env.initialize_contract( QualifiedContractIdentifier::local("target-contract").unwrap(), target_contract, + ASTRules::PrecheckSize, ) .unwrap(); } diff --git a/src/chainstate/burn/db/sortdb.rs b/src/chainstate/burn/db/sortdb.rs index cdfafa945..f4b07728c 100644 --- a/src/chainstate/burn/db/sortdb.rs +++ b/src/chainstate/burn/db/sortdb.rs @@ -59,6 +59,7 @@ use crate::chainstate::stacks::index::{Error as MARFError, MarfTrieId}; use crate::chainstate::stacks::StacksPublicKey; use crate::chainstate::stacks::*; use crate::chainstate::ChainstateDB; +use crate::core::AST_RULES_PRECHECK_SIZE; use crate::core::FIRST_BURNCHAIN_CONSENSUS_HASH; use crate::core::FIRST_STACKS_BLOCK_HASH; use crate::core::{StacksEpoch, StacksEpochId, STACKS_EPOCH_MAX}; @@ -72,6 +73,7 @@ use crate::util_lib::db::{ db_mkdirs, query_count, query_row, query_row_columns, query_row_panic, query_rows, sql_pragma, u64_to_sql, DBConn, FromColumn, FromRow, IndexDBConn, IndexDBTx, }; +use clarity::vm::ast::ASTRules; use clarity::vm::representations::{ClarityName, ContractName}; use clarity::vm::types::Value; use stacks_common::address::AddressHashMode; @@ -385,6 +387,22 @@ impl FromRow for TransferStxOp { } } +impl FromColumn for ASTRules { + fn from_column<'a>(row: &'a Row, column_name: &str) -> Result { + let x: u8 = row.get_unwrap(column_name); + let ast_rules = ASTRules::from_u8(x).ok_or(db_error::ParseError)?; + Ok(ast_rules) + } +} + +impl FromRow<(ASTRules, u64)> for (ASTRules, u64) { + fn from_row<'a>(row: &'a Row) -> Result<(ASTRules, u64), db_error> { + let ast_rules = ASTRules::from_column(row, "ast_rule_id")?; + let height = u64::from_column(row, "block_height")?; + Ok((ast_rules, height)) + } +} + struct AcceptedStacksBlockHeader { pub tip_consensus_hash: ConsensusHash, // PoX tip pub consensus_hash: ConsensusHash, // stacks block consensus hash @@ -434,7 +452,7 @@ impl FromRow for StacksEpoch { } } -pub const SORTITION_DB_VERSION: &'static str = "3"; +pub const SORTITION_DB_VERSION: &'static str = "4"; const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[ r#" @@ -612,6 +630,12 @@ const SORTITION_DB_SCHEMA_3: &'static [&'static str] = &[r#" FOREIGN KEY(block_commit_txid,block_commit_sortition_id) REFERENCES block_commits(txid,sortition_id) );"#]; +const SORTITION_DB_SCHEMA_4: &'static [&'static str] = &[r#" + CREATE TABLE ast_rule_heights ( + ast_rule_id INTEGER PRIMAR KEY NOT NULL, + block_height INTEGER NOT NULL + );"#]; + // update this to add new indexes const LAST_SORTITION_DB_INDEX: &'static str = "index_parent_sortition_id"; @@ -2288,19 +2312,9 @@ impl SortitionDB { for row_text in SORTITION_DB_INITIAL_SCHEMA { db_tx.execute_batch(row_text)?; } - for row_text in SORTITION_DB_SCHEMA_2 { - db_tx.execute_batch(row_text)?; - } - for row_text in SORTITION_DB_SCHEMA_3 { - db_tx.execute_batch(row_text)?; - } - - SortitionDB::validate_and_insert_epochs(&db_tx, epochs_ref)?; - - db_tx.execute( - "INSERT OR REPLACE INTO db_config (version) VALUES (?1)", - &[&SORTITION_DB_VERSION], - )?; + SortitionDB::apply_schema_2(&db_tx, epochs_ref)?; + SortitionDB::apply_schema_3(&db_tx)?; + SortitionDB::apply_schema_4(&db_tx)?; db_tx.instantiate_index()?; @@ -2462,7 +2476,7 @@ impl SortitionDB { match epoch { StacksEpochId::Epoch10 => false, StacksEpochId::Epoch20 => (version == "1" || version == "2" || version == "3"), - StacksEpochId::Epoch2_05 => (version == "2" || version == "3"), + StacksEpochId::Epoch2_05 => (version == "2" || version == "3" || version == "4"), } } @@ -2504,6 +2518,33 @@ impl SortitionDB { Ok(()) } + fn apply_schema_4(tx: &DBTx) -> Result<(), db_error> { + for sql_exec in SORTITION_DB_SCHEMA_4 { + tx.execute_batch(sql_exec)?; + } + + let typical_rules: &[&dyn ToSql] = &[&(ASTRules::Typical as u8), &0i64]; + + let precheck_size_rules: &[&dyn ToSql] = &[ + &(ASTRules::PrecheckSize as u8), + &u64_to_sql(AST_RULES_PRECHECK_SIZE)?, + ]; + + tx.execute( + "INSERT INTO ast_rule_heights (ast_rule_id,block_height) VALUES (?1, ?2)", + typical_rules, + )?; + tx.execute( + "INSERT INTO ast_rule_heights (ast_rule_id,block_height) VALUES (?1, ?2)", + precheck_size_rules, + )?; + tx.execute( + "INSERT OR REPLACE INTO db_config (version) VALUES (?1)", + &["4"], + )?; + Ok(()) + } + fn check_schema_version_or_error(&mut self) -> Result<(), db_error> { match SortitionDB::get_schema_version(self.conn()) { Ok(Some(version)) => { @@ -2538,6 +2579,10 @@ impl SortitionDB { let tx = self.tx_begin()?; SortitionDB::apply_schema_3(&tx.deref())?; tx.commit()?; + } else if version == "3" { + let tx = self.tx_begin()?; + SortitionDB::apply_schema_4(&tx.deref())?; + tx.commit()?; } else if version == expected_version { return Ok(()); } else { @@ -2587,6 +2632,52 @@ impl SortitionDB { } Ok(()) } + + #[cfg(any(test, feature = "testing"))] + pub fn override_ast_rule_height<'a>( + tx: &mut DBTx<'a>, + ast_rules: ASTRules, + height: u64, + ) -> Result<(), db_error> { + let rules: &[&dyn ToSql] = &[&u64_to_sql(height)?, &(ast_rules as u8)]; + + tx.execute( + "UPDATE ast_rule_heights SET block_height = ?1 WHERE ast_rule_id = ?2", + rules, + )?; + Ok(()) + } + + #[cfg(not(any(test, feature = "testing")))] + pub fn override_ast_rule_height<'a>( + _tx: &mut DBTx<'a>, + _ast_rules: ASTRules, + _height: u64, + ) -> Result<(), db_error> { + Ok(()) + } + + /// What's the default AST rules at the given block height? + pub fn get_ast_rules(conn: &DBConn, height: u64) -> Result { + let ast_rule_sets: Vec<(ASTRules, u64)> = query_rows( + conn, + "SELECT * FROM ast_rule_heights ORDER BY block_height ASC", + NO_PARAMS, + )?; + + assert!(ast_rule_sets.len() > 0); + let mut last_height = ast_rule_sets[0].1; + let mut last_rules = ast_rule_sets[0].0; + for (ast_rules, ast_rule_height) in ast_rule_sets.into_iter() { + if last_height <= height && height < ast_rule_height { + return Ok(last_rules); + } + last_height = ast_rule_height; + last_rules = ast_rules; + } + + return Ok(last_rules); + } } impl<'a> SortitionDBConn<'a> { diff --git a/src/chainstate/coordinator/tests.rs b/src/chainstate/coordinator/tests.rs index 0af4c1ac8..bce281e83 100644 --- a/src/chainstate/coordinator/tests.rs +++ b/src/chainstate/coordinator/tests.rs @@ -469,12 +469,15 @@ fn make_genesis_block_with_recipients( let iconn = sort_db.index_conn(); let mut miner_epoch_info = builder.pre_epoch_begin(state, &iconn).unwrap(); + let ast_rules = miner_epoch_info.ast_rules.clone(); let mut epoch_tx = builder .epoch_begin(&iconn, &mut miner_epoch_info) .unwrap() .0; - builder.try_mine_tx(&mut epoch_tx, &coinbase_op).unwrap(); + builder + .try_mine_tx(&mut epoch_tx, &coinbase_op, ast_rules) + .unwrap(); let block = builder.mine_anchored_block(&mut epoch_tx); builder.epoch_finish(epoch_tx); @@ -684,12 +687,15 @@ fn make_stacks_block_with_input( ) .unwrap(); let mut miner_epoch_info = builder.pre_epoch_begin(state, &iconn).unwrap(); + let ast_rules = miner_epoch_info.ast_rules.clone(); let mut epoch_tx = builder .epoch_begin(&iconn, &mut miner_epoch_info) .unwrap() .0; - builder.try_mine_tx(&mut epoch_tx, &coinbase_op).unwrap(); + builder + .try_mine_tx(&mut epoch_tx, &coinbase_op, ast_rules) + .unwrap(); let block = builder.mine_anchored_block(&mut epoch_tx); builder.epoch_finish(epoch_tx); diff --git a/src/chainstate/stacks/boot/contract_tests.rs b/src/chainstate/stacks/boot/contract_tests.rs index 0692f7b0a..e45dd6bcc 100644 --- a/src/chainstate/stacks/boot/contract_tests.rs +++ b/src/chainstate/stacks/boot/contract_tests.rs @@ -20,6 +20,7 @@ use crate::core::{ use crate::util_lib::db::{DBConn, FromRow}; use clarity::vm::analysis::arithmetic_checker::ArithmeticOnlyChecker; use clarity::vm::analysis::mem_type_check; +use clarity::vm::ast::ASTRules; use clarity::vm::contexts::OwnedEnvironment; use clarity::vm::contracts::Contract; use clarity::vm::costs::CostOverflowingMath; @@ -281,8 +282,12 @@ fn recency_tests() { let delegator = StacksPrivateKey::new(); sim.execute_next_block(|env| { - env.initialize_contract(POX_CONTRACT_TESTNET.clone(), &BOOT_CODE_POX_TESTNET) - .unwrap() + env.initialize_contract( + POX_CONTRACT_TESTNET.clone(), + &BOOT_CODE_POX_TESTNET, + ASTRules::PrecheckSize, + ) + .unwrap() }); sim.execute_next_block(|env| { // try to issue a far future stacking tx @@ -350,8 +355,12 @@ fn delegation_tests() { const REWARD_CYCLE_LENGTH: u128 = 1050; sim.execute_next_block(|env| { - env.initialize_contract(POX_CONTRACT_TESTNET.clone(), &BOOT_CODE_POX_TESTNET) - .unwrap() + env.initialize_contract( + POX_CONTRACT_TESTNET.clone(), + &BOOT_CODE_POX_TESTNET, + ASTRules::PrecheckSize, + ) + .unwrap() }); sim.execute_next_block(|env| { assert_eq!( @@ -899,8 +908,12 @@ fn test_vote_withdrawal() { let mut sim = ClarityTestSim::new(); sim.execute_next_block(|env| { - env.initialize_contract(COST_VOTING_CONTRACT_TESTNET.clone(), &BOOT_CODE_COST_VOTING) - .unwrap(); + env.initialize_contract( + COST_VOTING_CONTRACT_TESTNET.clone(), + &BOOT_CODE_COST_VOTING, + ASTRules::PrecheckSize, + ) + .unwrap(); // Submit a proposal assert_eq!( @@ -1078,8 +1091,12 @@ fn test_vote_fail() { // Test voting in a proposal sim.execute_next_block(|env| { - env.initialize_contract(COST_VOTING_CONTRACT_TESTNET.clone(), &BOOT_CODE_COST_VOTING) - .unwrap(); + env.initialize_contract( + COST_VOTING_CONTRACT_TESTNET.clone(), + &BOOT_CODE_COST_VOTING, + ASTRules::PrecheckSize, + ) + .unwrap(); // Submit a proposal assert_eq!( @@ -1279,8 +1296,12 @@ fn test_vote_confirm() { let mut sim = ClarityTestSim::new(); sim.execute_next_block(|env| { - env.initialize_contract(COST_VOTING_CONTRACT_TESTNET.clone(), &BOOT_CODE_COST_VOTING) - .unwrap(); + env.initialize_contract( + COST_VOTING_CONTRACT_TESTNET.clone(), + &BOOT_CODE_COST_VOTING, + ASTRules::PrecheckSize, + ) + .unwrap(); // Submit a proposal assert_eq!( @@ -1392,8 +1413,12 @@ fn test_vote_too_many_confirms() { let MAX_CONFIRMATIONS_PER_BLOCK = 10; sim.execute_next_block(|env| { - env.initialize_contract(COST_VOTING_CONTRACT_TESTNET.clone(), &BOOT_CODE_COST_VOTING) - .unwrap(); + env.initialize_contract( + COST_VOTING_CONTRACT_TESTNET.clone(), + &BOOT_CODE_COST_VOTING, + ASTRules::PrecheckSize, + ) + .unwrap(); // Submit a proposal for i in 0..(MAX_CONFIRMATIONS_PER_BLOCK + 1) { diff --git a/src/chainstate/stacks/boot/mod.rs b/src/chainstate/stacks/boot/mod.rs index 821e79049..5a138359c 100644 --- a/src/chainstate/stacks/boot/mod.rs +++ b/src/chainstate/stacks/boot/mod.rs @@ -28,6 +28,7 @@ use crate::chainstate::stacks::index::marf::MarfConnection; use crate::chainstate::stacks::Error; use crate::clarity_vm::clarity::ClarityConnection; use crate::core::{POX_MAXIMAL_SCALING, POX_THRESHOLD_STEPS_USTX}; +use clarity::vm::ast::ASTRules; use clarity::vm::contexts::ContractContext; use clarity::vm::costs::{ cost_functions::ClarityCostFunction, ClarityCostFunctionReference, CostStateSummary, @@ -153,6 +154,7 @@ impl StacksChainState { &iconn, &boot::boot_code_id(boot_contract_name, self.mainnet), code, + ASTRules::PrecheckSize, ) .map_err(Error::ClarityError) } diff --git a/src/chainstate/stacks/db/blocks.rs b/src/chainstate/stacks/db/blocks.rs index 14f8c34ea..c36e9bf51 100644 --- a/src/chainstate/stacks/db/blocks.rs +++ b/src/chainstate/stacks/db/blocks.rs @@ -24,6 +24,7 @@ use std::io::prelude::*; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; +use clarity::vm::ast::ASTRules; use rand::thread_rng; use rand::Rng; use rand::RngCore; @@ -63,7 +64,6 @@ use crate::util_lib::db::{ use crate::util_lib::strings::StacksString; pub use clarity::vm::analysis::errors::{CheckError, CheckErrors}; use clarity::vm::analysis::run_analysis; -use clarity::vm::ast::build_ast; use clarity::vm::clarity::TransactionConnection; use clarity::vm::contexts::AssetMap; use clarity::vm::contracts::Contract; @@ -2060,7 +2060,6 @@ impl StacksChainState { /// The query takes the consensus hash and block hash of a block that _produced_ this stream. /// Return Some(processed) if the microblock is queued up. /// Return None if the microblock is not queued up. - #[cfg(test)] pub fn get_microblock_status( &self, parent_consensus_hash: &ConsensusHash, @@ -4473,6 +4472,7 @@ impl StacksChainState { pub fn process_microblocks_transactions( clarity_tx: &mut ClarityTx, microblocks: &Vec, + ast_rules: ASTRules, ) -> Result<(u128, u128, Vec), (Error, BlockHeaderHash)> { let mut fees = 0u128; let mut burns = 0u128; @@ -4481,7 +4481,7 @@ impl StacksChainState { debug!("Process microblock {}", µblock.block_hash()); for (tx_index, tx) in microblock.txs.iter().enumerate() { let (tx_fee, mut tx_receipt) = - StacksChainState::process_transaction(clarity_tx, tx, false) + StacksChainState::process_transaction(clarity_tx, tx, false, ast_rules) .map_err(|e| (e, microblock.block_hash()))?; tx_receipt.microblock_header = Some(microblock.header.clone()); @@ -4680,13 +4680,14 @@ impl StacksChainState { clarity_tx: &mut ClarityTx, block: &StacksBlock, mut tx_index: u32, + ast_rules: ASTRules, ) -> Result<(u128, u128, Vec), Error> { let mut fees = 0u128; let mut burns = 0u128; let mut receipts = vec![]; for tx in block.txs.iter() { let (tx_fee, mut tx_receipt) = - StacksChainState::process_transaction(clarity_tx, tx, false)?; + StacksChainState::process_transaction(clarity_tx, tx, false, ast_rules)?; fees = fees.checked_add(tx_fee as u128).expect("Fee overflow"); tx_receipt.tx_index = tx_index; burns = burns @@ -4862,6 +4863,12 @@ impl StacksChainState { let parent_index_hash = StacksBlockHeader::make_index_block_hash(&parent_consensus_hash, &parent_header_hash); + let parent_burn_height = + SortitionDB::get_block_snapshot_consensus(conn, &parent_consensus_hash)? + .expect("Failed to get snapshot for parent's sortition") + .block_height; + let microblock_ast_rules = SortitionDB::get_ast_rules(conn, parent_burn_height)?; + // find matured miner rewards, so we can grant them within the Clarity DB tx. let (latest_matured_miners, matured_miner_parent) = { let latest_miners = StacksChainState::get_scheduled_block_rewards( @@ -4952,6 +4959,7 @@ impl StacksChainState { match StacksChainState::process_microblocks_transactions( &mut clarity_tx, &parent_microblocks, + microblock_ast_rules, ) { Ok((fees, burns, events)) => (fees, burns, events), Err((e, mblock_header_hash)) => { @@ -5114,6 +5122,9 @@ impl StacksChainState { block.txs.len() ); + let ast_rules = + SortitionDB::get_ast_rules(burn_dbconn.tx(), chain_tip_burn_header_height.into())?; + let mainnet = chainstate_tx.get_config().mainnet; let next_block_height = block.header.total_work.work; @@ -5304,6 +5315,7 @@ impl StacksChainState { &mut clarity_tx, &block, microblock_txs_receipts.len() as u32, + ast_rules, ) { Err(e) => { let msg = format!("Invalid Stacks block {}: {:?}", block.block_hash(), &e); @@ -6413,6 +6425,7 @@ pub mod test { use super::*; + use clarity::vm::ast::ASTRules; use clarity::vm::types::StacksAddressExtensions; use serde_json; @@ -10468,6 +10481,7 @@ pub mod test { µblock_privkey, &anchored_block.0.block_hash(), microblocks.last().map(|mblock| &mblock.header), + ASTRules::PrecheckSize, ) .unwrap(); microblocks.push(microblock); diff --git a/src/chainstate/stacks/db/contracts.rs b/src/chainstate/stacks/db/contracts.rs index d727dc2e4..14f9f0802 100644 --- a/src/chainstate/stacks/db/contracts.rs +++ b/src/chainstate/stacks/db/contracts.rs @@ -42,7 +42,6 @@ use clarity::vm::types::{PrincipalData, QualifiedContractIdentifier, StandardPri use clarity::vm::contexts::{AssetMap, OwnedEnvironment}; use clarity::vm::analysis::run_analysis; -use clarity::vm::ast::build_ast; use clarity::vm::types::{AssetIdentifier, Value}; pub use clarity::vm::analysis::errors::CheckErrors; diff --git a/src/chainstate/stacks/db/mod.rs b/src/chainstate/stacks/db/mod.rs index bf101282b..c8ab50d54 100644 --- a/src/chainstate/stacks/db/mod.rs +++ b/src/chainstate/stacks/db/mod.rs @@ -22,6 +22,7 @@ use std::io::prelude::*; use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; +use clarity::vm::ast::ASTRules; use rusqlite::types::ToSql; use rusqlite::Connection; use rusqlite::OpenFlags; @@ -68,7 +69,6 @@ use crate::util_lib::db::{ }; use clarity::vm::analysis::analysis_db::AnalysisDatabase; use clarity::vm::analysis::run_analysis; -use clarity::vm::ast::build_ast; use clarity::vm::clarity::TransactionConnection; use clarity::vm::contexts::OwnedEnvironment; use clarity::vm::costs::{ExecutionCost, LimitedCostTracker}; @@ -1120,6 +1120,7 @@ impl StacksChainState { clarity, &boot_code_smart_contract, &boot_code_account, + ASTRules::PrecheckSize, ) })?; receipts.push(tx_receipt); @@ -1715,28 +1716,11 @@ impl StacksChainState { burn_dbconn, contract, code, + ASTRules::PrecheckSize, ); result.unwrap() } - pub fn clarity_eval_read_only_checked( - &mut self, - burn_dbconn: &dyn BurnStateDB, - parent_id_bhh: &StacksBlockId, - contract: &QualifiedContractIdentifier, - code: &str, - ) -> Result { - self.clarity_state - .eval_read_only( - parent_id_bhh, - &HeadersDBConn(self.state_index.sqlite_conn()), - burn_dbconn, - contract, - code, - ) - .map_err(Error::ClarityError) - } - pub fn db(&self) -> &DBConn { self.state_index.sqlite_conn() } diff --git a/src/chainstate/stacks/db/transactions.rs b/src/chainstate/stacks/db/transactions.rs index bcc4250e4..51d6e2bf1 100644 --- a/src/chainstate/stacks/db/transactions.rs +++ b/src/chainstate/stacks/db/transactions.rs @@ -33,13 +33,13 @@ use crate::clarity_vm::clarity::{ use crate::net::Error as net_error; use crate::util_lib::db::Error as db_error; use crate::util_lib::db::{query_count, query_rows, DBConn}; +use clarity::vm::ast::ASTRules; use stacks_common::util::hash::to_hex; use crate::util_lib::strings::{StacksString, VecDisplay}; pub use clarity::vm::analysis::errors::CheckErrors; use clarity::vm::analysis::run_analysis; use clarity::vm::analysis::types::ContractAnalysis; -use clarity::vm::ast::build_ast; use clarity::vm::clarity::TransactionConnection; use clarity::vm::contexts::{AssetMap, AssetMapEntry, Environment}; use clarity::vm::contracts::Contract; @@ -58,6 +58,7 @@ use clarity::vm::types::{ }; use crate::chainstate::stacks::StacksMicroblockHeader; +use clarity::vm::ast::errors::ParseErrors; use clarity::vm::types::StacksAddressExtensions as ClarityStacksAddressExt; impl StacksTransactionReceipt { @@ -823,6 +824,7 @@ impl StacksChainState { clarity_tx: &mut ClarityTransactionConnection, tx: &StacksTransaction, origin_account: &StacksAccount, + ast_rules: ASTRules, ) -> Result { match tx.payload { TransactionPayload::TokenTransfer(ref addr, ref amount, ref _memo) => { @@ -973,7 +975,7 @@ impl StacksChainState { // The reason for this is that analyzing the transaction is itself an expensive // operation, and the paying account will need to be debited the fee regardless. let analysis_resp = - clarity_tx.analyze_smart_contract(&contract_id, &contract_code_str); + clarity_tx.analyze_smart_contract(&contract_id, &contract_code_str, ast_rules); let (contract_ast, contract_analysis) = match analysis_resp { Ok(x) => x, Err(e) => { @@ -986,16 +988,32 @@ impl StacksChainState { budget.clone(), )); } - _ => { + other_error => { + if ast_rules == ASTRules::PrecheckSize { + // a [Vary]ExpressionDepthTooDeep error in this situation + // invalidates the block, since this should have prevented the + // block from getting relayed in the first place + if let clarity_error::Parse(ref parse_error) = &other_error { + match parse_error.err { + ParseErrors::ExpressionStackDepthTooDeep + | ParseErrors::VaryExpressionStackDepthTooDeep => { + info!("Transaction {} is problematic and should have prevented this block from being relayed", tx.txid()); + return Err(Error::ClarityError(other_error)); + } + _ => {} + } + } + } // this analysis isn't free -- convert to runtime error let mut analysis_cost = clarity_tx.cost_so_far(); analysis_cost .sub(&cost_before) .expect("BUG: total block cost decreased"); - error!( + warn!( "Runtime error in contract analysis for {}: {:?}", - &contract_id, &e + &contract_id, &other_error; + "AST rules" => %format!("{:?}", &ast_rules) ); let receipt = StacksTransactionReceipt::from_analysis_failure( tx.clone(), @@ -1128,6 +1146,7 @@ impl StacksChainState { clarity_block: &mut ClarityTx, tx: &StacksTransaction, quiet: bool, + ast_rules: ASTRules, ) -> Result<(u64, StacksTransactionReceipt), Error> { debug!("Process transaction {} ({})", tx.txid(), tx.payload.name()); @@ -1137,8 +1156,12 @@ impl StacksChainState { let (origin_account, payer_account) = StacksChainState::check_transaction_nonces(&mut transaction, tx, quiet)?; - let tx_receipt = - StacksChainState::process_transaction_payload(&mut transaction, tx, &origin_account)?; + let tx_receipt = StacksChainState::process_transaction_payload( + &mut transaction, + tx, + &origin_account, + ast_rules, + )?; let new_payer_account = StacksChainState::get_payer_account(&mut transaction, tx); let fee = tx.get_tx_fee(); @@ -1239,7 +1262,13 @@ pub mod test { StacksChainState::account_credit(tx, &addr.to_account_principal(), 223) }); - let (fee, _) = StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); let account_after = StacksChainState::get_account(&mut conn, &addr.to_account_principal()); assert_eq!(account_after.nonce, 1); @@ -1283,7 +1312,13 @@ pub mod test { assert_eq!(recv_account.stx_balance.amount_unlocked, 0); assert_eq!(recv_account.nonce, 0); - let (fee, _) = StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); let account_after = StacksChainState::get_account(&mut conn, &addr.to_account_principal()); assert_eq!(account_after.nonce, 2); @@ -1470,7 +1505,12 @@ pub mod test { assert_eq!(account.stx_balance.amount_unlocked, 123); assert_eq!(account.nonce, 0); - let res = StacksChainState::process_transaction(&mut conn, &signed_tx, false); + let res = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ); assert!(res.is_err()); match res { @@ -1567,7 +1607,13 @@ pub mod test { StacksChainState::account_credit(tx, &addr.to_account_principal(), 123) }); - let (fee, _) = StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); let account_after = StacksChainState::get_account(&mut conn, &addr.to_account_principal()); assert_eq!(account_after.nonce, 1); @@ -1642,7 +1688,13 @@ pub mod test { let account = StacksChainState::get_account(&mut conn, &addr.to_account_principal()); assert_eq!(account.nonce, 0); - let (fee, _) = StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); let account = StacksChainState::get_account(&mut conn, &addr.to_account_principal()); assert_eq!(account.nonce, 1); @@ -1730,7 +1782,12 @@ pub mod test { let account = StacksChainState::get_account(&mut conn, &addr.to_account_principal()); assert_eq!(account.nonce, next_nonce); - let res = StacksChainState::process_transaction(&mut conn, &signed_tx, false); + let res = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ); if expected_behavior[i] { assert!(res.is_ok()); @@ -1833,8 +1890,13 @@ pub mod test { assert_eq!(account.nonce, i as u64); // runtime error should be handled - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); // account nonce should increment let account = StacksChainState::get_account(&mut conn, &addr.to_account_principal()); @@ -1917,7 +1979,13 @@ pub mod test { StacksChainState::get_account(&mut conn, &addr_sponsor.to_account_principal()); assert_eq!(account.nonce, 0); - let (fee, _) = StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); let account = StacksChainState::get_account(&mut conn, &addr.to_account_principal()); assert_eq!(account.nonce, 1); @@ -2024,14 +2092,25 @@ pub mod test { StacksChainState::get_data_var(&mut conn, &contract_id, "bar").unwrap(); assert!(var_before_res.is_none()); - let (fee, _) = StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); let var_before_set_res = StacksChainState::get_data_var(&mut conn, &contract_id, "bar").unwrap(); assert_eq!(var_before_set_res, Some(Value::Int(0))); - let (fee_2, _) = - StacksChainState::process_transaction(&mut conn, &signed_tx_2, false).unwrap(); + let (fee_2, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx_2, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); let account = StacksChainState::get_account(&mut conn, &addr.to_account_principal()); assert_eq!(account.nonce, 1); @@ -2104,8 +2183,13 @@ pub mod test { StandardPrincipalData::from(addr.clone()), ContractName::from("hello-world"), ); - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); // contract-calls that don't commit let contract_calls = vec![ @@ -2150,8 +2234,13 @@ pub mod test { StacksChainState::get_account(&mut conn, &addr_2.to_account_principal()); assert_eq!(account_2.nonce, next_nonce); - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &signed_tx_2, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx_2, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); // nonce should have incremented next_nonce += 1; @@ -2211,8 +2300,13 @@ pub mod test { &ConsensusHash([1u8; 20]), &BlockHeaderHash([1u8; 32]), ); - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); conn.commit_block(); } @@ -2273,8 +2367,13 @@ pub mod test { &ConsensusHash([1u8; 20]), &BlockHeaderHash([1u8; 32]), ); - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); // invalid contract-calls let contract_calls = vec![ @@ -2346,7 +2445,12 @@ pub mod test { assert_eq!(account_2.nonce, next_nonce); // transaction is invalid, and won't be mined - let res = StacksChainState::process_transaction(&mut conn, &signed_tx_2, false); + let res = StacksChainState::process_transaction( + &mut conn, + &signed_tx_2, + false, + ASTRules::PrecheckSize, + ); assert!(res.is_err()); // nonce should NOT have incremented @@ -2470,7 +2574,13 @@ pub mod test { StacksChainState::get_data_var(&mut conn, &contract_id, "bar").unwrap(); assert!(var_before_res.is_none()); - let (fee, _) = StacksChainState::process_transaction(&mut conn, &signed_tx, false).unwrap(); + let (fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); let account_publisher = StacksChainState::get_account(&mut conn, &addr_publisher.to_account_principal()); @@ -2480,8 +2590,13 @@ pub mod test { StacksChainState::get_data_var(&mut conn, &contract_id, "bar").unwrap(); assert_eq!(var_before_set_res, Some(Value::Int(0))); - let (fee_2, _) = - StacksChainState::process_transaction(&mut conn, &signed_tx_2, false).unwrap(); + let (fee_2, _) = StacksChainState::process_transaction( + &mut conn, + &signed_tx_2, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); let account_origin = StacksChainState::get_account(&mut conn, &addr_origin.to_account_principal()); @@ -2987,8 +3102,13 @@ pub mod test { .unwrap_err(); // publish contract - let _ = - StacksChainState::process_transaction(&mut conn, &signed_contract_tx, false).unwrap(); + let _ = StacksChainState::process_transaction( + &mut conn, + &signed_contract_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); // no initial stackaroos balance let account_stackaroos_balance = StacksChainState::get_account_ft( @@ -3007,8 +3127,13 @@ pub mod test { let mut expected_next_name: u64 = 0; for tx_pass in post_conditions_pass.iter() { - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_pass, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_pass, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_stackaroos_balance += 100; expected_nonce += 1; @@ -3030,8 +3155,13 @@ pub mod test { } for tx_pass in post_conditions_pass_payback.iter() { - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_pass, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_pass, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_stackaroos_balance -= 100; expected_payback_stackaroos_balance += 100; expected_recv_nonce += 1; @@ -3070,8 +3200,13 @@ pub mod test { } for (_i, tx_pass) in post_conditions_pass_nft.iter().enumerate() { - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_pass, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_pass, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_nonce += 1; let expected_value = @@ -3093,8 +3228,13 @@ pub mod test { } for tx_fail in post_conditions_fail.iter() { - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_fail, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_fail, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_nonce += 1; // no change in balance @@ -3129,8 +3269,13 @@ pub mod test { } for tx_fail in post_conditions_fail_payback.iter() { - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_fail, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_fail, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_recv_nonce += 1; // no change in balance @@ -3170,8 +3315,13 @@ pub mod test { } for (_i, tx_fail) in post_conditions_fail_nft.iter().enumerate() { - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_fail, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_fail, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_nonce += 1; // nft shouldn't exist -- the nft-mint! should have been rolled back @@ -3659,8 +3809,13 @@ pub mod test { .unwrap_err(); // publish contract - let _ = - StacksChainState::process_transaction(&mut conn, &signed_contract_tx, false).unwrap(); + let _ = StacksChainState::process_transaction( + &mut conn, + &signed_contract_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); // no initial stackaroos balance let account_stackaroos_balance = StacksChainState::get_account_ft( @@ -3678,8 +3833,13 @@ pub mod test { let mut expected_payback_stackaroos_balance = 0; for (_i, tx_pass) in post_conditions_pass.iter().enumerate() { - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_pass, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_pass, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_stackaroos_balance += 100; expected_nonce += 1; @@ -3718,8 +3878,13 @@ pub mod test { } for (_i, tx_pass) in post_conditions_pass_payback.iter().enumerate() { - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_pass, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_pass, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_stackaroos_balance -= 100; expected_payback_stackaroos_balance += 100; expected_recv_nonce += 1; @@ -3777,8 +3942,13 @@ pub mod test { } for (_i, tx_fail) in post_conditions_fail.iter().enumerate() { - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_fail, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_fail, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_nonce += 1; // no change in balance @@ -3828,8 +3998,13 @@ pub mod test { for (_i, tx_fail) in post_conditions_fail_payback.iter().enumerate() { eprintln!("tx fail {:?}", &tx_fail); - let (_fee, _) = - StacksChainState::process_transaction(&mut conn, &tx_fail, false).unwrap(); + let (_fee, _) = StacksChainState::process_transaction( + &mut conn, + &tx_fail, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); expected_recv_nonce += 1; // no change in balance @@ -3989,11 +4164,21 @@ pub mod test { ); // publish contract - let _ = - StacksChainState::process_transaction(&mut conn, &signed_contract_tx, false).unwrap(); + let _ = StacksChainState::process_transaction( + &mut conn, + &signed_contract_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); - let (_fee, receipt) = - StacksChainState::process_transaction(&mut conn, &contract_call_tx, false).unwrap(); + let (_fee, receipt) = StacksChainState::process_transaction( + &mut conn, + &contract_call_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); assert_eq!(receipt.post_condition_aborted, true); assert_eq!(receipt.result.to_string(), "(ok (err u1))"); @@ -7105,10 +7290,20 @@ pub mod test { &ConsensusHash([1u8; 20]), &BlockHeaderHash([1u8; 32]), ); - let (fee, _) = - StacksChainState::process_transaction(&mut conn, &signed_contract_tx, false).unwrap(); - let err = StacksChainState::process_transaction(&mut conn, &signed_contract_call_tx, false) - .unwrap_err(); + let (fee, _) = StacksChainState::process_transaction( + &mut conn, + &signed_contract_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); + let err = StacksChainState::process_transaction( + &mut conn, + &signed_contract_call_tx, + false, + ASTRules::PrecheckSize, + ) + .unwrap_err(); conn.commit_block(); @@ -7246,9 +7441,13 @@ pub mod test { let signed_tx_poison_microblock = signer.get_tx().unwrap(); // process it! - let (fee, receipt) = - StacksChainState::process_transaction(&mut conn, &signed_tx_poison_microblock, false) - .unwrap(); + let (fee, receipt) = StacksChainState::process_transaction( + &mut conn, + &signed_tx_poison_microblock, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); // there must be a poison record for this microblock, from the reporter, for the microblock // sequence. @@ -7357,9 +7556,13 @@ pub mod test { // should fail to process -- the transaction is invalid if it doesn't point to a known // microblock pubkey hash. - let err = - StacksChainState::process_transaction(&mut conn, &signed_tx_poison_microblock, false) - .unwrap_err(); + let err = StacksChainState::process_transaction( + &mut conn, + &signed_tx_poison_microblock, + false, + ASTRules::PrecheckSize, + ) + .unwrap_err(); if let Error::ClarityError(clarity_error::BadTransaction(msg)) = err { assert!(msg.find("never seen in this fork").is_some()); } else { @@ -7474,9 +7677,13 @@ pub mod test { let signed_tx_poison_microblock_2 = signer.get_tx().unwrap(); // process it! - let (fee, receipt) = - StacksChainState::process_transaction(&mut conn, &signed_tx_poison_microblock_1, false) - .unwrap(); + let (fee, receipt) = StacksChainState::process_transaction( + &mut conn, + &signed_tx_poison_microblock_1, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); // there must be a poison record for this microblock, from the reporter, for the microblock // sequence. @@ -7484,9 +7691,13 @@ pub mod test { assert_eq!(report_opt.unwrap(), (reporter_addr_1, 123)); // process the second one! - let (fee, receipt) = - StacksChainState::process_transaction(&mut conn, &signed_tx_poison_microblock_2, false) - .unwrap(); + let (fee, receipt) = StacksChainState::process_transaction( + &mut conn, + &signed_tx_poison_microblock_2, + false, + ASTRules::PrecheckSize, + ) + .unwrap(); // there must be a poison record for this microblock, from the reporter, for the microblock // sequence. Moreover, since the fork was earlier in the stream, the second reporter gets diff --git a/src/chainstate/stacks/db/unconfirmed.rs b/src/chainstate/stacks/db/unconfirmed.rs index a88eac538..b973152d1 100644 --- a/src/chainstate/stacks/db/unconfirmed.rs +++ b/src/chainstate/stacks/db/unconfirmed.rs @@ -195,6 +195,8 @@ impl UnconfirmedState { .get_burn_block_time_for_block(&self.confirmed_chain_tip) .expect("BUG: unable to get burn block timestamp based on chain tip"); + let ast_rules = burn_dbconn.get_ast_rules(burn_block_height); + let mut last_mblock = self.last_mblock.take(); let mut last_mblock_seq = self.last_mblock_seq; let db_config = chainstate.config(); @@ -253,6 +255,7 @@ impl UnconfirmedState { match StacksChainState::process_microblocks_transactions( &mut clarity_tx, &vec![mblock.clone()], + ast_rules, ) { Ok(x) => x, Err((e, _)) => { diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index a94e696fc..87026e8b4 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -41,6 +41,7 @@ use crate::core::mempool::*; use crate::core::*; use crate::cost_estimates::metrics::CostMetric; use crate::cost_estimates::CostEstimator; +use crate::net::relay::Relayer; use crate::net::Error as net_error; use crate::types::StacksPublicKeyBuffer; use clarity::vm::database::BurnStateDB; @@ -61,6 +62,8 @@ use crate::types::chainstate::StacksBlockId; use crate::types::chainstate::TrieHash; use crate::types::chainstate::{BlockHeaderHash, StacksAddress, StacksWorkScore}; use clarity::vm::analysis::{CheckError, CheckErrors}; +use clarity::vm::ast::errors::ParseErrors; +use clarity::vm::ast::ASTRules; use clarity::vm::clarity::TransactionConnection; use clarity::vm::errors::Error as InterpreterError; use clarity::vm::types::TypeSignature; @@ -121,6 +124,7 @@ pub struct MinerEpochInfo<'a> { pub burn_tip_height: u32, pub parent_microblocks: Vec, pub mainnet: bool, + pub ast_rules: ASTRules, } impl From<&UnconfirmedState> for MicroblockMinerRuntime { @@ -442,7 +446,20 @@ impl TransactionResult { return (true, Error::ClarityError(e)); } // recover original ClarityError - ClarityRuntimeTxError::Acceptable { error, .. } => Error::ClarityError(error), + ClarityRuntimeTxError::Acceptable { error, .. } => { + if let clarity_error::Parse(ref parse_err) = error { + info!("Parse error: {}", parse_err); + match &parse_err.err { + ParseErrors::ExpressionStackDepthTooDeep + | ParseErrors::VaryExpressionStackDepthTooDeep => { + info!("Problematic transaction failed AST depth check"; "txid" => %tx.txid()); + return (true, Error::ClarityError(error)); + } + _ => {} + } + } + Error::ClarityError(error) + } ClarityRuntimeTxError::CostError(cost, budget) => { Error::ClarityError(clarity_error::CostError(cost, budget)) } @@ -483,6 +500,7 @@ pub struct StacksMicroblockBuilder<'a> { unconfirmed: bool, runtime: MicroblockMinerRuntime, settings: BlockBuilderSettings, + ast_rules: ASTRules, } impl<'a> StacksMicroblockBuilder<'a> { @@ -501,7 +519,7 @@ impl<'a> StacksMicroblockBuilder<'a> { }; let (header_reader, _) = chainstate.reopen()?; - let anchor_block_height = StacksChainState::get_anchored_block_header_info( + let anchor_block_header = StacksChainState::get_anchored_block_header_info( header_reader.db(), &anchor_block_consensus_hash, &anchor_block, @@ -512,8 +530,10 @@ impl<'a> StacksMicroblockBuilder<'a> { &anchor_block_consensus_hash, &anchor_block ); Error::NoSuchBlockError - })? - .stacks_block_height; + })?; + let anchor_block_height = anchor_block_header.stacks_block_height; + let burn_height = anchor_block_header.burn_header_height; + let ast_rules = burn_dbconn.get_ast_rules(burn_height); // when we drop the miner, the underlying clarity instance will be rolled back chainstate.set_unconfirmed_dirty(true); @@ -552,6 +572,7 @@ impl<'a> StacksMicroblockBuilder<'a> { header_reader, unconfirmed: false, settings: settings, + ast_rules, }) } @@ -571,30 +592,36 @@ impl<'a> StacksMicroblockBuilder<'a> { }; let (header_reader, _) = chainstate.reopen()?; - let (anchored_consensus_hash, anchored_block_hash, anchored_block_height) = - if let Some(unconfirmed) = chainstate.unconfirmed_state.as_ref() { - let header_info = - StacksChainState::get_stacks_block_header_info_by_index_block_hash( - chainstate.db(), - &unconfirmed.confirmed_chain_tip, - )? - .ok_or_else(|| { - warn!( - "No such confirmed block {}", - &unconfirmed.confirmed_chain_tip - ); - Error::NoSuchBlockError - })?; - ( - header_info.consensus_hash, - header_info.anchored_header.block_hash(), - header_info.stacks_block_height, - ) - } else { - // unconfirmed state needs to be initialized - debug!("Unconfirmed chainstate not initialized"); - return Err(Error::NoSuchBlockError)?; - }; + let ( + anchored_consensus_hash, + anchored_block_hash, + anchored_block_height, + anchored_burn_height, + ) = if let Some(unconfirmed) = chainstate.unconfirmed_state.as_ref() { + let header_info = StacksChainState::get_stacks_block_header_info_by_index_block_hash( + chainstate.db(), + &unconfirmed.confirmed_chain_tip, + )? + .ok_or_else(|| { + warn!( + "No such confirmed block {}", + &unconfirmed.confirmed_chain_tip + ); + Error::NoSuchBlockError + })?; + ( + header_info.consensus_hash, + header_info.anchored_header.block_hash(), + header_info.stacks_block_height, + header_info.burn_header_height, + ) + } else { + // unconfirmed state needs to be initialized + debug!("Unconfirmed chainstate not initialized"); + return Err(Error::NoSuchBlockError)?; + }; + + let ast_rules = burn_dbconn.get_ast_rules(anchored_burn_height); let mut clarity_tx = chainstate.begin_unconfirmed(burn_dbconn).ok_or_else(|| { warn!( @@ -623,6 +650,7 @@ impl<'a> StacksMicroblockBuilder<'a> { header_reader, unconfirmed: true, settings: settings, + ast_rules, }) } @@ -633,6 +661,7 @@ impl<'a> StacksMicroblockBuilder<'a> { miner_key: &Secp256k1PrivateKey, parent_anchor_block_hash: &BlockHeaderHash, prev_microblock_header: Option<&StacksMicroblockHeader>, + ast_rules: ASTRules, ) -> Result { let miner_pubkey_hash = Hash160::from_node_public_key(&StacksPublicKey::from_private(miner_key)); @@ -652,6 +681,10 @@ impl<'a> StacksMicroblockBuilder<'a> { StacksMicroblockHeader::first_unsigned(parent_anchor_block_hash, &tx_merkle_root) }; + if ast_rules != ASTRules::Typical { + next_microblock_header.version = STACKS_BLOCK_VERSION_AST_PRECHECK_SIZE; + } + next_microblock_header.sign(miner_key).unwrap(); next_microblock_header.verify(&miner_pubkey_hash).unwrap(); Ok(StacksMicroblock { @@ -675,6 +708,7 @@ impl<'a> StacksMicroblockBuilder<'a> { miner_key, &self.anchor_block, self.runtime.prev_microblock_header.as_ref(), + self.ast_rules, )?; self.runtime.prev_microblock_header = Some(microblock.header.clone()); @@ -721,6 +755,7 @@ impl<'a> StacksMicroblockBuilder<'a> { tx_len: u64, bytes_so_far: u64, limit_behavior: &BlockLimitFunction, + ast_rules: ASTRules, ) -> Result { if tx.anchor_mode != TransactionAnchorMode::OffChainOnly && tx.anchor_mode != TransactionAnchorMode::Any @@ -775,8 +810,19 @@ impl<'a> StacksMicroblockBuilder<'a> { BlockLimitFunction::NO_LIMIT_HIT => {} }; + // preemptively skip problematic transactions + if let Err(e) = + Relayer::static_check_problematic_relayed_tx(clarity_tx.config.mainnet, &tx, ast_rules) + { + info!( + "Detected problematic tx {} while mining; dropping from mempool", + tx.txid() + ); + return Ok(TransactionResult::problematic(&tx, Error::NetError(e))); + } + let quiet = !cfg!(test); - match StacksChainState::process_transaction(clarity_tx, &tx, quiet) { + match StacksChainState::process_transaction(clarity_tx, &tx, quiet, ast_rules) { Ok((fee, receipt)) => Ok(TransactionResult::success(&tx, fee, receipt)), Err(e) => { let (is_problematic, e) = TransactionResult::is_problematic(&tx, e); @@ -822,6 +868,7 @@ impl<'a> StacksMicroblockBuilder<'a> { } /// NOTE: this is only used in integration tests. + #[cfg(any(test, feature = "testing"))] pub fn mine_next_microblock_from_txs( &mut self, txs_and_lens: Vec<(StacksTransaction, u64)>, @@ -859,6 +906,7 @@ impl<'a> StacksMicroblockBuilder<'a> { tx_len, bytes_so_far, &block_limit_hit, + self.ast_rules, ) { Ok(tx_result) => { tx_events.push(tx_result.convert_to_event()); @@ -1007,6 +1055,7 @@ impl<'a> StacksMicroblockBuilder<'a> { mempool_tx.metadata.len, bytes_so_far, &block_limit_hit, + self.ast_rules.clone(), ) { Ok(tx_result) => { let result_event = tx_result.convert_to_event(); @@ -1338,9 +1387,16 @@ impl StacksBlockBuilder { &mut self, clarity_tx: &mut ClarityTx, tx: &StacksTransaction, + ast_rules: ASTRules, ) -> Result { let tx_len = tx.tx_len(); - match self.try_mine_tx_with_len(clarity_tx, tx, tx_len, &BlockLimitFunction::NO_LIMIT_HIT) { + match self.try_mine_tx_with_len( + clarity_tx, + tx, + tx_len, + &BlockLimitFunction::NO_LIMIT_HIT, + ast_rules, + ) { TransactionResult::Success(s) => Ok(TransactionResult::Success(s)), TransactionResult::Skipped(TransactionSkipped { error, .. }) | TransactionResult::ProcessingError(TransactionError { error, .. }) => Err(error), @@ -1358,6 +1414,7 @@ impl StacksBlockBuilder { tx: &StacksTransaction, tx_len: u64, limit_behavior: &BlockLimitFunction, + ast_rules: ASTRules, ) -> TransactionResult { if self.bytes_so_far + tx_len >= MAX_EPOCH_SIZE.into() { return TransactionResult::skipped_due_to_error(&tx, Error::BlockTooBigError); @@ -1409,8 +1466,21 @@ impl StacksBlockBuilder { ); } - let (fee, receipt) = match StacksChainState::process_transaction(clarity_tx, tx, quiet) - { + // preemptively skip problematic transactions + if let Err(e) = Relayer::static_check_problematic_relayed_tx( + clarity_tx.config.mainnet, + &tx, + ast_rules, + ) { + info!( + "Detected problematic tx {} while mining; dropping from mempool", + tx.txid() + ); + return TransactionResult::problematic(&tx, Error::NetError(e)); + } + let (fee, receipt) = match StacksChainState::process_transaction( + clarity_tx, tx, quiet, ast_rules, + ) { Ok((fee, receipt)) => (fee, receipt), Err(e) => { let (is_problematic, e) = TransactionResult::is_problematic(&tx, e); @@ -1475,8 +1545,21 @@ impl StacksBlockBuilder { ); } - let (fee, receipt) = match StacksChainState::process_transaction(clarity_tx, tx, quiet) - { + // preemptively skip problematic transactions + if let Err(e) = Relayer::static_check_problematic_relayed_tx( + clarity_tx.config.mainnet, + &tx, + ast_rules, + ) { + info!( + "Detected problematic tx {} while mining; dropping from mempool", + tx.txid() + ); + return TransactionResult::problematic(&tx, Error::NetError(e)); + } + let (fee, receipt) = match StacksChainState::process_transaction( + clarity_tx, tx, quiet, ast_rules, + ) { Ok((fee, receipt)) => (fee, receipt), Err(e) => { let (is_problematic, e) = TransactionResult::is_problematic(&tx, e); @@ -1558,7 +1641,7 @@ impl StacksBlockBuilder { let quiet = !cfg!(test); if !self.anchored_done { // save - match StacksChainState::process_transaction(clarity_tx, tx, quiet) { + match StacksChainState::process_transaction(clarity_tx, tx, quiet, ASTRules::Typical) { Ok((fee, receipt)) => { self.total_anchored_fees += fee; } @@ -1569,7 +1652,7 @@ impl StacksBlockBuilder { self.txs.push(tx.clone()); } else { - match StacksChainState::process_transaction(clarity_tx, tx, quiet) { + match StacksChainState::process_transaction(clarity_tx, tx, quiet, ASTRules::Typical) { Ok((fee, receipt)) => { self.total_streamed_fees += fee; } @@ -1827,6 +1910,9 @@ impl StacksBlockBuilder { let (chainstate_tx, clarity_instance) = chainstate.chainstate_tx_begin()?; + let ast_rules = + SortitionDB::get_ast_rules(burn_dbconn.conn(), (burn_tip_height + 1).into())?; + Ok(MinerEpochInfo { chainstate_tx, clarity_instance, @@ -1834,6 +1920,7 @@ impl StacksBlockBuilder { burn_tip_height: burn_tip_height + 1, parent_microblocks, mainnet, + ast_rules, }) } @@ -1915,9 +2002,10 @@ impl StacksBlockBuilder { debug!("Build anchored block from {} transactions", txs.len()); let (mut chainstate, _) = chainstate_handle.reopen()?; let mut miner_epoch_info = builder.pre_epoch_begin(&mut chainstate, burn_dbconn)?; + let ast_rules = miner_epoch_info.ast_rules; let (mut epoch_tx, _) = builder.epoch_begin(burn_dbconn, &mut miner_epoch_info)?; for tx in txs.drain(..) { - match builder.try_mine_tx(&mut epoch_tx, &tx) { + match builder.try_mine_tx(&mut epoch_tx, &tx, ast_rules.clone()) { Ok(_) => { debug!("Included {}", &tx.txid()); } @@ -2094,6 +2182,11 @@ impl StacksBlockBuilder { let ts_start = get_epoch_time_ms(); let mut miner_epoch_info = builder.pre_epoch_begin(&mut chainstate, burn_dbconn)?; + let ast_rules = miner_epoch_info.ast_rules; + if ast_rules != ASTRules::Typical { + builder.header.version = STACKS_BLOCK_VERSION_AST_PRECHECK_SIZE; + } + let (mut epoch_tx, confirmed_mblock_cost) = builder.epoch_begin(burn_dbconn, &mut miner_epoch_info)?; let stacks_epoch_id = epoch_tx.get_epoch(); @@ -2104,7 +2197,7 @@ impl StacksBlockBuilder { let mut tx_events = Vec::new(); tx_events.push( builder - .try_mine_tx(&mut epoch_tx, coinbase_tx)? + .try_mine_tx(&mut epoch_tx, coinbase_tx, ast_rules.clone())? .convert_to_event(), ); @@ -2201,6 +2294,7 @@ impl StacksBlockBuilder { &txinfo.tx, txinfo.metadata.len, &block_limit_hit, + ast_rules, ); let result_event = tx_result.convert_to_event(); @@ -6016,7 +6110,7 @@ pub mod test { let tx_coinbase_signed = make_coinbase(miner, burnchain_height); builder - .try_mine_tx(clarity_tx, &tx_coinbase_signed) + .try_mine_tx(clarity_tx, &tx_coinbase_signed, ASTRules::PrecheckSize) .unwrap(); let stacks_block = builder.mine_anchored_block(clarity_tx); @@ -6051,7 +6145,7 @@ pub mod test { let tx_coinbase_signed = make_coinbase(miner, burnchain_height); builder - .try_mine_tx(clarity_tx, &tx_coinbase_signed) + .try_mine_tx(clarity_tx, &tx_coinbase_signed, ASTRules::PrecheckSize) .unwrap(); let stacks_block = builder.mine_anchored_block(clarity_tx); @@ -6086,7 +6180,7 @@ pub mod test { let tx_coinbase_signed = make_coinbase(miner, burnchain_height); builder - .try_mine_tx(clarity_tx, &tx_coinbase_signed) + .try_mine_tx(clarity_tx, &tx_coinbase_signed, ASTRules::PrecheckSize) .unwrap(); let stacks_block = builder.mine_anchored_block(clarity_tx); @@ -6227,7 +6321,7 @@ pub mod test { // make a coinbase for this miner let tx_coinbase_signed = make_coinbase(miner, burnchain_height); builder - .try_mine_tx(clarity_tx, &tx_coinbase_signed) + .try_mine_tx(clarity_tx, &tx_coinbase_signed, ASTRules::PrecheckSize) .unwrap(); let recipient = @@ -6305,7 +6399,7 @@ pub mod test { // make a coinbase for this miner let tx_coinbase_signed = make_coinbase(miner, burnchain_height); builder - .try_mine_tx(clarity_tx, &tx_coinbase_signed) + .try_mine_tx(clarity_tx, &tx_coinbase_signed, ASTRules::PrecheckSize) .unwrap(); // make a smart contract @@ -6315,7 +6409,7 @@ pub mod test { builder.header.total_work.work as usize, ); builder - .try_mine_tx(clarity_tx, &tx_contract_signed) + .try_mine_tx(clarity_tx, &tx_contract_signed, ASTRules::PrecheckSize) .unwrap(); // make a contract call @@ -6327,7 +6421,7 @@ pub mod test { 2, ); builder - .try_mine_tx(clarity_tx, &tx_contract_call_signed) + .try_mine_tx(clarity_tx, &tx_contract_call_signed, ASTRules::PrecheckSize) .unwrap(); let stacks_block = builder.mine_anchored_block(clarity_tx); @@ -6382,7 +6476,7 @@ pub mod test { // make a coinbase for this miner let tx_coinbase_signed = make_coinbase(miner, burnchain_height); builder - .try_mine_tx(clarity_tx, &tx_coinbase_signed) + .try_mine_tx(clarity_tx, &tx_coinbase_signed, ASTRules::PrecheckSize) .unwrap(); // make a smart contract @@ -6392,7 +6486,7 @@ pub mod test { builder.header.total_work.work as usize, ); builder - .try_mine_tx(clarity_tx, &tx_contract_signed) + .try_mine_tx(clarity_tx, &tx_contract_signed, ASTRules::PrecheckSize) .unwrap(); let stacks_block = builder.mine_anchored_block(clarity_tx); @@ -6469,7 +6563,7 @@ pub mod test { // make a coinbase for this miner let tx_coinbase_signed = make_coinbase(miner, burnchain_height); builder - .try_mine_tx(clarity_tx, &tx_coinbase_signed) + .try_mine_tx(clarity_tx, &tx_coinbase_signed, ASTRules::PrecheckSize) .unwrap(); // make a smart contract @@ -6479,7 +6573,7 @@ pub mod test { builder.header.total_work.work as usize, ); builder - .try_mine_tx(clarity_tx, &tx_contract_signed) + .try_mine_tx(clarity_tx, &tx_contract_signed, ASTRules::PrecheckSize) .unwrap(); let stacks_block = builder.mine_anchored_block(clarity_tx); diff --git a/src/chainstate/stacks/mod.rs b/src/chainstate/stacks/mod.rs index bd5420884..291edd930 100644 --- a/src/chainstate/stacks/mod.rs +++ b/src/chainstate/stacks/mod.rs @@ -80,6 +80,7 @@ pub use stacks_common::address::{ }; pub const STACKS_BLOCK_VERSION: u8 = 0; +pub const STACKS_BLOCK_VERSION_AST_PRECHECK_SIZE: u8 = 1; pub const STACKS_MICROBLOCK_VERSION: u8 = 0; pub const MAX_BLOCK_LEN: u32 = 2 * 1024 * 1024; diff --git a/src/chainstate/stacks/transaction.rs b/src/chainstate/stacks/transaction.rs index 1b57f9a65..d61ce8fde 100644 --- a/src/chainstate/stacks/transaction.rs +++ b/src/chainstate/stacks/transaction.rs @@ -24,7 +24,6 @@ use crate::chainstate::stacks::*; use crate::core::*; use crate::net::Error as net_error; use crate::types::StacksPublicKeyBuffer; -use clarity::vm::ast::build_ast; use clarity::vm::representations::{ClarityName, ContractName}; use clarity::vm::types::serialization::SerializationError as clarity_serialization_error; use clarity::vm::types::{QualifiedContractIdentifier, StandardPrincipalData}; diff --git a/src/clarity_cli.rs b/src/clarity_cli.rs index 4ad3c906b..9566d137d 100644 --- a/src/clarity_cli.rs +++ b/src/clarity_cli.rs @@ -43,7 +43,8 @@ use crate::clarity::{ vm::analysis::contract_interface_builder::build_contract_interface, vm::analysis::{errors::CheckError, errors::CheckResult, AnalysisDatabase, ContractAnalysis}, vm::ast, - vm::ast::build_ast, + vm::ast::build_ast_with_rules, + vm::ast::ASTRules, vm::contexts::GlobalContext, vm::contexts::{AssetMap, OwnedEnvironment}, vm::costs::ExecutionCost, @@ -69,7 +70,7 @@ use crate::util_lib::boot::{boot_code_addr, boot_code_id}; use crate::burnchains::Address; use crate::chainstate::stacks::index::ClarityMarfTrieId; -use crate::core::BLOCK_LIMIT_MAINNET_20; +use crate::core::BLOCK_LIMIT_MAINNET_205; use crate::core::HELIUM_BLOCK_LIMIT_20; use crate::util_lib::strings::StacksString; @@ -155,8 +156,13 @@ fn parse( contract_identifier: &QualifiedContractIdentifier, source_code: &str, ) -> Result, Error> { - let ast = build_ast(contract_identifier, source_code, &mut ()) - .map_err(|e| RuntimeErrorType::ASTError(e))?; + let ast = build_ast_with_rules( + contract_identifier, + source_code, + &mut (), + ASTRules::PrecheckSize, + ) + .map_err(|e| RuntimeErrorType::ASTError(e))?; Ok(ast.expressions) } @@ -223,7 +229,7 @@ fn run_analysis( let cost_track = LimitedCostTracker::new( mainnet, if mainnet { - BLOCK_LIMIT_MAINNET_20.clone() + BLOCK_LIMIT_MAINNET_205.clone() } else { HELIUM_BLOCK_LIMIT_20.clone() }, @@ -397,7 +403,7 @@ where let cost_track = LimitedCostTracker::new( mainnet, if mainnet { - BLOCK_LIMIT_MAINNET_20.clone() + BLOCK_LIMIT_MAINNET_205.clone() } else { HELIUM_BLOCK_LIMIT_20.clone() }, @@ -426,7 +432,8 @@ pub fn vm_execute(program: &str) -> Result, Error> { ); global_context.coverage_reporting = Some(CoverageReporter::new()); global_context.execute(|g| { - let parsed = ast::build_ast(&contract_id, program, &mut ())?.expressions; + let parsed = ast::build_ast_with_rules(&contract_id, program, &mut (), ASTRules::Typical)? + .expressions; eval_all(&parsed, &mut contract_context, g) }) } @@ -777,7 +784,11 @@ fn install_boot_code(header_db: &CLIHeadersDB, marf: &mut C) let db = marf.get_clarity_db(header_db, &NULL_BURN_STATE_DB); let mut vm_env = OwnedEnvironment::new_free(mainnet, db, DEFAULT_CLI_EPOCH); vm_env - .initialize_contract(contract_identifier, &contract_content) + .initialize_contract( + contract_identifier, + &contract_content, + ASTRules::PrecheckSize, + ) .unwrap(); } Err(_) => { @@ -1140,13 +1151,14 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option val, - Err(error) => { - println!("Execution error:\n{}", error); - continue; - } - }; + let eval_result = + match exec_env.eval_raw_with_rules(&content, ASTRules::PrecheckSize) { + Ok(val) => val, + Err(error) => { + println!("Execution error:\n{}", error); + continue; + } + }; println!("{}", eval_result); } @@ -1172,7 +1184,9 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option { - let result = vm_env.get_exec_environment(None).eval_raw(&content); + let result = vm_env + .get_exec_environment(None) + .eval_raw_with_rules(&content, ASTRules::PrecheckSize); match result { Ok(x) => ( 0, @@ -1221,9 +1235,11 @@ pub fn invoke_command(invoked_by: &str, args: &[String]) -> (i32, Option (i32, Option (i32, Option (i32, Option Result { let mut read_only_conn = self.datastore.begin_read_only(Some(at_block)); let mut clarity_db = read_only_conn.as_clarity_db(header_db, burn_state_db); @@ -492,7 +502,7 @@ impl ClarityInstance { }; let mut env = OwnedEnvironment::new_free(self.mainnet, clarity_db, epoch_id); - env.eval_read_only(contract, program) + env.eval_read_only_with_rules(contract, program, ast_rules) .map(|(x, _, _)| x) .map_err(Error::from) } @@ -707,6 +717,7 @@ impl<'a, 'b> ClarityBlockConnection<'a, 'b> { tx_conn, &costs_2_contract_tx, &boot_code_account, + ASTRules::PrecheckSize, ) .expect("FATAL: Failed to process PoX 2 contract initialization"); @@ -1039,13 +1050,25 @@ mod tests { let contract = "(define-public (foo (x int) (y uint)) (ok (+ x y)))"; let _e = conn - .as_transaction(|tx| tx.analyze_smart_contract(&contract_identifier, &contract)) + .as_transaction(|tx| { + tx.analyze_smart_contract( + &contract_identifier, + &contract, + ASTRules::PrecheckSize, + ) + }) .unwrap_err(); // okay, let's try it again: let _e = conn - .as_transaction(|tx| tx.analyze_smart_contract(&contract_identifier, &contract)) + .as_transaction(|tx| { + tx.analyze_smart_contract( + &contract_identifier, + &contract, + ASTRules::PrecheckSize, + ) + }) .unwrap_err(); conn.commit_block(); @@ -1087,7 +1110,7 @@ mod tests { conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); conn.initialize_smart_contract(&contract_identifier, &ct_ast, &contract, |_, _| { false @@ -1130,7 +1153,7 @@ mod tests { let mut tx = conn.start_transaction_processing(); let (ct_ast, ct_analysis) = tx - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); tx.initialize_smart_contract(&contract_identifier, &ct_ast, &contract, |_, _| { false @@ -1148,7 +1171,7 @@ mod tests { let contract = "(define-public (foo (x int) (y int)) (ok (+ x y)))"; let (ct_ast, ct_analysis) = tx - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); tx.initialize_smart_contract(&contract_identifier, &ct_ast, &contract, |_, _| { false @@ -1168,7 +1191,7 @@ mod tests { let contract = "(define-public (foo (x int) (y int)) (ok (+ x y)))"; let (ct_ast, _ct_analysis) = tx - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); assert!(format!( "{}", @@ -1215,7 +1238,7 @@ mod tests { conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); conn.initialize_smart_contract(&contract_identifier, &ct_ast, &contract, |_, _| { false @@ -1264,7 +1287,7 @@ mod tests { conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); conn.initialize_smart_contract(&contract_identifier, &ct_ast, &contract, |_, _| { false @@ -1345,7 +1368,7 @@ mod tests { conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); conn.initialize_smart_contract(&contract_identifier, &ct_ast, &contract, |_, _| { false @@ -1466,7 +1489,7 @@ mod tests { conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); conn.initialize_smart_contract(&contract_identifier, &ct_ast, &contract, |_, _| { false @@ -1656,19 +1679,33 @@ mod tests { ); conn.as_transaction(|clarity_tx| { - let receipt = - StacksChainState::process_transaction_payload(clarity_tx, &tx1, &account) - .unwrap(); + let receipt = StacksChainState::process_transaction_payload( + clarity_tx, + &tx1, + &account, + ASTRules::PrecheckSize, + ) + .unwrap(); assert_eq!(receipt.post_condition_aborted, true); }); conn.as_transaction(|clarity_tx| { - StacksChainState::process_transaction_payload(clarity_tx, &tx2, &account).unwrap(); + StacksChainState::process_transaction_payload( + clarity_tx, + &tx2, + &account, + ASTRules::PrecheckSize, + ) + .unwrap(); }); conn.as_transaction(|clarity_tx| { - let receipt = - StacksChainState::process_transaction_payload(clarity_tx, &tx3, &account) - .unwrap(); + let receipt = StacksChainState::process_transaction_payload( + clarity_tx, + &tx3, + &account, + ASTRules::PrecheckSize, + ) + .unwrap(); assert_eq!(receipt.post_condition_aborted, true); }); @@ -1722,6 +1759,9 @@ mod tests { ) -> Option { self.get_stacks_epoch(0) } + fn get_ast_rules(&self, height: u32) -> ASTRules { + ASTRules::Typical + } } let burn_state_db = BlockLimitBurnStateDB {}; @@ -1753,7 +1793,7 @@ mod tests { conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); conn.initialize_smart_contract(&contract_identifier, &ct_ast, &contract, |_, _| { false diff --git a/src/clarity_vm/database/mod.rs b/src/clarity_vm/database/mod.rs index d4cdab1c6..c4d122068 100644 --- a/src/clarity_vm/database/mod.rs +++ b/src/clarity_vm/database/mod.rs @@ -185,6 +185,10 @@ impl BurnStateDB for SortitionHandleTx<'_> { SortitionDB::get_stacks_epoch_by_epoch_id(self.tx(), epoch_id) .expect("BUG: failed to get epoch for epoch id") } + + fn get_ast_rules(&self, height: u32) -> clarity::vm::ast::ASTRules { + SortitionDB::get_ast_rules(self.tx(), height.into()).expect("BUG: failed to get AST rules") + } } impl BurnStateDB for SortitionDBConn<'_> { @@ -216,6 +220,11 @@ impl BurnStateDB for SortitionDBConn<'_> { SortitionDB::get_stacks_epoch_by_epoch_id(self.conn(), epoch_id) .expect("BUG: failed to get epoch for epoch id") } + + fn get_ast_rules(&self, height: u32) -> clarity::vm::ast::ASTRules { + SortitionDB::get_ast_rules(self.conn(), height.into()) + .expect("BUG: failed to get AST rules") + } } pub struct MemoryBackingStore { diff --git a/src/clarity_vm/tests/analysis_costs.rs b/src/clarity_vm/tests/analysis_costs.rs index 58a104918..d334c6804 100644 --- a/src/clarity_vm/tests/analysis_costs.rs +++ b/src/clarity_vm/tests/analysis_costs.rs @@ -16,6 +16,7 @@ use crate::chainstate::stacks::index::storage::TrieFileStorage; use crate::clarity_vm::clarity::ClarityInstance; +use clarity::vm::ast::ASTRules; use clarity::vm::clarity::TransactionConnection; use clarity::vm::contexts::Environment; use clarity::vm::contexts::{AssetMap, AssetMapEntry, GlobalContext, OwnedEnvironment}; @@ -117,7 +118,7 @@ pub fn test_tracked_costs(prog: &str, use_mainnet: bool, epoch: StacksEpochId) - conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&trait_contract_id, contract_trait) + .analyze_smart_contract(&trait_contract_id, contract_trait, ASTRules::PrecheckSize) .unwrap(); conn.initialize_smart_contract(&trait_contract_id, &ct_ast, contract_trait, |_, _| { false @@ -139,7 +140,7 @@ pub fn test_tracked_costs(prog: &str, use_mainnet: bool, epoch: StacksEpochId) - ); conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&other_contract_id, contract_other) + .analyze_smart_contract(&other_contract_id, contract_other, ASTRules::PrecheckSize) .unwrap(); conn.initialize_smart_contract(&other_contract_id, &ct_ast, contract_other, |_, _| { false @@ -162,7 +163,7 @@ pub fn test_tracked_costs(prog: &str, use_mainnet: bool, epoch: StacksEpochId) - conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&self_contract_id, &contract_self) + .analyze_smart_contract(&self_contract_id, &contract_self, ASTRules::PrecheckSize) .unwrap(); conn.initialize_smart_contract(&self_contract_id, &ct_ast, &contract_self, |_, _| { false diff --git a/src/clarity_vm/tests/costs.rs b/src/clarity_vm/tests/costs.rs index c4b584388..e18908f16 100644 --- a/src/clarity_vm/tests/costs.rs +++ b/src/clarity_vm/tests/costs.rs @@ -23,6 +23,7 @@ use crate::types::chainstate::BlockHeaderHash; use crate::types::chainstate::StacksBlockId; use crate::types::StacksEpochId; use crate::util_lib::boot::boot_code_id; +use clarity::vm::ast::ASTRules; use clarity::vm::clarity::TransactionConnection; use clarity::vm::contexts::Environment; use clarity::vm::contexts::{AssetMap, AssetMapEntry, GlobalContext, OwnedEnvironment}; @@ -204,7 +205,7 @@ fn exec_cost(contract: &str, use_mainnet: bool, epoch: StacksEpochId) -> Executi with_owned_env(epoch, use_mainnet, |mut owned_env| { owned_env - .initialize_contract(contract_id.clone(), contract) + .initialize_contract(contract_id.clone(), contract, ASTRules::PrecheckSize) .unwrap(); let cost_before = owned_env.get_cost_total(); @@ -830,13 +831,25 @@ fn test_tracked_costs(prog: &str, use_mainnet: bool, epoch: StacksEpochId) -> Ex with_owned_env(epoch, use_mainnet, |mut owned_env| { owned_env - .initialize_contract(trait_contract_id.clone(), contract_trait) + .initialize_contract( + trait_contract_id.clone(), + contract_trait, + ASTRules::PrecheckSize, + ) .unwrap(); owned_env - .initialize_contract(other_contract_id.clone(), contract_other) + .initialize_contract( + other_contract_id.clone(), + contract_other, + ASTRules::PrecheckSize, + ) .unwrap(); owned_env - .initialize_contract(self_contract_id.clone(), &contract_self) + .initialize_contract( + self_contract_id.clone(), + &contract_self, + ASTRules::PrecheckSize, + ) .unwrap(); let target_contract = Value::from(PrincipalData::Contract(other_contract_id.clone())); @@ -969,7 +982,7 @@ fn test_cost_contract_short_circuits(use_mainnet: bool) { { block_conn.as_transaction(|tx| { let (ast, analysis) = tx - .analyze_smart_contract(contract_name, contract_src) + .analyze_smart_contract(contract_name, contract_src, ASTRules::PrecheckSize) .unwrap(); tx.initialize_smart_contract(contract_name, &ast, contract_src, |_, _| false) .unwrap(); @@ -1231,7 +1244,7 @@ fn test_cost_voting_integration(use_mainnet: bool) { { block_conn.as_transaction(|tx| { let (ast, analysis) = tx - .analyze_smart_contract(contract_name, contract_src) + .analyze_smart_contract(contract_name, contract_src, ASTRules::PrecheckSize) .unwrap(); tx.initialize_smart_contract(contract_name, &ast, contract_src, |_, _| false) .unwrap(); diff --git a/src/clarity_vm/tests/forking.rs b/src/clarity_vm/tests/forking.rs index 13dc4d269..3b915bf58 100644 --- a/src/clarity_vm/tests/forking.rs +++ b/src/clarity_vm/tests/forking.rs @@ -17,6 +17,7 @@ use crate::chainstate::stacks::index::storage::TrieFileStorage; use crate::chainstate::stacks::index::ClarityMarfTrieId; use clarity::vm::analysis::errors::CheckErrors; +use clarity::vm::ast::ASTRules; use clarity::vm::contexts::OwnedEnvironment; use clarity::vm::database::ClarityDatabase; use clarity::vm::errors::{Error, InterpreterResult as Result, RuntimeErrorType}; @@ -65,7 +66,9 @@ fn test_at_block_mutations() { (ok (at-block 0x0101010101010101010101010101010101010101010101010101010101010101 (var-get datum)))))"; eprintln!("Initializing contract..."); - owned_env.initialize_contract(c.clone(), &contract).unwrap(); + owned_env + .initialize_contract(c.clone(), &contract, ASTRules::PrecheckSize) + .unwrap(); } fn branch( @@ -137,7 +140,9 @@ fn test_at_block_good() { (ok (var-get datum))))"; eprintln!("Initializing contract..."); - owned_env.initialize_contract(c.clone(), &contract).unwrap(); + owned_env + .initialize_contract(c.clone(), &contract, ASTRules::PrecheckSize) + .unwrap(); } fn branch( @@ -205,7 +210,7 @@ fn test_at_block_missing_defines() { eprintln!("Initializing contract..."); owned_env - .initialize_contract(c_a.clone(), &contract) + .initialize_contract(c_a.clone(), &contract, ASTRules::PrecheckSize) .unwrap(); } @@ -220,7 +225,7 @@ fn test_at_block_missing_defines() { eprintln!("Initializing contract..."); let e = owned_env - .initialize_contract(c_b.clone(), &contract) + .initialize_contract(c_b.clone(), &contract, ASTRules::PrecheckSize) .unwrap_err(); e } @@ -327,7 +332,7 @@ fn initialize_contract(owned_env: &mut OwnedEnvironment) { let contract_identifier = QualifiedContractIdentifier::new(p1_address, "tokens".into()); owned_env - .initialize_contract(contract_identifier, &contract) + .initialize_contract(contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); } diff --git a/src/clarity_vm/tests/large_contract.rs b/src/clarity_vm/tests/large_contract.rs index 5088b5353..d3ffdf561 100644 --- a/src/clarity_vm/tests/large_contract.rs +++ b/src/clarity_vm/tests/large_contract.rs @@ -16,13 +16,16 @@ use crate::chainstate::stacks::index::ClarityMarfTrieId; use crate::clarity_vm::clarity::{ClarityInstance, Error as ClarityError}; +use crate::core::StacksEpochId; use crate::types::chainstate::BlockHeaderHash; use crate::types::chainstate::StacksBlockId; -use clarity::vm::ast; +use clarity::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER; +use clarity::vm::ast::{self, ASTRules}; use clarity::vm::contexts::{Environment, GlobalContext, OwnedEnvironment}; use clarity::vm::contracts::Contract; use clarity::vm::costs::ExecutionCost; use clarity::vm::database::ClarityDatabase; +use clarity::vm::errors::Error as InterpreterError; use clarity::vm::errors::{CheckErrors, Error, RuntimeErrorType}; use clarity::vm::representations::SymbolicExpression; use clarity::vm::test_util::*; @@ -30,6 +33,7 @@ use clarity::vm::types::{ OptionalData, PrincipalData, QualifiedContractIdentifier, ResponseData, StandardPrincipalData, TypeSignature, Value, }; +use clarity::vm::MAX_CALL_STACK_DEPTH; use stacks_common::util::hash::hex_bytes; use crate::clarity_vm::database::marf::MarfedKV; @@ -39,6 +43,11 @@ fn test_block_headers(n: u8) -> StacksBlockId { StacksBlockId([n as u8; 32]) } +pub const TEST_BURN_STATE_DB_AST_PRECHECK: UnitTestBurnStateDB = UnitTestBurnStateDB { + epoch_id: StacksEpochId::Epoch20, + ast_rules: ast::ASTRules::PrecheckSize, +}; + const SIMPLE_TOKENS: &str = "(define-map tokens { account: principal } { balance: uint }) (define-read-only (my-get-token-balance (account principal)) (default-to u0 (get balance (map-get? tokens (tuple (account account)))))) @@ -352,7 +361,7 @@ pub fn rollback_log_memory_test() { conn.as_transaction(|conn| { let (ct_ast, _ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); assert!(format!( "{:?}", @@ -418,7 +427,7 @@ pub fn let_memory_test() { conn.as_transaction(|conn| { let (ct_ast, _ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); assert!(format!( "{:?}", @@ -482,7 +491,7 @@ pub fn argument_memory_test() { conn.as_transaction(|conn| { let (ct_ast, _ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract(&contract_identifier, &contract, ASTRules::PrecheckSize) .unwrap(); assert!(format!( "{:?}", @@ -565,7 +574,7 @@ pub fn fcall_memory_test() { conn.as_transaction(|conn| { let (ct_ast, _ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract_ok) + .analyze_smart_contract(&contract_identifier, &contract_ok, ASTRules::PrecheckSize) .unwrap(); assert!(match conn .initialize_smart_contract( @@ -584,7 +593,7 @@ pub fn fcall_memory_test() { conn.as_transaction(|conn| { let (ct_ast, _ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract_err) + .analyze_smart_contract(&contract_identifier, &contract_err, ASTRules::PrecheckSize) .unwrap(); assert!(format!( "{:?}", @@ -664,7 +673,11 @@ pub fn ccall_memory_test() { if i < (CONTRACTS - 1) { conn.as_transaction(|conn| { let (ct_ast, ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract( + &contract_identifier, + &contract, + ASTRules::PrecheckSize, + ) .unwrap(); conn.initialize_smart_contract( &contract_identifier, @@ -679,7 +692,11 @@ pub fn ccall_memory_test() { } else { conn.as_transaction(|conn| { let (ct_ast, _ct_analysis) = conn - .analyze_smart_contract(&contract_identifier, &contract) + .analyze_smart_contract( + &contract_identifier, + &contract, + ASTRules::PrecheckSize, + ) .unwrap(); assert!(format!( "{:?}", diff --git a/src/clarity_vm/tests/simple_tests.rs b/src/clarity_vm/tests/simple_tests.rs index d8eac3d25..e0c2d60c5 100644 --- a/src/clarity_vm/tests/simple_tests.rs +++ b/src/clarity_vm/tests/simple_tests.rs @@ -54,6 +54,7 @@ fn test_at_unknown_block() { .initialize_contract( QualifiedContractIdentifier::local("contract").unwrap(), &contract, + clarity::vm::ast::ASTRules::PrecheckSize, ) .unwrap_err(); eprintln!("{}", err); diff --git a/src/core/mod.rs b/src/core/mod.rs index 78290aad0..6941558ce 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -141,6 +141,9 @@ pub const POX_THRESHOLD_STEPS_USTX: u128 = 10_000 * (MICROSTACKS_PER_STACKS as u pub const POX_MAX_NUM_CYCLES: u8 = 12; +/// Burn block height at which the ASTRules::PrecheckSize becomes the default behavior on mainnet +pub const AST_RULES_PRECHECK_SIZE: u64 = 752000; // on or about Aug 30 2022 + // Stacks 1.0 did not allow smart contracts so all limits are 0. pub const BLOCK_LIMIT_MAINNET_10: ExecutionCost = ExecutionCost { write_length: 0, diff --git a/src/net/http.rs b/src/net/http.rs index 24230a368..600dc9895 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -171,9 +171,9 @@ pub(crate) enum HttpReservedHeader { /// Stacks block accepted struct #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -struct StacksBlockAcceptedData { - stacks_block_id: StacksBlockId, - accepted: bool, +pub struct StacksBlockAcceptedData { + pub stacks_block_id: StacksBlockId, + pub accepted: bool, } impl FromStr for PeerHost { diff --git a/src/net/mod.rs b/src/net/mod.rs index 3bc121979..e9b1e5d09 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -58,6 +58,7 @@ use crate::core::mempool::*; use crate::core::POX_REWARD_CYCLE_LENGTH; use crate::net::atlas::{Attachment, AttachmentInstance}; use crate::net::http::HttpReservedHeader; +pub use crate::net::http::StacksBlockAcceptedData; use crate::util_lib::bloom::{BloomFilter, BloomNodeHasher}; use crate::util_lib::boot::boot_code_tx_auth; use crate::util_lib::db::DBConn; @@ -2053,6 +2054,7 @@ pub mod test { use std::sync::Mutex; use std::thread; + use clarity::vm::ast::ASTRules; use mio; use rand; use rand::RngCore; @@ -2668,6 +2670,7 @@ pub mod test { clarity, &boot_code_smart_contract, &boot_code_account, + ASTRules::PrecheckSize, ) .unwrap() }); diff --git a/src/net/p2p.rs b/src/net/p2p.rs index 889c270ca..72698c475 100644 --- a/src/net/p2p.rs +++ b/src/net/p2p.rs @@ -86,6 +86,8 @@ use stacks_common::util::secp256k1::Secp256k1PublicKey; use crate::chainstate::stacks::StacksBlockHeader; use crate::types::chainstate::{PoxId, SortitionId}; +use clarity::vm::ast::ASTRules; + /// inter-thread request to send a p2p message from another thread in this program. #[derive(Debug)] pub enum NetworkRequest { @@ -236,6 +238,7 @@ pub struct PeerNetwork { pub chain_view: BurnchainView, pub burnchain_tip: BlockSnapshot, pub chain_view_stable_consensus_hash: ConsensusHash, + pub ast_rules: ASTRules, // handles to p2p databases pub peerdb: PeerDB, @@ -306,6 +309,8 @@ pub struct PeerNetwork { mempool_state: MempoolSyncState, mempool_sync_deadline: u64, mempool_sync_timeout: u64, + mempool_sync_completions: u64, + mempool_sync_txs: u64, // how often we pruned a given inbound/outbound peer pub prune_outbound_counts: HashMap, @@ -385,6 +390,7 @@ impl PeerNetwork { local_peer: local_peer, chain_view: chain_view, chain_view_stable_consensus_hash: ConsensusHash([0u8; 20]), + ast_rules: ASTRules::Typical, burnchain_tip: BlockSnapshot::initial( first_block_height, &first_burn_header_hash, @@ -435,6 +441,8 @@ impl PeerNetwork { mempool_state: MempoolSyncState::PickOutboundPeer, mempool_sync_deadline: 0, mempool_sync_timeout: 0, + mempool_sync_completions: 0, + mempool_sync_txs: 0, prune_outbound_counts: HashMap::new(), prune_inbound_counts: HashMap::new(), @@ -2173,6 +2181,8 @@ impl PeerNetwork { self.mempool_sync_deadline = get_epoch_time_secs() + self.connection_opts.mempool_sync_interval; + self.mempool_sync_completions = self.mempool_sync_completions.saturating_add(1); + self.mempool_sync_txs = self.mempool_sync_txs.saturating_add(txs.len() as u64); return Some(txs); } else { return None; @@ -2187,6 +2197,7 @@ impl PeerNetwork { txs.len() ); + self.mempool_sync_txs = self.mempool_sync_txs.saturating_add(txs.len() as u64); return Some(txs); } else { return None; @@ -4951,6 +4962,9 @@ impl PeerNetwork { // update cached burnchain view for /v2/info self.chain_view = new_chain_view; self.chain_view_stable_consensus_hash = new_chain_view_stable_consensus_hash; + + // update tx validation information + self.ast_rules = SortitionDB::get_ast_rules(sortdb.conn(), sn.block_height)?; } if sn.burn_header_hash != self.burnchain_tip.burn_header_hash { @@ -5358,6 +5372,9 @@ mod test { use stacks_common::util::log; use stacks_common::util::sleep_ms; + use clarity::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER; + use clarity::vm::MAX_CALL_STACK_DEPTH; + use super::*; fn make_random_peer_address() -> PeerAddress { @@ -6170,9 +6187,9 @@ mod test { // peer 1 gets some transactions; peer 2 blacklists some of them; // verify peer 2 gets only the non-blacklisted ones. let mut peer_1_config = - TestPeerConfig::new("test_mempool_sync_2_peers_paginated", 2218, 2219); + TestPeerConfig::new("test_mempool_sync_2_peers_blacklisted", 2218, 2219); let mut peer_2_config = - TestPeerConfig::new("test_mempool_sync_2_peers_paginated", 2220, 2221); + TestPeerConfig::new("test_mempool_sync_2_peers_blacklisted", 2220, 2221); peer_1_config.add_neighbor(&peer_2_config.to_neighbor()); peer_2_config.add_neighbor(&peer_1_config.to_neighbor()); diff --git a/src/net/relay.rs b/src/net/relay.rs index 6d3acfe84..7766df86b 100644 --- a/src/net/relay.rs +++ b/src/net/relay.rs @@ -30,11 +30,14 @@ use rand::Rng; use crate::burnchains::Burnchain; use crate::burnchains::BurnchainView; use crate::chainstate::burn::db::sortdb::{SortitionDB, SortitionDBConn, SortitionHandleConn}; +use crate::chainstate::burn::BlockSnapshot; use crate::chainstate::burn::ConsensusHash; use crate::chainstate::coordinator::comm::CoordinatorChannels; use crate::chainstate::stacks::db::{StacksChainState, StacksEpochReceipt, StacksHeaderInfo}; use crate::chainstate::stacks::events::StacksTransactionReceipt; use crate::chainstate::stacks::StacksBlockHeader; +use crate::chainstate::stacks::TransactionPayload; +use crate::clarity_vm::clarity::Error as clarity_error; use crate::core::mempool::MemPoolDB; use crate::core::mempool::*; use crate::net::chat::*; @@ -47,7 +50,11 @@ use crate::net::rpc::*; use crate::net::Error as net_error; use crate::net::*; use crate::types::chainstate::StacksBlockId; +use clarity::vm::ast::errors::{ParseError, ParseErrors}; +use clarity::vm::ast::{ast_check_size, ASTRules}; use clarity::vm::costs::ExecutionCost; +use clarity::vm::errors::RuntimeErrorType; +use clarity::vm::types::{QualifiedContractIdentifier, StacksAddressExtensions}; use stacks_common::util::get_epoch_time_secs; use stacks_common::util::hash::Sha512Trunc256Sum; @@ -488,7 +495,46 @@ impl Relayer { Ok(()) } - /// Insert a staging block + /// Get the snapshot of the parent of a given Stacks block + pub fn get_parent_stacks_block_snapshot( + sort_handle: &SortitionHandleConn, + consensus_hash: &ConsensusHash, + block_hash: &BlockHeaderHash, + ) -> Result { + let parent_block_snapshot = match sort_handle + .get_block_snapshot_of_parent_stacks_block(consensus_hash, block_hash) + { + Ok(Some((_, sn))) => { + debug!( + "Parent of {}/{} is {}/{}", + consensus_hash, block_hash, sn.consensus_hash, sn.winning_stacks_block_hash + ); + sn + } + Ok(None) => { + debug!( + "Received block with unknown parent snapshot: {}/{}", + consensus_hash, block_hash + ); + return Err(chainstate_error::NoSuchBlockError); + } + Err(db_error::InvalidPoxSortition) => { + warn!( + "Received block {}/{} on a non-canonical PoX sortition", + consensus_hash, block_hash + ); + return Err(chainstate_error::DBError(db_error::InvalidPoxSortition)); + } + Err(e) => { + return Err(e.into()); + } + }; + Ok(parent_block_snapshot) + } + + /// Insert a staging block that got relayed to us somehow -- e.g. uploaded via http, downloaded + /// by us, or pushed via p2p. + /// Return Ok(true) if we stored it, Ok(false) if we didn't pub fn process_new_anchored_block( sort_ic: &SortitionDBConn, chainstate: &mut StacksChainState, @@ -496,41 +542,44 @@ impl Relayer { block: &StacksBlock, download_time: u64, ) -> Result { + debug!( + "Handle incoming block {}/{}", + consensus_hash, + &block.block_hash() + ); + // find the snapshot of the parent of this block let db_handle = SortitionHandleConn::open_reader_consensus(sort_ic, consensus_hash)?; - let parent_block_snapshot = match db_handle - .get_block_snapshot_of_parent_stacks_block(consensus_hash, &block.block_hash()) - { - Ok(Some((_, sn))) => { - debug!( - "Parent of {}/{} is {}/{}", - consensus_hash, - block.block_hash(), - sn.consensus_hash, - sn.winning_stacks_block_hash - ); - sn - } - Ok(None) => { - debug!( - "Received block with unknown parent snapshot: {}/{}", - consensus_hash, - &block.block_hash() - ); - return Ok(false); - } - Err(db_error::InvalidPoxSortition) => { - warn!( - "Received block {}/{} on a non-canonical PoX sortition", - consensus_hash, - &block.block_hash() - ); - return Ok(false); - } - Err(e) => { - return Err(e.into()); - } - }; + let parent_block_snapshot = Relayer::get_parent_stacks_block_snapshot( + &db_handle, + consensus_hash, + &block.block_hash(), + )?; + + // don't relay this block if it's using the wrong AST rules (this would render at least one of its + // txs problematic). + let block_sn = SortitionDB::get_block_snapshot_consensus(sort_ic, consensus_hash)? + .ok_or(chainstate_error::DBError(db_error::NotFoundError))?; + let ast_rules = SortitionDB::get_ast_rules(sort_ic, block_sn.block_height)?; + debug!( + "Current AST rules for block {}/{} height {} sortitioned at {} is {:?}", + consensus_hash, + &block.block_hash(), + block.header.total_work.work, + &block_sn.block_height, + &ast_rules + ); + if !Relayer::static_check_problematic_relayed_block(chainstate.mainnet, block, ast_rules) { + warn!( + "Block is problematic; will not store or relay"; + "stacks_block_hash" => %block.block_hash(), + "consensus_hash" => %consensus_hash, + "burn_height" => block.header.total_work.work, + "sortition_height" => block_sn.block_height, + "ast_rules" => ?ast_rules, + ); + return Ok(false); + } chainstate.preprocess_anchored_block( sort_ic, @@ -629,6 +678,11 @@ impl Relayer { let mut new_blocks = HashMap::new(); for (consensus_hash, block, download_time) in network_result.blocks.iter() { + debug!( + "Received downloaded block {}/{}", + consensus_hash, + &block.block_hash() + ); match Relayer::process_new_anchored_block( sort_ic, chainstate, @@ -638,7 +692,18 @@ impl Relayer { ) { Ok(accepted) => { if accepted { + debug!( + "Accepted downloaded block {}/{}", + consensus_hash, + &block.block_hash() + ); new_blocks.insert((*consensus_hash).clone(), block.clone()); + } else { + debug!( + "Rejected downloaded block {}/{}", + consensus_hash, + &block.block_hash() + ); } } Err(chainstate_error::InvalidStacksBlock(msg)) => { @@ -727,6 +792,11 @@ impl Relayer { &consensus_hash, &bhh, &neighbor_key ); new_blocks.insert(consensus_hash.clone(), block.clone()); + } else { + debug!( + "Rejected block {}/{} from {}", + &consensus_hash, &bhh, &neighbor_key + ); } } Err(chainstate_error::InvalidStacksBlock(msg)) => { @@ -758,6 +828,7 @@ impl Relayer { /// Does not fail on invalid blocks; just logs a warning. /// Returns the consensus hashes for the sortitions that elected the stacks anchored blocks that produced these streams. fn preprocess_downloaded_microblocks( + sort_ic: &SortitionDBConn, network_result: &mut NetworkResult, chainstate: &mut StacksChainState, ) -> HashMap)> { @@ -770,13 +841,54 @@ impl Relayer { } let anchored_block_hash = microblock_stream[0].header.prev_block.clone(); + let block_snapshot = + match SortitionDB::get_block_snapshot_consensus(sort_ic, consensus_hash) { + Ok(Some(sn)) => sn, + Ok(None) => { + warn!( + "Failed to load parent anchored block snapshot for {}/{}", + consensus_hash, &anchored_block_hash + ); + continue; + } + Err(e) => { + warn!("Failed to load parent stacks block snapshot: {:?}", &e); + continue; + } + }; + + let ast_rules = match SortitionDB::get_ast_rules(sort_ic, block_snapshot.block_height) { + Ok(rules) => rules, + Err(e) => { + error!("Failed to load current AST rules: {:?}", &e); + continue; + } + }; + + let mut stored = false; for mblock in microblock_stream.iter() { + debug!( + "Preprocess downloaded microblock {}/{}-{}", + consensus_hash, + &anchored_block_hash, + &mblock.block_hash() + ); + if !Relayer::static_check_problematic_relayed_microblock( + chainstate.mainnet, + mblock, + ast_rules, + ) { + info!("Microblock {} from {}/{} is problematic; will not store or relay it, nor its descendants", &mblock.block_hash(), consensus_hash, &anchored_block_hash); + break; + } match chainstate.preprocess_streamed_microblock( consensus_hash, &anchored_block_hash, mblock, ) { - Ok(_) => {} + Ok(s) => { + stored = s; + } Err(e) => { warn!( "Invalid downloaded microblock {}/{}-{}: {:?}", @@ -789,12 +901,15 @@ impl Relayer { } } - let index_block_hash = - StacksBlockHeader::make_index_block_hash(consensus_hash, &anchored_block_hash); - ret.insert( - (*consensus_hash).clone(), - (index_block_hash, microblock_stream.clone()), - ); + // if we did indeed store this microblock (i.e. we didn't have it), then we can relay it + if stored { + let index_block_hash = + StacksBlockHeader::make_index_block_hash(consensus_hash, &anchored_block_hash); + ret.insert( + (*consensus_hash).clone(), + (index_block_hash, microblock_stream.clone()), + ); + } } ret } @@ -803,6 +918,7 @@ impl Relayer { /// Return the list of MicroblockData messages we need to broadcast to our neighbors, as well /// as the list of neighbors we need to ban because they sent us invalid microblocks. fn preprocess_pushed_microblocks( + sort_ic: &SortitionDBConn, network_result: &mut NetworkResult, chainstate: &mut StacksChainState, ) -> Result<(Vec<(Vec, MicroblocksData)>, Vec), net_error> { @@ -829,7 +945,27 @@ impl Relayer { } }; let index_block_hash = mblock_data.index_anchor_block.clone(); + + let block_snapshot = + SortitionDB::get_block_snapshot_consensus(sort_ic, &consensus_hash)? + .ok_or(net_error::DBError(db_error::NotFoundError))?; + let ast_rules = SortitionDB::get_ast_rules(sort_ic, block_snapshot.block_height)?; + for mblock in mblock_data.microblocks.iter() { + debug!( + "Preprocess downloaded microblock {}/{}-{}", + &consensus_hash, + &anchored_block_hash, + &mblock.block_hash() + ); + if !Relayer::static_check_problematic_relayed_microblock( + chainstate.mainnet, + mblock, + ast_rules, + ) { + info!("Microblock {} from {}/{} is problematic; will not store or relay it, nor its descendants", &mblock.block_hash(), &consensus_hash, &anchored_block_hash); + continue; + } let need_relay = !chainstate.has_descendant_microblock_indexed( &index_block_hash, &mblock.block_hash(), @@ -885,20 +1021,57 @@ impl Relayer { } } - // process uploaded microblocks. We will have already stored them, so just reconstruct the + // process uploaded microblocks. We may have already stored them, so just reconstruct the // data we need to forward them to neighbors. for uploaded_mblock in network_result.uploaded_microblocks.iter() { for mblock in uploaded_mblock.microblocks.iter() { - if let Some((_, mblocks_map)) = - new_microblocks.get_mut(&uploaded_mblock.index_anchor_block) + // is this microblock actually stored? i.e. it wasn't problematic? + let (consensus_hash, block_hash) = + match chainstate.get_block_header_hashes(&uploaded_mblock.index_anchor_block) { + Ok(Some((ch, bhh))) => (ch, bhh), + Ok(None) => { + warn!("No such block {}", &uploaded_mblock.index_anchor_block); + continue; + } + Err(e) => { + warn!( + "Failed to look up hashes for {}: {:?}", + &uploaded_mblock.index_anchor_block, &e + ); + continue; + } + }; + if chainstate + .get_microblock_status(&consensus_hash, &block_hash, &mblock.block_hash()) + .unwrap_or(None) + .is_some() { - mblocks_map.insert(mblock.block_hash(), (*mblock).clone()); + // yup, stored! + debug!( + "Preprocessed uploaded microblock {}/{}-{}", + &consensus_hash, + &block_hash, + &mblock.block_hash() + ); + if let Some((_, mblocks_map)) = + new_microblocks.get_mut(&uploaded_mblock.index_anchor_block) + { + mblocks_map.insert(mblock.block_hash(), (*mblock).clone()); + } else { + let mut mblocks_map = HashMap::new(); + mblocks_map.insert(mblock.block_hash(), (*mblock).clone()); + new_microblocks.insert( + uploaded_mblock.index_anchor_block.clone(), + (vec![], mblocks_map), + ); + } } else { - let mut mblocks_map = HashMap::new(); - mblocks_map.insert(mblock.block_hash(), (*mblock).clone()); - new_microblocks.insert( - uploaded_mblock.index_anchor_block.clone(), - (vec![], mblocks_map), + // nope + debug!( + "Did NOT preprocess uploaded microblock {}/{}-{}", + &consensus_hash, + &block_hash, + &mblock.block_hash() ); } } @@ -908,6 +1081,149 @@ impl Relayer { Ok((mblock_datas, bad_neighbors)) } + /// Verify that a relayed transaction is not problematic. This is a static check -- we only + /// look at the tx contents. + /// + /// Return true if the check passes -- i.e. it's not problematic + /// Return false if the check fails -- i.e. it is problematic + pub fn static_check_problematic_relayed_tx( + mainnet: bool, + tx: &StacksTransaction, + ast_rules: ASTRules, + ) -> Result<(), Error> { + debug!( + "Check {} to see if it is problematic in {:?}", + &tx.txid(), + &ast_rules + ); + match tx.payload { + TransactionPayload::SmartContract(ref smart_contract) => { + if ast_rules == ASTRules::PrecheckSize { + let origin = tx.get_origin(); + let issuer_principal = { + let addr = if mainnet { + origin.address_mainnet() + } else { + origin.address_testnet() + }; + addr.to_account_principal() + }; + let issuer_principal = if let PrincipalData::Standard(data) = issuer_principal { + data + } else { + // not possible + panic!("Transaction had a contract principal origin"); + }; + + let contract_id = QualifiedContractIdentifier::new( + issuer_principal, + smart_contract.name.clone(), + ); + let contract_code_str = smart_contract.code_body.to_string(); + + // make sure that the AST isn't unreasonably big + debug!("ast_size_check on {}", &contract_id); + let ast_res = ast_check_size(&contract_id, &contract_code_str); + debug!("ast_size_check on {} was {:?}", &contract_id, &ast_res); + match ast_res { + Ok(_) => {} + Err(parse_error) => match parse_error.err { + ParseErrors::ExpressionStackDepthTooDeep + | ParseErrors::VaryExpressionStackDepthTooDeep => { + // don't include this block + info!("Transaction {} is problematic and will not be included, relayed, or built upon", &tx.txid()); + return Err(Error::ClarityError(parse_error.into())); + } + _ => {} + }, + } + } + } + _ => {} + } + Ok(()) + } + + /// Verify that a relayed block is not problematic -- i.e. it doesn't contain any problematic + /// transactions. This is a static check -- we only look at the block contents. + /// + /// Returns true if the check passed -- i.e. no problems. + /// Returns false if not + pub fn static_check_problematic_relayed_block( + mainnet: bool, + block: &StacksBlock, + ast_rules: ASTRules, + ) -> bool { + for tx in block.txs.iter() { + if !Relayer::static_check_problematic_relayed_tx(mainnet, tx, ast_rules).is_ok() { + info!( + "Block {} with tx {} will not be stored or relayed", + block.block_hash(), + tx.txid() + ); + return false; + } + } + true + } + + /// Verify that a relayed microblock is not problematic -- i.e. it doesn't contain any + /// problematic transactions. This is a static check -- we only look at the microblock + /// contents. + /// + /// Returns true if the check passed -- i.e. no problems. + /// Returns false if not + pub fn static_check_problematic_relayed_microblock( + mainnet: bool, + mblock: &StacksMicroblock, + ast_rules: ASTRules, + ) -> bool { + for tx in mblock.txs.iter() { + if !Relayer::static_check_problematic_relayed_tx(mainnet, tx, ast_rules).is_ok() { + info!( + "Microblock {} with tx {} will not be stored relayed", + mblock.block_hash(), + tx.txid() + ); + return false; + } + } + true + } + + /// Should we apply static checks against problematic blocks and microblocks? + #[cfg(any(test, feature = "testing"))] + pub fn do_static_problematic_checks() -> bool { + std::env::var("STACKS_DISABLE_TX_PROBLEMATIC_CHECK") != Ok("1".into()) + } + + /// Should we apply static checks against problematic blocks and microblocks? + #[cfg(not(any(test, feature = "testing")))] + pub fn do_static_problematic_checks() -> bool { + true + } + + /// Should we store and process problematic blocks and microblocks to staging that we mined? + #[cfg(any(test, feature = "testing"))] + pub fn process_mined_problematic_blocks( + cur_ast_rules: ASTRules, + processed_ast_rules: ASTRules, + ) -> bool { + std::env::var("STACKS_PROCESS_PROBLEMATIC_BLOCKS") != Ok("1".into()) + || cur_ast_rules != processed_ast_rules + } + + /// Should we store and process problematic blocks and microblocks to staging that we mined? + /// We should do this only if we used a different ruleset than the active one. If it was + /// problematic with the currently-active rules, then obviously it shouldn't be processed. + #[cfg(not(any(test, feature = "testing")))] + pub fn process_mined_problematic_blocks( + cur_ast_rules: ASTRules, + processed_ast_rules: ASTRules, + ) -> bool { + cur_ast_rules != processed_ast_rules + } + /// Process blocks and microblocks that we recieved, both downloaded (confirmed) and streamed /// (unconfirmed). Returns: /// * set of consensus hashes that elected the newly-discovered blocks, and the blocks, so we can turn them into BlocksAvailable / BlocksData messages @@ -931,41 +1247,52 @@ impl Relayer { let mut new_blocks = HashMap::new(); let mut bad_neighbors = vec![]; - { - let sort_ic = sortdb.index_conn(); + let sort_ic = sortdb.index_conn(); - // process blocks we downloaded - let new_dled_blocks = - Relayer::preprocess_downloaded_blocks(&sort_ic, network_result, chainstate); - for (new_dled_block_ch, block_data) in new_dled_blocks.into_iter() { - debug!( - "Received downloaded block for {}/{}", - &new_dled_block_ch, - &block_data.block_hash(); - "consensus_hash" => %new_dled_block_ch, - "block_hash" => %block_data.block_hash() - ); - new_blocks.insert(new_dled_block_ch, block_data); - } + // process blocks we downloaded + let new_dled_blocks = + Relayer::preprocess_downloaded_blocks(&sort_ic, network_result, chainstate); + for (new_dled_block_ch, block_data) in new_dled_blocks.into_iter() { + debug!( + "Received downloaded block for {}/{}", + &new_dled_block_ch, + &block_data.block_hash(); + "consensus_hash" => %new_dled_block_ch, + "block_hash" => %block_data.block_hash() + ); + new_blocks.insert(new_dled_block_ch, block_data); + } - // process blocks pushed to us - let (new_pushed_blocks, mut new_bad_neighbors) = - Relayer::preprocess_pushed_blocks(&sort_ic, network_result, chainstate)?; - for (new_pushed_block_ch, block_data) in new_pushed_blocks.into_iter() { - debug!( - "Received p2p-pushed block for {}/{}", - &new_pushed_block_ch, - &block_data.block_hash(); - "consensus_hash" => %new_pushed_block_ch, - "block_hash" => %block_data.block_hash() - ); - new_blocks.insert(new_pushed_block_ch, block_data); - } - bad_neighbors.append(&mut new_bad_neighbors); + // process blocks pushed to us + let (new_pushed_blocks, mut new_bad_neighbors) = + Relayer::preprocess_pushed_blocks(&sort_ic, network_result, chainstate)?; + for (new_pushed_block_ch, block_data) in new_pushed_blocks.into_iter() { + debug!( + "Received p2p-pushed block for {}/{}", + &new_pushed_block_ch, + &block_data.block_hash(); + "consensus_hash" => %new_pushed_block_ch, + "block_hash" => %block_data.block_hash() + ); + new_blocks.insert(new_pushed_block_ch, block_data); + } + bad_neighbors.append(&mut new_bad_neighbors); - // process blocks uploaded to us. They've already been stored - for block_data in network_result.uploaded_blocks.drain(..) { - for BlocksDatum(consensus_hash, block) in block_data.blocks.into_iter() { + // process blocks uploaded to us. They've already been stored, but we need to report them + // as available anyway so the callers of this method can know that they have shown up (e.g. + // so they can be relayed). + for block_data in network_result.uploaded_blocks.drain(..) { + for BlocksDatum(consensus_hash, block) in block_data.blocks.into_iter() { + // did we actually store it? + if StacksChainState::get_staging_block_status( + chainstate.db(), + &consensus_hash, + &block.block_hash(), + ) + .unwrap_or(None) + .is_some() + { + // block stored debug!( "Received http-uploaded block for {}/{}", &consensus_hash, @@ -978,11 +1305,13 @@ impl Relayer { // process microblocks we downloaded let new_confirmed_microblocks = - Relayer::preprocess_downloaded_microblocks(network_result, chainstate); + Relayer::preprocess_downloaded_microblocks(&sort_ic, network_result, chainstate); - // process microblocks pushed to us + // process microblocks pushed to us, as well as identify which ones were uploaded via http + // (these ones will have already been processed, but we need to report them as + // newly-available to the caller nevertheless) let (new_microblocks, mut new_bad_neighbors) = - Relayer::preprocess_pushed_microblocks(network_result, chainstate)?; + Relayer::preprocess_pushed_microblocks(&sort_ic, network_result, chainstate)?; bad_neighbors.append(&mut new_bad_neighbors); if new_blocks.len() > 0 || new_microblocks.len() > 0 || new_confirmed_microblocks.len() > 0 @@ -1027,6 +1356,62 @@ impl Relayer { Ok(ret) } + /// Filter out problematic transactions from the network result. + /// Modifies network_result in-place. + fn filter_problematic_transactions(network_result: &mut NetworkResult, mainnet: bool) { + // filter out transactions that prove problematic + let mut filtered_pushed_transactions = HashMap::new(); + let mut filtered_uploaded_transactions = vec![]; + for (nk, tx_data) in network_result.pushed_transactions.drain() { + let mut filtered_tx_data = vec![]; + for (relayers, tx) in tx_data.into_iter() { + if Relayer::do_static_problematic_checks() + && !Relayer::static_check_problematic_relayed_tx( + mainnet, + &tx, + ASTRules::PrecheckSize, + ) + .is_ok() + { + info!( + "Pushed transaction {} is problematic; will not store or relay", + &tx.txid() + ); + continue; + } + filtered_tx_data.push((relayers, tx)); + } + if filtered_tx_data.len() > 0 { + filtered_pushed_transactions.insert(nk, filtered_tx_data); + } + } + + for tx in network_result.uploaded_transactions.drain(..) { + if Relayer::do_static_problematic_checks() + && !Relayer::static_check_problematic_relayed_tx( + mainnet, + &tx, + ASTRules::PrecheckSize, + ) + .is_ok() + { + info!( + "Uploaded transaction {} is problematic; will not store or relay", + &tx.txid() + ); + continue; + } + filtered_uploaded_transactions.push(tx); + } + + network_result + .pushed_transactions + .extend(filtered_pushed_transactions); + network_result + .uploaded_transactions + .append(&mut filtered_uploaded_transactions); + } + /// Store all new transactions we received, and return the list of transactions that we need to /// forward (as well as their relay hints). Also, garbage-collect the mempool. fn process_transactions( @@ -1036,8 +1421,8 @@ impl Relayer { mempool: &mut MemPoolDB, event_observer: Option<&dyn MemPoolEventDispatcher>, ) -> Result, StacksTransaction)>, net_error> { - let chain_height = match chainstate.get_stacks_chain_tip(sortdb)? { - Some(tip) => tip.height, + let chain_tip = match chainstate.get_stacks_chain_tip(sortdb)? { + Some(tip) => tip, None => { debug!( "No Stacks chain tip; dropping {} transaction(s)", @@ -1047,6 +1432,9 @@ impl Relayer { } }; + let chain_height = chain_tip.height; + Relayer::filter_problematic_transactions(network_result, chainstate.mainnet); + if let Err(e) = PeerNetwork::store_transactions( mempool, chainstate, @@ -1759,14 +2147,14 @@ impl PeerNetwork { } #[cfg(test)] -mod test { +pub mod test { use std::cell::RefCell; use std::collections::HashMap; use crate::chainstate::stacks::db::blocks::MINIMUM_TX_FEE; use crate::chainstate::stacks::db::blocks::MINIMUM_TX_FEE_RATE_PER_BYTE; use crate::chainstate::stacks::test::*; - use crate::chainstate::stacks::*; + use crate::chainstate::stacks::Error as ChainstateError; use crate::chainstate::stacks::*; use crate::net::asn::*; use crate::net::chat::*; @@ -1784,9 +2172,25 @@ mod test { use super::*; use crate::clarity_vm::clarity::ClarityConnection; - use crate::core::StacksEpochExtension; use stacks_common::types::chainstate::BlockHeaderHash; + use clarity::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER; + use clarity::vm::ast::ASTRules; + use clarity::vm::MAX_CALL_STACK_DEPTH; + + use crate::chainstate::stacks::miner::test::make_coinbase; + use crate::chainstate::stacks::miner::test::make_user_stacks_transfer; + use crate::chainstate::stacks::miner::BlockBuilderSettings; + use crate::chainstate::stacks::miner::StacksMicroblockBuilder; + use crate::core::*; + use stacks_common::address::AddressHashMode; + use stacks_common::types::chainstate::StacksBlockId; + use stacks_common::types::chainstate::StacksWorkScore; + use stacks_common::types::chainstate::TrieHash; + use stacks_common::types::Address; + use stacks_common::util::hash::MerkleTree; + use stacks_common::util::vrf::VRFProof; + #[test] fn test_relayer_stats_add_relyed_messages() { let mut relay_stats = RelayerStats::new(); diff --git a/src/net/rpc.rs b/src/net/rpc.rs index 32cfb550a..9b2bfcdf2 100644 --- a/src/net/rpc.rs +++ b/src/net/rpc.rs @@ -96,6 +96,7 @@ use clarity::vm::database::clarity_store::make_contract_hash_key; use clarity::vm::types::TraitIdentifier; use clarity::vm::{ analysis::errors::CheckErrors, + ast::ASTRules, costs::{ExecutionCost, LimitedCostTracker}, database::{ clarity_store::ContractCommitment, BurnStateDB, ClarityDatabase, ClaritySerializable, @@ -1913,6 +1914,7 @@ impl ConversationHttp { attachment: Option, event_observer: Option<&dyn MemPoolEventDispatcher>, canonical_stacks_tip_height: u64, + ast_rules: ASTRules, ) -> Result { let txid = tx.txid(); let response_metadata = @@ -1924,40 +1926,55 @@ impl ConversationHttp { false, ) } else { - let tip = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn())?; - let stacks_epoch = sortdb - .index_conn() - .get_stacks_epoch(tip.block_height as u32) - .ok_or_else(|| { - warn!( - "Failed to store transaction because could not load Stacks epoch for canonical burn height = {}", - tip.block_height - ); - net_error::ChainstateError("Could not load Stacks epoch for canonical burn height".into()) - })?; + if Relayer::do_static_problematic_checks() + && !Relayer::static_check_problematic_relayed_tx(chainstate.mainnet, &tx, ast_rules) + .is_ok() + { + debug!( + "Transaction {} is problematic in rules {:?}; will not store or relay", + &tx.txid(), + ast_rules + ); + ( + HttpResponseType::TransactionID(response_metadata, txid), + false, + ) + } else { + let tip = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn())?; + let stacks_epoch = sortdb + .index_conn() + .get_stacks_epoch(tip.block_height as u32) + .ok_or_else(|| { + warn!( + "Failed to store transaction because could not load Stacks epoch for canonical burn height = {}", + tip.block_height + ); + net_error::ChainstateError("Could not load Stacks epoch for canonical burn height".into()) + })?; - match mempool.submit( - chainstate, - &consensus_hash, - &block_hash, - &tx, - event_observer, - &stacks_epoch.block_limit, - &stacks_epoch.epoch_id, - ) { - Ok(_) => { - debug!("Mempool accepted POSTed transaction {}", &txid); - ( - HttpResponseType::TransactionID(response_metadata, txid), - true, - ) - } - Err(e) => { - debug!("Mempool rejected POSTed transaction {}: {:?}", &txid, &e); - ( - HttpResponseType::BadRequestJSON(response_metadata, e.into_json(&txid)), - false, - ) + match mempool.submit( + chainstate, + &consensus_hash, + &block_hash, + &tx, + event_observer, + &stacks_epoch.block_limit, + &stacks_epoch.epoch_id, + ) { + Ok(_) => { + debug!("Mempool accepted POSTed transaction {}", &txid); + ( + HttpResponseType::TransactionID(response_metadata, txid), + true, + ) + } + Err(e) => { + debug!("Mempool rejected POSTed transaction {}: {:?}", &txid, &e); + ( + HttpResponseType::BadRequestJSON(response_metadata, e.into_json(&txid)), + false, + ) + } } } }; @@ -2106,13 +2123,54 @@ impl ConversationHttp { req: &HttpRequestType, consensus_hash: &ConsensusHash, block_hash: &BlockHeaderHash, + sortdb: &SortitionDB, chainstate: &mut StacksChainState, microblock: &StacksMicroblock, canonical_stacks_tip_height: u64, ) -> Result { let response_metadata = HttpResponseMetadata::from_http_request_type(req, Some(canonical_stacks_tip_height)); - let (response, accepted) = + + // make sure we can accept this + let ch_sn = match SortitionDB::get_block_snapshot_consensus(sortdb.conn(), consensus_hash) { + Ok(Some(sn)) => sn, + Ok(None) => { + let resp = HttpResponseType::NotFound( + response_metadata, + "No such consensus hash".to_string(), + ); + return resp.send(http, fd).and_then(|_| Ok(false)); + } + Err(e) => { + let resp = HttpResponseType::BadRequestJSON( + response_metadata, + chain_error::DBError(e).into_json(), + ); + return resp.send(http, fd).and_then(|_| Ok(false)); + } + }; + + let sort_handle = sortdb.index_handle(&ch_sn.sortition_id); + let parent_block_snapshot = + Relayer::get_parent_stacks_block_snapshot(&sort_handle, consensus_hash, block_hash)?; + let ast_rules = + SortitionDB::get_ast_rules(&sort_handle, parent_block_snapshot.block_height)?; + + let (response, accepted) = if !Relayer::static_check_problematic_relayed_microblock( + chainstate.mainnet, + microblock, + ast_rules, + ) { + info!("Microblock {} from {}/{} is problematic; will not store or relay it, nor its descendants", µblock.block_hash(), consensus_hash, &block_hash); + ( + // NOTE: txid is ignored in chainstate error .into_json() + HttpResponseType::BadRequestJSON( + response_metadata, + chain_error::ProblematicTransaction(Txid([0u8; 32])).into_json(), + ), + false, + ) + } else { match chainstate.preprocess_streamed_microblock(consensus_hash, block_hash, microblock) { Ok(accepted) => { @@ -2144,7 +2202,8 @@ impl ConversationHttp { HttpResponseType::BadRequestJSON(response_metadata, e.into_json()), false, ), - }; + } + }; response.send(http, fd).and_then(|_| Ok(accepted)) } @@ -2549,6 +2608,7 @@ impl ConversationHttp { attachment.clone(), handler_opts.event_observer.as_deref(), network.burnchain_tip.canonical_stacks_tip_height, + network.ast_rules, )?; if accepted { // forward to peer network @@ -2643,6 +2703,7 @@ impl ConversationHttp { &req, &consensus_hash, &block_hash, + sortdb, chainstate, mblock, network.burnchain_tip.canonical_stacks_tip_height, diff --git a/stacks-common/src/util/macros.rs b/stacks-common/src/util/macros.rs index 47c46b0d6..61bd9551a 100644 --- a/stacks-common/src/util/macros.rs +++ b/stacks-common/src/util/macros.rs @@ -209,7 +209,7 @@ macro_rules! guarded_string { macro_rules! define_u8_enum { ($Name:ident { $($Variant:ident = $Val:literal),+ }) => { - #[derive(Debug, Clone, PartialEq)] + #[derive(Debug, Clone, PartialEq, Copy)] #[repr(u8)] pub enum $Name { $($Variant = $Val),*, diff --git a/testnet/stacks-node/Cargo.toml b/testnet/stacks-node/Cargo.toml index 5785518b7..ee2572345 100644 --- a/testnet/stacks-node/Cargo.toml +++ b/testnet/stacks-node/Cargo.toml @@ -30,6 +30,9 @@ ring = "0.16.19" warp = "0.3" tokio = "1.15" reqwest = { version = "0.11", features = ["blocking", "json", "rustls"] } +clarity = { package = "clarity", path = "../../clarity/.", features = ["default", "testing"]} +stacks_common = { package = "stacks-common", path = "../../stacks-common/.", features = ["default", "testing"] } +stacks = { package = "blockstack-core", path = "../../.", features = ["default", "testing"] } [dev-dependencies.rusqlite] version = "=0.24.2" diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index 6eac291ed..7e449f9bb 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -573,6 +573,7 @@ impl Config { Some(epochs) => Some(epochs), None => default_burnchain_config.epochs, }, + ast_precheck_size_height: burnchain.ast_precheck_size_height, } } None => default_burnchain_config, @@ -1027,6 +1028,7 @@ pub struct BurnchainConfig { /// Custom override for the definitions of the epochs. This will only be applied for testnet and /// regtest nodes. pub epochs: Option>, + pub ast_precheck_size_height: Option, } impl BurnchainConfig { @@ -1055,6 +1057,7 @@ impl BurnchainConfig { block_commit_tx_estimated_size: BLOCK_COMMIT_TX_ESTIM_SIZE, rbf_fee_increment: DEFAULT_RBF_FEE_RATE_INCREMENT, epochs: None, + ast_precheck_size_height: None, } } @@ -1109,6 +1112,7 @@ pub struct BurnchainConfigFile { pub rbf_fee_increment: Option, pub max_rbf: Option, pub epochs: Option>, + pub ast_precheck_size_height: Option, } #[derive(Clone, Debug, Default)] diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index 1b8666287..156c05be9 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -3,7 +3,10 @@ use std::collections::HashMap; use std::collections::{HashSet, VecDeque}; use std::convert::{TryFrom, TryInto}; use std::default::Default; +use std::fs; +use std::io::Write; use std::net::SocketAddr; +use std::path::Path; use std::sync::mpsc::{sync_channel, Receiver, SyncSender, TrySendError}; use std::sync::{atomic::Ordering, Arc, Mutex}; use std::time::Duration; @@ -69,6 +72,8 @@ use super::{BurnchainController, BurnchainTip, Config, EventDispatcher, Keychain use crate::stacks::vm::database::BurnStateDB; use stacks::monitoring; +use clarity::vm::ast::ASTRules; + pub const RELAYER_MAX_BUFFER: usize = 100; struct AssembledAnchorBlock { @@ -174,10 +179,78 @@ fn inner_process_tenure( // already processed my tenure return Ok(true); } + let burn_height = SortitionDB::get_block_snapshot_consensus(burn_db.conn(), consensus_hash) + .map_err(|e| { + error!("Failed to find block snapshot for mined block: {}", e); + e + })? + .ok_or_else(|| { + error!("Failed to find block snapshot for mined block"); + ChainstateError::NoSuchBlockError + })? + .block_height; - let ic = burn_db.index_conn(); + let ast_rules = SortitionDB::get_ast_rules(burn_db.conn(), burn_height)?; + + // failsafe + if !Relayer::static_check_problematic_relayed_block( + chain_state.mainnet, + &anchored_block, + ASTRules::PrecheckSize, + ) { + // nope! + warn!( + "Our mined block {} was problematic", + &anchored_block.block_hash() + ); + #[cfg(any(test, feature = "testing"))] + { + if let Ok(path) = std::env::var("STACKS_BAD_BLOCKS_DIR") { + // record this block somewhere + if !fs::metadata(&path).is_ok() { + fs::create_dir_all(&path) + .expect(&format!("FATAL: could not create '{}'", &path)); + } + + let mut path = Path::new(&path); + let path = path.join(Path::new(&format!("{}", &anchored_block.block_hash()))); + let mut file = fs::File::create(&path) + .expect(&format!("FATAL: could not create '{:?}'", &path)); + + let block_bits = anchored_block.serialize_to_vec(); + let block_bits_hex = to_hex(&block_bits); + let block_json = format!( + r#"{{"block":"{}","consensus":"{}"}}"#, + &block_bits_hex, &consensus_hash + ); + file.write_all(&block_json.as_bytes()).expect(&format!( + "FATAL: failed to write block bits to '{:?}'", + &path + )); + info!( + "Fault injection: bad block {} saved to {}", + &anchored_block.block_hash(), + &path.to_str().unwrap() + ); + } + } + if !Relayer::process_mined_problematic_blocks(ast_rules, ASTRules::PrecheckSize) { + // don't process it + warn!( + "Will NOT process our problematic mined block {}", + &anchored_block.block_hash() + ); + return Err(ChainstateError::NoTransactionsToMine); + } else { + warn!( + "Will process our problematic mined block {}", + &anchored_block.block_hash() + ) + } + } // Preprocess the anchored block + let ic = burn_db.index_conn(); chain_state.preprocess_anchored_block( &ic, consensus_hash, @@ -339,6 +412,25 @@ fn mine_one_microblock( .unwrap_or(0) ); + let burn_height = SortitionDB::get_block_snapshot_consensus( + sortdb.conn(), + µblock_state.parent_consensus_hash, + ) + .map_err(|e| { + error!("Failed to find block snapshot for mined block: {}", e); + e + })? + .ok_or_else(|| { + error!("Failed to find block snapshot for mined block"); + ChainstateError::NoSuchBlockError + })? + .block_height; + + let ast_rules = SortitionDB::get_ast_rules(sortdb.conn(), burn_height).map_err(|e| { + error!("Failed to get AST rules for microblock: {}", e); + e + })?; + let mint_result = { let ic = sortdb.index_conn(); let mut microblock_miner = match StacksMicroblockBuilder::resume_unconfirmed( @@ -389,6 +481,67 @@ fn mine_one_microblock( } }; + // failsafe + if !Relayer::static_check_problematic_relayed_microblock( + chainstate.mainnet, + &mined_microblock, + ASTRules::PrecheckSize, + ) { + // nope! + warn!( + "Our mined microblock {} was problematic", + &mined_microblock.block_hash() + ); + + #[cfg(any(test, feature = "testing"))] + { + if let Ok(path) = std::env::var("STACKS_BAD_BLOCKS_DIR") { + // record this microblock somewhere + if !fs::metadata(&path).is_ok() { + fs::create_dir_all(&path) + .expect(&format!("FATAL: could not create '{}'", &path)); + } + + let mut path = Path::new(&path); + let path = path.join(Path::new(&format!("{}", &mined_microblock.block_hash()))); + let mut file = fs::File::create(&path) + .expect(&format!("FATAL: could not create '{:?}'", &path)); + + let mblock_bits = mined_microblock.serialize_to_vec(); + let mblock_bits_hex = to_hex(&mblock_bits); + + let mblock_json = format!( + r#"{{"microblock":"{}","parent_consensus":"{}","parent_block":"{}"}}"#, + &mblock_bits_hex, + µblock_state.parent_consensus_hash, + µblock_state.parent_block_hash + ); + file.write_all(&mblock_json.as_bytes()).expect(&format!( + "FATAL: failed to write microblock bits to '{:?}'", + &path + )); + info!( + "Fault injection: bad microblock {} saved to {}", + &mined_microblock.block_hash(), + &path.to_str().unwrap() + ); + } + } + if !Relayer::process_mined_problematic_blocks(ast_rules, ASTRules::PrecheckSize) { + // don't process it + warn!( + "Will NOT process our problematic mined microblock {}", + &mined_microblock.block_hash() + ); + return Err(ChainstateError::NoTransactionsToMine); + } else { + warn!( + "Will process our problematic mined microblock {}", + &mined_microblock.block_hash() + ) + } + } + // preprocess the microblock locally chainstate.preprocess_streamed_microblock( µblock_state.parent_consensus_hash, @@ -982,7 +1135,6 @@ fn spawn_miner_relayer( ); increment_stx_blocks_mined_counter(); - match inner_process_tenure( &mined_block, &consensus_hash, @@ -1240,7 +1392,7 @@ impl StacksNode { // we can call _open_ here rather than _connect_, since connect is first called in // make_genesis_block - let sortdb = SortitionDB::open(&config.get_burn_db_file_path(), false) + let mut sortdb = SortitionDB::open(&config.get_burn_db_file_path(), true) .expect("Error while instantiating sortition db"); let epochs = SortitionDB::get_stacks_epochs(sortdb.conn()) @@ -1252,6 +1404,25 @@ impl StacksNode { SortitionDB::get_burnchain_view(&sortdb.conn(), &burnchain, &sortition_tip).unwrap() }; + if let Some(ast_precheck_size_height) = config.burnchain.ast_precheck_size_height { + info!( + "Override burnchain height of {:?} to {}", + ASTRules::PrecheckSize, + ast_precheck_size_height + ); + let mut tx = sortdb + .tx_begin() + .expect("FATAL: failed to begin tx on sortition DB"); + SortitionDB::override_ast_rule_height( + &mut tx, + ASTRules::PrecheckSize, + ast_precheck_size_height, + ) + .expect("FATAL: failed to override AST PrecheckSize rule height"); + tx.commit() + .expect("FATAL: failed to commit sortition DB transaction"); + } + // create a new peerdb let data_url = UrlString::try_from(format!("{}", &config.node.data_url)).unwrap(); let initial_neighbors = config.node.bootstrap_node.clone(); @@ -1544,6 +1715,12 @@ impl StacksNode { true } + /// Determine where in the set of forks to attempt to mine the next anchored block. + /// `mine_tip_ch` and `mine_tip_bhh` identify the parent block on top of which to mine. + /// `check_burn_block` identifies what we believe to be the burn chain's sortition history tip. + /// This is used to mitigate (but not eliminate) a TOCTTOU issue with mining: the caller's + /// conception of the sortition history tip may have become stale by the time they call this + /// method, in which case, mining should *not* happen (since the block will be invalid). fn get_mining_tenure_information( chain_state: &mut StacksChainState, burn_db: &mut SortitionDB, diff --git a/testnet/stacks-node/src/run_loop/neon.rs b/testnet/stacks-node/src/run_loop/neon.rs index 8882ae1ac..68246da2d 100644 --- a/testnet/stacks-node/src/run_loop/neon.rs +++ b/testnet/stacks-node/src/run_loop/neon.rs @@ -133,6 +133,7 @@ pub struct RunLoop { pox_watchdog: Option, // can't be instantiated until .start() is called is_miner: Option, // not known until .start() is called burnchain: Option, // not known until .start() is called + pox_watchdog_comms: PoxSyncWatchdogComms, } /// Write to stderr in an async-safe manner. @@ -158,6 +159,7 @@ impl RunLoop { pub fn new(config: Config) -> Self { let channels = CoordinatorCommunication::instantiate(); let should_keep_running = Arc::new(AtomicBool::new(true)); + let pox_watchdog_comms = PoxSyncWatchdogComms::new(should_keep_running.clone()); let mut event_dispatcher = EventDispatcher::new(); for observer in config.events_observers.iter() { @@ -174,6 +176,7 @@ impl RunLoop { pox_watchdog: None, is_miner: None, burnchain: None, + pox_watchdog_comms, } } @@ -218,10 +221,7 @@ impl RunLoop { } pub fn get_pox_sync_comms(&self) -> PoxSyncWatchdogComms { - self.pox_watchdog - .as_ref() - .expect("FATAL: tried to get PoX watchdog before calling .start()") - .make_comms_handle() + self.pox_watchdog_comms.clone() } pub fn get_termination_switch(&self) -> Arc { @@ -474,7 +474,7 @@ impl RunLoop { /// Instantiate the PoX watchdog fn instantiate_pox_watchdog(&mut self) { - let pox_watchdog = PoxSyncWatchdog::new(&self.config, self.should_keep_running.clone()) + let pox_watchdog = PoxSyncWatchdog::new(&self.config, self.pox_watchdog_comms.clone()) .expect("FATAL: failed to instantiate PoX sync watchdog"); self.pox_watchdog = Some(pox_watchdog); } diff --git a/testnet/stacks-node/src/syncctl.rs b/testnet/stacks-node/src/syncctl.rs index 80afa35de..5d1c511ea 100644 --- a/testnet/stacks-node/src/syncctl.rs +++ b/testnet/stacks-node/src/syncctl.rs @@ -171,7 +171,7 @@ const PER_SAMPLE_WAIT_MS: u64 = 1000; impl PoxSyncWatchdog { pub fn new( config: &Config, - should_keep_running: Arc, + watchdog_comms: PoxSyncWatchdogComms, ) -> Result { let mainnet = config.is_mainnet(); let chain_id = config.burnchain.chain_id; @@ -208,7 +208,7 @@ impl PoxSyncWatchdog { steady_state_burnchain_sync_interval: burnchain_poll_time, steady_state_resync_ts: 0, chainstate: chainstate, - relayer_comms: PoxSyncWatchdogComms::new(should_keep_running), + relayer_comms: watchdog_comms, }) } diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 6438e67f6..d712fa142 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -1,9 +1,12 @@ use std::cmp; +use std::fs; +use std::path::Path; use std::sync::mpsc; use std::sync::Arc; use std::time::{Duration, Instant}; use std::{ collections::HashMap, + collections::HashSet, sync::atomic::{AtomicU64, Ordering}, }; use std::{env, thread}; @@ -14,14 +17,19 @@ use stacks::burnchains::bitcoin::address::{BitcoinAddress, BitcoinAddressType}; use stacks::burnchains::bitcoin::BitcoinNetworkType; use stacks::burnchains::Txid; use stacks::chainstate::burn::operations::{BlockstackOperationType, PreStxOp, TransferStxOp}; +use stacks::chainstate::coordinator::comm::CoordinatorChannels; use stacks::clarity_cli::vm_execute as execute; use stacks::codec::StacksMessageCodec; use stacks::core; -use stacks::core::{StacksEpoch, StacksEpochId, CHAIN_ID_TESTNET, PEER_VERSION_EPOCH_2_0}; +use stacks::core::{ + StacksEpoch, StacksEpochId, BLOCK_LIMIT_MAINNET_20, BLOCK_LIMIT_MAINNET_205, CHAIN_ID_TESTNET, + PEER_VERSION_EPOCH_2_0, PEER_VERSION_EPOCH_2_05, +}; use stacks::net::atlas::{AtlasConfig, AtlasDB, MAX_ATTACHMENT_INV_PAGES_PER_REQUEST}; use stacks::net::{ AccountEntryResponse, ContractSrcResponse, GetAttachmentResponse, GetAttachmentsInvResponse, - PostTransactionRequestBody, RPCPeerInfoData, + PostTransactionRequestBody, RPCPeerInfoData, StacksBlockAcceptedData, + UnconfirmedTransactionResponse, }; use stacks::types::chainstate::{ BlockHeaderHash, BurnchainHeaderHash, StacksAddress, StacksBlockId, @@ -57,7 +65,8 @@ use stacks::{ use crate::{ burnchains::bitcoin_regtest_controller::UTXO, config::EventKeyType, config::EventObserverConfig, config::InitialBalance, neon, operations::BurnchainOpSigner, - BitcoinRegtestController, BurnchainController, Config, ConfigFile, Keychain, + syncctl::PoxSyncWatchdogComms, BitcoinRegtestController, BurnchainController, Config, + ConfigFile, Keychain, }; use crate::util::hash::{MerkleTree, Sha512Trunc256Sum}; @@ -71,12 +80,15 @@ use super::{ make_microblock, make_stacks_transfer, make_stacks_transfer_mblock_only, to_addr, ADDR_4, SK_1, SK_2, }; +use crate::config::FeeEstimatorName; use crate::tests::SK_3; +use clarity::vm::ast::stack_depth_checker::AST_CALL_STACK_DEPTH_BUFFER; +use clarity::vm::ast::ASTRules; +use clarity::vm::MAX_CALL_STACK_DEPTH; +use stacks::chainstate::burn::db::sortdb::SortitionDB; use stacks::chainstate::stacks::miner::{ TransactionErrorEvent, TransactionEvent, TransactionSkippedEvent, TransactionSuccessEvent, }; - -use crate::config::FeeEstimatorName; use stacks::net::RPCFeeEstimateResponse; use stacks::vm::ClarityName; use stacks::vm::ContractName; From 9db371195eeafcc3e4ba2e56ebe7244ee50f7f51 Mon Sep 17 00:00:00 2001 From: CharlieC3 <2747302+CharlieC3@users.noreply.github.com> Date: Wed, 31 Aug 2022 14:17:17 -0400 Subject: [PATCH 016/178] ci: revert GH actions changes --- .github/workflows/ci.yml | 20 ++-- .github/workflows/clarity-js-sdk-pr.yml | 46 ++++----- .github/workflows/docker-platforms.yml | 18 ++-- .github/workflows/docs-pr.yml | 128 ++++++++++++------------ 4 files changed, 104 insertions(+), 108 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94f96496d..9f1aa4766 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,13 +151,13 @@ jobs: path: ${{ matrix.platform }}.zip call-docker-platforms-workflow: + if: ${{ github.event.inputs.tag != '' }} uses: stacks-network/stacks-blockchain/.github/workflows/docker-platforms.yml@master - secrets: inherit with: tag: ${{ github.event.inputs.tag }} secrets: - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} # Build docker image, tag it with the git tag and `latest` if running on master branch, and publish under the following conditions # Will publish if: @@ -185,12 +185,11 @@ jobs: type=ref,event=pr ${{ github.event.inputs.tag }} - - name: Login to Quay + - name: Login to DockerHub uses: docker/login-action@v1 with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build/Tag/Push Image uses: docker/build-push-action@v2 @@ -247,12 +246,11 @@ jobs: type=ref,event=pr ${{ env.STRETCH_TAG }} - - name: Login to Quay + - name: Login to DockerHub uses: docker/login-action@v1 with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build/Tag/Push Image uses: docker/build-push-action@v2 diff --git a/.github/workflows/clarity-js-sdk-pr.yml b/.github/workflows/clarity-js-sdk-pr.yml index e64a7f61b..fd28738cf 100644 --- a/.github/workflows/clarity-js-sdk-pr.yml +++ b/.github/workflows/clarity-js-sdk-pr.yml @@ -28,29 +28,29 @@ jobs: repository: ${{ env.CLARITY_JS_SDK_REPOSITORY }} ref: master - # - name: Determine Release Version - # run: | - # RELEASE_VERSION=$(echo ${GITHUB_REF#refs/*/} | tr / -) - # echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV + - name: Determine Release Version + run: | + RELEASE_VERSION=$(echo ${GITHUB_REF#refs/*/} | tr / -) + echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV - # - name: Update SDK Tag - # run: sed -i "s@CORE_SDK_TAG = \".*\"@CORE_SDK_TAG = \"$RELEASE_VERSION\"@g" packages/clarity-native-bin/src/index.ts + - name: Update SDK Tag + run: sed -i "s@CORE_SDK_TAG = \".*\"@CORE_SDK_TAG = \"$RELEASE_VERSION\"@g" packages/clarity-native-bin/src/index.ts - # - name: Create Pull Request - # uses: peter-evans/create-pull-request@v3 - # with: - # token: ${{ secrets.GH_TOKEN }} - # commit-message: "chore: update clarity-native-bin tag" - # committer: ${{ env.COMMIT_USER }} <${{ env.COMMIT_EMAIL }}> - # author: ${{ env.COMMIT_USER }} <${{ env.COMMIT_EMAIL }}> - # branch: auto/update-bin-tag - # delete-branch: true - # title: "clarity-native-bin tag update: ${{ env.RELEASE_VERSION }}" - # labels: | - # dependencies - # body: | - # :robot: This is an automated pull request created from a new release in [stacks-blockchain](https://github.com/blockstack/stacks-blockchain/releases). + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GH_TOKEN }} + commit-message: "chore: update clarity-native-bin tag" + committer: ${{ env.COMMIT_USER }} <${{ env.COMMIT_EMAIL }}> + author: ${{ env.COMMIT_USER }} <${{ env.COMMIT_EMAIL }}> + branch: auto/update-bin-tag + delete-branch: true + title: "clarity-native-bin tag update: ${{ env.RELEASE_VERSION }}" + labels: | + dependencies + body: | + :robot: This is an automated pull request created from a new release in [stacks-blockchain](https://github.com/blockstack/stacks-blockchain/releases). - # Updates the clarity-native-bin tag. - # assignees: zone117x - # reviewers: zone117x + Updates the clarity-native-bin tag. + assignees: zone117x + reviewers: zone117x diff --git a/.github/workflows/docker-platforms.yml b/.github/workflows/docker-platforms.yml index fb62a785e..7ee44b3ed 100644 --- a/.github/workflows/docker-platforms.yml +++ b/.github/workflows/docker-platforms.yml @@ -10,9 +10,9 @@ on: required: true type: string secrets: - QUAY_USERNAME: + DOCKERHUB_USERNAME: required: true - QUAY_PASSWORD: + DOCKERHUB_PASSWORD: required: true env: @@ -46,12 +46,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - name: Login to QUAY + - name: Login to DockerHub uses: docker/login-action@v1 with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build/Tag/Push Image uses: docker/build-push-action@v2 @@ -106,12 +105,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - name: Login to quay + - name: Login to DockerHub uses: docker/login-action@v1 with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build/Tag/Push Image uses: docker/build-push-action@v2 diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml index b143447f1..ad293191b 100644 --- a/.github/workflows/docs-pr.yml +++ b/.github/workflows/docs-pr.yml @@ -27,73 +27,73 @@ jobs: steps: - uses: actions/checkout@v2 - # - name: Build docs - # env: - # DOCKER_BUILDKIT: 1 - # run: rm -rf docs-output && docker build -o docs-output -f ./.github/actions/docsgen/Dockerfile.docsgen . + - name: Build docs + env: + DOCKER_BUILDKIT: 1 + run: rm -rf docs-output && docker build -o docs-output -f ./.github/actions/docsgen/Dockerfile.docsgen . - # - name: Checkout latest docs - # uses: actions/checkout@v2 - # with: - # token: ${{ secrets.DOCS_GITHUB_TOKEN }} - # repository: ${{ env.TARGET_REPOSITORY }} - # path: docs.blockstack + - name: Checkout latest docs + uses: actions/checkout@v2 + with: + token: ${{ secrets.DOCS_GITHUB_TOKEN }} + repository: ${{ env.TARGET_REPOSITORY }} + path: docs.blockstack - # - name: Branch and commit - # id: push - # run: | - # cd docs.blockstack - # git config user.email "kantai+robot@gmail.com" - # git config user.name "PR Robot" - # git fetch --unshallow - # git checkout -b $ROBOT_BRANCH - # cp ../docs-output/clarity-reference.json ./src/_data/clarity-reference.json - # cp ../docs-output/boot-contracts-reference.json ./src/_data/boot-contracts-reference.json - # git add src/_data/clarity-reference.json - # git add src/_data/boot-contracts-reference.json - # if $(git diff --staged --quiet --exit-code); then - # echo "No reference.json changes, stopping" - # echo "::set-output name=open_pr::0" - # else - # git remote add robot https://github.com/$ROBOT_OWNER/$ROBOT_REPO - # git commit -m "auto: update Clarity references JSONs from stacks-blockchain@${GITHUB_SHA}" - # git push robot $ROBOT_BRANCH - # echo "::set-output name=open_pr::1" - # fi - # - name: Open PR - # if: ${{ steps.push.outputs.open_pr == '1' }} - # uses: actions/github-script@v2 - # with: - # github-token: ${{ secrets.DOCS_GITHUB_TOKEN }} - # script: | - # // get env vars - # const process = require("process"); - # const robot_owner = process.env.ROBOT_OWNER; - # const robot_branch = process.env.ROBOT_BRANCH; - # const head = `${robot_owner}:${robot_branch}`; - # const owner = process.env.TARGET_OWNER; - # const repo = process.env.TARGET_REPO; + - name: Branch and commit + id: push + run: | + cd docs.blockstack + git config user.email "kantai+robot@gmail.com" + git config user.name "PR Robot" + git fetch --unshallow + git checkout -b $ROBOT_BRANCH + cp ../docs-output/clarity-reference.json ./src/_data/clarity-reference.json + cp ../docs-output/boot-contracts-reference.json ./src/_data/boot-contracts-reference.json + git add src/_data/clarity-reference.json + git add src/_data/boot-contracts-reference.json + if $(git diff --staged --quiet --exit-code); then + echo "No reference.json changes, stopping" + echo "::set-output name=open_pr::0" + else + git remote add robot https://github.com/$ROBOT_OWNER/$ROBOT_REPO + git commit -m "auto: update Clarity references JSONs from stacks-blockchain@${GITHUB_SHA}" + git push robot $ROBOT_BRANCH + echo "::set-output name=open_pr::1" + fi + - name: Open PR + if: ${{ steps.push.outputs.open_pr == '1' }} + uses: actions/github-script@v2 + with: + github-token: ${{ secrets.DOCS_GITHUB_TOKEN }} + script: | + // get env vars + const process = require("process"); + const robot_owner = process.env.ROBOT_OWNER; + const robot_branch = process.env.ROBOT_BRANCH; + const head = `${robot_owner}:${robot_branch}`; + const owner = process.env.TARGET_OWNER; + const repo = process.env.TARGET_REPO; - # console.log(`Checking PR with params: head= ${head} owner= ${owner} repo= ${repo}`); + console.log(`Checking PR with params: head= ${head} owner= ${owner} repo= ${repo}`); - # // check if a pull exists - # const existingPulls = await github.pulls.list({ - # owner, repo, state: "open" }); - # const myPulls = existingPulls.data.filter( pull => pull.user.login == robot_owner ); - # console.log(myPulls); + // check if a pull exists + const existingPulls = await github.pulls.list({ + owner, repo, state: "open" }); + const myPulls = existingPulls.data.filter( pull => pull.user.login == robot_owner ); + console.log(myPulls); - # for (myPull of myPulls) { - # // close any open PRs - # const pull_number = myPull.number; - # console.log(`Closing PR: ${ pull_number }`); - # await github.pulls.update({ owner, repo, pull_number, state: "closed" }); - # } + for (myPull of myPulls) { + // close any open PRs + const pull_number = myPull.number; + console.log(`Closing PR: ${ pull_number }`); + await github.pulls.update({ owner, repo, pull_number, state: "closed" }); + } - # // Open PR if one doesn't exist - # console.log("Opening the new PR."); - # let result = await github.pulls.create({ - # owner, repo, head, - # base: "master", - # title: "Auto: Update API documentation from stacks-blockchain", - # body: "Update API documentation from the latest in `stacks-blockchain`", - # }); + // Open PR if one doesn't exist + console.log("Opening the new PR."); + let result = await github.pulls.create({ + owner, repo, head, + base: "master", + title: "Auto: Update API documentation from stacks-blockchain", + body: "Update API documentation from the latest in `stacks-blockchain`", + }); From 74f0d997be3efab5c1213296813ca45870ec3ea9 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 12 Sep 2022 10:56:59 -0500 Subject: [PATCH 017/178] ci: use rust stable for code coverage tests --- .github/actions/bitcoin-int-tests/Dockerfile.code-cov | 7 +++---- .../bitcoin-int-tests/Dockerfile.generic.bitcoin-tests | 5 ++--- .github/actions/bitcoin-int-tests/Dockerfile.large-genesis | 5 ++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov index 733f879b7..209b80473 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov +++ b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov @@ -4,13 +4,12 @@ WORKDIR /build ENV CARGO_MANIFEST_DIR="$(pwd)" -RUN rustup override set nightly-2022-01-14 && \ - rustup component add llvm-tools-preview && \ +RUN rustup component add llvm-tools-preview && \ cargo install grcov -ENV RUSTFLAGS="-Zinstrument-coverage" \ +ENV RUSTFLAGS="-Cinstrument-coverage" \ LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" - + COPY . . RUN cargo build --workspace && \ diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests b/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests index 42a0235cf..2fd43a589 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests +++ b/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests @@ -6,11 +6,10 @@ COPY . . WORKDIR /src/testnet/stacks-node -RUN rustup override set nightly-2022-01-14 && \ - rustup component add llvm-tools-preview && \ +RUN rustup component add llvm-tools-preview && \ cargo install grcov -ENV RUSTFLAGS="-Zinstrument-coverage" \ +ENV RUSTFLAGS="-Cinstrument-coverage" \ LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" RUN cargo test --no-run && \ diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis b/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis index 4f96fd304..1350a6ed8 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis +++ b/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis @@ -9,11 +9,10 @@ RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/ -RUN rustup override set nightly-2022-01-14 && \ - rustup component add llvm-tools-preview && \ +RUN rustup component add llvm-tools-preview && \ cargo install grcov -ENV RUSTFLAGS="-Zinstrument-coverage" \ +ENV RUSTFLAGS="-Cinstrument-coverage" \ LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" RUN cargo test --no-run --workspace && \ From be09336d6d65829c517f23009843c68e6eabc9f3 Mon Sep 17 00:00:00 2001 From: omahs <73983677+omahs@users.noreply.github.com> Date: Thu, 29 Sep 2022 10:40:49 +0200 Subject: [PATCH 018/178] Fix: typos Fix: typos --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 33e71e855..8025bc7dd 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Stacks 2.0 is a layer-1 blockchain that connects to Bitcoin for security and ena ## Repository -| Blockstack Topic/Tech | Where to learn more more | +| Blockstack Topic/Tech | Where to learn more | | -------------------------- | --------------------------------------------------------------------------------- | | Stacks 2.0 | [master branch](https://github.com/blockstack/stacks-blockchain/tree/master) | | Stacks 1.0 | [legacy branch](https://github.com/blockstack/stacks-blockchain/tree/stacks-1.0) | @@ -55,7 +55,7 @@ to upgrade to `2.0.10.1.0` or `2.0.10.0.1`. However, upgrading to `2.0.11.0.0` w - [x] [SIP 001: Burn Election](https://github.com/stacksgov/sips/blob/main/sips/sip-001/sip-001-burn-election.md) - [x] [SIP 002: Clarity, a language for predictable smart contracts](https://github.com/stacksgov/sips/blob/main/sips/sip-002/sip-002-smart-contract-language.md) - [x] [SIP 003: Peer Network](https://github.com/stacksgov/sips/blob/main/sips/sip-003/sip-003-peer-network.md) -- [x] [SIP 004: Cryptographic Committment to Materialized Views](https://github.com/stacksgov/sips/blob/main/sips/sip-004/sip-004-materialized-view.md) +- [x] [SIP 004: Cryptographic Commitment to Materialized Views](https://github.com/stacksgov/sips/blob/main/sips/sip-004/sip-004-materialized-view.md) - [x] [SIP 005: Blocks, Transactions, and Accounts](https://github.com/stacksgov/sips/blob/main/sips/sip-005/sip-005-blocks-and-transactions.md) - [x] [SIP 006: Clarity Execution Cost Assessment](https://github.com/stacksgov/sips/blob/main/sips/sip-006/sip-006-runtime-cost-assessment.md) - [x] [SIP 007: Stacking Consensus](https://github.com/stacksgov/sips/blob/main/sips/sip-007/sip-007-stacking-consensus.md) @@ -357,7 +357,7 @@ INFO [1630127492.062652] [testnet/stacks-node/src/run_loop/neon.rs:164] [main] U ### Configuring Cost and Fee Estimation -Fee and cost estimators can be configure via the config section `[fee_estimation]`: +Fee and cost estimators can be configured via the config section `[fee_estimation]`: ``` [fee_estimation] @@ -404,7 +404,7 @@ that the change is not consensus-breaking. So, the release manager must first determine whether there are any "non-consensus-breaking changes that require a fresh chainstate". This means, in other words, that the database schema has changed, but an automatic migration was not implemented. Then, the release manager -should determine whether this is a feature release, as opposed to a hot fix or a +should determine whether this is a feature release, as opposed to a hotfix or a patch. Given the answers to these questions, the version number can be computed. 1. The release manager enumerates the PRs or issues that would _block_ From f74816fb1e86e21a7d06b351e275fbc1d1b222ce Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 30 Sep 2022 14:08:02 -0500 Subject: [PATCH 019/178] feat: denormalize mempool/fee_estimates table to remove join, only deser txs actually considered --- src/core/mempool.rs | 118 +++++++++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 55588a713..27b847032 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -241,6 +241,12 @@ pub struct MemPoolTxInfo { pub metadata: MemPoolTxMetadata, } +#[derive(Debug, PartialEq, Clone)] +pub enum MemPoolTxInfoPartial { + NeedsNonces { addrs_needed: Vec }, + HasNonces(MemPoolTxInfo), +} + #[derive(Debug, PartialEq, Clone)] pub struct MemPoolTxMetadata { pub txid: Txid, @@ -258,6 +264,19 @@ pub struct MemPoolTxMetadata { pub accept_time: u64, } +impl MemPoolTxMetadata { + pub fn get_unknown_nonces(&self) -> Vec { + let mut needs_nonces = vec![]; + if self.last_known_origin_nonce.is_none() { + needs_nonces.push(self.origin_address); + } + if self.last_known_sponsor_nonce.is_none() { + needs_nonces.push(self.sponsor_address); + } + needs_nonces + } +} + #[derive(Debug, Clone)] pub struct MemPoolWalkSettings { /// Minimum transaction fee that will be considered @@ -346,6 +365,30 @@ impl FromRow for MemPoolTxInfo { } } +impl FromRow for MemPoolTxInfoPartial { + fn from_row<'a>(row: &'a Row) -> Result { + let md = MemPoolTxMetadata::from_row(row)?; + let needs_nonces = md.get_unknown_nonces(); + let consider = if !needs_nonces.is_empty() { + MemPoolTxInfoPartial::NeedsNonces { + addrs_needed: needs_nonces, + } + } else { + let tx_bytes: Vec = row.get_unwrap("tx"); + let tx = StacksTransaction::consensus_deserialize(&mut &tx_bytes[..]) + .map_err(|_e| db_error::ParseError)?; + + if tx.txid() != md.txid { + return Err(db_error::ParseError); + } + + MemPoolTxInfoPartial::HasNonces(MemPoolTxInfo { tx, metadata: md }) + }; + + Ok(consider) + } +} + impl FromRow<(u64, u64)> for (u64, u64) { fn from_row<'a>(row: &'a Row) -> Result<(u64, u64), db_error> { let t1: i64 = row.get_unwrap(0); @@ -377,6 +420,19 @@ const MEMPOOL_INITIAL_SCHEMA: &'static [&'static str] = &[r#" ); "#]; +const MEMPOOL_SCHEMA_5: &'static [&'static str] = &[ + r#" + ALTER TABLE mempool ADD COLUMN fee_rate NUMBER; + "#, + r#" + CREATE INDEX IF NOT EXISTS by_fee_rate ON mempool(fee_rate); + "#, + r#" + UPDATE mempool + SET fee_rate = (SELECT f.fee_rate FROM fee_estimates as f WHERE f.txid = mempool.txid); + "#, +]; + const MEMPOOL_SCHEMA_2_COST_ESTIMATOR: &'static [&'static str] = &[ r#" CREATE TABLE fee_estimates( @@ -744,6 +800,9 @@ impl MemPoolDB { MemPoolDB::instantiate_tx_blacklist(tx)?; } 4 => { + MemPoolDB::denormalize_fee_rate(tx)?; + } + 5 => { break; } _ => { @@ -788,6 +847,15 @@ impl MemPoolDB { Ok(()) } + /// Denormalize fee rate schema 5 + fn denormalize_fee_rate(tx: &DBTx) -> Result<(), db_error> { + for sql_exec in MEMPOOL_SCHEMA_5 { + tx.execute_batch(sql_exec)?; + } + + Ok(()) + } + /// Instantiate the tx blacklist schema fn instantiate_tx_blacklist(tx: &DBTx) -> Result<(), db_error> { for sql_exec in MEMPOOL_SCHEMA_4_BLACKLIST { @@ -925,11 +993,11 @@ impl MemPoolDB { /// whether or not the miner should propagate transaction receipts back to the estimator. fn get_next_tx_to_consider_no_estimate( &self, - ) -> Result, db_error> { - let select_no_estimate = "SELECT * FROM mempool LEFT JOIN fee_estimates as f ON mempool.txid = f.txid WHERE + ) -> Result, db_error> { + let select_no_estimate = "SELECT * FROM mempool WHERE ((origin_nonce = last_known_origin_nonce AND sponsor_nonce = last_known_sponsor_nonce) OR (last_known_origin_nonce is NULL) OR (last_known_sponsor_nonce is NULL)) - AND f.fee_rate IS NULL ORDER BY tx_fee DESC LIMIT 1"; + AND fee_rate IS NULL ORDER BY tx_fee DESC LIMIT 1"; query_row(&self.db, select_no_estimate, rusqlite::NO_PARAMS) .map(|opt_tx| opt_tx.map(|tx| (tx, true))) } @@ -939,11 +1007,11 @@ impl MemPoolDB { /// whether or not the miner should propagate transaction receipts back to the estimator. fn get_next_tx_to_consider_with_estimate( &self, - ) -> Result, db_error> { - let select_estimate = "SELECT * FROM mempool LEFT OUTER JOIN fee_estimates as f ON mempool.txid = f.txid WHERE + ) -> Result, db_error> { + let select_estimate = "SELECT * FROM mempool WHERE ((origin_nonce = last_known_origin_nonce AND sponsor_nonce = last_known_sponsor_nonce) OR (last_known_origin_nonce is NULL) OR (last_known_sponsor_nonce is NULL)) - AND f.fee_rate IS NOT NULL ORDER BY f.fee_rate DESC LIMIT 1"; + AND fee_rate IS NOT NULL ORDER BY fee_rate DESC LIMIT 1"; query_row(&self.db, select_estimate, rusqlite::NO_PARAMS) .map(|opt_tx| opt_tx.map(|tx| (tx, false))) } @@ -956,7 +1024,7 @@ impl MemPoolDB { &self, start_with_no_estimate: bool, ) -> Result { - let (next_tx, update_estimate): (MemPoolTxInfo, bool) = if start_with_no_estimate { + let (next_tx, update_estimate): (MemPoolTxInfoPartial, bool) = if start_with_no_estimate { match self.get_next_tx_to_consider_no_estimate()? { Some(result) => result, None => match self.get_next_tx_to_consider_with_estimate()? { @@ -974,21 +1042,16 @@ impl MemPoolDB { } }; - let mut needs_nonces = vec![]; - if next_tx.metadata.last_known_origin_nonce.is_none() { - needs_nonces.push(next_tx.metadata.origin_address); - } - if next_tx.metadata.last_known_sponsor_nonce.is_none() { - needs_nonces.push(next_tx.metadata.sponsor_address); - } - - if !needs_nonces.is_empty() { - Ok(ConsiderTransactionResult::UpdateNonces(needs_nonces)) - } else { - Ok(ConsiderTransactionResult::Consider(ConsiderTransaction { - tx: next_tx, - update_estimate, - })) + match next_tx { + MemPoolTxInfoPartial::NeedsNonces { addrs_needed } => { + Ok(ConsiderTransactionResult::UpdateNonces(addrs_needed)) + } + MemPoolTxInfoPartial::HasNonces(tx) => { + Ok(ConsiderTransactionResult::Consider(ConsiderTransaction { + tx, + update_estimate, + })) + } } } @@ -1027,8 +1090,7 @@ impl MemPoolDB { let sql_tx = tx_begin_immediate(&mut self.db)?; let txs: Vec = query_rows( &sql_tx, - "SELECT * FROM mempool as m LEFT OUTER JOIN fee_estimates as f ON - m.txid = f.txid WHERE f.fee_rate IS NULL LIMIT ?", + "SELECT * FROM mempool as m WHERE m.fee_rate IS NULL LIMIT ?", &[max_updates], )?; let mut updated = 0; @@ -1053,8 +1115,8 @@ impl MemPoolDB { }; sql_tx.execute( - "INSERT OR REPLACE INTO fee_estimates(txid, fee_rate) VALUES (?, ?)", - rusqlite::params![&txid, fee_rate_f64], + "UPDATE mempool SET fee_rate = ? WHERE txid = ?", + rusqlite::params![fee_rate_f64, &txid], )?; updated += 1; } @@ -1603,8 +1665,8 @@ impl MemPoolDB { mempool_tx .execute( - "INSERT OR REPLACE INTO fee_estimates(txid, fee_rate) VALUES (?, ?)", - rusqlite::params![&txid, fee_rate_estimate], + "UPDATE mempool SET fee_rate = ? WHERE txid = ?", + rusqlite::params![fee_rate_estimate, &txid], ) .map_err(db_error::from)?; From 3b4ca5f3cf1fe61d124243ad20db4d6625c48666 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Fri, 30 Sep 2022 14:26:53 -0500 Subject: [PATCH 020/178] set mempool schema version to 5 --- src/core/mempool.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 27b847032..db9b95f32 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -420,19 +420,6 @@ const MEMPOOL_INITIAL_SCHEMA: &'static [&'static str] = &[r#" ); "#]; -const MEMPOOL_SCHEMA_5: &'static [&'static str] = &[ - r#" - ALTER TABLE mempool ADD COLUMN fee_rate NUMBER; - "#, - r#" - CREATE INDEX IF NOT EXISTS by_fee_rate ON mempool(fee_rate); - "#, - r#" - UPDATE mempool - SET fee_rate = (SELECT f.fee_rate FROM fee_estimates as f WHERE f.txid = mempool.txid); - "#, -]; - const MEMPOOL_SCHEMA_2_COST_ESTIMATOR: &'static [&'static str] = &[ r#" CREATE TABLE fee_estimates( @@ -516,6 +503,22 @@ const MEMPOOL_SCHEMA_4_BLACKLIST: &'static [&'static str] = &[ "#, ]; +const MEMPOOL_SCHEMA_5: &'static [&'static str] = &[ + r#" + ALTER TABLE mempool ADD COLUMN fee_rate NUMBER; + "#, + r#" + CREATE INDEX IF NOT EXISTS by_fee_rate ON mempool(fee_rate); + "#, + r#" + UPDATE mempool + SET fee_rate = (SELECT f.fee_rate FROM fee_estimates as f WHERE f.txid = mempool.txid); + "#, + r#" + INSERT INTO schema_version (version) VALUES (5) + "#, +]; + const MEMPOOL_INDEXES: &'static [&'static str] = &[ "CREATE INDEX IF NOT EXISTS by_txid ON mempool(txid);", "CREATE INDEX IF NOT EXISTS by_height ON mempool(height);", From cff3d1e47b245d3e01dba9a9f279450e2887d984 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 12 Sep 2022 10:56:59 -0500 Subject: [PATCH 021/178] ci: use rust stable for code coverage tests --- .github/actions/bitcoin-int-tests/Dockerfile.code-cov | 7 +++---- .../bitcoin-int-tests/Dockerfile.generic.bitcoin-tests | 5 ++--- .github/actions/bitcoin-int-tests/Dockerfile.large-genesis | 5 ++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov index 733f879b7..209b80473 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.code-cov +++ b/.github/actions/bitcoin-int-tests/Dockerfile.code-cov @@ -4,13 +4,12 @@ WORKDIR /build ENV CARGO_MANIFEST_DIR="$(pwd)" -RUN rustup override set nightly-2022-01-14 && \ - rustup component add llvm-tools-preview && \ +RUN rustup component add llvm-tools-preview && \ cargo install grcov -ENV RUSTFLAGS="-Zinstrument-coverage" \ +ENV RUSTFLAGS="-Cinstrument-coverage" \ LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" - + COPY . . RUN cargo build --workspace && \ diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests b/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests index 42a0235cf..2fd43a589 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests +++ b/.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests @@ -6,11 +6,10 @@ COPY . . WORKDIR /src/testnet/stacks-node -RUN rustup override set nightly-2022-01-14 && \ - rustup component add llvm-tools-preview && \ +RUN rustup component add llvm-tools-preview && \ cargo install grcov -ENV RUSTFLAGS="-Zinstrument-coverage" \ +ENV RUSTFLAGS="-Cinstrument-coverage" \ LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" RUN cargo test --no-run && \ diff --git a/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis b/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis index 4f96fd304..1350a6ed8 100644 --- a/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis +++ b/.github/actions/bitcoin-int-tests/Dockerfile.large-genesis @@ -9,11 +9,10 @@ RUN cd / && tar -xvzf bitcoin-0.20.0-x86_64-linux-gnu.tar.gz RUN ln -s /bitcoin-0.20.0/bin/bitcoind /bin/ -RUN rustup override set nightly-2022-01-14 && \ - rustup component add llvm-tools-preview && \ +RUN rustup component add llvm-tools-preview && \ cargo install grcov -ENV RUSTFLAGS="-Zinstrument-coverage" \ +ENV RUSTFLAGS="-Cinstrument-coverage" \ LLVM_PROFILE_FILE="stacks-blockchain-%p-%m.profraw" RUN cargo test --no-run --workspace && \ From 378fc1b80e161ce1b82cc98b3802565222100816 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Fri, 30 Sep 2022 19:24:27 -0400 Subject: [PATCH 022/178] fix: changelog entry for 2.05.0.4.0 --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b14cc2aac..008320906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to the versioning scheme outlined in the [README.md](README.md). +## [2.05.0.4.0] + +### Fixed + +- Denormalize the mempool database so as to remove a `LEFT JOIN` from the SQL + query for choosing transactions in order by estimated fee rate. This +drastically speeds up mempool transaction iteration in the miner (#3314) + + ## [2.05.0.3.0] ### Added From a2a451039a522dbdb051794b4edf8330dda6a765 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Tue, 4 Oct 2022 08:56:12 -0700 Subject: [PATCH 023/178] disable arm64 builds --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a48626df..0877b16cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,14 +141,14 @@ jobs: name: ${{ matrix.platform }} path: ${{ matrix.platform }}.zip - call-docker-platforms-workflow: - if: ${{ github.event.inputs.tag != '' }} - uses: stacks-network/stacks-blockchain/.github/workflows/docker-platforms.yml@master - with: - tag: ${{ github.event.inputs.tag }} - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} +# call-docker-platforms-workflow: +# if: ${{ github.event.inputs.tag != '' }} +# uses: stacks-network/stacks-blockchain/.github/workflows/docker-platforms.yml@master +# with: +# tag: ${{ github.event.inputs.tag }} +# secrets: +# DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} +# DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} # Build docker image, tag it with the git tag and `latest` if running on master branch, and publish under the following conditions # Will publish if: From 602317677b73f9c2d27b39c280ddcb4559510bc3 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:09:52 -0400 Subject: [PATCH 024/178] chore: clean up canonical stacks tip memo code to make it more comprehensible --- src/chainstate/burn/db/sortdb.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/chainstate/burn/db/sortdb.rs b/src/chainstate/burn/db/sortdb.rs index f4b07728c..10adea06d 100644 --- a/src/chainstate/burn/db/sortdb.rs +++ b/src/chainstate/burn/db/sortdb.rs @@ -89,7 +89,7 @@ use crate::chainstate::stacks::index::{ClarityMarfTrieId, MARFValue}; use stacks_common::types::chainstate::StacksAddress; use stacks_common::types::chainstate::TrieHash; use stacks_common::types::chainstate::{ - BlockHeaderHash, BurnchainHeaderHash, PoxId, SortitionId, VRFSeed, + BlockHeaderHash, BurnchainHeaderHash, PoxId, SortitionId, StacksBlockId, VRFSeed, }; const BLOCK_HEIGHT_MAX: u64 = ((1 as u64) << 63) - 1; @@ -1434,7 +1434,7 @@ impl<'a> SortitionHandleTx<'a> { )?; } else { // see if this block builds off of a Stacks block mined on this burnchain fork - let height_opt = match SortitionDB::get_accepted_stacks_block_pointer( + let parent_height_opt = match SortitionDB::get_accepted_stacks_block_pointer( self, &burn_tip.consensus_hash, parent_stacks_block_hash, @@ -1452,10 +1452,11 @@ impl<'a> SortitionHandleTx<'a> { } } }; - match height_opt { - Some(height) => { + match parent_height_opt { + Some(parent_height) => { if stacks_block_height > burn_tip.canonical_stacks_tip_height { - assert!(stacks_block_height > height, "BUG: DB corruption -- block height {} <= {} means we accepted a block out-of-order", stacks_block_height, height); + assert!(stacks_block_height > parent_height, "BUG: DB corruption -- block height {} <= {} means we accepted a block out-of-order", stacks_block_height, parent_height); + // This block builds off of a parent that is _concurrent_ with the memoized canonical stacks chain pointer. // i.e. this block will reorg the Stacks chain on the canonical burnchain fork. // Memoize this new stacks chain tip to the canonical burn chain snapshot. @@ -1463,7 +1464,7 @@ impl<'a> SortitionHandleTx<'a> { // are guaranteed by the Stacks chain state code that Stacks blocks in a given // Stacks fork will be marked as accepted in sequential order (i.e. at height h, h+1, // h+2, etc., without any gaps). - debug!("Accepted Stacks block {}/{} builds on a previous canonical Stacks tip on this burnchain fork ({})", consensus_hash, stacks_block_hash, &burn_tip.burn_header_hash); + debug!("Accepted Stacks block {}/{} ({}) builds on a previous canonical Stacks tip on this burnchain fork ({})", consensus_hash, stacks_block_hash, stacks_block_height, &burn_tip.burn_header_hash); let args: &[&dyn ToSql] = &[ consensus_hash, stacks_block_hash, @@ -1477,7 +1478,7 @@ impl<'a> SortitionHandleTx<'a> { // This block was mined on this fork, but it's acceptance doesn't overtake // the current stacks chain tip. Remember it so that we can process its children, // which might do so later. - debug!("Accepted Stacks block {}/{} builds on a non-canonical Stacks tip in this burnchain fork ({})", consensus_hash, stacks_block_hash, &burn_tip.burn_header_hash); + debug!("Accepted Stacks block {}/{} ({}) builds on a non-canonical Stacks tip in this burnchain fork ({} height {})", consensus_hash, stacks_block_hash, stacks_block_height, &burn_tip.burn_header_hash, burn_tip.canonical_stacks_tip_height); } SortitionDB::insert_accepted_stacks_block_pointer( self, @@ -2475,8 +2476,8 @@ impl SortitionDB { pub fn is_db_version_supported_in_epoch(epoch: StacksEpochId, version: &str) -> bool { match epoch { StacksEpochId::Epoch10 => false, - StacksEpochId::Epoch20 => (version == "1" || version == "2" || version == "3"), - StacksEpochId::Epoch2_05 => (version == "2" || version == "3" || version == "4"), + StacksEpochId::Epoch20 => version == "1" || version == "2" || version == "3", + StacksEpochId::Epoch2_05 => version == "2" || version == "3" || version == "4", } } From 5fe6ff7944dc856c18782c7550c20229a1ac3f01 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:10:20 -0400 Subject: [PATCH 025/178] feat: when the chains coordinator is about to start processing a burnchain or stacks block, signal to any running miner thread that it should cease what it's doing. This prevents the miner from holding open the write lock on the underlying DB, and in doing so, blocking the coordinator (whose operation takes precedence since the miner builds off of a chain tip produced by the coordinator) --- src/chainstate/coordinator/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/chainstate/coordinator/mod.rs b/src/chainstate/coordinator/mod.rs index aa9eae266..d1c16d6e8 100644 --- a/src/chainstate/coordinator/mod.rs +++ b/src/chainstate/coordinator/mod.rs @@ -19,6 +19,8 @@ use std::convert::{TryFrom, TryInto}; use std::fs; use std::path::PathBuf; use std::sync::mpsc::SyncSender; +use std::sync::Arc; +use std::sync::Mutex; use std::time::Duration; use crate::burnchains::{ @@ -39,6 +41,7 @@ use crate::chainstate::stacks::{ StacksHeaderInfo, }, events::{StacksTransactionEvent, StacksTransactionReceipt, TransactionOrigin}, + miner::{signal_mining_blocked, signal_mining_ready, MinerStatus}, Error as ChainstateError, StacksBlock, TransactionPayload, }; use crate::core::StacksEpoch; @@ -272,6 +275,7 @@ impl<'a, T: BlockEventDispatcher, CE: CostEstimator + ?Sized, FE: FeeEstimator + atlas_config: AtlasConfig, cost_estimator: Option<&mut CE>, fee_estimator: Option<&mut FE>, + miner_status: Arc>, ) where T: BlockEventDispatcher, { @@ -311,18 +315,23 @@ impl<'a, T: BlockEventDispatcher, CE: CostEstimator + ?Sized, FE: FeeEstimator + // timeout so that we handle Ctrl-C a little gracefully match comms.wait_on() { CoordinatorEvents::NEW_STACKS_BLOCK => { + signal_mining_blocked(miner_status.clone()); debug!("Received new stacks block notice"); if let Err(e) = inst.handle_new_stacks_block() { warn!("Error processing new stacks block: {:?}", e); } + signal_mining_ready(miner_status.clone()); } CoordinatorEvents::NEW_BURN_BLOCK => { + signal_mining_blocked(miner_status.clone()); debug!("Received new burn block notice"); if let Err(e) = inst.handle_new_burnchain_block() { warn!("Error processing new burn block: {:?}", e); } + signal_mining_ready(miner_status.clone()); } CoordinatorEvents::STOP => { + signal_mining_blocked(miner_status.clone()); debug!("Received stop notice"); return; } From 0fd8fae1c97810c9e2c0840fa09ee613e0922827 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:11:26 -0400 Subject: [PATCH 026/178] feat: add a query to see if there are any pending unprocessed blocks higher than a certain height; add a read-only DB query to see if we have already processed a Stacks block (so we don't need a write lock on the chainstate); catch microblock forks earlier by checking for common ancestors in addition to sequence numbers; --- src/chainstate/stacks/db/blocks.rs | 139 +++++++++++++++++++++-------- 1 file changed, 102 insertions(+), 37 deletions(-) diff --git a/src/chainstate/stacks/db/blocks.rs b/src/chainstate/stacks/db/blocks.rs index c36e9bf51..81b87cd51 100644 --- a/src/chainstate/stacks/db/blocks.rs +++ b/src/chainstate/stacks/db/blocks.rs @@ -1501,7 +1501,7 @@ impl StacksChainState { } } - test_debug!( + debug!( "Loaded microblock {}/{}-{} (parent={}, expect_seq={})", &parent_consensus_hash, &parent_anchored_block_hash, @@ -1533,6 +1533,16 @@ impl StacksChainState { } } ret.reverse(); + + if ret.len() > 0 { + // should start with 0 + if ret[0].header.sequence != 0 { + warn!("Invalid microblock stream from {}/{} to {}: sequence does not start with 0, but with {}", + parent_consensus_hash, parent_anchored_block_hash, tip_microblock_hash, ret[0].header.sequence); + + return Ok(None); + } + } Ok(Some(ret)) } @@ -1617,10 +1627,11 @@ impl StacksChainState { return Ok(None); } - let mut ret = vec![]; + let mut ret: Vec = vec![]; let mut tip: Option = None; let mut fork_poison = None; let mut expected_sequence = start_seq; + let mut parents: HashMap = HashMap::new(); // load associated staging microblock data, but best-effort. // Stop loading once we find a fork juncture. @@ -1657,6 +1668,21 @@ impl StacksChainState { break; } + if let Some(idx) = parents.get(&mblock.header.prev_block) { + let conflict = ret[*idx].clone(); + warn!( + "Microblock fork found: microblocks {} and {} share parent {}", + mblock.block_hash(), + conflict.block_hash(), + &mblock.header.prev_block + ); + fork_poison = Some(TransactionPayload::PoisonMicroblock( + mblock.header, + conflict.header, + )); + break; + } + // expect forks, so expected_sequence may not always increase expected_sequence = cmp::min(mblock.header.sequence, expected_sequence).saturating_add(1); @@ -1677,6 +1703,10 @@ impl StacksChainState { } tip = Some(mblock.clone()); + + let prev_block = mblock.header.prev_block.clone(); + parents.insert(prev_block, ret.len()); + ret.push(mblock); } if fork_poison.is_none() && ret.len() == 0 { @@ -3453,6 +3483,20 @@ impl StacksChainState { Ok(count - to_write) } + /// Check whether or not there exists a Stacks block at or higher than a given height that is + /// unprocessed. This is used by miners to determine whether or not the block-commit they're + /// about to send is about to be invalidated + pub fn has_higher_unprocessed_blocks(conn: &DBConn, height: u64) -> Result { + let sql = + "SELECT 1 FROM staging_blocks WHERE orphaned = 0 AND processed = 0 AND height >= ?1"; + let args: &[&dyn ToSql] = &[&u64_to_sql(height)?]; + let res = conn + .query_row(sql, args, |_r| Ok(())) + .optional() + .map(|x| x.is_some())?; + Ok(res) + } + fn extract_signed_microblocks( parent_anchored_block_header: &StacksBlockHeader, microblocks: &Vec, @@ -3793,6 +3837,49 @@ impl StacksChainState { Ok(Some((block_commit.burn_fee, sortition_burns))) } + /// Do we already have an anchored block? + pub fn has_anchored_block( + conn: &DBConn, + blocks_path: &str, + consensus_hash: &ConsensusHash, + block: &StacksBlock, + ) -> Result { + let index_block_hash = + StacksBlockHeader::make_index_block_hash(consensus_hash, &block.block_hash()); + if StacksChainState::has_stored_block( + &conn, + blocks_path, + consensus_hash, + &block.block_hash(), + )? { + debug!( + "Block already stored and processed: {}/{} ({})", + consensus_hash, + &block.block_hash(), + &index_block_hash + ); + return Ok(true); + } else if StacksChainState::has_staging_block(conn, consensus_hash, &block.block_hash())? { + debug!( + "Block already stored (but not processed): {}/{} ({})", + consensus_hash, + &block.block_hash(), + &index_block_hash + ); + return Ok(true); + } else if StacksChainState::has_block_indexed(&blocks_path, &index_block_hash)? { + debug!( + "Block already stored to chunk store: {}/{} ({})", + consensus_hash, + &block.block_hash(), + &index_block_hash + ); + return Ok(true); + } + + Ok(false) + } + /// Pre-process and store an anchored block to staging, queuing it up for /// subsequent processing once all of its ancestors have been processed. /// @@ -3828,43 +3915,21 @@ impl StacksChainState { let mainnet = self.mainnet; let chain_id = self.chain_id; let blocks_path = self.blocks_path.clone(); + + // optimistic check (before opening a tx): already in queue or already processed? + if StacksChainState::has_anchored_block( + self.db(), + &self.blocks_path, + consensus_hash, + block, + )? { + return Ok(false); + } + let mut block_tx = self.db_tx_begin()?; - // already in queue or already processed? - let index_block_hash = - StacksBlockHeader::make_index_block_hash(consensus_hash, &block.block_hash()); - if StacksChainState::has_stored_block( - &block_tx, - &blocks_path, - consensus_hash, - &block.block_hash(), - )? { - debug!( - "Block already stored and processed: {}/{} ({})", - consensus_hash, - &block.block_hash(), - &index_block_hash - ); - return Ok(false); - } else if StacksChainState::has_staging_block( - &block_tx, - consensus_hash, - &block.block_hash(), - )? { - debug!( - "Block already stored (but not processed): {}/{} ({})", - consensus_hash, - &block.block_hash(), - &index_block_hash - ); - return Ok(false); - } else if StacksChainState::has_block_indexed(&blocks_path, &index_block_hash)? { - debug!( - "Block already stored to chunk store: {}/{} ({})", - consensus_hash, - &block.block_hash(), - &index_block_hash - ); + // already in queue or already processed (within the tx; things might have changed) + if StacksChainState::has_anchored_block(&block_tx, &blocks_path, consensus_hash, block)? { return Ok(false); } From 0b8397b6f52b0f4e41e2f082431f53fb2b167358 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:12:15 -0400 Subject: [PATCH 027/178] chore: remove fmt warning --- src/chainstate/stacks/db/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chainstate/stacks/db/mod.rs b/src/chainstate/stacks/db/mod.rs index c8ab50d54..81efdc929 100644 --- a/src/chainstate/stacks/db/mod.rs +++ b/src/chainstate/stacks/db/mod.rs @@ -192,7 +192,7 @@ impl DBConfig { pub fn supports_epoch(&self, epoch_id: StacksEpochId) -> bool { match epoch_id { StacksEpochId::Epoch10 => false, - StacksEpochId::Epoch20 => (self.version == "1" || self.version == "2"), + StacksEpochId::Epoch20 => self.version == "1" || self.version == "2", StacksEpochId::Epoch2_05 => self.version == "2", } } From 6e81ee4bcc8d9ed5b620842f97db83ae18839905 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:12:27 -0400 Subject: [PATCH 028/178] feat: add make_readonly_owned() function that effectively takes a read-only snapshot of the the unconfirmed state. Used by a separate miner thread so it doesn't need to hold the relayer thread's chainstate --- src/chainstate/stacks/db/unconfirmed.rs | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/chainstate/stacks/db/unconfirmed.rs b/src/chainstate/stacks/db/unconfirmed.rs index b973152d1..1c1efe799 100644 --- a/src/chainstate/stacks/db/unconfirmed.rs +++ b/src/chainstate/stacks/db/unconfirmed.rs @@ -24,6 +24,7 @@ use crate::chainstate::stacks::db::accounts::*; use crate::chainstate::stacks::db::blocks::*; use crate::chainstate::stacks::db::*; use crate::chainstate::stacks::events::*; +use crate::chainstate::stacks::index::marf::MARFOpenOpts; use crate::chainstate::stacks::Error; use crate::chainstate::stacks::*; use crate::clarity_vm::clarity::{ClarityInstance, Error as clarity_error}; @@ -84,6 +85,10 @@ pub struct UnconfirmedState { num_mblocks_added: u64, have_state: bool, + mainnet: bool, + clarity_state_index_root: String, + marf_opts: Option, + // fault injection for testing pub disable_cost_check: bool, pub disable_bytes_check: bool, @@ -120,11 +125,51 @@ impl UnconfirmedState { num_mblocks_added: 0, have_state: false, + mainnet: chainstate.mainnet, + clarity_state_index_root: chainstate.clarity_state_index_root.clone(), + marf_opts: chainstate.marf_opts.clone(), + disable_cost_check: check_fault_injection(FAULT_DISABLE_MICROBLOCKS_COST_CHECK), disable_bytes_check: check_fault_injection(FAULT_DISABLE_MICROBLOCKS_BYTES_CHECK), }) } + /// Make a read-only copy of this unconfirmed state. The resulting unconfiremd state cannot + /// be refreshed, but it will represent a snapshot of the existing unconfirmed state. + pub fn make_readonly_owned(&self) -> Result { + let marf = MarfedKV::open_unconfirmed( + &self.clarity_state_index_root, + None, + self.marf_opts.clone(), + )?; + + let clarity_instance = ClarityInstance::new(self.mainnet, marf); + + Ok(UnconfirmedState { + confirmed_chain_tip: self.confirmed_chain_tip.clone(), + unconfirmed_chain_tip: self.unconfirmed_chain_tip.clone(), + clarity_inst: clarity_instance, + mined_txs: self.mined_txs.clone(), + cost_so_far: self.cost_so_far.clone(), + bytes_so_far: self.bytes_so_far, + + last_mblock: self.last_mblock.clone(), + last_mblock_seq: self.last_mblock_seq, + + readonly: true, + dirty: false, + num_mblocks_added: self.num_mblocks_added, + have_state: self.have_state, + + mainnet: self.mainnet, + clarity_state_index_root: self.clarity_state_index_root.clone(), + marf_opts: self.marf_opts.clone(), + + disable_cost_check: self.disable_cost_check, + disable_bytes_check: self.disable_bytes_check, + }) + } + /// Make a new unconfirmed state, but don't do anything with it yet, and deny refreshes. fn new_readonly( chainstate: &StacksChainState, @@ -157,6 +202,10 @@ impl UnconfirmedState { num_mblocks_added: 0, have_state: false, + mainnet: chainstate.mainnet, + clarity_state_index_root: chainstate.clarity_state_index_root.clone(), + marf_opts: chainstate.marf_opts.clone(), + disable_cost_check: check_fault_injection(FAULT_DISABLE_MICROBLOCKS_COST_CHECK), disable_bytes_check: check_fault_injection(FAULT_DISABLE_MICROBLOCKS_BYTES_CHECK), }) From d47df1fb9f5da9fcbf6cec8a61d72a6b86461227 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:13:14 -0400 Subject: [PATCH 029/178] chore: remove fmt warnings --- src/chainstate/stacks/index/storage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/chainstate/stacks/index/storage.rs b/src/chainstate/stacks/index/storage.rs index c38ca09ad..0b8bf3744 100644 --- a/src/chainstate/stacks/index/storage.rs +++ b/src/chainstate/stacks/index/storage.rs @@ -1827,7 +1827,7 @@ impl<'a, T: MarfTrieId> TrieStorageTransaction<'a, T> { let size_hint = match self.data.uncommitted_writes { Some((_, ref trie_storage)) => 2 * trie_storage.size_hint(), - None => (1024), // don't try to guess _byte_ allocation here. + None => 1024, // don't try to guess _byte_ allocation here. }; let trie_buf = TrieRAM::new(bhh, size_hint, &self.data.cur_block); @@ -1869,7 +1869,7 @@ impl<'a, T: MarfTrieId> TrieStorageTransaction<'a, T> { // new trie let size_hint = match self.data.uncommitted_writes { Some((_, ref trie_storage)) => 2 * trie_storage.size_hint(), - None => (1024), // don't try to guess _byte_ allocation here. + None => 1024, // don't try to guess _byte_ allocation here. }; ( From b1ceaba5e0b15022f970d2bdcc9e848ec88ab416 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:13:33 -0400 Subject: [PATCH 030/178] feat: make the miner loop interruptable by checking the status of a shared mutex. Mining is blocked as long as at least one thread wants it blocked, and is unblocked if no threads want it blocked. Blocking/unblocking is idempotent when done from the same thread. --- src/chainstate/stacks/miner.rs | 126 ++++++++++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 3 deletions(-) diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index 87026e8b4..ba6fdd87e 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -19,6 +19,11 @@ use std::collections::HashSet; use std::convert::From; use std::fs; use std::mem; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::sync::Mutex; +use std::thread::ThreadId; use crate::burnchains::PrivateKey; use crate::burnchains::PublicKey; @@ -68,10 +73,72 @@ use clarity::vm::clarity::TransactionConnection; use clarity::vm::errors::Error as InterpreterError; use clarity::vm::types::TypeSignature; +/// System status for mining. +/// The miner can be Ready, in which case a miner is allowed to run +/// The miner can be Blocked, in which case the miner *should not start* and/or *should terminate* +/// if running. +/// The inner u64 is a per-thread ID that lets threads querying the miner status identify whether +/// or not they or another thread were the last to modify the state. +#[derive(Debug, Clone, PartialEq)] +pub struct MinerStatus { + blockers: HashSet, +} + +impl MinerStatus { + pub fn make_ready() -> MinerStatus { + MinerStatus { + blockers: HashSet::new(), + } + } + + pub fn add_blocked(&mut self) { + self.blockers.insert(std::thread::current().id()); + } + + pub fn remove_blocked(&mut self) { + self.blockers.remove(&std::thread::current().id()); + } + + pub fn is_blocked(&self) -> bool { + self.blockers.len() > 0 + } +} + +impl std::fmt::Display for MinerStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", &self) + } +} + +/// halt mining +pub fn signal_mining_blocked(miner_status: Arc>) { + match miner_status.lock() { + Ok(mut status) => { + status.add_blocked(); + } + Err(_e) => { + panic!("FATAL: mutex poisoned"); + } + } +} + +/// resume mining if we blocked it earlier +pub fn signal_mining_ready(miner_status: Arc>) { + match miner_status.lock() { + Ok(mut status) => { + status.remove_blocked(); + } + Err(_e) => { + panic!("FATAL: mutex poisoned"); + } + } +} + #[derive(Debug, Clone)] pub struct BlockBuilderSettings { pub max_miner_time_ms: u64, pub mempool_settings: MemPoolWalkSettings, + pub miner_status: Arc>, } impl BlockBuilderSettings { @@ -79,6 +146,7 @@ impl BlockBuilderSettings { BlockBuilderSettings { max_miner_time_ms: u64::max_value(), mempool_settings: MemPoolWalkSettings::default(), + miner_status: Arc::new(Mutex::new(MinerStatus::make_ready())), } } @@ -86,6 +154,7 @@ impl BlockBuilderSettings { BlockBuilderSettings { max_miner_time_ms: u64::max_value(), mempool_settings: MemPoolWalkSettings::zero(), + miner_status: Arc::new(Mutex::new(MinerStatus::make_ready())), } } } @@ -1021,6 +1090,8 @@ impl<'a> StacksMicroblockBuilder<'a> { "Microblock transaction selection begins (child of {}), bytes so far: {}", &self.anchor_block, bytes_so_far ); + let mut blocked = false; + let result = { let mut intermediate_result; loop { @@ -1042,6 +1113,12 @@ impl<'a> StacksMicroblockBuilder<'a> { return Ok(None); } + blocked = (*self.settings.miner_status.lock().expect("FATAL: mutex poisoned")).is_blocked(); + if blocked { + debug!("Microblock miner stopping due to preemption"); + return Ok(None); + } + if considered.contains(&mempool_tx.tx.txid()) { return Ok(Some(TransactionResult::skipped( &mempool_tx.tx, "Transaction already considered.".to_string()).convert_to_event())); @@ -1154,8 +1231,9 @@ impl<'a> StacksMicroblockBuilder<'a> { } intermediate_result }; + debug!( - "Microblock transaction selection finished (child of {}); {} transactions selected", + "Miner: Microblock transaction selection finished (child of {}); {} transactions selected", &self.anchor_block, num_selected ); @@ -1178,6 +1256,14 @@ impl<'a> StacksMicroblockBuilder<'a> { event_dispatcher.mempool_txs_dropped(invalidated_txs, MemPoolDropReason::TOO_EXPENSIVE); event_dispatcher.mempool_txs_dropped(to_drop_and_blacklist, MemPoolDropReason::PROBLEMATIC); + if blocked { + debug!( + "Miner: Microblock transaction selection aborted (child of {}); {} transactions selected", + &self.anchor_block, num_selected + ); + return Err(Error::MinerAborted); + } + match result { Ok(_) => {} Err(e) => { @@ -1794,7 +1880,6 @@ impl StacksBlockBuilder { chainstate: &mut StacksChainState, parent_consensus_hash: &ConsensusHash, parent_header_hash: &BlockHeaderHash, - parent_index_hash: &StacksBlockId, ) -> Result, Error> { if let Some(microblock_parent_hash) = self.parent_microblock_hash.as_ref() { // load up a microblock fork @@ -1806,9 +1891,20 @@ impl StacksBlockBuilder { )? .ok_or(Error::NoSuchBlockError)?; + debug!( + "Loaded {} microblocks made by {}/{} tipped at {}", + microblocks.len(), + &parent_consensus_hash, + &parent_header_hash, + µblock_parent_hash + ); Ok(microblocks) } else { // apply all known parent microblocks before beginning our tenure + let parent_index_hash = StacksBlockHeader::make_index_block_hash( + &self.parent_consensus_hash, + &self.parent_header_hash, + ); let (parent_microblocks, _) = match StacksChainState::load_descendant_staging_microblock_stream_with_poison( &chainstate.db(), @@ -1819,6 +1915,13 @@ impl StacksBlockBuilder { Some(x) => x, None => (vec![], None), }; + + debug!( + "Loaded {} microblocks made by {}/{}", + parent_microblocks.len(), + &parent_consensus_hash, + &parent_header_hash + ); Ok(parent_microblocks) } } @@ -1873,7 +1976,6 @@ impl StacksBlockBuilder { chainstate, &self.parent_consensus_hash.clone(), &self.parent_header_hash.clone(), - &parent_index_hash, ) { Ok(x) => x, Err(e) => { @@ -1908,6 +2010,7 @@ impl StacksBlockBuilder { let mainnet = chainstate.config().mainnet; + // data won't be committed, so do a concurrent transaction let (chainstate_tx, clarity_instance) = chainstate.chainstate_tx_begin()?; let ast_rules = @@ -2215,6 +2318,7 @@ impl StacksBlockBuilder { let mut block_limit_hit = BlockLimitFunction::NO_LIMIT_HIT; let deadline = ts_start + (max_miner_time_ms as u128); let mut num_txs = 0; + let mut blocked = false; debug!( "Anchored block transaction selection begins (child of {})", @@ -2230,6 +2334,14 @@ impl StacksBlockBuilder { tip_height, mempool_settings.clone(), |epoch_tx, to_consider, estimator| { + // first, have we been preempted? + blocked = (*settings.miner_status.lock().expect("FATAL: mutex poisoned")) + .is_blocked(); + if blocked { + debug!("Miner stopping due to preemption"); + return Ok(None); + } + let txinfo = &to_consider.tx; let update_estimator = to_consider.update_estimate; @@ -2404,6 +2516,14 @@ impl StacksBlockBuilder { } } + if blocked { + debug!( + "Miner: Anchored block transaction selection aborted (child of {})", + &parent_stacks_header.anchored_header.block_hash() + ); + return Err(Error::MinerAborted); + } + // the prior do_rebuild logic wasn't necessary // a transaction that caused a budget exception is rolled back in process_transaction From ab0d95a95a27aef3ff048a1c43e0c04ddbbc1576 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:14:38 -0400 Subject: [PATCH 031/178] chore: add MinerAborted error variant --- src/chainstate/stacks/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/chainstate/stacks/mod.rs b/src/chainstate/stacks/mod.rs index 291edd930..fe35b8192 100644 --- a/src/chainstate/stacks/mod.rs +++ b/src/chainstate/stacks/mod.rs @@ -120,6 +120,7 @@ pub enum Error { PoxInsufficientBalance, PoxNoRewardCycle, ProblematicTransaction(Txid), + MinerAborted, } impl From for Error { @@ -195,6 +196,7 @@ impl fmt::Display for Error { "Transaction {} is problematic and will not be mined again", txid ), + Error::MinerAborted => write!(f, "Mining attempt aborted by signal"), } } } @@ -229,6 +231,7 @@ impl error::Error for Error { Error::PoxNoRewardCycle => None, Error::StacksTransactionSkipped(ref _r) => None, Error::ProblematicTransaction(ref _txid) => None, + Error::MinerAborted => None, } } } @@ -263,6 +266,7 @@ impl Error { Error::PoxNoRewardCycle => "PoxNoRewardCycle", Error::StacksTransactionSkipped(ref _r) => "StacksTransactionSkipped", Error::ProblematicTransaction(ref _txid) => "ProblematicTransaction", + Error::MinerAborted => "MinerAborted", } } From 0ea6586c79300ac5a2d17f933d9dbee86d91d2eb Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:14:52 -0400 Subject: [PATCH 032/178] feat: add a miner-driven mempool submission function so that the miner can add its own transactions while bypassing the usual checks that would prevent them (such as fees or nonce conflicts). Used to submit poison-microblock transactions --- src/core/mempool.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index db9b95f32..93e8b6cc1 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -1732,6 +1732,34 @@ impl MemPoolDB { Ok(()) } + /// Miner-driven submit (e.g. for poison microblocks), where no checks are performed + pub fn miner_submit( + &mut self, + chainstate: &mut StacksChainState, + consensus_hash: &ConsensusHash, + block_hash: &BlockHeaderHash, + tx: &StacksTransaction, + event_observer: Option<&dyn MemPoolEventDispatcher>, + miner_estimate: f64, + ) -> Result<(), MemPoolRejection> { + let mut mempool_tx = self.tx_begin().map_err(MemPoolRejection::DBError)?; + + let fee_estimate = Some(miner_estimate); + + MemPoolDB::tx_submit( + &mut mempool_tx, + chainstate, + consensus_hash, + block_hash, + tx, + false, + event_observer, + fee_estimate, + )?; + mempool_tx.commit().map_err(MemPoolRejection::DBError)?; + Ok(()) + } + /// Directly submit to the mempool, and don't do any admissions checks. /// This method is only used during testing, but because it is used by the /// integration tests, it cannot be marked #[cfg(test)]. From 6b0befd79f270a4dda4343a65042c5f04219ac79 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:15:36 -0400 Subject: [PATCH 033/178] feat: NetworkResult now reports the p2p thread's burn height --- src/net/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/net/mod.rs b/src/net/mod.rs index e9b1e5d09..24df9a6ab 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -1901,6 +1901,7 @@ pub struct NetworkResult { pub num_state_machine_passes: u64, pub num_inv_sync_passes: u64, pub num_download_passes: u64, + pub burn_height: u64, } impl NetworkResult { @@ -1908,6 +1909,7 @@ impl NetworkResult { num_state_machine_passes: u64, num_inv_sync_passes: u64, num_download_passes: u64, + burn_height: u64, ) -> NetworkResult { NetworkResult { unhandled_messages: HashMap::new(), @@ -1925,6 +1927,7 @@ impl NetworkResult { num_state_machine_passes: num_state_machine_passes, num_inv_sync_passes: num_inv_sync_passes, num_download_passes: num_download_passes, + burn_height, } } From d2c1c59bf87089f940a2dd989211ec13073d2fc7 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:15:50 -0400 Subject: [PATCH 034/178] feat: use new NetworkResult contructor --- src/net/p2p.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/net/p2p.rs b/src/net/p2p.rs index 72698c475..bd14b5158 100644 --- a/src/net/p2p.rs +++ b/src/net/p2p.rs @@ -5282,12 +5282,6 @@ impl PeerNetwork { .remove(&self.http_network_handle) .expect("BUG: no poll state for http network handle"); - let mut network_result = NetworkResult::new( - self.num_state_machine_passes, - self.num_inv_sync_passes, - self.num_downloader_passes, - ); - // update local-peer state self.refresh_local_peer() .expect("FATAL: failed to read local peer from the peer DB"); @@ -5297,6 +5291,13 @@ impl PeerNetwork { .refresh_burnchain_view(sortdb, chainstate, ibd) .expect("FATAL: failed to refresh burnchain view"); + let mut network_result = NetworkResult::new( + self.num_state_machine_passes, + self.num_inv_sync_passes, + self.num_downloader_passes, + self.chain_view.burn_block_height, + ); + network_result.consume_unsolicited(unsolicited_buffered_messages); // update PoX view, before handling any HTTP connections From a27d9f42693c111ab4c3af8d8678646b0e0bc8a0 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:16:00 -0400 Subject: [PATCH 035/178] chore: exit_at_block_height is now a bare Option instead of a borrowed Option<&'a u64> --- src/net/rpc.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/net/rpc.rs b/src/net/rpc.rs index 9b2bfcdf2..cce01f265 100644 --- a/src/net/rpc.rs +++ b/src/net/rpc.rs @@ -131,7 +131,7 @@ pub const STREAM_CHUNK_SIZE: u64 = 4096; #[derive(Default)] pub struct RPCHandlerArgs<'a> { - pub exit_at_block_height: Option<&'a u64>, + pub exit_at_block_height: Option, pub genesis_chainstate_hash: Sha256Sum, pub event_observer: Option<&'a dyn MemPoolEventDispatcher>, pub cost_estimator: Option<&'a dyn CostEstimator>, @@ -207,7 +207,7 @@ impl RPCPeerInfoData { pub fn from_network( network: &PeerNetwork, chainstate: &StacksChainState, - exit_at_block_height: &Option<&u64>, + exit_at_block_height: Option, genesis_chainstate_hash: &Sha256Sum, ) -> RPCPeerInfoData { let server_version = version_string( @@ -251,7 +251,7 @@ impl RPCPeerInfoData { .clone(), unanchored_tip: unconfirmed_tip, unanchored_seq: unconfirmed_seq, - exit_at_block_height: exit_at_block_height.cloned(), + exit_at_block_height: exit_at_block_height, genesis_chainstate_hash: genesis_chainstate_hash.clone(), node_public_key: Some(public_key_buf), node_public_key_hash: Some(public_key_hash), @@ -636,7 +636,7 @@ impl ConversationHttp { let pi = RPCPeerInfoData::from_network( network, chainstate, - &handler_args.exit_at_block_height, + handler_args.exit_at_block_height.clone(), &handler_args.genesis_chainstate_hash, ); let response = HttpResponseType::PeerInfo(response_metadata, pi); From 46a76c68529575b0bfb19ee528c3ada6ad94c50c Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:16:24 -0400 Subject: [PATCH 036/178] feat: make it possible to create a burnchain tx submission client from ongoing commit state (so another thread can be used to send the burnchain commit) --- .../burnchains/bitcoin_regtest_controller.rs | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs b/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs index 6a181505a..f401a26ff 100644 --- a/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs +++ b/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs @@ -80,7 +80,8 @@ pub struct BitcoinRegtestController { should_keep_running: Option>, } -struct OngoingBlockCommit { +#[derive(Clone)] +pub struct OngoingBlockCommit { payload: LeaderBlockCommitOp, utxos: UTXOSet, fees: LeaderBlockCommitFees, @@ -309,6 +310,23 @@ impl BitcoinRegtestController { } } + /// Creates a dummy bitcoin regtest controller, with the given ongoing block-commits + pub fn new_ongoing_dummy(config: Config, ongoing: Option) -> Self { + let mut ret = Self::new_dummy(config); + ret.ongoing_block_commit = ongoing; + ret + } + + /// Get an owned copy of the ongoing block commit state + pub fn get_ongoing_commit(&self) -> Option { + self.ongoing_block_commit.clone() + } + + /// Set the ongoing block commit state + pub fn set_ongoing_commit(&mut self, ongoing: Option) { + self.ongoing_block_commit = ongoing; + } + fn default_burnchain(&self) -> Burnchain { let (network_name, _network_type) = self.config.burnchain.get_bitcoin_network(); match &self.burnchain_config { @@ -1446,6 +1464,18 @@ impl BitcoinRegtestController { } } } + + #[cfg(test)] + pub fn get_mining_pubkey(&self) -> Option { + self.config.burnchain.local_mining_public_key.clone() + } + + #[cfg(test)] + pub fn set_mining_pubkey(&mut self, pubkey: String) -> Option { + let old_key = self.config.burnchain.local_mining_public_key.take(); + self.config.burnchain.local_mining_public_key = Some(pubkey); + old_key + } } impl BurnchainController for BitcoinRegtestController { From 567303dd4877495e2ed1da4aefa8b88ddfe7cfcf Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:17:06 -0400 Subject: [PATCH 037/178] chre: make it so the test framework can avoid the need for a downloader pass to unblock mining (since tests tend to only have one node, and thus no downloader passes complete) --- testnet/stacks-node/src/config.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index 7e449f9bb..0e8e20104 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -2,6 +2,8 @@ use std::convert::TryInto; use std::fs; use std::net::{SocketAddr, ToSocketAddrs}; use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; use rand::RngCore; @@ -10,6 +12,7 @@ use stacks::burnchains::{MagicBytes, BLOCKSTACK_MAGIC_MAINNET}; use stacks::chainstate::stacks::index::marf::MARFOpenOpts; use stacks::chainstate::stacks::index::storage::TrieHashCalculationMode; use stacks::chainstate::stacks::miner::BlockBuilderSettings; +use stacks::chainstate::stacks::miner::MinerStatus; use stacks::chainstate::stacks::MAX_BLOCK_LEN; use stacks::core::mempool::MemPoolWalkSettings; use stacks::core::StacksEpoch; @@ -595,6 +598,7 @@ impl Config { probability_pick_no_estimate_tx: miner .probability_pick_no_estimate_tx .unwrap_or(miner_default_config.probability_pick_no_estimate_tx), + wait_for_block_download: miner_default_config.wait_for_block_download, }, None => miner_default_config, }; @@ -947,6 +951,7 @@ impl Config { &self, attempt: u64, microblocks: bool, + miner_status: Arc>, ) -> BlockBuilderSettings { BlockBuilderSettings { max_miner_time_ms: if microblocks { @@ -971,6 +976,7 @@ impl Config { }, consider_no_estimate_tx_prob: self.miner.probability_pick_no_estimate_tx, }, + miner_status, } } } @@ -1327,7 +1333,7 @@ impl FeeEstimationConfig { } } - pub fn make_scalar_fee_estimator( + pub fn make_scalar_fee_estimator( &self, mut estimates_path: PathBuf, metric: CM, @@ -1345,7 +1351,7 @@ impl FeeEstimationConfig { // Creates a fuzzed WeightedMedianFeeRateEstimator with window_size 5. The fuzz // is uniform with bounds [+/- 0.5]. - pub fn make_fuzzed_weighted_median_fee_estimator( + pub fn make_fuzzed_weighted_median_fee_estimator( &self, mut estimates_path: PathBuf, metric: CM, @@ -1450,6 +1456,7 @@ impl NodeConfig { let (pubkey_str, hostport) = (parts[0], parts[1]); let pubkey = Secp256k1PublicKey::from_hex(pubkey_str) .expect(&format!("Invalid public key '{}'", pubkey_str)); + debug!("Resolve '{}'", &hostport); let sockaddr = hostport.to_socket_addrs().unwrap().next().unwrap(); let neighbor = NodeConfig::default_neighbor(sockaddr, pubkey, chain_id, peer_version); self.bootstrap_node.push(neighbor); @@ -1514,6 +1521,9 @@ pub struct MinerConfig { pub subsequent_attempt_time_ms: u64, pub microblock_attempt_time_ms: u64, pub probability_pick_no_estimate_tx: u8, + /// Wait for a downloader pass before mining. + /// This can only be disabled in testing; it can't be changed in the config file. + pub wait_for_block_download: bool, } impl MinerConfig { @@ -1524,6 +1534,7 @@ impl MinerConfig { subsequent_attempt_time_ms: 30_000, microblock_attempt_time_ms: 30_000, probability_pick_no_estimate_tx: 5, + wait_for_block_download: true, } } } From 9d6027551919450cb0d4eae8b3c5b27a42362330 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:17:43 -0400 Subject: [PATCH 038/178] fix: accommodate poison-microblock transaction evaluation results --- testnet/stacks-node/src/event_dispatcher.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/testnet/stacks-node/src/event_dispatcher.rs b/testnet/stacks-node/src/event_dispatcher.rs index 615bbb0c2..e290fb201 100644 --- a/testnet/stacks-node/src/event_dispatcher.rs +++ b/testnet/stacks-node/src/event_dispatcher.rs @@ -38,6 +38,7 @@ use super::config::{EventKeyType, EventObserverConfig}; use stacks::chainstate::burn::ConsensusHash; use stacks::chainstate::stacks::db::unconfirmed::ProcessedUnconfirmedState; use stacks::chainstate::stacks::miner::TransactionEvent; +use stacks::chainstate::stacks::TransactionPayload; #[derive(Debug, Clone)] struct EventObserver { @@ -205,7 +206,17 @@ impl EventObserver { } } (true, Value::Response(_)) => STATUS_RESP_POST_CONDITION, - _ => unreachable!(), // Transaction results should always be a Value::Response type + _ => { + if let TransactionOrigin::Stacks(inner_tx) = &tx { + if let TransactionPayload::PoisonMicroblock(..) = &inner_tx.payload { + STATUS_RESP_TRUE + } else { + unreachable!() // Transaction results should otherwise always be a Value::Response type + } + } else { + unreachable!() // Transaction results should always be a Value::Response type + } + } }; let (txid, raw_tx) = match tx { From 361a3f29fbeda66fe8bc550191b9330e2e7c60d0 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:19:37 -0400 Subject: [PATCH 039/178] feat: untangle and restructure the hairball that was neon_node. Now, all the state for all the threads is clearly identified and encapsulated in structs, as is thread-shared global state. Also, everything is now documented. Neon node should be much more approachable now. --- testnet/stacks-node/src/neon_node.rs | 5293 ++++++++++++++++---------- 1 file changed, 3291 insertions(+), 2002 deletions(-) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index 156c05be9..cfa5afbf1 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -3,12 +3,11 @@ use std::collections::HashMap; use std::collections::{HashSet, VecDeque}; use std::convert::{TryFrom, TryInto}; use std::default::Default; -use std::fs; -use std::io::Write; +use std::mem; use std::net::SocketAddr; use std::path::Path; -use std::sync::mpsc::{sync_channel, Receiver, SyncSender, TrySendError}; -use std::sync::{atomic::Ordering, Arc, Mutex}; +use std::sync::mpsc::{Receiver, SyncSender, TrySendError}; +use std::sync::{atomic::AtomicBool, atomic::Ordering, Arc, Mutex}; use std::time::Duration; use std::{thread, thread::JoinHandle}; @@ -27,8 +26,8 @@ use stacks::chainstate::stacks::db::{StacksChainState, MINER_REWARD_MATURITY}; use stacks::chainstate::stacks::Error as ChainstateError; use stacks::chainstate::stacks::StacksPublicKey; use stacks::chainstate::stacks::{ - miner::BlockBuilderSettings, miner::StacksMicroblockBuilder, StacksBlockBuilder, - StacksBlockHeader, + miner::signal_mining_blocked, miner::signal_mining_ready, miner::BlockBuilderSettings, + miner::MinerStatus, miner::StacksMicroblockBuilder, StacksBlockBuilder, StacksBlockHeader, }; use stacks::chainstate::stacks::{ CoinbasePayload, StacksBlock, StacksMicroblock, StacksTransaction, StacksTransactionSigner, @@ -38,12 +37,15 @@ use stacks::codec::StacksMessageCodec; use stacks::core::mempool::MemPoolDB; use stacks::core::FIRST_BURNCHAIN_CONSENSUS_HASH; use stacks::core::STACKS_EPOCH_2_05_MARKER; +use stacks::cost_estimates::metrics::CostMetric; use stacks::cost_estimates::metrics::UnitMetric; use stacks::cost_estimates::UnitEstimator; +use stacks::cost_estimates::{CostEstimator, FeeEstimator}; use stacks::monitoring::{increment_stx_blocks_mined_counter, update_active_miners_count_gauge}; use stacks::net::{ atlas::{AtlasConfig, AtlasDB, AttachmentInstance}, db::{LocalPeer, PeerDB}, + dns::DNSClient, dns::DNSResolver, p2p::PeerNetwork, relay::Relayer, @@ -63,58 +65,250 @@ use stacks::vm::costs::ExecutionCost; use stacks::{burnchains::BurnchainSigner, chainstate::stacks::db::StacksHeaderInfo}; use crate::burnchains::bitcoin_regtest_controller::BitcoinRegtestController; +use crate::burnchains::bitcoin_regtest_controller::OngoingBlockCommit; use crate::run_loop::neon::Counters; use crate::run_loop::neon::RunLoop; use crate::run_loop::RegisteredKey; use crate::ChainTip; -use super::{BurnchainController, BurnchainTip, Config, EventDispatcher, Keychain}; +use super::{BurnchainController, Config, EventDispatcher, Keychain}; use crate::stacks::vm::database::BurnStateDB; +use crate::syncctl::PoxSyncWatchdogComms; use stacks::monitoring; +use stacks_common::types::chainstate::StacksPrivateKey; +use stacks_common::util::vrf::VRFProof; + use clarity::vm::ast::ASTRules; pub const RELAYER_MAX_BUFFER: usize = 100; +type MinedBlocks = HashMap; + +/// Result of running the miner thread. It could produce a Stacks block or a microblock. +enum MinerThreadResult { + Block( + AssembledAnchorBlock, + Keychain, + Secp256k1PrivateKey, + Option, + ), + Microblock(Result, NetError>, MinerTip), +} + +/// Fully-assembled Stacks anchored, block as well as some extra metadata pertaining to how it was +/// linked to the burnchain and what view(s) the miner had of the burnchain before and after +/// completing the block. +#[derive(Clone)] struct AssembledAnchorBlock { + /// Consensus hash of the parent Stacks block parent_consensus_hash: ConsensusHash, + /// Burnchain tip's block hash when we finished mining my_burn_hash: BurnchainHeaderHash, + /// Burnchain tip's block height when we finished mining + my_block_height: u64, + /// Burnchain tip's block hash when we started mining (could be different) + orig_burn_hash: BurnchainHeaderHash, + /// The block we produced anchored_block: StacksBlock, + /// The attempt count of this block (multiple blocks will be attempted per burnchain block) attempt: u64, + /// Epoch timestamp in milliseconds when we started producing the block. + tenure_begin: u128, } -struct MicroblockMinerState { - parent_consensus_hash: ConsensusHash, - parent_block_hash: BlockHeaderHash, - miner_key: Secp256k1PrivateKey, - frequency: u64, - last_mined: u128, - quantity: u64, - cost_so_far: ExecutionCost, - settings: BlockBuilderSettings, -} - -enum RelayerDirective { +/// Command types for the relayer thread, issued to it by other threads +pub enum RelayerDirective { + /// Handle some new data that arrived on the network (such as blocks, transactions, and + /// microblocks) HandleNetResult(NetworkResult), + /// Announce a new sortition. Process and broadcast the block if we won. ProcessTenure(ConsensusHash, BurnchainHeaderHash, BlockHeaderHash), + /// Try to mine a block RunTenure(RegisteredKey, BlockSnapshot, u128), // (vrf key, chain tip, time of issuance in ms) + /// Try to register a VRF public key RegisterKey(BlockSnapshot), + /// Try to mine a microblock RunMicroblockTenure(BlockSnapshot, u128), // time of issuance in ms + /// Stop the relayer thread Exit, } -pub struct StacksNode { - config: Config, - relay_channel: SyncSender, +/// Inter-thread communication structure, shared between threads +#[derive(Clone)] +pub struct Globals { + /// Last sortition processed last_sortition: Arc>>, - burnchain_signer: BurnchainSigner, - is_miner: bool, + /// Status of the miner + miner_status: Arc>, + /// Communication link to the coordinator thread + coord_comms: CoordinatorChannels, + /// Unconfirmed transactions (shared between the relayer and p2p threads) + unconfirmed_txs: Arc>, + /// Writer endpoint to the relayer thread + relay_send: SyncSender, + /// Cointer state in the main thread + counters: Counters, + /// Connection to the PoX sync watchdog + sync_comms: PoxSyncWatchdogComms, + /// Global flag to see if we should keep running + pub should_keep_running: Arc, +} + +/// Miner chain tip, on top of which to build microblocks +#[derive(Debug, Clone, PartialEq)] +pub struct MinerTip { + /// tip's consensus hash + consensus_hash: ConsensusHash, + /// tip's Stacks block header hash + block_hash: BlockHeaderHash, + /// Microblock private key to use to sign microblocks + microblock_privkey: Secp256k1PrivateKey, +} + +impl MinerTip { + pub fn new(ch: ConsensusHash, bh: BlockHeaderHash, pk: Secp256k1PrivateKey) -> MinerTip { + MinerTip { + consensus_hash: ch, + block_hash: bh, + microblock_privkey: pk, + } + } +} + +impl Globals { + pub fn new( + coord_comms: CoordinatorChannels, + miner_status: Arc>, + relay_send: SyncSender, + counters: Counters, + sync_comms: PoxSyncWatchdogComms, + should_keep_running: Arc, + ) -> Globals { + Globals { + last_sortition: Arc::new(Mutex::new(None)), + miner_status, + coord_comms, + unconfirmed_txs: Arc::new(Mutex::new(UnconfirmedTxMap::new())), + relay_send, + counters, + sync_comms, + should_keep_running, + } + } + + /// Get the last sortition processed by the relayer thread + pub fn get_last_sortition(&self) -> Option { + match self.last_sortition.lock() { + Ok(sort_opt) => sort_opt.clone(), + Err(_) => { + error!("Sortition mutex poisoned!"); + panic!(); + } + } + } + + /// Set the last sortition processed + pub fn set_last_sortition(&self, block_snapshot: BlockSnapshot) { + match self.last_sortition.lock() { + Ok(mut sortition_opt) => { + sortition_opt.replace(block_snapshot); + } + Err(_) => { + error!("Sortition mutex poisoned!"); + panic!(); + } + }; + } + + /// Get the status of the miner (blocked or ready) + pub fn get_miner_status(&self) -> Arc> { + self.miner_status.clone() + } + + /// Get the main thread's counters + pub fn get_counters(&self) -> Counters { + self.counters.clone() + } + + /// Called by the relayer to pass unconfirmed txs to the p2p thread, so the p2p thread doesn't + /// need to do the disk I/O needed to instantiate the unconfirmed state trie they represent. + /// Clears the unconfirmed transactions, and replaces them with the chainstate's. + pub fn send_unconfirmed_txs(&self, chainstate: &StacksChainState) { + if let Some(ref unconfirmed) = chainstate.unconfirmed_state { + match self.unconfirmed_txs.lock() { + Ok(mut txs) => { + txs.clear(); + txs.extend(unconfirmed.mined_txs.clone()); + } + Err(e) => { + // can only happen due to a thread panic in the relayer + error!("FATAL: unconfirmed tx arc mutex is poisoned: {:?}", &e); + panic!(); + } + }; + } + } + + /// Called by the p2p thread to accept the unconfirmed tx state processed by the relayer. + /// Puts the shared unconfirmed transactions to chainstate. + pub fn recv_unconfirmed_txs(&self, chainstate: &mut StacksChainState) { + if let Some(ref mut unconfirmed) = chainstate.unconfirmed_state { + match self.unconfirmed_txs.lock() { + Ok(txs) => { + unconfirmed.mined_txs.clear(); + unconfirmed.mined_txs.extend(txs.clone()); + } + Err(e) => { + // can only happen due to a thread panic in the relayer + error!("FATAL: unconfirmed arc mutex is poisoned: {:?}", &e); + panic!(); + } + }; + } + } + + /// Signal system-wide stop + pub fn signal_stop(&self) { + self.should_keep_running.store(false, Ordering::SeqCst); + } + + /// Should we keep running? + pub fn keep_running(&self) -> bool { + self.should_keep_running.load(Ordering::SeqCst) + } + + /// Get the handle to the coordinator + pub fn coord(&self) -> &CoordinatorChannels { + &self.coord_comms + } +} + +/// Node implementation for both miners and followers. +/// This struct is used to set up the node proper and launch the p2p thread and relayer thread. +/// It is further used by the main thread to communicate with these two threads. +pub struct StacksNode { + /// Node configuration + config: Config, + /// Atlas network configuration pub atlas_config: AtlasConfig, + /// Global inter-thread communication handle + pub globals: Globals, + /// Stringy representation of our keychain (the authoritative keychain is stored in the + /// subordinate RelayerThread instance) + burnchain_signer: BurnchainSigner, + /// True if we're a miner + is_miner: bool, + /// VRF public key registration state machine leader_key_registration_state: LeaderKeyRegistrationState, + /// handle to the p2p thread pub p2p_thread_handle: JoinHandle<()>, + /// handle to the relayer thread pub relayer_thread_handle: JoinHandle<()>, } +/// Fault injection logic to artificially increase the length of a tenure. +/// Only used in testing #[cfg(test)] fn fault_injection_long_tenure() { // simulated slow block @@ -139,14 +333,21 @@ fn fault_injection_long_tenure() { #[cfg(not(test))] fn fault_injection_long_tenure() {} +/// Types of errors that can arise during mining enum Error { + /// Can't find the header record for the chain tip HeaderNotFoundForChainTip, + /// Can't find the stacks block's offset in the burnchain block WinningVtxNotFoundForChainTip, + /// Can't find the block sortition snapshot for the chain tip SnapshotNotFoundForChainTip, + /// The burnchain tip changed while this operation was in progress BurnchainTipChanged, } -struct MiningTenureInformation { +/// Metadata required for beginning a new tenure +struct ParentStacksBlockInfo { + /// Header metadata for the Stacks block we're going to build on top of stacks_parent_header: StacksHeaderInfo, /// the consensus hash of the sortition that selected the Stacks block parent parent_consensus_hash: ConsensusHash, @@ -154,1394 +355,154 @@ struct MiningTenureInformation { parent_block_burn_height: u64, /// the total amount burned in the sortition that selected the Stacks block parent parent_block_total_burn: u64, + /// offset in the burnchain block where the parent's block-commit was parent_winning_vtxindex: u16, + /// nonce to use for this new block's coinbase transaction coinbase_nonce: u64, } -/// Process artifacts from the tenure. -/// At this point, we're modifying the chainstate, and merging the artifacts from the previous tenure. -fn inner_process_tenure( - anchored_block: &StacksBlock, - consensus_hash: &ConsensusHash, - parent_consensus_hash: &ConsensusHash, - burn_db: &mut SortitionDB, - chain_state: &mut StacksChainState, - coord_comms: &CoordinatorChannels, -) -> Result { - let stacks_blocks_processed = coord_comms.get_stacks_blocks_processed(); - - if StacksChainState::has_stored_block( - &chain_state.db(), - &chain_state.blocks_path, - consensus_hash, - &anchored_block.block_hash(), - )? { - // already processed my tenure - return Ok(true); - } - let burn_height = SortitionDB::get_block_snapshot_consensus(burn_db.conn(), consensus_hash) - .map_err(|e| { - error!("Failed to find block snapshot for mined block: {}", e); - e - })? - .ok_or_else(|| { - error!("Failed to find block snapshot for mined block"); - ChainstateError::NoSuchBlockError - })? - .block_height; - - let ast_rules = SortitionDB::get_ast_rules(burn_db.conn(), burn_height)?; - - // failsafe - if !Relayer::static_check_problematic_relayed_block( - chain_state.mainnet, - &anchored_block, - ASTRules::PrecheckSize, - ) { - // nope! - warn!( - "Our mined block {} was problematic", - &anchored_block.block_hash() - ); - #[cfg(any(test, feature = "testing"))] - { - if let Ok(path) = std::env::var("STACKS_BAD_BLOCKS_DIR") { - // record this block somewhere - if !fs::metadata(&path).is_ok() { - fs::create_dir_all(&path) - .expect(&format!("FATAL: could not create '{}'", &path)); - } - - let mut path = Path::new(&path); - let path = path.join(Path::new(&format!("{}", &anchored_block.block_hash()))); - let mut file = fs::File::create(&path) - .expect(&format!("FATAL: could not create '{:?}'", &path)); - - let block_bits = anchored_block.serialize_to_vec(); - let block_bits_hex = to_hex(&block_bits); - let block_json = format!( - r#"{{"block":"{}","consensus":"{}"}}"#, - &block_bits_hex, &consensus_hash - ); - file.write_all(&block_json.as_bytes()).expect(&format!( - "FATAL: failed to write block bits to '{:?}'", - &path - )); - info!( - "Fault injection: bad block {} saved to {}", - &anchored_block.block_hash(), - &path.to_str().unwrap() - ); - } - } - if !Relayer::process_mined_problematic_blocks(ast_rules, ASTRules::PrecheckSize) { - // don't process it - warn!( - "Will NOT process our problematic mined block {}", - &anchored_block.block_hash() - ); - return Err(ChainstateError::NoTransactionsToMine); - } else { - warn!( - "Will process our problematic mined block {}", - &anchored_block.block_hash() - ) - } - } - - // Preprocess the anchored block - let ic = burn_db.index_conn(); - chain_state.preprocess_anchored_block( - &ic, - consensus_hash, - &anchored_block, - &parent_consensus_hash, - 0, - )?; - - if !coord_comms.announce_new_stacks_block() { - return Ok(false); - } - if !coord_comms.wait_for_stacks_blocks_processed(stacks_blocks_processed, 15000) { - warn!("ChainsCoordinator timed out while waiting for new stacks block to be processed"); - } - - Ok(true) -} - -fn inner_generate_coinbase_tx( - keychain: &mut Keychain, - nonce: u64, - is_mainnet: bool, - chain_id: u32, -) -> StacksTransaction { - let mut tx_auth = keychain.get_transaction_auth().unwrap(); - tx_auth.set_origin_nonce(nonce); - - let version = if is_mainnet { - TransactionVersion::Mainnet - } else { - TransactionVersion::Testnet - }; - let mut tx = StacksTransaction::new( - version, - tx_auth, - TransactionPayload::Coinbase(CoinbasePayload([0u8; 32])), - ); - tx.chain_id = chain_id; - tx.anchor_mode = TransactionAnchorMode::OnChainOnly; - let mut tx_signer = StacksTransactionSigner::new(&tx); - keychain.sign_as_origin(&mut tx_signer); - - tx_signer.get_tx().unwrap() -} - -fn inner_generate_poison_microblock_tx( - keychain: &mut Keychain, - nonce: u64, - poison_payload: TransactionPayload, - is_mainnet: bool, - chain_id: u32, -) -> StacksTransaction { - let mut tx_auth = keychain.get_transaction_auth().unwrap(); - tx_auth.set_origin_nonce(nonce); - - let version = if is_mainnet { - TransactionVersion::Mainnet - } else { - TransactionVersion::Testnet - }; - let mut tx = StacksTransaction::new(version, tx_auth, poison_payload); - tx.chain_id = chain_id; - tx.anchor_mode = TransactionAnchorMode::OnChainOnly; - let mut tx_signer = StacksTransactionSigner::new(&tx); - keychain.sign_as_origin(&mut tx_signer); - - tx_signer.get_tx().unwrap() -} - -/// Constructs and returns a LeaderKeyRegisterOp out of the provided params -fn inner_generate_leader_key_register_op( - address: StacksAddress, - vrf_public_key: VRFPublicKey, - consensus_hash: &ConsensusHash, -) -> BlockstackOperationType { - BlockstackOperationType::LeaderKeyRegister(LeaderKeyRegisterOp { - public_key: vrf_public_key, - memo: vec![], - address, - consensus_hash: consensus_hash.clone(), - vtxindex: 0, - txid: Txid([0u8; 32]), - block_height: 0, - burn_header_hash: BurnchainHeaderHash::zero(), - }) -} - -fn rotate_vrf_and_register( - is_mainnet: bool, - keychain: &mut Keychain, - burn_block: &BlockSnapshot, - btc_controller: &mut BitcoinRegtestController, -) -> bool { - let vrf_pk = keychain.rotate_vrf_keypair(burn_block.block_height); - let burnchain_tip_consensus_hash = &burn_block.consensus_hash; - let op = inner_generate_leader_key_register_op( - keychain.get_address(is_mainnet), - vrf_pk, - burnchain_tip_consensus_hash, - ); - - let mut one_off_signer = keychain.generate_op_signer(); - btc_controller.submit_operation(op, &mut one_off_signer, 1) -} - -/// Constructs and returns a LeaderBlockCommitOp out of the provided params -fn inner_generate_block_commit_op( - sender: BurnchainSigner, - block_header_hash: BlockHeaderHash, - burn_fee: u64, - key: &RegisteredKey, - parent_burnchain_height: u32, - parent_winning_vtx: u16, - vrf_seed: VRFSeed, - commit_outs: Vec, - sunset_burn: u64, - current_burn_height: u64, -) -> BlockstackOperationType { - let (parent_block_ptr, parent_vtxindex) = (parent_burnchain_height, parent_winning_vtx); - let burn_parent_modulus = (current_burn_height % BURN_BLOCK_MINED_AT_MODULUS) as u8; - - BlockstackOperationType::LeaderBlockCommit(LeaderBlockCommitOp { - sunset_burn, - block_header_hash, - burn_fee, - input: (Txid([0; 32]), 0), - apparent_sender: sender, - key_block_ptr: key.block_height as u32, - key_vtxindex: key.op_vtxindex as u16, - memo: vec![STACKS_EPOCH_2_05_MARKER], - new_seed: vrf_seed, - parent_block_ptr, - parent_vtxindex, - vtxindex: 0, - txid: Txid([0u8; 32]), - block_height: 0, - burn_header_hash: BurnchainHeaderHash::zero(), - burn_parent_modulus, - commit_outs, - }) -} - -/// Mine and broadcast a single microblock, unconditionally. -fn mine_one_microblock( - microblock_state: &mut MicroblockMinerState, - sortdb: &SortitionDB, - chainstate: &mut StacksChainState, - mempool: &mut MemPoolDB, - event_dispatcher: &EventDispatcher, -) -> Result { - debug!( - "Try to mine one microblock off of {}/{} (total: {})", - µblock_state.parent_consensus_hash, - µblock_state.parent_block_hash, - chainstate - .unconfirmed_state - .as_ref() - .map(|us| us.num_microblocks()) - .unwrap_or(0) - ); - - let burn_height = SortitionDB::get_block_snapshot_consensus( - sortdb.conn(), - µblock_state.parent_consensus_hash, - ) - .map_err(|e| { - error!("Failed to find block snapshot for mined block: {}", e); - e - })? - .ok_or_else(|| { - error!("Failed to find block snapshot for mined block"); - ChainstateError::NoSuchBlockError - })? - .block_height; - - let ast_rules = SortitionDB::get_ast_rules(sortdb.conn(), burn_height).map_err(|e| { - error!("Failed to get AST rules for microblock: {}", e); - e - })?; - - let mint_result = { - let ic = sortdb.index_conn(); - let mut microblock_miner = match StacksMicroblockBuilder::resume_unconfirmed( - chainstate, - &ic, - µblock_state.cost_so_far, - microblock_state.settings.clone(), - ) { - Ok(x) => x, - Err(e) => { - let msg = format!( - "Failed to create a microblock miner at chaintip {}/{}: {:?}", - µblock_state.parent_consensus_hash, - µblock_state.parent_block_hash, - &e - ); - error!("{}", msg); - return Err(e); - } - }; - - let t1 = get_epoch_time_ms(); - - let mblock = microblock_miner.mine_next_microblock( - mempool, - µblock_state.miner_key, - event_dispatcher, - )?; - let new_cost_so_far = microblock_miner.get_cost_so_far().expect("BUG: cannot read cost so far from miner -- indicates that the underlying Clarity Tx is somehow in use still."); - let t2 = get_epoch_time_ms(); - - info!( - "Mined microblock {} ({}) with {} transactions in {}ms", - mblock.block_hash(), - mblock.header.sequence, - mblock.txs.len(), - t2.saturating_sub(t1) - ); - - Ok((mblock, new_cost_so_far)) - }; - - let (mined_microblock, new_cost) = match mint_result { - Ok(x) => x, - Err(e) => { - warn!("Failed to mine microblock: {}", e); - return Err(e); - } - }; - - // failsafe - if !Relayer::static_check_problematic_relayed_microblock( - chainstate.mainnet, - &mined_microblock, - ASTRules::PrecheckSize, - ) { - // nope! - warn!( - "Our mined microblock {} was problematic", - &mined_microblock.block_hash() - ); - - #[cfg(any(test, feature = "testing"))] - { - if let Ok(path) = std::env::var("STACKS_BAD_BLOCKS_DIR") { - // record this microblock somewhere - if !fs::metadata(&path).is_ok() { - fs::create_dir_all(&path) - .expect(&format!("FATAL: could not create '{}'", &path)); - } - - let mut path = Path::new(&path); - let path = path.join(Path::new(&format!("{}", &mined_microblock.block_hash()))); - let mut file = fs::File::create(&path) - .expect(&format!("FATAL: could not create '{:?}'", &path)); - - let mblock_bits = mined_microblock.serialize_to_vec(); - let mblock_bits_hex = to_hex(&mblock_bits); - - let mblock_json = format!( - r#"{{"microblock":"{}","parent_consensus":"{}","parent_block":"{}"}}"#, - &mblock_bits_hex, - µblock_state.parent_consensus_hash, - µblock_state.parent_block_hash - ); - file.write_all(&mblock_json.as_bytes()).expect(&format!( - "FATAL: failed to write microblock bits to '{:?}'", - &path - )); - info!( - "Fault injection: bad microblock {} saved to {}", - &mined_microblock.block_hash(), - &path.to_str().unwrap() - ); - } - } - if !Relayer::process_mined_problematic_blocks(ast_rules, ASTRules::PrecheckSize) { - // don't process it - warn!( - "Will NOT process our problematic mined microblock {}", - &mined_microblock.block_hash() - ); - return Err(ChainstateError::NoTransactionsToMine); - } else { - warn!( - "Will process our problematic mined microblock {}", - &mined_microblock.block_hash() - ) - } - } - - // preprocess the microblock locally - chainstate.preprocess_streamed_microblock( - µblock_state.parent_consensus_hash, - µblock_state.parent_block_hash, - &mined_microblock, - )?; - - // update unconfirmed state cost - microblock_state.cost_so_far = new_cost; - microblock_state.quantity += 1; - return Ok(mined_microblock); -} - -fn try_mine_microblock( - config: &Config, - microblock_miner_state: &mut Option, - chainstate: &mut StacksChainState, - sortdb: &SortitionDB, - mem_pool: &mut MemPoolDB, - winning_tip: (ConsensusHash, BlockHeaderHash, Secp256k1PrivateKey), - event_dispatcher: &EventDispatcher, -) -> Result, NetError> { - let ch = winning_tip.0; - let bhh = winning_tip.1; - let microblock_privkey = winning_tip.2; - - let mut next_microblock = None; - if microblock_miner_state.is_none() { - debug!( - "Instantiate microblock mining state off of {}/{}", - &ch, &bhh - ); - // we won a block! proceed to build a microblock tail if we've stored it - match StacksChainState::get_anchored_block_header_info(chainstate.db(), &ch, &bhh) { - Ok(Some(_)) => { - let parent_index_hash = StacksBlockHeader::make_index_block_hash(&ch, &bhh); - let cost_so_far = StacksChainState::get_stacks_block_anchored_cost( - chainstate.db(), - &parent_index_hash, - )? - .ok_or(NetError::NotFoundError)?; - microblock_miner_state.replace(MicroblockMinerState { - parent_consensus_hash: ch.clone(), - parent_block_hash: bhh.clone(), - miner_key: microblock_privkey.clone(), - frequency: config.node.microblock_frequency, - last_mined: 0, - quantity: 0, - cost_so_far: cost_so_far, - settings: config.make_block_builder_settings(0, true), - }); - } - Ok(None) => { - warn!( - "No such anchored block: {}/{}. Cannot mine microblocks", - ch, bhh - ); - } - Err(e) => { - warn!( - "Failed to get anchored block cost for {}/{}: {:?}", - ch, bhh, &e - ); - } - } - } - - if let Some(mut microblock_miner) = microblock_miner_state.take() { - if microblock_miner.parent_consensus_hash == ch && microblock_miner.parent_block_hash == bhh - { - if microblock_miner.last_mined + (microblock_miner.frequency as u128) - < get_epoch_time_ms() - { - // opportunistically try and mine, but only if there are no attachable blocks in - // recent history (i.e. in the last 10 minutes) - let num_attachable = StacksChainState::count_attachable_staging_blocks( - chainstate.db(), - 1, - get_epoch_time_secs() - 600, - )?; - if num_attachable == 0 { - match mine_one_microblock( - &mut microblock_miner, - sortdb, - chainstate, - mem_pool, - event_dispatcher, - ) { - Ok(microblock) => { - // will need to relay this - next_microblock = Some(microblock); - } - Err(ChainstateError::NoTransactionsToMine) => { - info!("Will keep polling mempool for transactions to include in a microblock"); - } - Err(e) => { - warn!("Failed to mine one microblock: {:?}", &e); - } - } - } else { - debug!("Will not mine microblocks yet -- have {} attachable blocks that arrived in the last 10 minutes", num_attachable); - } - } - microblock_miner.last_mined = get_epoch_time_ms(); - microblock_miner_state.replace(microblock_miner); - } - // otherwise, we're not the sortition winner, and the microblock miner state can be - // discarded. - } - - Ok(next_microblock) -} - -fn run_microblock_tenure( - config: &Config, - microblock_miner_state: &mut Option, - chainstate: &mut StacksChainState, - sortdb: &mut SortitionDB, - mem_pool: &mut MemPoolDB, - relayer: &mut Relayer, - miner_tip: (ConsensusHash, BlockHeaderHash, Secp256k1PrivateKey), - counters: &Counters, - event_dispatcher: &EventDispatcher, -) { - // TODO: this is sensitive to poll latency -- can we call this on a fixed - // schedule, regardless of network activity? - let parent_consensus_hash = &miner_tip.0; - let parent_block_hash = &miner_tip.1; - - debug!( - "Run microblock tenure for {}/{}", - parent_consensus_hash, parent_block_hash - ); - - // Mine microblocks, if we're active - let next_microblock_opt = match try_mine_microblock( - &config, - microblock_miner_state, - chainstate, - sortdb, - mem_pool, - miner_tip.clone(), - event_dispatcher, - ) { - Ok(x) => x, - Err(e) => { - warn!("Failed to mine next microblock: {:?}", &e); - None - } - }; - - // did we mine anything? - if let Some(next_microblock) = next_microblock_opt { - // apply it - let microblock_hash = next_microblock.block_hash(); - - let processed_unconfirmed_state = Relayer::refresh_unconfirmed(chainstate, sortdb); - let num_mblocks = chainstate - .unconfirmed_state - .as_ref() - .map(|ref unconfirmed| unconfirmed.num_microblocks()) - .unwrap_or(0); - - info!( - "Mined one microblock: {} seq {} (total processed: {})", - µblock_hash, next_microblock.header.sequence, num_mblocks - ); - counters.set_microblocks_processed(num_mblocks); - - let parent_index_block_hash = - StacksBlockHeader::make_index_block_hash(parent_consensus_hash, parent_block_hash); - event_dispatcher - .process_new_microblocks(parent_index_block_hash, processed_unconfirmed_state); - - // send it off - if let Err(e) = - relayer.broadcast_microblock(parent_consensus_hash, parent_block_hash, next_microblock) - { - error!( - "Failure trying to broadcast microblock {}: {}", - microblock_hash, e - ); - } - } -} - -/// Grant the p2p thread a copy of the unconfirmed microblock transaction list, so it can serve it -/// out via the unconfirmed transaction API. -/// Not the prettiest way to do this, but the least disruptive way to do this. -fn send_unconfirmed_txs( - chainstate: &StacksChainState, - unconfirmed_txs: Arc>, -) { - if let Some(ref unconfirmed) = chainstate.unconfirmed_state { - match unconfirmed_txs.lock() { - Ok(mut txs) => { - txs.clear(); - txs.extend(unconfirmed.mined_txs.clone()); - } - Err(e) => { - // can only happen due to a thread panic in the relayer - error!("FATAL: unconfirmed tx arc mutex is poisoned: {:?}", &e); - panic!(); - } - }; - } -} - -/// Have the p2p thread receive unconfirmed txs -fn recv_unconfirmed_txs( - chainstate: &mut StacksChainState, - unconfirmed_txs: Arc>, -) { - if let Some(ref mut unconfirmed) = chainstate.unconfirmed_state { - match unconfirmed_txs.lock() { - Ok(txs) => { - unconfirmed.mined_txs.clear(); - unconfirmed.mined_txs.extend(txs.clone()); - } - Err(e) => { - // can only happen due to a thread panic in the relayer - error!("FATAL: unconfirmed arc mutex is poisoned: {:?}", &e); - panic!(); - } - }; - } -} - -fn spawn_peer( - runloop: &RunLoop, - mut this: PeerNetwork, - p2p_sock: &SocketAddr, - rpc_sock: &SocketAddr, - poll_timeout: u64, - relay_channel: SyncSender, - attachments_rx: Receiver>, - unconfirmed_txs: Arc>, -) -> Result, NetError> { - let config = runloop.config().clone(); - let mut sync_comms = runloop.get_pox_sync_comms(); - let event_dispatcher = runloop.get_event_dispatcher(); - let should_keep_running = runloop.get_termination_switch(); - - let is_mainnet = config.is_mainnet(); - let burn_db_path = config.get_burn_db_file_path(); - let stacks_chainstate_path = config.get_chainstate_path_str(); - let exit_at_block_height = config.burnchain.process_exit_at_block_height; - - this.bind(p2p_sock, rpc_sock).unwrap(); - let (mut dns_resolver, mut dns_client) = DNSResolver::new(10); - let sortdb = SortitionDB::open(&burn_db_path, false).map_err(NetError::DBError)?; - - let (mut chainstate, _) = StacksChainState::open( - is_mainnet, - config.burnchain.chain_id, - &stacks_chainstate_path, - Some(config.node.get_marf_opts()), - ) - .map_err(|e| NetError::ChainstateError(e.to_string()))?; - - // buffer up blocks to store without stalling the p2p thread - let mut results_with_data = VecDeque::new(); - - let server_thread = thread::Builder::new() - .name("p2p".to_string()) - .spawn(move || { - // create estimators, metric instances for RPC handler - let cost_estimator = config - .make_cost_estimator() - .unwrap_or_else(|| Box::new(UnitEstimator)); - let metric = config - .make_cost_metric() - .unwrap_or_else(|| Box::new(UnitMetric)); - let fee_estimator = config.make_fee_estimator(); - - let mut mem_pool = MemPoolDB::open( - is_mainnet, - config.burnchain.chain_id, - &stacks_chainstate_path, - cost_estimator, - metric, - ) - .expect("Database failure opening mempool"); - - let cost_estimator = config - .make_cost_estimator() - .unwrap_or_else(|| Box::new(UnitEstimator)); - let metric = config - .make_cost_metric() - .unwrap_or_else(|| Box::new(UnitMetric)); - - let handler_args = RPCHandlerArgs { - exit_at_block_height: exit_at_block_height.as_ref(), - genesis_chainstate_hash: Sha256Sum::from_hex(stx_genesis::GENESIS_CHAINSTATE_HASH) - .unwrap(), - event_observer: Some(&event_dispatcher), - cost_estimator: Some(cost_estimator.as_ref()), - cost_metric: Some(metric.as_ref()), - fee_estimator: fee_estimator.as_ref().map(|x| x.as_ref()), - ..RPCHandlerArgs::default() - }; - - let mut num_p2p_state_machine_passes = 0; - let mut num_inv_sync_passes = 0; - let mut num_download_passes = 0; - let mut mblock_deadline = 0; - - while should_keep_running.load(Ordering::SeqCst) { - // initial block download? - let ibd = sync_comms.get_ibd(); - let download_backpressure = results_with_data.len() > 0; - let poll_ms = if !download_backpressure && this.has_more_downloads() { - // keep getting those blocks -- drive the downloader state-machine - debug!( - "P2P: backpressure: {}, more downloads: {}", - download_backpressure, - this.has_more_downloads() - ); - 1 - } else { - cmp::min(poll_timeout, config.node.microblock_frequency) - }; - - let mut expected_attachments = match attachments_rx.try_recv() { - Ok(expected_attachments) => { - debug!("Atlas: received attachments: {:?}", &expected_attachments); - expected_attachments - } - _ => { - debug!("Atlas: attachment channel is empty"); - HashSet::new() - } - }; - - let _ = Relayer::setup_unconfirmed_state_readonly(&mut chainstate, &sortdb); - recv_unconfirmed_txs(&mut chainstate, unconfirmed_txs.clone()); - - match this.run( - &sortdb, - &mut chainstate, - &mut mem_pool, - Some(&mut dns_client), - download_backpressure, - ibd, - poll_ms, - &handler_args, - &mut expected_attachments, - ) { - Ok(network_result) => { - if num_p2p_state_machine_passes < network_result.num_state_machine_passes { - // p2p state-machine did a full pass. Notify anyone listening. - sync_comms.notify_p2p_state_pass(); - num_p2p_state_machine_passes = network_result.num_state_machine_passes; - } - - if num_inv_sync_passes < network_result.num_inv_sync_passes { - // inv-sync state-machine did a full pass. Notify anyone listening. - sync_comms.notify_inv_sync_pass(); - num_inv_sync_passes = network_result.num_inv_sync_passes; - } - - if num_download_passes < network_result.num_download_passes { - // download state-machine did a full pass. Notify anyone listening. - sync_comms.notify_download_pass(); - num_download_passes = network_result.num_download_passes; - } - - if network_result.has_data_to_store() { - results_with_data - .push_back(RelayerDirective::HandleNetResult(network_result)); - } - - // only do this on the Ok() path, even if we're mining, because an error in - // network dispatching is likely due to resource exhaustion - if mblock_deadline < get_epoch_time_ms() { - debug!("P2P: schedule microblock tenure"); - results_with_data.push_back(RelayerDirective::RunMicroblockTenure( - this.burnchain_tip.clone(), - get_epoch_time_ms(), - )); - mblock_deadline = - get_epoch_time_ms() + (config.node.microblock_frequency as u128); - } - } - Err(e) => { - // this is only reachable if the network is not instantiated correctly -- - // i.e. you didn't connect it - panic!("P2P: Failed to process network dispatch: {:?}", &e); - } - }; - - while let Some(next_result) = results_with_data.pop_front() { - // have blocks, microblocks, and/or transactions (don't care about anything else), - // or a directive to mine microblocks - if let Err(e) = relay_channel.try_send(next_result) { - debug!( - "P2P: {:?}: download backpressure detected", - &this.local_peer - ); - match e { - TrySendError::Full(directive) => { - if let RelayerDirective::RunMicroblockTenure(..) = directive { - // can drop this - } else if let RelayerDirective::RunTenure(..) = directive { - // can drop this - } else { - // don't lose this data -- just try it again - results_with_data.push_front(directive); - } - break; - } - TrySendError::Disconnected(_) => { - info!("P2P: Relayer hang up with p2p channel"); - should_keep_running.store(false, Ordering::SeqCst); - break; - } - } - } else { - debug!("P2P: Dispatched result to Relayer!"); - } - } - } - - while let Err(TrySendError::Full(_)) = relay_channel.try_send(RelayerDirective::Exit) { - warn!("Failed to direct relayer thread to exit, sleeping and trying again"); - thread::sleep(Duration::from_secs(5)); - } - info!("P2P thread exit!"); - }) - .unwrap(); - - let _jh = thread::Builder::new() - .name("dns-resolver".to_string()) - .spawn(move || { - dns_resolver.thread_main(); - }) - .unwrap(); - - Ok(server_thread) -} - -fn get_last_sortition(last_sortition: &Arc>>) -> Option { - match last_sortition.lock() { - Ok(sort_opt) => sort_opt.clone(), - Err(_) => { - error!("Sortition mutex poisoned!"); - panic!(); - } - } -} - -fn set_last_sortition( - last_sortition: &mut Arc>>, - block_snapshot: BlockSnapshot, -) { - match last_sortition.lock() { - Ok(mut sortition_opt) => { - sortition_opt.replace(block_snapshot); - } - Err(_) => { - error!("Sortition mutex poisoned!"); - panic!(); - } - }; -} - -fn spawn_miner_relayer( - runloop: &RunLoop, - mut relayer: Relayer, - local_peer: LocalPeer, - mut keychain: Keychain, - relay_channel: Receiver, - last_sortition: Arc>>, - coord_comms: CoordinatorChannels, - unconfirmed_txs: Arc>, -) -> Result, NetError> { - let config = runloop.config().clone(); - let event_dispatcher = runloop.get_event_dispatcher(); - let counters = runloop.get_counters(); - let sync_comms = runloop.get_pox_sync_comms(); - let burnchain = runloop.get_burnchain(); - - let is_mainnet = config.is_mainnet(); - let chain_id = config.burnchain.chain_id; - let burn_db_path = config.get_burn_db_file_path(); - let stacks_chainstate_path = config.get_chainstate_path_str(); - - // Note: the chainstate coordinator is *the* block processor, it is responsible for writes to - // the chainstate -- eventually, no other codepaths should be writing to it. - // - // the relayer _should not_ be modifying the sortdb, - // however, it needs a mut reference to create read TXs. - // should address via #1449 - let mut sortdb = SortitionDB::open(&burn_db_path, true).map_err(NetError::DBError)?; - - let (mut chainstate, _) = StacksChainState::open( - is_mainnet, - chain_id, - &stacks_chainstate_path, - Some(config.node.get_marf_opts()), - ) - .map_err(|e| NetError::ChainstateError(e.to_string()))?; - - let mut last_mined_blocks: HashMap< - BurnchainHeaderHash, - Vec<(AssembledAnchorBlock, Secp256k1PrivateKey)>, - > = HashMap::new(); - let burn_fee_cap = config.burnchain.burn_fee_cap; - - let mut bitcoin_controller = BitcoinRegtestController::new_dummy(config.clone()); - let mut microblock_miner_state: Option = None; - let mut miner_tip = None; // only set if we won the last sortition - let mut last_microblock_tenure_time = 0; - let mut last_tenure_issue_time = 0; - - let relayer_handle = thread::Builder::new().name("relayer".to_string()).spawn(move || { - let cost_estimator = config.make_cost_estimator() - .unwrap_or_else(|| Box::new(UnitEstimator)); - let metric = config.make_cost_metric() - .unwrap_or_else(|| Box::new(UnitMetric)); - - let mut mem_pool = MemPoolDB::open(is_mainnet, chain_id, &stacks_chainstate_path, cost_estimator, metric) - .expect("Database failure opening mempool"); - - while let Ok(mut directive) = relay_channel.recv() { - match directive { - RelayerDirective::HandleNetResult(ref mut net_result) => { - debug!("Relayer: Handle network result"); - let net_receipts = relayer - .process_network_result( - &local_peer, - net_result, - &mut sortdb, - &mut chainstate, - &mut mem_pool, - sync_comms.get_ibd(), - Some(&coord_comms), - Some(&event_dispatcher), - ) - .expect("BUG: failure processing network results"); - - let mempool_txs_added = net_receipts.mempool_txs_added.len(); - if mempool_txs_added > 0 { - event_dispatcher.process_new_mempool_txs(net_receipts.mempool_txs_added); - } - - let num_unconfirmed_microblock_tx_receipts = net_receipts.processed_unconfirmed_state.receipts.len(); - if num_unconfirmed_microblock_tx_receipts > 0 { - if let Some(unconfirmed_state) = chainstate.unconfirmed_state.as_ref() { - let canonical_tip = unconfirmed_state.confirmed_chain_tip.clone(); - event_dispatcher.process_new_microblocks(canonical_tip, net_receipts.processed_unconfirmed_state); - } else { - warn!("Relayer: oops, unconfirmed state is uninitialized but there are microblock events"); - } - } - - // Dispatch retrieved attachments, if any. - if net_result.has_attachments() { - event_dispatcher.process_new_attachments(&net_result.attachments); - } - - // synchronize unconfirmed tx index to p2p thread - send_unconfirmed_txs(&chainstate, unconfirmed_txs.clone()); - } - RelayerDirective::ProcessTenure(consensus_hash, burn_hash, block_header_hash) => { - debug!( - "Relayer: Process tenure {}/{} in {}", - &consensus_hash, &block_header_hash, &burn_hash - ); - if let Some(last_mined_blocks_at_burn_hash) = - last_mined_blocks.remove(&burn_hash) - { - for (last_mined_block, microblock_privkey) in - last_mined_blocks_at_burn_hash.into_iter() - { - let AssembledAnchorBlock { - parent_consensus_hash, - anchored_block: mined_block, - my_burn_hash: mined_burn_hash, - attempt: _, - } = last_mined_block; - if mined_block.block_hash() == block_header_hash - && burn_hash == mined_burn_hash - { - // we won! - let reward_block_height = mined_block.header.total_work.work + MINER_REWARD_MATURITY; - info!("Won sortition! Mining reward will be received in {} blocks (block #{})", MINER_REWARD_MATURITY, reward_block_height); - debug!("Won sortition!"; - "stacks_header" => %block_header_hash, - "burn_hash" => %mined_burn_hash, - ); - - increment_stx_blocks_mined_counter(); - match inner_process_tenure( - &mined_block, - &consensus_hash, - &parent_consensus_hash, - &mut sortdb, - &mut chainstate, - &coord_comms, - ) { - Ok(coordinator_running) => { - if !coordinator_running { - warn!( - "Coordinator stopped, stopping relayer thread..." - ); - return; - } - } - Err(e) => { - warn!( - "Error processing my tenure, bad block produced: {}", - e - ); - warn!( - "Bad block"; - "stacks_header" => %block_header_hash, - "data" => %to_hex(&mined_block.serialize_to_vec()), - ); - continue; - } - }; - - // advertize _and_ push blocks for now - let blocks_available = Relayer::load_blocks_available_data( - &sortdb, - vec![consensus_hash.clone()], - ) - .expect("Failed to obtain block information for a block we mined."); - - let block_data = { - let mut bd = HashMap::new(); - bd.insert(consensus_hash.clone(), mined_block.clone()); - bd - }; - - if let Err(e) = relayer.advertize_blocks(blocks_available, block_data) { - warn!("Failed to advertise new block: {}", e); - } - - let snapshot = SortitionDB::get_block_snapshot_consensus( - sortdb.conn(), - &consensus_hash, - ) - .expect("Failed to obtain snapshot for block") - .expect("Failed to obtain snapshot for block"); - if !snapshot.pox_valid { - warn!( - "Snapshot for {} is no longer valid; discarding {}...", - &consensus_hash, - &mined_block.block_hash() - ); - miner_tip = None; - - } else { - let ch = snapshot.consensus_hash.clone(); - let bh = mined_block.block_hash(); - - if let Err(e) = relayer - .broadcast_block(snapshot.consensus_hash, mined_block) - { - warn!("Failed to push new block: {}", e); - } - - // proceed to mine microblocks - debug!( - "Microblock miner tip is now {}/{} ({})", - &consensus_hash, &block_header_hash, StacksBlockHeader::make_index_block_hash(&consensus_hash, &block_header_hash) - ); - miner_tip = Some((ch, bh, microblock_privkey)); - - Relayer::refresh_unconfirmed(&mut chainstate, &mut sortdb); - send_unconfirmed_txs(&chainstate, unconfirmed_txs.clone()); - } - } else { - debug!("Did not win sortition, my blocks [burn_hash= {}, block_hash= {}], their blocks [parent_consenus_hash= {}, burn_hash= {}, block_hash ={}]", - mined_burn_hash, mined_block.block_hash(), parent_consensus_hash, burn_hash, block_header_hash); - - miner_tip = None; - } - } - } - } - RelayerDirective::RunTenure(registered_key, last_burn_block, issue_timestamp_ms) => { - if let Some(cur_sortition) = get_last_sortition(&last_sortition) { - if last_burn_block.sortition_id != cur_sortition.sortition_id { - debug!("Drop stale RunTenure for {}: current sortition is for {}", &last_burn_block.burn_header_hash, &cur_sortition.burn_header_hash); - counters.bump_missed_tenures(); - continue; - } - } - - let burn_header_hash = last_burn_block.burn_header_hash.clone(); - let burn_chain_sn = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) - .expect("FATAL: failed to query sortition DB for canonical burn chain tip"); - - let burn_chain_tip = burn_chain_sn - .burn_header_hash - .clone(); - - let mut burn_tenure_snapshot = last_burn_block.clone(); - if burn_chain_tip == burn_header_hash { - // no burnchain change, so only re-run block tenure every so often in order - // to give microblocks a chance to collect - if issue_timestamp_ms < last_tenure_issue_time + (config.node.wait_time_for_microblocks as u128) { - debug!("Relayer: will NOT run tenure since issuance at {} is too fresh (wait until {} + {} = {})", - issue_timestamp_ms / 1000, last_tenure_issue_time / 1000, config.node.wait_time_for_microblocks / 1000, (last_tenure_issue_time + (config.node.wait_time_for_microblocks as u128)) / 1000); - continue; - } - } - else { - // burnchain has changed since this directive was sent, so mine immediately - burn_tenure_snapshot = burn_chain_sn; - if issue_timestamp_ms + (config.node.wait_time_for_microblocks as u128) < get_epoch_time_ms() { - // still waiting for microblocks to arrive - debug!("Relayer: will NOT run tenure since still waiting for microblocks to arrive ({} <= {})", (issue_timestamp_ms + (config.node.wait_time_for_microblocks as u128)) / 1000, get_epoch_time_secs()); - continue; - } - debug!("Relayer: burnchain has advanced from {} to {}", &burn_header_hash, &burn_chain_tip); - } - - debug!( - "Relayer: Run tenure"; - "height" => last_burn_block.block_height, - "burn_header_hash" => %burn_chain_tip, - "last_burn_header_hash" => %burn_header_hash - ); - - let tenure_begin = get_epoch_time_ms(); - fault_injection_long_tenure(); - - let mut last_mined_blocks_vec = last_mined_blocks - .remove(&burn_header_hash) - .unwrap_or_default(); - - let last_mined_block_opt = StacksNode::relayer_run_tenure( - &config, - registered_key, - &mut chainstate, - &mut sortdb, - &burnchain, - burn_tenure_snapshot, - &mut keychain, - &mut mem_pool, - burn_fee_cap, - &mut bitcoin_controller, - &last_mined_blocks_vec.iter().map(|(blk, _)| blk).collect(), - &event_dispatcher, - ); - if let Some((last_mined_block, microblock_privkey)) = last_mined_block_opt { - if last_mined_blocks_vec.len() == 0 { - counters.bump_blocks_processed(); - } - last_mined_blocks_vec.push((last_mined_block, microblock_privkey)); - } - last_mined_blocks.insert(burn_header_hash, last_mined_blocks_vec); - - last_tenure_issue_time = get_epoch_time_ms(); - debug!("Relayer: RunTenure finished at {} (in {}ms)", last_tenure_issue_time, last_tenure_issue_time.saturating_sub(tenure_begin)); - } - RelayerDirective::RegisterKey(ref last_burn_block) => { - rotate_vrf_and_register( - is_mainnet, - &mut keychain, - last_burn_block, - &mut bitcoin_controller, - ); - counters.bump_blocks_processed(); - } - RelayerDirective::RunMicroblockTenure(burnchain_tip, tenure_issue_ms) => { - if last_microblock_tenure_time > tenure_issue_ms { - // stale request - continue; - } - if let Some(cur_sortition) = get_last_sortition(&last_sortition) { - if burnchain_tip.sortition_id != cur_sortition.sortition_id { - debug!("Drop stale RunMicroblockTenure for {}/{}: current sortition is for {} ({})", &burnchain_tip.consensus_hash, &burnchain_tip.winning_stacks_block_hash, &cur_sortition.consensus_hash, &cur_sortition.burn_header_hash); - continue; - } - } - - debug!("Relayer: Run microblock tenure"); - - // unconfirmed state must be consistent with the chain tip, as must the - // microblock mining state. - if let Some((ch, bh, mblock_pkey)) = miner_tip.clone() { - if let Some(miner_state) = microblock_miner_state.take() { - if miner_state.parent_consensus_hash == ch || miner_state.parent_block_hash == bh { - // preserve -- chaintip is unchanged - microblock_miner_state = Some(miner_state); - } - else { - debug!("Relayer: reset microblock miner state"); - microblock_miner_state = None; - counters.set_microblocks_processed(0); - } - } - - run_microblock_tenure( - &config, - &mut microblock_miner_state, - &mut chainstate, - &mut sortdb, - &mut mem_pool, - &mut relayer, - (ch, bh, mblock_pkey), - &counters, - &event_dispatcher, - ); - - // synchronize unconfirmed tx index to p2p thread - send_unconfirmed_txs(&chainstate, unconfirmed_txs.clone()); - last_microblock_tenure_time = get_epoch_time_ms(); - } - else { - debug!("Relayer: reset unconfirmed state to 0 microblocks"); - counters.set_microblocks_processed(0); - microblock_miner_state = None; - } - } - RelayerDirective::Exit => break - } - } - debug!("Relayer exit!"); - }).unwrap(); - - Ok(relayer_handle) -} - +/// States we can be in when registering a leader VRF key enum LeaderKeyRegistrationState { + /// Not started yet Inactive, + /// Waiting for burnchain confirmation Pending, + /// Ready to go! Active(RegisteredKey), } -impl StacksNode { - pub fn spawn( - runloop: &RunLoop, - last_burn_block: Option, - coord_comms: CoordinatorChannels, - attachments_rx: Receiver>, - ) -> StacksNode { - let config = runloop.config().clone(); - let miner = runloop.is_miner(); - let burnchain = runloop.get_burnchain(); - let atlas_config = AtlasConfig::default(config.is_mainnet()); - let mut keychain = Keychain::default(config.node.seed.clone()); +/// Relayer thread +/// * accepts network results and stores blocks and microblocks +/// * forwards new blocks, microblocks, and transactions to the p2p thread +/// * processes burnchain state +/// * if mining, runs the miner and broadcasts blocks (via a subordinate MinerThread) +pub struct RelayerThread { + /// Node config + config: Config, + /// Handle to the sortition DB (optional so we can take/replace it) + sortdb: Option, + /// Handle to the chainstate DB (optional so we can take/replace it) + chainstate: Option, + /// Handle to the mempool DB (optional so we can take/replace it) + mempool: Option, + /// Handle to global state and inter-thread communication channels + globals: Globals, + /// Authoritative copy of the keychain state + keychain: Keychain, + /// Burnchian configuration + burnchain: Burnchain, + /// Set of blocks that we have mined, but are still potentially-broadcastable + last_mined_blocks: MinedBlocks, + /// client to the burnchain (used only for sending block-commits) + bitcoin_controller: BitcoinRegtestController, + /// client to the event dispatcher + event_dispatcher: EventDispatcher, - // we can call _open_ here rather than _connect_, since connect is first called in - // make_genesis_block - let mut sortdb = SortitionDB::open(&config.get_burn_db_file_path(), true) - .expect("Error while instantiating sortition db"); + /// copy of the local peer state + local_peer: LocalPeer, + /// last time we tried to mine a block (in millis) + last_tenure_issue_time: u128, + /// last observed burnchain block height from the p2p thread (obtained from network results) + last_network_block_height: u64, + /// last observed number of downloader state-machine passes from the p2p thread (obtained from + /// network results) + last_network_download_passes: u64, + /// minimum number of downloader state-machine passes that must take place before mining (this + /// is used to ensure that the p2p thread attempts to download new Stacks block data before + /// this thread tries to mine a block) + min_network_download_passes: u64, - let epochs = SortitionDB::get_stacks_epochs(sortdb.conn()) - .expect("Error while loading stacks epochs"); + /// consensus hash of the last sortition we saw, even if we weren't the winner + last_tenure_consensus_hash: Option, + /// tip of last tenure we won (used for mining microblocks) + miner_tip: Option, + /// last time we mined a microblock, in millis + last_microblock_tenure_time: u128, - let view = { - let sortition_tip = SortitionDB::get_canonical_burn_chain_tip(&sortdb.conn()) - .expect("Failed to get sortition tip"); - SortitionDB::get_burnchain_view(&sortdb.conn(), &burnchain, &sortition_tip).unwrap() - }; + /// Inner relayer instance for forwarding broadcasted data back to the p2p thread for dispatch + /// to neighbors + relayer: Relayer, - if let Some(ast_precheck_size_height) = config.burnchain.ast_precheck_size_height { - info!( - "Override burnchain height of {:?} to {}", - ASTRules::PrecheckSize, - ast_precheck_size_height - ); - let mut tx = sortdb - .tx_begin() - .expect("FATAL: failed to begin tx on sortition DB"); - SortitionDB::override_ast_rule_height( - &mut tx, - ASTRules::PrecheckSize, - ast_precheck_size_height, - ) - .expect("FATAL: failed to override AST PrecheckSize rule height"); - tx.commit() - .expect("FATAL: failed to commit sortition DB transaction"); - } + /// handle to the subordinate miner thread + miner_thread: Option>>, + /// if true, then the last time the miner thread was launched, it was used to mine a Stacks + /// block (used to alternate between mining microblocks and Stacks blocks that confirm them) + mined_stacks_block: bool, +} - // create a new peerdb - let data_url = UrlString::try_from(format!("{}", &config.node.data_url)).unwrap(); - let initial_neighbors = config.node.bootstrap_node.clone(); - if initial_neighbors.len() > 0 { - info!( - "Will bootstrap from peers {}", - VecDisplay(&initial_neighbors) - ); - } else { - warn!("Without a peer to bootstrap from, the node will start mining a new chain"); - } +struct BlockMinerThread { + /// node config struct + config: Config, + /// handle to global state + globals: Globals, + /// copy of the node's keychain + keychain: Keychain, + /// burnchain configuration + burnchain: Burnchain, + /// Set of blocks that we have mined, but are still potentially-broadcastable + /// (copied from RelayerThread since we need the info to determine the strategy for mining the + /// next block during this tenure). + last_mined_blocks: MinedBlocks, + /// Copy of the node's last ongoing block commit from the last time this thread was run + ongoing_commit: Option, + /// Copy of the node's registered VRF key + registered_key: RegisteredKey, + /// Burnchain block snapshot at the time this thread was initialized + burn_block: BlockSnapshot, + /// Handle to the node's event dispatcher + event_dispatcher: EventDispatcher, +} - let p2p_sock: SocketAddr = config.node.p2p_bind.parse().expect(&format!( - "Failed to parse socket: {}", - &config.node.p2p_bind - )); - let rpc_sock = config.node.rpc_bind.parse().expect(&format!( - "Failed to parse socket: {}", - &config.node.rpc_bind - )); - let p2p_addr: SocketAddr = config.node.p2p_address.parse().expect(&format!( - "Failed to parse socket: {}", - &config.node.p2p_address - )); - let node_privkey = { - let mut re_hashed_seed = config.node.local_peer_seed.clone(); - let my_private_key = loop { - match Secp256k1PrivateKey::from_slice(&re_hashed_seed[..]) { - Ok(sk) => break sk, - Err(_) => { - re_hashed_seed = Sha256Sum::from_data(&re_hashed_seed[..]) - .as_bytes() - .to_vec() - } - } - }; - my_private_key - }; +/// State representing the microblock miner. +struct MicroblockMinerThread { + /// handle to global state + globals: Globals, + /// handle to chainstate DB (optional so we can take/replace it) + chainstate: Option, + /// handle to sortition DB (optional so we can take/replace it) + sortdb: Option, + /// handle to mempool DB (optional so we can take/replace it) + mempool: Option, + /// Handle to the node's event dispatcher + event_dispatcher: EventDispatcher, + /// Stacks block's sortition's consensus hash + parent_consensus_hash: ConsensusHash, + /// Stacks block's hash + parent_block_hash: BlockHeaderHash, + /// Microblock signing key + miner_key: Secp256k1PrivateKey, + /// How often to make microblocks, in milliseconds + frequency: u64, + /// Epoch timestamp, in milliseconds, when the last microblock was produced + last_mined: u128, + /// How many microblocks produced so far + quantity: u64, + /// Block budget consumed so far by this tenure (initialized to the cost of the Stacks block + /// itself; microblocks fill up the remaining budget) + cost_so_far: ExecutionCost, + /// Block builder settings for the microblock miner. + settings: BlockBuilderSettings, +} - let mut peerdb = PeerDB::connect( - &config.get_peer_db_file_path(), - true, - config.burnchain.chain_id, - burnchain.network_id, - Some(node_privkey), - config.connection_options.private_key_lifetime.clone(), - PeerAddress::from_socketaddr(&p2p_addr), - p2p_sock.port(), - data_url, - &vec![], - Some(&initial_neighbors), - ) - .map_err(|e| { - eprintln!( - "Failed to open {}: {:?}", - &config.get_peer_db_file_path(), - &e - ); - panic!(); - }) - .unwrap(); - - { - // bootstrap nodes *always* allowed - let mut tx = peerdb.tx_begin().unwrap(); - for initial_neighbor in initial_neighbors.iter() { - // update peer in case public key changed - PeerDB::update_peer(&mut tx, &initial_neighbor).unwrap(); - PeerDB::set_allow_peer( - &mut tx, - initial_neighbor.addr.network_id, - &initial_neighbor.addr.addrbytes, - initial_neighbor.addr.port, - -1, - ) - .unwrap(); +impl MicroblockMinerThread { + /// Instantiate the miner thread state from the relayer thread. + /// May fail if: + /// * we didn't win the last sortition + /// * we couldn't open or read the DBs for some reason + /// * we couldn't find the anchored block (i.e. it's not processed yet) + pub fn from_relayer_thread(relayer_thread: &RelayerThread) -> Option { + let globals = relayer_thread.globals.clone(); + let config = relayer_thread.config.clone(); + let miner_tip = match relayer_thread.miner_tip.clone() { + Some(tip) => tip, + None => { + debug!("Relayer: cannot instantiate microblock miner: did not win Stacks tip sortition"); + return None; } - tx.commit().unwrap(); - } - - if !config.node.deny_nodes.is_empty() { - warn!("Will ignore nodes {:?}", &config.node.deny_nodes); - } - - { - let mut tx = peerdb.tx_begin().unwrap(); - for denied in config.node.deny_nodes.iter() { - PeerDB::set_deny_peer( - &mut tx, - denied.addr.network_id, - &denied.addr.addrbytes, - denied.addr.port, - get_epoch_time_secs() + 24 * 365 * 3600, - ) - .unwrap(); - } - tx.commit().unwrap(); - } - - // update services to indicate we can support mempool sync - { - let mut tx = peerdb.tx_begin().unwrap(); - PeerDB::set_local_services( - &mut tx, - (ServiceFlags::RPC as u16) | (ServiceFlags::RELAY as u16), - ) - .unwrap(); - tx.commit().unwrap(); - } - - let atlasdb = - AtlasDB::connect(atlas_config.clone(), &config.get_atlas_db_file_path(), true).unwrap(); - - let local_peer = match PeerDB::get_local_peer(peerdb.conn()) { - Ok(local_peer) => local_peer, - _ => panic!("Unable to retrieve local peer"), }; - // force early mempool instantiation + let stacks_chainstate_path = config.get_chainstate_path_str(); + let burn_db_path = config.get_burn_db_file_path(); let cost_estimator = config .make_cost_estimator() .unwrap_or_else(|| Box::new(UnitEstimator)); @@ -1549,186 +510,2230 @@ impl StacksNode { .make_cost_metric() .unwrap_or_else(|| Box::new(UnitMetric)); - let _ = MemPoolDB::open( + // NOTE: read-write access is needed in order to be able to query the recipient set. + // This is an artifact of the way the MARF is built (see #1449) + let sortdb = SortitionDB::open(&burn_db_path, true) + .map_err(|e| { + error!( + "Relayer: Could not open sortdb '{}' ({:?}); skipping tenure", + &burn_db_path, &e + ); + e + }) + .ok()?; + + let (mut chainstate, _) = StacksChainState::open( config.is_mainnet(), config.burnchain.chain_id, - &config.get_chainstate_path_str(), + &stacks_chainstate_path, + Some(config.node.get_marf_opts()), + ) + .map_err(|e| { + error!( + "Relayer: Could not open chainstate '{}' ({:?}); skipping microblock tenure", + &stacks_chainstate_path, &e + ); + e + }) + .ok()?; + + let mempool = MemPoolDB::open( + config.is_mainnet(), + config.burnchain.chain_id, + &stacks_chainstate_path, cost_estimator, metric, ) - .expect("BUG: failed to instantiate mempool"); + .expect("Database failure opening mempool"); - // now we're ready to instantiate a p2p network object, the relayer, and the event dispatcher - let mut p2p_net = PeerNetwork::new( - peerdb, - atlasdb, - local_peer.clone(), - config.burnchain.peer_version, - burnchain.clone(), - view, - config.connection_options.clone(), - epochs, + let MinerTip { + consensus_hash: ch, + block_hash: bhh, + microblock_privkey: miner_key, + } = miner_tip; + + debug!( + "Relayer: Instantiate microblock mining state off of {}/{}", + &ch, &bhh ); - // setup the relayer channel - let (relay_send, relay_recv) = sync_channel(RELAYER_MAX_BUFFER); + // we won a block! proceed to build a microblock tail if we've stored it + match StacksChainState::get_anchored_block_header_info(chainstate.db(), &ch, &bhh) { + Ok(Some(_)) => { + let parent_index_hash = StacksBlockHeader::make_index_block_hash(&ch, &bhh); + let cost_so_far = StacksChainState::get_stacks_block_anchored_cost( + chainstate.db(), + &parent_index_hash, + ) + .expect("FATAL: failed to get anchored block cost") + .expect("FATAL: no anchored block cost stored for processed anchored block"); - let last_sortition = Arc::new(Mutex::new(last_burn_block)); + let frequency = config.node.microblock_frequency; + let settings = + config.make_block_builder_settings(0, true, globals.get_miner_status()); - let burnchain_signer = keychain.get_burnchain_signer(); - match monitoring::set_burnchain_signer(burnchain_signer.clone()) { - Err(e) => { - warn!("Failed to set global burnchain signer: {:?}", &e); + // port over unconfirmed state to this thread + chainstate.unconfirmed_state = if let Some(unconfirmed_state) = + relayer_thread.chainstate_ref().unconfirmed_state.as_ref() + { + Some(unconfirmed_state.make_readonly_owned().ok()?) + } else { + None + }; + + Some(MicroblockMinerThread { + globals, + chainstate: Some(chainstate), + sortdb: Some(sortdb), + mempool: Some(mempool), + event_dispatcher: relayer_thread.event_dispatcher.clone(), + parent_consensus_hash: ch.clone(), + parent_block_hash: bhh.clone(), + miner_key, + frequency, + last_mined: 0, + quantity: 0, + cost_so_far: cost_so_far, + settings, + }) + } + Ok(None) => { + warn!( + "Relayer: No such anchored block: {}/{}. Cannot mine microblocks", + ch, bhh + ); + None + } + Err(e) => { + warn!( + "Relayer: Failed to get anchored block cost for {}/{}: {:?}", + ch, bhh, &e + ); + None } - _ => {} } + } - let relayer = Relayer::from_p2p(&mut p2p_net); - let shared_unconfirmed_txs = Arc::new(Mutex::new(UnconfirmedTxMap::new())); + /// Do something with the inner chainstate DBs (borrowed mutably). + /// Used to fool the borrow-checker. + /// NOT COMPOSIBLE - WILL PANIC IF CALLED FROM WITHIN ITSELF. + fn with_chainstate(&mut self, func: F) -> R + where + F: FnOnce(&mut Self, &mut SortitionDB, &mut StacksChainState, &mut MemPoolDB) -> R, + { + let mut sortdb = self.sortdb.take().expect("FATAL: already took sortdb"); + let mut chainstate = self + .chainstate + .take() + .expect("FATAL: already took chainstate"); + let mut mempool = self.mempool.take().expect("FATAL: already took mempool"); - let leader_key_registration_state = if config.node.mock_mining { - // mock mining, pretend to have a registered key - let vrf_public_key = keychain.rotate_vrf_keypair(1); - LeaderKeyRegistrationState::Active(RegisteredKey { - block_height: 1, - op_vtxindex: 1, - vrf_public_key, - }) - } else { - LeaderKeyRegistrationState::Inactive + let res = func(self, &mut sortdb, &mut chainstate, &mut mempool); + + self.sortdb = Some(sortdb); + self.chainstate = Some(chainstate); + self.mempool = Some(mempool); + + res + } + + /// Unconditionally mine one microblock. + /// Can fail if the miner thread gets cancelled (most likely cause), or if there's some kind of + /// DB error. + fn inner_mine_one_microblock( + &mut self, + sortdb: &SortitionDB, + chainstate: &mut StacksChainState, + mempool: &mut MemPoolDB, + ) -> Result { + debug!( + "Try to mine one microblock off of {}/{} (total: {})", + &self.parent_consensus_hash, + &self.parent_block_hash, + chainstate + .unconfirmed_state + .as_ref() + .map(|us| us.num_microblocks()) + .unwrap_or(0) + ); + + let burn_height = + SortitionDB::get_block_snapshot_consensus(sortdb.conn(), &self.parent_consensus_hash) + .map_err(|e| { + error!("Failed to find block snapshot for mined block: {}", e); + e + })? + .ok_or_else(|| { + error!("Failed to find block snapshot for mined block"); + ChainstateError::NoSuchBlockError + })? + .block_height; + + let ast_rules = SortitionDB::get_ast_rules(sortdb.conn(), burn_height).map_err(|e| { + error!("Failed to get AST rules for microblock: {}", e); + e + })?; + + let mint_result = { + let ic = sortdb.index_conn(); + let mut microblock_miner = match StacksMicroblockBuilder::resume_unconfirmed( + chainstate, + &ic, + &self.cost_so_far, + self.settings.clone(), + ) { + Ok(x) => x, + Err(e) => { + let msg = format!( + "Failed to create a microblock miner at chaintip {}/{}: {:?}", + &self.parent_consensus_hash, &self.parent_block_hash, &e + ); + error!("{}", msg); + return Err(e); + } + }; + + let t1 = get_epoch_time_ms(); + + let mblock = microblock_miner.mine_next_microblock( + mempool, + &self.miner_key, + &self.event_dispatcher, + )?; + let new_cost_so_far = microblock_miner.get_cost_so_far().expect("BUG: cannot read cost so far from miner -- indicates that the underlying Clarity Tx is somehow in use still."); + let t2 = get_epoch_time_ms(); + + info!( + "Mined microblock {} ({}) with {} transactions in {}ms", + mblock.block_hash(), + mblock.header.sequence, + mblock.txs.len(), + t2.saturating_sub(t1) + ); + + Ok((mblock, new_cost_so_far)) }; - let relayer_thread_handle = spawn_miner_relayer( - runloop, - relayer, - local_peer, - keychain, - relay_recv, - last_sortition.clone(), - coord_comms, - shared_unconfirmed_txs.clone(), - ) - .expect("Failed to initialize mine/relay thread"); - - let p2p_thread_handle = spawn_peer( - runloop, - p2p_net, - &p2p_sock, - &rpc_sock, - 5000, - relay_send.clone(), - attachments_rx, - shared_unconfirmed_txs, - ) - .expect("Failed to initialize p2p thread"); - - info!("Start HTTP server on: {}", &config.node.rpc_bind); - info!("Start P2P server on: {}", &config.node.p2p_bind); - - let is_miner = miner; - - StacksNode { - config, - relay_channel: relay_send, - last_sortition, - burnchain_signer, - is_miner, - atlas_config, - leader_key_registration_state, - p2p_thread_handle, - relayer_thread_handle, - } - } - - /// Tell the relayer to fire off a tenure and a block commit op, - /// if it is time to do so. - pub fn relayer_issue_tenure(&mut self) -> bool { - if !self.is_miner { - // node is a follower, don't try to issue a tenure - return true; - } - - if let Some(burnchain_tip) = get_last_sortition(&self.last_sortition) { - match self.leader_key_registration_state { - LeaderKeyRegistrationState::Active(ref key) => { - debug!( - "Tenure: Using key {:?} off of {}", - &key.vrf_public_key, &burnchain_tip.burn_header_hash - ); - - self.relay_channel - .send(RelayerDirective::RunTenure( - key.clone(), - burnchain_tip, - get_epoch_time_ms(), - )) - .is_ok() - } - LeaderKeyRegistrationState::Inactive => { - warn!( - "Tenure: skipped tenure because no active VRF key. Trying to register one." - ); - self.leader_key_registration_state = LeaderKeyRegistrationState::Pending; - self.relay_channel - .send(RelayerDirective::RegisterKey(burnchain_tip)) - .is_ok() - } - LeaderKeyRegistrationState::Pending => true, + let (mined_microblock, new_cost) = match mint_result { + Ok(x) => x, + Err(e) => { + warn!("Failed to mine microblock: {}", e); + return Err(e); } - } else { - warn!("Tenure: Do not know the last burn block. As a miner, this is bad."); - true - } - } + }; - /// Notify the relayer of a sortition, telling it to process the block - /// and advertize it if it was mined by the node. - /// returns _false_ if the relayer hung up the channel. - pub fn relayer_sortition_notify(&self) -> bool { - if !self.is_miner { - // node is a follower, don't try to process my own tenure. - return true; - } - - if let Some(snapshot) = get_last_sortition(&self.last_sortition) { - debug!( - "Tenure: Notify sortition!"; - "consensus_hash" => %snapshot.consensus_hash, - "burn_block_hash" => %snapshot.burn_header_hash, - "winning_stacks_block_hash" => %snapshot.winning_stacks_block_hash, - "burn_block_height" => &snapshot.block_height, - "sortition_id" => %snapshot.sortition_id + // failsafe + if !Relayer::static_check_problematic_relayed_microblock( + chainstate.mainnet, + &mined_microblock, + ASTRules::PrecheckSize, + ) { + // nope! + warn!( + "Our mined microblock {} was problematic", + &mined_microblock.block_hash() ); - if snapshot.sortition { - return self - .relay_channel - .send(RelayerDirective::ProcessTenure( - snapshot.consensus_hash.clone(), - snapshot.parent_burn_header_hash.clone(), - snapshot.winning_stacks_block_hash.clone(), - )) - .is_ok(); + + #[cfg(any(test, feature = "testing"))] + { + use std::fs; + use std::io::Write; + if let Ok(path) = std::env::var("STACKS_BAD_BLOCKS_DIR") { + // record this microblock somewhere + if !fs::metadata(&path).is_ok() { + fs::create_dir_all(&path) + .expect(&format!("FATAL: could not create '{}'", &path)); + } + + let path = Path::new(&path); + let path = path.join(Path::new(&format!("{}", &mined_microblock.block_hash()))); + let mut file = fs::File::create(&path) + .expect(&format!("FATAL: could not create '{:?}'", &path)); + + let mblock_bits = mined_microblock.serialize_to_vec(); + let mblock_bits_hex = to_hex(&mblock_bits); + + let mblock_json = format!( + r#"{{"microblock":"{}","parent_consensus":"{}","parent_block":"{}"}}"#, + &mblock_bits_hex, &self.parent_consensus_hash, &self.parent_block_hash + ); + file.write_all(&mblock_json.as_bytes()).expect(&format!( + "FATAL: failed to write microblock bits to '{:?}'", + &path + )); + info!( + "Fault injection: bad microblock {} saved to {}", + &mined_microblock.block_hash(), + &path.to_str().unwrap() + ); + } + } + if !Relayer::process_mined_problematic_blocks(ast_rules, ASTRules::PrecheckSize) { + // don't process it + warn!( + "Will NOT process our problematic mined microblock {}", + &mined_microblock.block_hash() + ); + return Err(ChainstateError::NoTransactionsToMine); + } else { + warn!( + "Will process our problematic mined microblock {}", + &mined_microblock.block_hash() + ) + } + } + + // cancelled? + let is_miner_blocked = self + .globals + .get_miner_status() + .lock() + .expect("FATAL: mutex poisoned") + .is_blocked(); + if is_miner_blocked { + return Err(ChainstateError::MinerAborted); + } + + // preprocess the microblock locally + chainstate.preprocess_streamed_microblock( + &self.parent_consensus_hash, + &self.parent_block_hash, + &mined_microblock, + )?; + + // update unconfirmed state cost + self.cost_so_far = new_cost; + self.quantity += 1; + return Ok(mined_microblock); + } + + /// Can this microblock miner mine off of this given tip? + pub fn can_mine_on_tip( + &self, + consensus_hash: &ConsensusHash, + block_hash: &BlockHeaderHash, + ) -> bool { + self.parent_consensus_hash == *consensus_hash && self.parent_block_hash == *block_hash + } + + /// Body of try_mine_microblock() + fn inner_try_mine_microblock( + &mut self, + miner_tip: MinerTip, + sortdb: &SortitionDB, + chainstate: &mut StacksChainState, + mem_pool: &mut MemPoolDB, + ) -> Result, NetError> { + if !self.can_mine_on_tip(&self.parent_consensus_hash, &self.parent_block_hash) { + // not configured to mine on this tip + return Ok(None); + } + if !self.can_mine_on_tip(&miner_tip.consensus_hash, &miner_tip.block_hash) { + // this tip isn't what this miner is meant to mine on + return Ok(None); + } + + if self.last_mined + (self.frequency as u128) >= get_epoch_time_ms() { + // too soon to mine + return Ok(None); + } + + let mut next_microblock = None; + + // opportunistically try and mine, but only if there are no attachable blocks in + // recent history (i.e. in the last 10 minutes) + let num_attachable = StacksChainState::count_attachable_staging_blocks( + chainstate.db(), + 1, + get_epoch_time_secs() - 600, + )?; + if num_attachable == 0 { + match self.inner_mine_one_microblock(sortdb, chainstate, mem_pool) { + Ok(microblock) => { + // will need to relay this + next_microblock = Some(microblock); + } + Err(ChainstateError::NoTransactionsToMine) => { + info!("Will keep polling mempool for transactions to include in a microblock"); + } + Err(e) => { + warn!("Failed to mine one microblock: {:?}", &e); + } } } else { - debug!("Tenure: Notify sortition! No last burn block"); + debug!("Will not mine microblocks yet -- have {} attachable blocks that arrived in the last 10 minutes", num_attachable); } + + self.last_mined = get_epoch_time_ms(); + + Ok(next_microblock) + } + + /// Try to mine one microblock, given the current chain tip and access to the chain state DBs. + /// If we succeed, return the microblock and log the tx events to the given event dispatcher. + /// May return None if any of the following are true: + /// * `miner_tip` does not match this miner's miner tip + /// * it's been too soon (less than microblock_frequency milliseconds) since we tried this call + /// * there are simply no transactions to mine + /// * there are still stacks blocks to be processed in the staging db + /// * the miner thread got cancelled + pub fn try_mine_microblock( + &mut self, + cur_tip: MinerTip, + ) -> Result, NetError> { + self.with_chainstate(|mblock_miner, sortdb, chainstate, mempool| { + mblock_miner.inner_try_mine_microblock(cur_tip, sortdb, chainstate, mempool) + }) + } +} + +impl BlockMinerThread { + /// Instantiate the miner thread from its parent RelayerThread + pub fn from_relayer_thread( + rt: &RelayerThread, + registered_key: RegisteredKey, + burn_block: BlockSnapshot, + ) -> BlockMinerThread { + BlockMinerThread { + config: rt.config.clone(), + globals: rt.globals.clone(), + keychain: rt.keychain.clone(), + burnchain: rt.burnchain.clone(), + last_mined_blocks: rt.last_mined_blocks.clone(), + ongoing_commit: rt.bitcoin_controller.get_ongoing_commit(), + registered_key, + burn_block, + event_dispatcher: rt.event_dispatcher.clone(), + } + } + + /// Create a coinbase transaction. + fn inner_generate_coinbase_tx(&mut self, nonce: u64) -> StacksTransaction { + let is_mainnet = self.config.is_mainnet(); + let chain_id = self.config.burnchain.chain_id; + let mut tx_auth = self.keychain.get_transaction_auth().unwrap(); + tx_auth.set_origin_nonce(nonce); + + let version = if is_mainnet { + TransactionVersion::Mainnet + } else { + TransactionVersion::Testnet + }; + let mut tx = StacksTransaction::new( + version, + tx_auth, + TransactionPayload::Coinbase(CoinbasePayload([0u8; 32])), + ); + tx.chain_id = chain_id; + tx.anchor_mode = TransactionAnchorMode::OnChainOnly; + let mut tx_signer = StacksTransactionSigner::new(&tx); + self.keychain.sign_as_origin(&mut tx_signer); + + tx_signer.get_tx().unwrap() + } + + /// Create a poison microblock transaction. + fn inner_generate_poison_microblock_tx( + &mut self, + nonce: u64, + poison_payload: TransactionPayload, + ) -> StacksTransaction { + let is_mainnet = self.config.is_mainnet(); + let chain_id = self.config.burnchain.chain_id; + let mut tx_auth = self.keychain.get_transaction_auth().unwrap(); + tx_auth.set_origin_nonce(nonce); + + let version = if is_mainnet { + TransactionVersion::Mainnet + } else { + TransactionVersion::Testnet + }; + let mut tx = StacksTransaction::new(version, tx_auth, poison_payload); + tx.chain_id = chain_id; + tx.anchor_mode = TransactionAnchorMode::OnChainOnly; + let mut tx_signer = StacksTransactionSigner::new(&tx); + self.keychain.sign_as_origin(&mut tx_signer); + + tx_signer.get_tx().unwrap() + } + + /// Constructs and returns a LeaderBlockCommitOp out of the provided params. + fn inner_generate_block_commit_op( + &self, + block_header_hash: BlockHeaderHash, + burn_fee: u64, + key: &RegisteredKey, + parent_burnchain_height: u32, + parent_winning_vtx: u16, + vrf_seed: VRFSeed, + commit_outs: Vec, + sunset_burn: u64, + current_burn_height: u64, + ) -> BlockstackOperationType { + let (parent_block_ptr, parent_vtxindex) = (parent_burnchain_height, parent_winning_vtx); + let burn_parent_modulus = (current_burn_height % BURN_BLOCK_MINED_AT_MODULUS) as u8; + let sender = self.keychain.get_burnchain_signer(); + BlockstackOperationType::LeaderBlockCommit(LeaderBlockCommitOp { + sunset_burn, + block_header_hash, + burn_fee, + input: (Txid([0; 32]), 0), + apparent_sender: sender, + key_block_ptr: key.block_height as u32, + key_vtxindex: key.op_vtxindex as u16, + memo: vec![STACKS_EPOCH_2_05_MARKER], + new_seed: vrf_seed, + parent_block_ptr, + parent_vtxindex, + vtxindex: 0, + txid: Txid([0u8; 32]), + block_height: 0, + burn_header_hash: BurnchainHeaderHash::zero(), + burn_parent_modulus, + commit_outs, + }) + } + + /// Get references to the inner assembled anchor block data we've produced for a given burnchain block height + fn find_inflight_mined_blocks( + burn_height: u64, + last_mined_blocks: &MinedBlocks, + ) -> Vec<&AssembledAnchorBlock> { + let mut ret = vec![]; + for (_, (assembled_block, _)) in last_mined_blocks.iter() { + if assembled_block.my_block_height >= burn_height { + ret.push(assembled_block); + } + } + ret + } + + /// Load up the parent block info for mining. + /// If there's no parent because this is the first block, then return the genesis block's info. + /// If we can't find the parent in the DB but we expect one, return None. + fn load_block_parent_info( + &self, + burn_db: &mut SortitionDB, + chain_state: &mut StacksChainState, + ) -> Option { + if let Some(stacks_tip) = chain_state + .get_stacks_chain_tip(burn_db) + .expect("FATAL: could not query chain tip") + { + let miner_address = self + .keychain + .origin_address(self.config.is_mainnet()) + .unwrap(); + ParentStacksBlockInfo::lookup( + chain_state, + burn_db, + &self.burn_block, + miner_address, + &stacks_tip.consensus_hash, + &stacks_tip.anchored_block_hash, + ) + .ok() + } else { + debug!("No Stacks chain tip known, will return a genesis block"); + let (network, _) = self.config.burnchain.get_bitcoin_network(); + let burnchain_params = + BurnchainParameters::from_params(&self.config.burnchain.chain, &network) + .expect("Bitcoin network unsupported"); + + let chain_tip = ChainTip::genesis( + &burnchain_params.first_block_hash, + burnchain_params.first_block_height.into(), + burnchain_params.first_block_timestamp.into(), + ); + + Some(ParentStacksBlockInfo { + stacks_parent_header: chain_tip.metadata, + parent_consensus_hash: FIRST_BURNCHAIN_CONSENSUS_HASH.clone(), + parent_block_burn_height: 0, + parent_block_total_burn: 0, + parent_winning_vtxindex: 0, + coinbase_nonce: 0, + }) + } + } + + /// Determine which attempt this will be when mining a block, and whether or not an attempt + /// should even be made. + /// Returns Some(attempt) if we should attempt to mine (and what attempt it will be) + /// Returns None if we should not mine. + fn get_mine_attempt( + &self, + chain_state: &StacksChainState, + parent_block_info: &ParentStacksBlockInfo, + ) -> Option { + let parent_consensus_hash = &parent_block_info.parent_consensus_hash; + let stacks_parent_header = &parent_block_info.stacks_parent_header; + let parent_block_burn_height = parent_block_info.parent_block_burn_height; + + let last_mined_blocks = + Self::find_inflight_mined_blocks(self.burn_block.block_height, &self.last_mined_blocks); + + // has the tip changed from our previously-mined block for this epoch? + let attempt = if last_mined_blocks.len() <= 1 { + // always mine if we've not mined a block for this epoch yet, or + // if we've mined just one attempt, unconditionally try again (so we + // can use `subsequent_miner_time_ms` in this attempt) + if last_mined_blocks.len() == 1 { + debug!("Have only attempted one block; unconditionally trying again"); + } + last_mined_blocks.len() as u64 + 1 + } else { + let mut best_attempt = 0; + debug!( + "Consider {} in-flight Stacks tip(s)", + &last_mined_blocks.len() + ); + for prev_block in last_mined_blocks.iter() { + debug!( + "Consider in-flight block {} on Stacks tip {}/{} in {} with {} txs", + &prev_block.anchored_block.block_hash(), + &prev_block.parent_consensus_hash, + &prev_block.anchored_block.header.parent_block, + &prev_block.my_burn_hash, + &prev_block.anchored_block.txs.len() + ); + + if prev_block.anchored_block.txs.len() == 1 && prev_block.attempt == 1 { + // Don't let the fact that we've built an empty block during this sortition + // prevent us from trying again. + best_attempt = 1; + continue; + } + if prev_block.parent_consensus_hash == *parent_consensus_hash + && prev_block.my_burn_hash == self.burn_block.burn_header_hash + && prev_block.anchored_block.header.parent_block + == stacks_parent_header.anchored_header.block_hash() + { + // the anchored chain tip hasn't changed since we attempted to build a block. + // But, have discovered any new microblocks worthy of being mined? + if let Ok(Some(stream)) = + StacksChainState::load_descendant_staging_microblock_stream( + chain_state.db(), + &StacksBlockHeader::make_index_block_hash( + &prev_block.parent_consensus_hash, + &stacks_parent_header.anchored_header.block_hash(), + ), + 0, + u16::MAX, + ) + { + if (prev_block.anchored_block.header.parent_microblock + == BlockHeaderHash([0u8; 32]) + && stream.len() == 0) + || (prev_block.anchored_block.header.parent_microblock + != BlockHeaderHash([0u8; 32]) + && stream.len() + <= (prev_block.anchored_block.header.parent_microblock_sequence + as usize) + + 1) + { + // the chain tip hasn't changed since we attempted to build a block. Use what we + // already have. + debug!("Relayer: Stacks tip is unchanged since we last tried to mine a block off of {}/{} at height {} with {} txs, in {} at burn height {}, and no new microblocks ({} <= {})", + &prev_block.parent_consensus_hash, &prev_block.anchored_block.header.parent_block, prev_block.anchored_block.header.total_work.work, + prev_block.anchored_block.txs.len(), prev_block.my_burn_hash, parent_block_burn_height, stream.len(), prev_block.anchored_block.header.parent_microblock_sequence); + + return None; + } else { + // there are new microblocks! + // TODO: only consider rebuilding our anchored block if we (a) have + // time, and (b) the new microblocks are worth more than the new BTC + // fee minus the old BTC fee + debug!("Relayer: Stacks tip is unchanged since we last tried to mine a block off of {}/{} at height {} with {} txs, in {} at burn height {}, but there are new microblocks ({} > {})", + &prev_block.parent_consensus_hash, &prev_block.anchored_block.header.parent_block, prev_block.anchored_block.header.total_work.work, + prev_block.anchored_block.txs.len(), prev_block.my_burn_hash, parent_block_burn_height, stream.len(), prev_block.anchored_block.header.parent_microblock_sequence); + + best_attempt = cmp::max(best_attempt, prev_block.attempt); + } + } else { + // no microblock stream to confirm, and the stacks tip hasn't changed + debug!("Relayer: Stacks tip is unchanged since we last tried to mine a block off of {}/{} at height {} with {} txs, in {} at burn height {}, and no microblocks present", + &prev_block.parent_consensus_hash, &prev_block.anchored_block.header.parent_block, prev_block.anchored_block.header.total_work.work, + prev_block.anchored_block.txs.len(), prev_block.my_burn_hash, parent_block_burn_height); + + return None; + } + } else { + if self.burn_block.burn_header_hash == prev_block.my_burn_hash { + // only try and re-mine if there was no sortition since the last chain tip + debug!("Relayer: Stacks tip has changed to {}/{} since we last tried to mine a block in {} at burn height {}; attempt was {} (for Stacks tip {}/{})", + parent_consensus_hash, stacks_parent_header.anchored_header.block_hash(), prev_block.my_burn_hash, parent_block_burn_height, prev_block.attempt, &prev_block.parent_consensus_hash, &prev_block.anchored_block.header.parent_block); + best_attempt = cmp::max(best_attempt, prev_block.attempt); + } else { + debug!("Relayer: Burn tip has changed to {} ({}) since we last tried to mine a block in {}", + &self.burn_block.burn_header_hash, self.burn_block.block_height, &prev_block.my_burn_hash); + } + } + } + best_attempt + 1 + }; + Some(attempt) + } + + /// Generate the VRF proof for the block we're going to build. + /// Returns Some(proof) if we could make the proof + /// Return None if we could not make the proof + fn make_vrf_proof(&mut self) -> Option { + // Generates a proof out of the sortition hash provided in the params. + let vrf_proof = match self.keychain.generate_proof( + &self.registered_key.vrf_public_key, + self.burn_block.sortition_hash.as_bytes(), + ) { + Some(vrfp) => vrfp, + None => { + // Try to recover a key registered in a former session. + // registered_key.block_height gives us a pointer to the height of the block + // holding the key register op, but the VRF was derived using the height of one + // of the parents blocks. + let _ = self + .keychain + .rotate_vrf_keypair(self.registered_key.block_height - 1); + match self.keychain.generate_proof( + &self.registered_key.vrf_public_key, + self.burn_block.sortition_hash.as_bytes(), + ) { + Some(vrfp) => vrfp, + None => { + error!( + "Relayer: Failed to generate proof with {:?}", + &self.registered_key.vrf_public_key + ); + return None; + } + } + } + }; + + debug!( + "Generated VRF Proof: {} over {} with key {}", + vrf_proof.to_hex(), + &self.burn_block.sortition_hash, + &self.registered_key.vrf_public_key.to_hex() + ); + Some(vrf_proof) + } + + /// Get the microblock private key we'll be using for this tenure, should we win. + /// Return the private key on success + /// return None if we were unable to generate the key. + fn make_microblock_private_key(&mut self, attempt: u64) -> Option { + // Generates a new secret key for signing the trail of microblocks + // of the upcoming tenure. + let microblock_secret_key = if attempt > 1 { + match self.keychain.get_microblock_key() { + Some(k) => k, + None => { + error!( + "Relayer: Failed to obtain microblock key for mining attempt"; + "attempt" => %attempt + ); + return None; + } + } + } else { + // NOTE: this is a no-op if run in a separate thread with a moved copy of the keychain + self.keychain + .rotate_microblock_keypair(self.burn_block.block_height) + }; + + Some(microblock_secret_key) + } + + /// Load the parent microblock stream and vet it for the absence of forks. + /// If there is a fork, then mine and relay a poison microblock transaction. + /// Update stacks_parent_header's microblock tail to point to the end of the stream we load. + /// Return the microblocks we'll confirm, if there are any. + fn load_and_vet_parent_microblocks( + &mut self, + burn_db: &SortitionDB, + chain_state: &mut StacksChainState, + mem_pool: &mut MemPoolDB, + parent_block_info: &mut ParentStacksBlockInfo, + ) -> Option> { + let parent_consensus_hash = &parent_block_info.parent_consensus_hash; + let stacks_parent_header = &mut parent_block_info.stacks_parent_header; + + let microblock_info_opt = + match StacksChainState::load_descendant_staging_microblock_stream_with_poison( + chain_state.db(), + &StacksBlockHeader::make_index_block_hash( + parent_consensus_hash, + &stacks_parent_header.anchored_header.block_hash(), + ), + 0, + u16::MAX, + ) { + Ok(x) => { + let num_mblocks = x.as_ref().map(|(mblocks, ..)| mblocks.len()).unwrap_or(0); + debug!( + "Loaded {} microblocks descending from {}/{}", + num_mblocks, + parent_consensus_hash, + &stacks_parent_header.anchored_header.block_hash() + ); + x + } + Err(e) => { + warn!( + "Failed to load descendant microblock stream from {}/{}: {:?}", + parent_consensus_hash, + &stacks_parent_header.anchored_header.block_hash(), + &e + ); + None + } + }; + + if let Some((ref microblocks, ref poison_opt)) = µblock_info_opt { + if let Some(ref tail) = microblocks.last() { + debug!( + "Confirm microblock stream tailed at {} (seq {})", + &tail.block_hash(), + tail.header.sequence + ); + } + + // try and confirm as many microblocks as we can (but note that the stream itself may + // be too long; we'll try again if that happens). + stacks_parent_header.microblock_tail = + microblocks.last().clone().map(|blk| blk.header.clone()); + + if let Some(poison_payload) = poison_opt { + debug!("Detected poisoned microblock fork: {:?}", &poison_payload); + + // submit it multiple times with different nonces, so it'll have a good chance of + // eventually getting picked up (even if the miner sends other transactions from + // the same address) + for i in 0..10 { + let poison_microblock_tx = self.inner_generate_poison_microblock_tx( + parent_block_info.coinbase_nonce + 1 + i, + poison_payload.clone(), + ); + + let stacks_epoch = burn_db + .index_conn() + .get_stacks_epoch(self.burn_block.block_height as u32) + .expect("Could not find a stacks epoch."); + + // submit the poison payload, privately, so we'll mine it when building the + // anchored block. + if let Err(e) = mem_pool.miner_submit( + chain_state, + &parent_consensus_hash, + &stacks_parent_header.anchored_header.block_hash(), + &poison_microblock_tx, + Some(&self.event_dispatcher), + 1_000_000_000.0, // prioritize this for inclusion + ) { + warn!( + "Detected but failed to mine poison-microblock transaction: {:?}", + &e + ); + } else { + debug!( + "Submit poison-microblock transaction {:?}", + &poison_microblock_tx + ); + } + } + } + } + + microblock_info_opt.map(|(stream, _)| stream) + } + + /// Produce the block-commit for this anchored block, if we can. + /// Returns the op on success + /// Returns None if we fail somehow. + pub fn make_block_commit( + &self, + burn_db: &mut SortitionDB, + chain_state: &mut StacksChainState, + block_hash: BlockHeaderHash, + parent_block_burn_height: u64, + parent_winning_vtxindex: u16, + vrf_proof: &VRFProof, + ) -> Option { + // let's figure out the recipient set! + let recipients = match get_next_recipients( + &self.burn_block, + chain_state, + burn_db, + &self.burnchain, + &OnChainRewardSetProvider(), + ) { + Ok(x) => x, + Err(e) => { + error!("Relayer: Failure fetching recipient set: {:?}", e); + return None; + } + }; + + let burn_fee_cap = self.config.burnchain.burn_fee_cap; + let sunset_burn = self + .burnchain + .expected_sunset_burn(self.burn_block.block_height + 1, burn_fee_cap); + let rest_commit = burn_fee_cap - sunset_burn; + + let commit_outs = if self.burn_block.block_height + 1 + < self.burnchain.pox_constants.sunset_end + && !self + .burnchain + .is_in_prepare_phase(self.burn_block.block_height + 1) + { + RewardSetInfo::into_commit_outs(recipients, self.config.is_mainnet()) + } else { + vec![StacksAddress::burn_address(self.config.is_mainnet())] + }; + + // let's commit, but target the current burnchain tip with our modulus + let op = self.inner_generate_block_commit_op( + block_hash, + rest_commit, + &self.registered_key, + parent_block_burn_height + .try_into() + .expect("Could not convert parent block height into u32"), + parent_winning_vtxindex, + VRFSeed::from_proof(vrf_proof), + commit_outs, + sunset_burn, + self.burn_block.block_height, + ); + Some(op) + } + + /// Try to mine a Stacks block by assembling one from mempool transactions and sending a + /// burnchain block-commit transaction. If we succeed, then return the assembled block data as + /// well as the microblock private key to use to produce microblocks. + /// Return None if we couldn't build a block for whatever reason. + pub fn run_tenure(&mut self) -> Option { + fault_injection_long_tenure(); + + let burn_db_path = self.config.get_burn_db_file_path(); + let stacks_chainstate_path = self.config.get_chainstate_path_str(); + + let cost_estimator = self + .config + .make_cost_estimator() + .unwrap_or_else(|| Box::new(UnitEstimator)); + let metric = self + .config + .make_cost_metric() + .unwrap_or_else(|| Box::new(UnitMetric)); + + let mut bitcoin_controller = BitcoinRegtestController::new_ongoing_dummy( + self.config.clone(), + self.ongoing_commit.clone(), + ); + + // NOTE: read-write access is needed in order to be able to query the recipient set. + // This is an artifact of the way the MARF is built (see #1449) + let mut burn_db = + SortitionDB::open(&burn_db_path, true).expect("FATAL: could not open sortition DB"); + + let (mut chain_state, _) = StacksChainState::open( + self.config.is_mainnet(), + self.config.burnchain.chain_id, + &stacks_chainstate_path, + Some(self.config.node.get_marf_opts()), + ) + .expect("FATAL: could not open chainstate DB"); + + let mut mem_pool = MemPoolDB::open( + self.config.is_mainnet(), + self.config.burnchain.chain_id, + &stacks_chainstate_path, + cost_estimator, + metric, + ) + .expect("Database failure opening mempool"); + + let tenure_begin = get_epoch_time_ms(); + + let mut parent_block_info = self.load_block_parent_info(&mut burn_db, &mut chain_state)?; + let attempt = self.get_mine_attempt(&chain_state, &parent_block_info)?; + let vrf_proof = self.make_vrf_proof()?; + + // Generates a new secret key for signing the trail of microblocks + // of the upcoming tenure. + let microblock_private_key = self.make_microblock_private_key(attempt)?; + let mblock_pubkey_hash = + Hash160::from_node_public_key(&StacksPublicKey::from_private(µblock_private_key)); + + // create our coinbase + let coinbase_tx = self.inner_generate_coinbase_tx(parent_block_info.coinbase_nonce); + + // find the longest microblock tail we can build off of. + // target it to the microblock tail in parent_block_info + let microblocks_opt = self.load_and_vet_parent_microblocks( + &burn_db, + &mut chain_state, + &mut mem_pool, + &mut parent_block_info, + ); + + // build the block itself + let (anchored_block, _, _) = match StacksBlockBuilder::build_anchored_block( + &chain_state, + &burn_db.index_conn(), + &mut mem_pool, + &parent_block_info.stacks_parent_header, + parent_block_info.parent_block_total_burn, + vrf_proof.clone(), + mblock_pubkey_hash, + &coinbase_tx, + self.config.make_block_builder_settings( + attempt, + false, + self.globals.get_miner_status(), + ), + Some(&self.event_dispatcher), + ) { + Ok(block) => block, + Err(ChainstateError::InvalidStacksMicroblock(msg, mblock_header_hash)) => { + // part of the parent microblock stream is invalid, so try again + info!("Parent microblock stream is invalid; trying again without the offender {} (msg: {})", &mblock_header_hash, &msg); + + // truncate the stream + parent_block_info.stacks_parent_header.microblock_tail = match microblocks_opt { + Some(microblocks) => { + let mut tail = None; + for mblock in microblocks.into_iter() { + if mblock.block_hash() == mblock_header_hash { + break; + } + tail = Some(mblock); + } + if let Some(ref t) = &tail { + debug!( + "New parent microblock stream tail is {} (seq {})", + t.block_hash(), + t.header.sequence + ); + } + tail.map(|t| t.header) + } + None => None, + }; + + // try again + match StacksBlockBuilder::build_anchored_block( + &chain_state, + &burn_db.index_conn(), + &mut mem_pool, + &parent_block_info.stacks_parent_header, + parent_block_info.parent_block_total_burn, + vrf_proof.clone(), + mblock_pubkey_hash, + &coinbase_tx, + self.config.make_block_builder_settings( + attempt, + false, + self.globals.get_miner_status(), + ), + Some(&self.event_dispatcher), + ) { + Ok(block) => block, + Err(e) => { + error!("Relayer: Failure mining anchor block even after removing offending microblock {}: {}", &mblock_header_hash, &e); + return None; + } + } + } + Err(e) => { + error!("Relayer: Failure mining anchored block: {}", e); + return None; + } + }; + + info!( + "Relayer: Succeeded assembling {} block #{}: {}, with {} txs, attempt {}", + if parent_block_info.parent_block_total_burn == 0 { + "Genesis" + } else { + "Stacks" + }, + anchored_block.header.total_work.work, + anchored_block.block_hash(), + anchored_block.txs.len(), + attempt + ); + + // let's commit + let op = self.make_block_commit( + &mut burn_db, + &mut chain_state, + anchored_block.block_hash(), + parent_block_info.parent_block_burn_height, + parent_block_info.parent_winning_vtxindex, + &vrf_proof, + )?; + + // last chance -- confirm that the stacks tip is unchanged (since it could have taken long + // enough to build this block that another block could have arrived), and confirm that all + // Stacks blocks with heights higher than the canoincal tip are processed. + let cur_burn_chain_tip = SortitionDB::get_canonical_burn_chain_tip(burn_db.conn()) + .expect("FATAL: failed to query sortition DB for canonical burn chain tip"); + + if let Some(stacks_tip) = chain_state + .get_stacks_chain_tip(&burn_db) + .expect("FATAL: could not query chain tip") + { + let is_miner_blocked = self + .globals + .get_miner_status() + .lock() + .expect("FATAL: mutex poisoned") + .is_blocked(); + let has_unprocessed = StacksChainState::has_higher_unprocessed_blocks( + chain_state.db(), + stacks_tip.height, + ) + .expect("FATAL: failed to query staging blocks"); + if stacks_tip.anchored_block_hash != anchored_block.header.parent_block + || parent_block_info.parent_consensus_hash != stacks_tip.consensus_hash + || cur_burn_chain_tip.burn_header_hash != self.burn_block.burn_header_hash + || is_miner_blocked + || has_unprocessed + { + debug!( + "Relayer: Cancel block-commit; chain tip(s) have changed or cancelled"; + "block_hash" => %anchored_block.block_hash(), + "tx_count" => anchored_block.txs.len(), + "target_height" => %anchored_block.header.total_work.work, + "parent_consensus_hash" => %parent_block_info.parent_consensus_hash, + "parent_block_hash" => %anchored_block.header.parent_block, + "parent_microblock_hash" => %anchored_block.header.parent_microblock, + "parent_microblock_seq" => anchored_block.header.parent_microblock_sequence, + "old_tip_burn_block_hash" => %self.burn_block.burn_header_hash, + "old_tip_burn_block_height" => self.burn_block.block_height, + "old_tip_burn_block_sortition_id" => %self.burn_block.sortition_id, + "attempt" => attempt, + "new_stacks_tip_block_hash" => %stacks_tip.anchored_block_hash, + "new_stacks_tip_consensus_hash" => %stacks_tip.consensus_hash, + "new_tip_burn_block_height" => cur_burn_chain_tip.block_height, + "new_tip_burn_block_sortition_id" => %cur_burn_chain_tip.sortition_id, + "new_burn_block_sortition_id" => %cur_burn_chain_tip.sortition_id, + "miner_blocked" => %is_miner_blocked, + "has_unprocessed" => %has_unprocessed + ); + return None; + } + } + + let mut op_signer = self.keychain.generate_op_signer(); + debug!( + "Relayer: Submit block-commit"; + "block_hash" => %anchored_block.block_hash(), + "tx_count" => anchored_block.txs.len(), + "target_height" => anchored_block.header.total_work.work, + "parent_consensus_hash" => %parent_block_info.parent_consensus_hash, + "parent_block_hash" => %anchored_block.header.parent_block, + "parent_microblock_hash" => %anchored_block.header.parent_microblock, + "parent_microblock_seq" => anchored_block.header.parent_microblock_sequence, + "tip_burn_block_hash" => %self.burn_block.burn_header_hash, + "tip_burn_block_height" => self.burn_block.block_height, + "tip_burn_block_sortition_id" => %self.burn_block.sortition_id, + "cur_burn_block_hash" => %cur_burn_chain_tip.burn_header_hash, + "cur_burn_block_height" => %cur_burn_chain_tip.block_height, + "cur_burn_block_sortition_id" => %cur_burn_chain_tip.sortition_id, + "attempt" => attempt + ); + + let res = bitcoin_controller.submit_operation(op, &mut op_signer, attempt); + if !res { + if !self.config.node.mock_mining { + warn!("Relayer: Failed to submit Bitcoin transaction"); + return None; + } else { + debug!("Relayer: Mock-mining enabled; not sending Bitcoin transaction"); + } + } + + Some(MinerThreadResult::Block( + AssembledAnchorBlock { + parent_consensus_hash: parent_block_info.parent_consensus_hash, + my_burn_hash: cur_burn_chain_tip.burn_header_hash, + my_block_height: cur_burn_chain_tip.block_height, + orig_burn_hash: self.burn_block.burn_header_hash, + anchored_block, + attempt, + tenure_begin, + }, + self.keychain.clone(), + microblock_private_key, + bitcoin_controller.get_ongoing_commit(), + )) + } +} + +impl RelayerThread { + /// Instantiate off of a StacksNode, a runloop, and a relayer. + pub fn new(runloop: &RunLoop, local_peer: LocalPeer, relayer: Relayer) -> RelayerThread { + let config = runloop.config().clone(); + let globals = runloop.get_globals(); + let burn_db_path = config.get_burn_db_file_path(); + let stacks_chainstate_path = config.get_chainstate_path_str(); + let is_mainnet = config.is_mainnet(); + let chain_id = config.burnchain.chain_id; + + let sortdb = + SortitionDB::open(&burn_db_path, true).expect("FATAL: failed to open burnchain DB"); + + let (chainstate, _) = StacksChainState::open( + is_mainnet, + chain_id, + &stacks_chainstate_path, + Some(config.node.get_marf_opts()), + ) + .expect("FATAL: failed to open chainstate DB"); + + let cost_estimator = config + .make_cost_estimator() + .unwrap_or_else(|| Box::new(UnitEstimator)); + let metric = config + .make_cost_metric() + .unwrap_or_else(|| Box::new(UnitMetric)); + + let mempool = MemPoolDB::open( + is_mainnet, + chain_id, + &stacks_chainstate_path, + cost_estimator, + metric, + ) + .expect("Database failure opening mempool"); + + let keychain = Keychain::default(config.node.seed.clone()); + let bitcoin_controller = BitcoinRegtestController::new_dummy(config.clone()); + + RelayerThread { + config: config.clone(), + sortdb: Some(sortdb), + chainstate: Some(chainstate), + mempool: Some(mempool), + globals, + keychain, + burnchain: runloop.get_burnchain(), + last_mined_blocks: MinedBlocks::new(), + bitcoin_controller, + event_dispatcher: runloop.get_event_dispatcher(), + local_peer, + + last_tenure_issue_time: 0, + last_network_block_height: 0, + last_network_download_passes: 0, + min_network_download_passes: 0, + + last_tenure_consensus_hash: None, + miner_tip: None, + last_microblock_tenure_time: 0, + + relayer, + + miner_thread: None, + mined_stacks_block: false, + } + } + + /// Get an immutible ref to the sortdb + pub fn sortdb_ref(&self) -> &SortitionDB { + self.sortdb + .as_ref() + .expect("FATAL: tried to access sortdb while taken") + } + + /// Get an immutible ref to the chainstate + pub fn chainstate_ref(&self) -> &StacksChainState { + self.chainstate + .as_ref() + .expect("FATAL: tried to access chainstate while it was taken") + } + + /// Fool the borrow checker into letting us do something with the chainstate databases. + /// DOES NOT COMPOSE -- do NOT call this, or self.sortdb_ref(), or self.chainstate_ref(), within + /// `func`. You will get a runtime panic. + pub fn with_chainstate(&mut self, func: F) -> R + where + F: FnOnce(&mut RelayerThread, &mut SortitionDB, &mut StacksChainState, &mut MemPoolDB) -> R, + { + let mut sortdb = self + .sortdb + .take() + .expect("FATAL: tried to take sortdb while taken"); + let mut chainstate = self + .chainstate + .take() + .expect("FATAL: tried to take chainstate while taken"); + let mut mempool = self + .mempool + .take() + .expect("FATAL: tried to take mempool while taken"); + let res = func(self, &mut sortdb, &mut chainstate, &mut mempool); + self.sortdb = Some(sortdb); + self.chainstate = Some(chainstate); + self.mempool = Some(mempool); + res + } + + /// Handle a NetworkResult from the p2p/http state machine. Usually this is the act of + /// * preprocessing and storing new blocks and microblocks + /// * relaying blocks, microblocks, and transacctions + /// * updating unconfirmed state views + pub fn process_network_result(&mut self, mut net_result: NetworkResult) { + debug!( + "Relayer: Handle network result (from {})", + net_result.burn_height + ); + + // if there are any blocks or microblocks, then stop mining so we can process + // them. + if net_result.has_blocks() || net_result.has_microblocks() { + // temporarily halt mining + debug!("Relayer: block mining to process newly-arrived blocks or microblocks"); + signal_mining_blocked(self.globals.get_miner_status()); + } + + if self.last_network_block_height != net_result.burn_height { + // burnchain advanced; disable mining until we also do a download pass + self.last_network_block_height = net_result.burn_height; + self.min_network_download_passes = net_result.num_download_passes + 1; + + debug!( + "Relayer: block mining until the next download pass {}", + self.min_network_download_passes + ); + signal_mining_blocked(self.globals.get_miner_status()); + } + + let net_receipts = self.with_chainstate(|relayer_thread, sortdb, chainstate, mempool| { + relayer_thread + .relayer + .process_network_result( + &relayer_thread.local_peer, + &mut net_result, + sortdb, + chainstate, + mempool, + relayer_thread.globals.sync_comms.get_ibd(), + Some(&relayer_thread.globals.coord_comms), + Some(&relayer_thread.event_dispatcher), + ) + .expect("BUG: failure processing network results") + }); + + let mempool_txs_added = net_receipts.mempool_txs_added.len(); + if mempool_txs_added > 0 { + self.event_dispatcher + .process_new_mempool_txs(net_receipts.mempool_txs_added); + } + + let num_unconfirmed_microblock_tx_receipts = + net_receipts.processed_unconfirmed_state.receipts.len(); + if num_unconfirmed_microblock_tx_receipts > 0 { + if let Some(unconfirmed_state) = self.chainstate_ref().unconfirmed_state.as_ref() { + let canonical_tip = unconfirmed_state.confirmed_chain_tip.clone(); + self.event_dispatcher.process_new_microblocks( + canonical_tip, + net_receipts.processed_unconfirmed_state, + ); + } else { + warn!("Relayer: oops, unconfirmed state is uninitialized but there are microblock events"); + } + } + + // Dispatch retrieved attachments, if any. + if net_result.has_attachments() { + self.event_dispatcher + .process_new_attachments(&net_result.attachments); + } + + // synchronize unconfirmed tx index to p2p thread + self.with_chainstate(|relayer_thread, _sortdb, chainstate, _mempool| { + relayer_thread.globals.send_unconfirmed_txs(chainstate); + }); + + // resume mining if we blocked it, and if we've done the requisite download + // passes + self.last_network_download_passes = net_result.num_download_passes; + if self.min_network_download_passes <= self.last_network_download_passes + || !self.config.miner.wait_for_block_download + { + debug!("Relayer: did a download pass, so unblocking mining"); + signal_mining_ready(self.globals.get_miner_status()); + } + } + + /// Process the block and microblocks from a sortition that we won. + /// At this point, we're modifying the chainstate, and merging the artifacts from the previous tenure. + /// Blocks until the given stacks block is processed + fn accept_winning_tenure( + &mut self, + anchored_block: &StacksBlock, + consensus_hash: &ConsensusHash, + parent_consensus_hash: &ConsensusHash, + ) -> Result { + let stacks_blocks_processed = self.globals.coord_comms.get_stacks_blocks_processed(); + + if StacksChainState::has_stored_block( + self.chainstate_ref().db(), + &self.chainstate_ref().blocks_path, + consensus_hash, + &anchored_block.block_hash(), + )? { + // already processed my tenure + return Ok(true); + } + let burn_height = + SortitionDB::get_block_snapshot_consensus(self.sortdb_ref().conn(), consensus_hash) + .map_err(|e| { + error!("Failed to find block snapshot for mined block: {}", e); + e + })? + .ok_or_else(|| { + error!("Failed to find block snapshot for mined block"); + ChainstateError::NoSuchBlockError + })? + .block_height; + + let ast_rules = SortitionDB::get_ast_rules(self.sortdb_ref().conn(), burn_height)?; + + // failsafe + if !Relayer::static_check_problematic_relayed_block( + self.chainstate_ref().mainnet, + &anchored_block, + ASTRules::PrecheckSize, + ) { + // nope! + warn!( + "Our mined block {} was problematic", + &anchored_block.block_hash() + ); + #[cfg(any(test, feature = "testing"))] + { + use std::fs; + use std::io::Write; + if let Ok(path) = std::env::var("STACKS_BAD_BLOCKS_DIR") { + // record this block somewhere + if !fs::metadata(&path).is_ok() { + fs::create_dir_all(&path) + .expect(&format!("FATAL: could not create '{}'", &path)); + } + + let path = Path::new(&path); + let path = path.join(Path::new(&format!("{}", &anchored_block.block_hash()))); + let mut file = fs::File::create(&path) + .expect(&format!("FATAL: could not create '{:?}'", &path)); + + let block_bits = anchored_block.serialize_to_vec(); + let block_bits_hex = to_hex(&block_bits); + let block_json = format!( + r#"{{"block":"{}","consensus":"{}"}}"#, + &block_bits_hex, &consensus_hash + ); + file.write_all(&block_json.as_bytes()).expect(&format!( + "FATAL: failed to write block bits to '{:?}'", + &path + )); + info!( + "Fault injection: bad block {} saved to {}", + &anchored_block.block_hash(), + &path.to_str().unwrap() + ); + } + } + if !Relayer::process_mined_problematic_blocks(ast_rules, ASTRules::PrecheckSize) { + // don't process it + warn!( + "Will NOT process our problematic mined block {}", + &anchored_block.block_hash() + ); + return Err(ChainstateError::NoTransactionsToMine); + } else { + warn!( + "Will process our problematic mined block {}", + &anchored_block.block_hash() + ) + } + } + + // Preprocess the anchored block + self.with_chainstate(|_relayer_thread, sort_db, chainstate, _mempool| { + let ic = sort_db.index_conn(); + chainstate.preprocess_anchored_block( + &ic, + consensus_hash, + &anchored_block, + &parent_consensus_hash, + 0, + ) + })?; + + if !self.globals.coord_comms.announce_new_stacks_block() { + return Ok(false); + } + if !self + .globals + .coord_comms + .wait_for_stacks_blocks_processed(stacks_blocks_processed, u64::MAX) + { + warn!("ChainsCoordinator timed out while waiting for new stacks block to be processed"); + } + + Ok(true) + } + + /// Given the pointer to a recently-discovered tenure, see if we won the sortition and if so, + /// store it, preprocess it, and forward it to our neighbors. All the while, keep track of the + /// latest Stacks mining tip we have produced so far. + /// + /// Returns (true, Some(tip)) if the coordinator is still running and we have a miner tip to + /// build on (i.e. we won this last sortition). + /// + /// Returns (true, None) if the coordinator is still running, and we do NOT have a miner tip to + /// build on (i.e. we did not win this last sortition) + /// + /// Returns (false, _) if the coordinator could not be reached, meaning this thread should die. + pub fn process_one_tenure( + &mut self, + consensus_hash: ConsensusHash, + block_header_hash: BlockHeaderHash, + burn_hash: BurnchainHeaderHash, + ) -> (bool, Option) { + let miner_tip; + let sn = + SortitionDB::get_block_snapshot_consensus(self.sortdb_ref().conn(), &consensus_hash) + .expect("FATAL: failed to query sortition DB") + .expect("FATAL: unknown consensus hash"); + + debug!( + "Relayer: Process tenure {}/{} in {} burn height {}", + &consensus_hash, &block_header_hash, &burn_hash, sn.block_height + ); + + if let Some((last_mined_block_data, microblock_privkey)) = + self.last_mined_blocks.remove(&block_header_hash) + { + // we won! + let AssembledAnchorBlock { + parent_consensus_hash, + anchored_block: mined_block, + my_burn_hash: mined_burn_hash, + attempt: _, + .. + } = last_mined_block_data; + + let reward_block_height = mined_block.header.total_work.work + MINER_REWARD_MATURITY; + info!( + "Relayer: Won sortition! Mining reward will be received in {} blocks (block #{})", + MINER_REWARD_MATURITY, reward_block_height + ); + debug!("Relayer: Won sortition!"; + "stacks_header" => %block_header_hash, + "burn_hash" => %mined_burn_hash, + ); + + increment_stx_blocks_mined_counter(); + match self.accept_winning_tenure(&mined_block, &consensus_hash, &parent_consensus_hash) + { + Ok(coordinator_running) => { + if !coordinator_running { + warn!("Coordinator stopped, stopping relayer thread..."); + return (false, None); + } + } + Err(e) => { + warn!("Error processing my tenure, bad block produced: {}", e); + warn!( + "Bad block"; + "stacks_header" => %block_header_hash, + "data" => %to_hex(&mined_block.serialize_to_vec()), + ); + return (true, None); + } + }; + + // advertize _and_ push blocks for now + let blocks_available = Relayer::load_blocks_available_data( + self.sortdb_ref(), + vec![consensus_hash.clone()], + ) + .expect("Failed to obtain block information for a block we mined."); + + let block_data = { + let mut bd = HashMap::new(); + bd.insert(consensus_hash.clone(), mined_block.clone()); + bd + }; + + if let Err(e) = self.relayer.advertize_blocks(blocks_available, block_data) { + warn!("Failed to advertise new block: {}", e); + } + + let snapshot = SortitionDB::get_block_snapshot_consensus( + self.sortdb_ref().conn(), + &consensus_hash, + ) + .expect("Failed to obtain snapshot for block") + .expect("Failed to obtain snapshot for block"); + + if !snapshot.pox_valid { + warn!( + "Snapshot for {} is no longer valid; discarding {}...", + &consensus_hash, + &mined_block.block_hash() + ); + miner_tip = None; + } else { + let ch = snapshot.consensus_hash.clone(); + let bh = mined_block.block_hash(); + + if let Err(e) = self + .relayer + .broadcast_block(snapshot.consensus_hash, mined_block) + { + warn!("Failed to push new block: {}", e); + } + + // proceed to mine microblocks + miner_tip = Some(MinerTip::new(ch, bh, microblock_privkey)); + } + } else { + debug!( + "Relayer: Did not win sortition in {}, winning block was {}/{}", + &burn_hash, &consensus_hash, &block_header_hash + ); + miner_tip = None; + } + + (true, miner_tip) + } + + /// Process all new tenures that we're aware of. + /// Clear out stale tenure artifacts as well. + /// Update the miner tip if we won the highest tenure (or clear it if we didn't). + /// If we won any sortitions, send the block and microblock data to the p2p thread. + /// Return true if we can still continue to run; false if not. + pub fn process_new_tenures( + &mut self, + consensus_hash: ConsensusHash, + burn_hash: BurnchainHeaderHash, + block_header_hash: BlockHeaderHash, + ) -> bool { + let mut miner_tip = None; + + // process all sortitions between the last-processed consensus hash and this + // one. ProcessTenure(..) messages can get lost. + let burn_tip = SortitionDB::get_canonical_burn_chain_tip(self.sortdb_ref().conn()) + .expect("FATAL: failed to read current burnchain tip"); + + let tenures = if let Some(last_ch) = self.last_tenure_consensus_hash.as_ref() { + let mut tenures = vec![]; + let last_sn = + SortitionDB::get_block_snapshot_consensus(self.sortdb_ref().conn(), &last_ch) + .expect("FATAL: failed to query sortition DB") + .expect("FATAL: unknown prior consensus hash"); + + debug!( + "Relayer: query tenures between burn block heights {} and {}", + last_sn.block_height + 1, + burn_tip.block_height + 1 + ); + for block_to_process in (last_sn.block_height + 1)..(burn_tip.block_height + 1) { + let sn = { + let ic = self.sortdb_ref().index_conn(); + SortitionDB::get_ancestor_snapshot( + &ic, + block_to_process, + &burn_tip.sortition_id, + ) + .expect("FATAL: failed to read ancestor snapshot from sortition DB") + .expect("Failed to find block in fork processed by burnchain indexer") + }; + if !sn.sortition { + debug!( + "Relayer: Skipping tenure {}/{} at burn hash/height {},{} -- no sortition", + &sn.consensus_hash, + &sn.winning_stacks_block_hash, + &sn.burn_header_hash, + sn.block_height + ); + continue; + } + debug!( + "Relayer: Will process tenure {}/{} at burn hash/height {},{}", + &sn.consensus_hash, + &sn.winning_stacks_block_hash, + &sn.burn_header_hash, + sn.block_height + ); + tenures.push(( + sn.consensus_hash, + sn.burn_header_hash, + sn.winning_stacks_block_hash, + )); + } + tenures + } else { + // first-ever tenure processed + vec![(consensus_hash, burn_hash, block_header_hash)] + }; + + debug!("Relayer: will process {} tenures", &tenures.len()); + let num_tenures = tenures.len(); + if num_tenures > 0 { + // temporarily halt mining + signal_mining_blocked(self.globals.get_miner_status()); + } + + for (consensus_hash, burn_hash, block_header_hash) in tenures.into_iter() { + self.miner_thread_try_join(); + let (continue_thread, new_miner_tip) = + self.process_one_tenure(consensus_hash, block_header_hash, burn_hash); + if !continue_thread { + // coordinator thread hang-up + return false; + } + miner_tip = new_miner_tip; + + // clear all blocks up to this consensus hash + let this_burn_tip = SortitionDB::get_block_snapshot_consensus( + self.sortdb_ref().conn(), + &consensus_hash, + ) + .expect("FATAL: failed to query sortition DB") + .expect("FATAL: no snapshot for consensus hash"); + + let old_last_mined_blocks = + mem::replace(&mut self.last_mined_blocks, MinedBlocks::new()); + self.last_mined_blocks = + Self::clear_stale_mined_blocks(this_burn_tip.block_height, old_last_mined_blocks); + + // update last-tenure pointer + self.last_tenure_consensus_hash = Some(consensus_hash); + } + + if let Some(miner_tip) = miner_tip.as_ref() { + debug!( + "Relayer: Microblock miner tip is now {}/{} ({})", + miner_tip.consensus_hash, + miner_tip.block_hash, + StacksBlockHeader::make_index_block_hash( + &miner_tip.consensus_hash, + &miner_tip.block_hash + ) + ); + + self.with_chainstate(|relayer_thread, sortdb, chainstate, _mempool| { + Relayer::refresh_unconfirmed(chainstate, sortdb); + relayer_thread.globals.send_unconfirmed_txs(chainstate); + }); + } + + // resume mining if we blocked it + if num_tenures > 0 { + signal_mining_ready(self.globals.get_miner_status()); + } + + // update state + self.miner_tip = miner_tip; + true } + /// Constructs and returns a LeaderKeyRegisterOp out of the provided params + fn inner_generate_leader_key_register_op( + address: StacksAddress, + vrf_public_key: VRFPublicKey, + consensus_hash: &ConsensusHash, + ) -> BlockstackOperationType { + BlockstackOperationType::LeaderKeyRegister(LeaderKeyRegisterOp { + public_key: vrf_public_key, + memo: vec![], + address, + consensus_hash: consensus_hash.clone(), + vtxindex: 0, + txid: Txid([0u8; 32]), + block_height: 0, + burn_header_hash: BurnchainHeaderHash::zero(), + }) + } + + /// Create and broadcast a VRF public key registration transaction. + /// Returns true if we succeed in doing so; false if not. + pub fn rotate_vrf_and_register(&mut self, burn_block: &BlockSnapshot) -> bool { + let is_mainnet = self.config.is_mainnet(); + let vrf_pk = self.keychain.rotate_vrf_keypair(burn_block.block_height); + let burnchain_tip_consensus_hash = &burn_block.consensus_hash; + let op = Self::inner_generate_leader_key_register_op( + self.keychain.get_address(is_mainnet), + vrf_pk, + burnchain_tip_consensus_hash, + ); + + let mut one_off_signer = self.keychain.generate_op_signer(); + self.bitcoin_controller + .submit_operation(op, &mut one_off_signer, 1) + } + + /// Remove any block state we've mined for the given burnchain height. + /// Return the filtered `last_mined_blocks` + fn clear_stale_mined_blocks(burn_height: u64, last_mined_blocks: MinedBlocks) -> MinedBlocks { + let mut ret = HashMap::new(); + for (stacks_bhh, (assembled_block, microblock_privkey)) in last_mined_blocks.into_iter() { + if assembled_block.my_block_height < burn_height { + debug!( + "Stale mined block: {} (as of {},{})", + &stacks_bhh, &assembled_block.my_burn_hash, assembled_block.my_block_height + ); + continue; + } + debug!( + "Mined block in-flight: {} (as of {},{})", + &stacks_bhh, &assembled_block.my_burn_hash, assembled_block.my_block_height + ); + ret.insert(stacks_bhh, (assembled_block, microblock_privkey)); + } + ret + } + + /// Create the block miner thread state + fn create_block_miner( + &mut self, + registered_key: RegisteredKey, + last_burn_block: BlockSnapshot, + issue_timestamp_ms: u128, + ) -> Option { + if self + .globals + .get_miner_status() + .lock() + .expect("FATAL: mutex poisoned") + .is_blocked() + { + debug!( + "Relayer: miner is blocked as of {}; cannot mine Stacks block at this time", + &last_burn_block.burn_header_hash + ); + return None; + } + + // start a new tenure + if let Some(cur_sortition) = self.globals.get_last_sortition() { + if last_burn_block.sortition_id != cur_sortition.sortition_id { + debug!( + "Relayer: Drop stale RunTenure for {}: current sortition is for {}", + &last_burn_block.burn_header_hash, &cur_sortition.burn_header_hash + ); + self.globals.counters.bump_missed_tenures(); + return None; + } + } + + let burn_header_hash = last_burn_block.burn_header_hash.clone(); + let burn_chain_sn = SortitionDB::get_canonical_burn_chain_tip(self.sortdb_ref().conn()) + .expect("FATAL: failed to query sortition DB for canonical burn chain tip"); + + let burn_chain_tip = burn_chain_sn.burn_header_hash.clone(); + + if burn_chain_tip != burn_header_hash { + debug!( + "Relayer: Drop stale RunTenure for {}: current sortition is for {}", + &burn_header_hash, &burn_chain_tip + ); + self.globals.counters.bump_missed_tenures(); + return None; + } + + if let Some(stacks_tip) = self + .chainstate_ref() + .get_stacks_chain_tip(self.sortdb_ref()) + .expect("FATAL: could not query chain tip") + { + let has_unprocessed = StacksChainState::has_higher_unprocessed_blocks( + self.chainstate_ref().db(), + stacks_tip.height, + ) + .expect("FATAL: failed to query staging blocks"); + if has_unprocessed { + debug!( + "Relayer: Drop RunTenure for {} because there are pending blocks", + &burn_header_hash + ); + return None; + } + } + + if burn_chain_sn.block_height == self.last_network_block_height + && (self.last_network_download_passes < self.min_network_download_passes + && self.config.miner.wait_for_block_download) + { + debug!("Relayer: network has not had a chance to process in-flight blocks ({} == {} && {} < {})", + burn_chain_sn.block_height, self.last_network_block_height, self.last_network_download_passes, self.min_network_download_passes); + return None; + } + + let tenure_cooldown = if self.config.node.mine_microblocks { + self.config.node.wait_time_for_microblocks as u128 + } else { + 0 + }; + + // no burnchain change, so only re-run block tenure every so often in order + // to give microblocks a chance to collect + if issue_timestamp_ms < self.last_tenure_issue_time + tenure_cooldown { + debug!("Relayer: will NOT run tenure since issuance at {} is too fresh (wait until {} + {} = {})", + issue_timestamp_ms / 1000, self.last_tenure_issue_time / 1000, tenure_cooldown / 1000, (self.last_tenure_issue_time + tenure_cooldown) / 1000); + return None; + } + + // if we're still mining on this burn block, then do nothing + if self.miner_thread.is_some() { + debug!("Relayer: will NOT run tenure since miner thread is already running for burn tip {}", &burn_chain_tip); + return None; + } + + debug!( + "Relayer: Spawn tenure thread"; + "height" => last_burn_block.block_height, + "burn_header_hash" => %burn_chain_tip, + "last_burn_header_hash" => %burn_header_hash + ); + + let miner_thread_state = + BlockMinerThread::from_relayer_thread(self, registered_key, last_burn_block); + Some(miner_thread_state) + } + + /// Try to start up a block miner thread with this given VRF key and current burnchain tip. + /// Returns true if the thread was started; false if it was not (for any reason) + pub fn block_miner_thread_try_start( + &mut self, + registered_key: RegisteredKey, + last_burn_block: BlockSnapshot, + issue_timestamp_ms: u128, + ) -> bool { + if !self.miner_thread_try_join() { + return false; + } + + let mut miner_thread_state = + match self.create_block_miner(registered_key, last_burn_block, issue_timestamp_ms) { + Some(state) => state, + None => { + return false; + } + }; + + if let Ok(miner_handle) = thread::Builder::new() + .name(format!("miner-{}", self.local_peer.data_url)) + .spawn(move || miner_thread_state.run_tenure()) + .map_err(|e| { + error!("Relayer: Failed to start tenure thread: {:?}", &e); + e + }) + { + self.miner_thread = Some(miner_handle); + } + + true + } + + /// Start up a microblock miner thread if we can: + /// * no miner thread must be running already + /// * the miner must not be blocked + /// * we must have won the sortition on the stacks chain tip + /// Returns true if the thread was started; false if not. + pub fn microblock_miner_thread_try_start( + &mut self, + burnchain_tip: BlockSnapshot, + tenure_issue_ms: u128, + ) -> bool { + if !self.config.node.mine_microblocks { + // node will not mine microblocks + return false; + } + if self.last_microblock_tenure_time > tenure_issue_ms { + // stale request + return false; + } + if !self.mined_stacks_block { + // have not tried to mine a stacks block yet that confirms previously-mined unconfirmed + // state (or have not tried to mine a new Stacks block yet for this active tenure); + return false; + } + if let Some(cur_sortition) = self.globals.get_last_sortition() { + if burnchain_tip.sortition_id != cur_sortition.sortition_id { + debug!("Relayer: Drop stale RunMicroblockTenure for {}/{}: current sortition is for {} ({})", &burnchain_tip.consensus_hash, &burnchain_tip.winning_stacks_block_hash, &cur_sortition.consensus_hash, &cur_sortition.burn_header_hash); + return false; + } + } + let miner_tip = match self.miner_tip.as_ref() { + Some(tip) => tip.clone(), + None => { + debug!("Relayer: did not win last block, so cannot mine microblocks"); + return false; + } + }; + if !self.miner_thread_try_join() { + // already running (for an anchored block or microblock) + return false; + } + if self + .globals + .get_miner_status() + .lock() + .expect("FATAL: mutex poisoned") + .is_blocked() + { + debug!( + "Relayer: miner is blocked as of {}; cannot mine microblock at this time", + &burnchain_tip.burn_header_hash + ); + self.globals.counters.set_microblocks_processed(0); + return false; + } + + let parent_consensus_hash = &miner_tip.consensus_hash; + let parent_block_hash = &miner_tip.block_hash; + + debug!( + "Relayer: Run microblock tenure for {}/{}", + parent_consensus_hash, parent_block_hash + ); + + let mut microblock_thread_state = match MicroblockMinerThread::from_relayer_thread(self) { + Some(ts) => ts, + None => { + return false; + } + }; + + if let Ok(miner_handle) = thread::Builder::new() + .name(format!("miner-{}", self.local_peer.data_url)) + .spawn(move || { + Some(MinerThreadResult::Microblock( + microblock_thread_state.try_mine_microblock(miner_tip.clone()), + miner_tip, + )) + }) + .map_err(|e| { + error!("Relayer: Failed to start tenure thread: {:?}", &e); + e + }) + { + self.miner_thread = Some(miner_handle); + } + + true + } + + /// Inner body of Self::miner_thread_try_join + fn inner_miner_thread_try_join( + &mut self, + thread_handle: JoinHandle>, + ) -> Option>> { + // tenure run already in progress; try and join + if !thread_handle.is_finished() { + debug!("Relayer: RunTenure thread not finished / is in-progress"); + return Some(thread_handle); + } + let last_mined_block_opt = thread_handle + .join() + .expect("FATAL: failed to join miner thread"); + if let Some(miner_result) = last_mined_block_opt { + match miner_result { + MinerThreadResult::Block( + last_mined_block, + modified_keychain, + microblock_privkey, + ongoing_commit_opt, + ) => { + // finished mining a block + if self.last_mined_blocks.len() == 0 { + self.globals.counters.bump_blocks_processed(); + } + + debug!( + "Relayer: RunTenure thread joined; got Stacks block {}", + &last_mined_block.anchored_block.block_hash() + ); + + let bhh = last_mined_block.my_burn_hash.clone(); + let orig_bhh = last_mined_block.orig_burn_hash.clone(); + let tenure_begin = last_mined_block.tenure_begin; + + // keep our keychain up-to-date with the miner's progress + self.keychain = modified_keychain; + + self.last_mined_blocks.insert( + last_mined_block.anchored_block.block_hash(), + (last_mined_block, microblock_privkey), + ); + + self.last_tenure_issue_time = get_epoch_time_ms(); + self.bitcoin_controller + .set_ongoing_commit(ongoing_commit_opt); + + debug!( + "Relayer: RunTenure finished at {} (in {}ms) targeting {} (originally {})", + self.last_tenure_issue_time, + self.last_tenure_issue_time.saturating_sub(tenure_begin), + &bhh, + &orig_bhh + ); + + // this stacks block confirms all in-flight microblocks we know about, + // including the ones we produced. + self.mined_stacks_block = true; + } + MinerThreadResult::Microblock(microblock_result, miner_tip) => { + // finished mining a microblock + match microblock_result { + Ok(Some(next_microblock)) => { + // apply it + let microblock_hash = next_microblock.block_hash(); + + let (processed_unconfirmed_state, num_mblocks) = self.with_chainstate( + |_relayer_thread, sortdb, chainstate, _mempool| { + let processed_unconfirmed_state = + Relayer::refresh_unconfirmed(chainstate, sortdb); + let num_mblocks = chainstate + .unconfirmed_state + .as_ref() + .map(|ref unconfirmed| unconfirmed.num_microblocks()) + .unwrap_or(0); + + (processed_unconfirmed_state, num_mblocks) + }, + ); + + info!( + "Mined one microblock: {} seq {} txs {} (total processed: {})", + µblock_hash, + next_microblock.header.sequence, + next_microblock.txs.len(), + num_mblocks + ); + self.globals.counters.set_microblocks_processed(num_mblocks); + + let parent_index_block_hash = StacksBlockHeader::make_index_block_hash( + &miner_tip.consensus_hash, + &miner_tip.block_hash, + ); + self.event_dispatcher.process_new_microblocks( + parent_index_block_hash, + processed_unconfirmed_state, + ); + + // send it off + if let Err(e) = self.relayer.broadcast_microblock( + &miner_tip.consensus_hash, + &miner_tip.block_hash, + next_microblock, + ) { + error!( + "Failure trying to broadcast microblock {}: {}", + microblock_hash, e + ); + } + + // synchronise state + self.with_chainstate( + |relayer_thread, _sortdb, chainstate, _mempool| { + relayer_thread.globals.send_unconfirmed_txs(chainstate); + }, + ); + self.last_microblock_tenure_time = get_epoch_time_ms(); + + // have not yet mined a stacks block that confirms this microblock, so + // do that on the next run + self.mined_stacks_block = false; + } + Ok(None) => { + debug!("Relayer: did not mine microblock in this tenure") + } + Err(e) => { + warn!("Relayer: Failed to mine next microblock: {:?}", &e); + } + } + } + } + } + None + } + + /// Try to join with the miner thread. If we succeed, join the thread and return true. + /// Otherwise, if the thread is still running, return false; + /// Updates internal state gleaned from the miner, such as: + /// * new stacks block data + /// * new keychain state + /// * new metrics + /// * new unconfirmed state + /// Returns true if joined; false if not. + pub fn miner_thread_try_join(&mut self) -> bool { + if let Some(thread_handle) = self.miner_thread.take() { + let new_thread_handle = self.inner_miner_thread_try_join(thread_handle); + self.miner_thread = new_thread_handle; + } + self.miner_thread.is_none() + } + + /// Top-level dispatcher + pub fn handle_directive(&mut self, directive: RelayerDirective) -> bool { + debug!("Relayer: received next directive"); + let continue_running = match directive { + RelayerDirective::HandleNetResult(net_result) => { + self.process_network_result(net_result); + true + } + RelayerDirective::RegisterKey(last_burn_block) => { + self.rotate_vrf_and_register(&last_burn_block); + self.globals.counters.bump_blocks_processed(); + true + } + RelayerDirective::ProcessTenure(consensus_hash, burn_hash, block_header_hash) => { + self.process_new_tenures(consensus_hash, burn_hash, block_header_hash) + } + RelayerDirective::RunTenure(registered_key, last_burn_block, issue_timestamp_ms) => { + self.block_miner_thread_try_start( + registered_key, + last_burn_block, + issue_timestamp_ms, + ); + true + } + RelayerDirective::RunMicroblockTenure(burnchain_tip, tenure_issue_ms) => { + self.microblock_miner_thread_try_start(burnchain_tip, tenure_issue_ms); + true + } + RelayerDirective::Exit => false, + }; + continue_running + } +} + +impl ParentStacksBlockInfo { /// Determine where in the set of forks to attempt to mine the next anchored block. /// `mine_tip_ch` and `mine_tip_bhh` identify the parent block on top of which to mine. /// `check_burn_block` identifies what we believe to be the burn chain's sortition history tip. /// This is used to mitigate (but not eliminate) a TOCTTOU issue with mining: the caller's /// conception of the sortition history tip may have become stale by the time they call this /// method, in which case, mining should *not* happen (since the block will be invalid). - fn get_mining_tenure_information( + pub fn lookup( chain_state: &mut StacksChainState, burn_db: &mut SortitionDB, check_burn_block: &BlockSnapshot, miner_address: StacksAddress, mine_tip_ch: &ConsensusHash, mine_tip_bh: &BlockHeaderHash, - ) -> Result { + ) -> Result { let stacks_tip_header = StacksChainState::get_anchored_block_header_info( chain_state.db(), &mine_tip_ch, @@ -1806,7 +2811,7 @@ impl StacksNode { account.nonce }; - Ok(MiningTenureInformation { + Ok(ParentStacksBlockInfo { stacks_parent_header: stacks_tip_header, parent_consensus_hash: mine_tip_ch.clone(), parent_block_burn_height: parent_block.block_height, @@ -1815,503 +2820,787 @@ impl StacksNode { coinbase_nonce, }) } +} - /// Return the assembled anchor block info and microblock private key on success. - /// Return None if we couldn't build a block for whatever reason - fn relayer_run_tenure( - config: &Config, - registered_key: RegisteredKey, - chain_state: &mut StacksChainState, - burn_db: &mut SortitionDB, - burnchain: &Burnchain, - burn_block: BlockSnapshot, - keychain: &mut Keychain, - mem_pool: &mut MemPoolDB, - burn_fee_cap: u64, - bitcoin_controller: &mut BitcoinRegtestController, - last_mined_blocks: &Vec<&AssembledAnchorBlock>, - event_dispatcher: &EventDispatcher, - ) -> Option<(AssembledAnchorBlock, Secp256k1PrivateKey)> { - let MiningTenureInformation { - mut stacks_parent_header, - parent_consensus_hash, - parent_block_burn_height, - parent_block_total_burn, - parent_winning_vtxindex, - coinbase_nonce, - } = if let Some(stacks_tip) = chain_state - .get_stacks_chain_tip(burn_db) - .expect("FATAL: could not query chain tip") - { - let miner_address = keychain.origin_address(config.is_mainnet()).unwrap(); - Self::get_mining_tenure_information( - chain_state, - burn_db, - &burn_block, - miner_address, - &stacks_tip.consensus_hash, - &stacks_tip.anchored_block_hash, - ) - .ok()? - } else { - debug!("No Stacks chain tip known, will return a genesis block"); - let (network, _) = config.burnchain.get_bitcoin_network(); - let burnchain_params = - BurnchainParameters::from_params(&config.burnchain.chain, &network) - .expect("Bitcoin network unsupported"); +/// Thread that runs the network state machine, handling both p2p and http requests. +pub struct PeerThread { + /// Node config + config: Config, + /// instance of the peer network. Made optional in order to trick the borrow checker. + net: Option, + /// handle to global inter-thread comms + globals: Globals, + /// how long to wait for network messages on each poll, in millis + poll_timeout: u64, + /// receiver for attachments discovered by the chains coordinator thread + attachments_rx: Receiver>, + /// handle to the sortition DB (optional so we can take/replace it) + sortdb: Option, + /// handle to the chainstate DB (optional so we can take/replace it) + chainstate: Option, + /// handle to the mempool DB (optional so we can take/replace it) + mempool: Option, + /// buffer of relayer commands with block data that couldn't be sent to the relayer just yet + /// (i.e. due to backpressure). We track this separately, instead of just using a bigger + /// channel, because we need to know when backpressure occurs in order to throttle the p2p + /// thread's downloader. + results_with_data: VecDeque, + /// total number of p2p state-machine passes so far. Used to signal when to download the next + /// reward cycle of blocks + num_p2p_state_machine_passes: u64, + /// total number of inventory state-machine passes so far. Used to signal when to download the + /// next reward cycle of blocks. + num_inv_sync_passes: u64, + /// total number of download state-machine passes so far. Used to signal when to download the + /// next reward cycle of blocks. + num_download_passes: u64, + /// when should we queue up the next request to run a microblock tenure? (epoch time in millis) + mblock_deadline: u128, +} - let chain_tip = ChainTip::genesis( - &burnchain_params.first_block_hash, - burnchain_params.first_block_height.into(), - burnchain_params.first_block_timestamp.into(), - ); +impl PeerThread { + /// set up the mempool DB connection + fn connect_mempool_db(config: &Config) -> MemPoolDB { + // create estimators, metric instances for RPC handler + let cost_estimator = config + .make_cost_estimator() + .unwrap_or_else(|| Box::new(UnitEstimator)); + let metric = config + .make_cost_metric() + .unwrap_or_else(|| Box::new(UnitMetric)); - MiningTenureInformation { - stacks_parent_header: chain_tip.metadata, - parent_consensus_hash: FIRST_BURNCHAIN_CONSENSUS_HASH.clone(), - parent_block_burn_height: 0, - parent_block_total_burn: 0, - parent_winning_vtxindex: 0, - coinbase_nonce: 0, - } - }; - - // has the tip changed from our previously-mined block for this epoch? - let attempt = if last_mined_blocks.len() <= 1 { - // always mine if we've not mined a block for this epoch yet, or - // if we've mined just one attempt, unconditionally try again (so we - // can use `subsequent_miner_time_ms` in this attempt) - if last_mined_blocks.len() == 1 { - debug!("Have only attempted one block; unconditionally trying again"); - } - last_mined_blocks.len() as u64 + 1 - } else { - let mut best_attempt = 0; - debug!( - "Consider {} in-flight Stacks tip(s)", - &last_mined_blocks.len() - ); - for prev_block in last_mined_blocks.iter() { - debug!( - "Consider in-flight block {} on Stacks tip {}/{} in {} with {} txs", - &prev_block.anchored_block.block_hash(), - &prev_block.parent_consensus_hash, - &prev_block.anchored_block.header.parent_block, - &prev_block.my_burn_hash, - &prev_block.anchored_block.txs.len() - ); - - if prev_block.anchored_block.txs.len() == 1 && prev_block.attempt == 1 { - // Don't let the fact that we've built an empty block during this sortition - // prevent us from trying again. - best_attempt = 1; - continue; - } - if prev_block.parent_consensus_hash == parent_consensus_hash - && prev_block.my_burn_hash == burn_block.burn_header_hash - && prev_block.anchored_block.header.parent_block - == stacks_parent_header.anchored_header.block_hash() - { - // the anchored chain tip hasn't changed since we attempted to build a block. - // But, have discovered any new microblocks worthy of being mined? - if let Ok(Some(stream)) = - StacksChainState::load_descendant_staging_microblock_stream( - chain_state.db(), - &StacksBlockHeader::make_index_block_hash( - &prev_block.parent_consensus_hash, - &stacks_parent_header.anchored_header.block_hash(), - ), - 0, - u16::MAX, - ) - { - if (prev_block.anchored_block.header.parent_microblock - == BlockHeaderHash([0u8; 32]) - && stream.len() == 0) - || (prev_block.anchored_block.header.parent_microblock - != BlockHeaderHash([0u8; 32]) - && stream.len() - <= (prev_block.anchored_block.header.parent_microblock_sequence - as usize) - + 1) - { - // the chain tip hasn't changed since we attempted to build a block. Use what we - // already have. - debug!("Stacks tip is unchanged since we last tried to mine a block off of {}/{} at height {} with {} txs, in {} at burn height {}, and no new microblocks ({} <= {})", - &prev_block.parent_consensus_hash, &prev_block.anchored_block.header.parent_block, prev_block.anchored_block.header.total_work.work, - prev_block.anchored_block.txs.len(), prev_block.my_burn_hash, parent_block_burn_height, stream.len(), prev_block.anchored_block.header.parent_microblock_sequence); - - return None; - } else { - // there are new microblocks! - // TODO: only consider rebuilding our anchored block if we (a) have - // time, and (b) the new microblocks are worth more than the new BTC - // fee minus the old BTC fee - debug!("Stacks tip is unchanged since we last tried to mine a block off of {}/{} at height {} with {} txs, in {} at burn height {}, but there are new microblocks ({} > {})", - &prev_block.parent_consensus_hash, &prev_block.anchored_block.header.parent_block, prev_block.anchored_block.header.total_work.work, - prev_block.anchored_block.txs.len(), prev_block.my_burn_hash, parent_block_burn_height, stream.len(), prev_block.anchored_block.header.parent_microblock_sequence); - - best_attempt = cmp::max(best_attempt, prev_block.attempt); - } - } else { - // no microblock stream to confirm, and the stacks tip hasn't changed - debug!("Stacks tip is unchanged since we last tried to mine a block off of {}/{} at height {} with {} txs, in {} at burn height {}, and no microblocks present", - &prev_block.parent_consensus_hash, &prev_block.anchored_block.header.parent_block, prev_block.anchored_block.header.total_work.work, - prev_block.anchored_block.txs.len(), prev_block.my_burn_hash, parent_block_burn_height); - - return None; - } - } else { - if burn_block.burn_header_hash == prev_block.my_burn_hash { - // only try and re-mine if there was no sortition since the last chain tip - debug!("Stacks tip has changed to {}/{} since we last tried to mine a block in {} at burn height {}; attempt was {} (for Stacks tip {}/{})", - parent_consensus_hash, stacks_parent_header.anchored_header.block_hash(), prev_block.my_burn_hash, parent_block_burn_height, prev_block.attempt, &prev_block.parent_consensus_hash, &prev_block.anchored_block.header.parent_block); - best_attempt = cmp::max(best_attempt, prev_block.attempt); - } else { - debug!("Burn tip has changed to {} ({}) since we last tried to mine a block in {}", - &burn_block.burn_header_hash, burn_block.block_height, &prev_block.my_burn_hash); - } - } - } - best_attempt + 1 - }; - - // Generates a proof out of the sortition hash provided in the params. - let vrf_proof = match keychain.generate_proof( - ®istered_key.vrf_public_key, - burn_block.sortition_hash.as_bytes(), - ) { - Some(vrfp) => vrfp, - None => { - // Try to recover a key registered in a former session. - // registered_key.block_height gives us a pointer to the height of the block - // holding the key register op, but the VRF was derived using the height of one - // of the parents blocks. - let _ = keychain.rotate_vrf_keypair(registered_key.block_height - 1); - match keychain.generate_proof( - ®istered_key.vrf_public_key, - burn_block.sortition_hash.as_bytes(), - ) { - Some(vrfp) => vrfp, - None => { - error!( - "Failed to generate proof with {:?}", - ®istered_key.vrf_public_key - ); - return None; - } - } - } - }; - - debug!( - "Generated VRF Proof: {} over {} with key {}", - vrf_proof.to_hex(), - &burn_block.sortition_hash, - ®istered_key.vrf_public_key.to_hex() - ); - - // Generates a new secret key for signing the trail of microblocks - // of the upcoming tenure. - let microblock_secret_key = if attempt > 1 { - match keychain.get_microblock_key() { - Some(k) => k, - None => { - error!( - "Failed to obtain microblock key for mining attempt"; - "attempt" => %attempt - ); - return None; - } - } - } else { - keychain.rotate_microblock_keypair(burn_block.block_height) - }; - let mblock_pubkey_hash = - Hash160::from_node_public_key(&StacksPublicKey::from_private(µblock_secret_key)); - - let coinbase_tx = inner_generate_coinbase_tx( - keychain, - coinbase_nonce, + let mempool = MemPoolDB::open( config.is_mainnet(), config.burnchain.chain_id, - ); + &config.get_chainstate_path_str(), + cost_estimator, + metric, + ) + .expect("Database failure opening mempool"); - // find the longest microblock tail we can build off of - let microblock_info_opt = - match StacksChainState::load_descendant_staging_microblock_stream_with_poison( - chain_state.db(), - &StacksBlockHeader::make_index_block_hash( - &parent_consensus_hash, - &stacks_parent_header.anchored_header.block_hash(), - ), - 0, - u16::MAX, - ) { - Ok(x) => { - let num_mblocks = x.as_ref().map(|(mblocks, ..)| mblocks.len()).unwrap_or(0); - debug!( - "Loaded {} microblocks descending from {}/{}", - num_mblocks, - &parent_consensus_hash, - &stacks_parent_header.anchored_header.block_hash() - ); - x + mempool + } + + /// Instantiate the p2p thread. + /// Binds the addresses in the config (which may panic if the port is blocked). + /// This is so the node will crash "early" before any new threads start if there's going to be + /// a bind error anyway. + pub fn new( + runloop: &RunLoop, + mut net: PeerNetwork, + attachments_rx: Receiver>, + ) -> PeerThread { + let config = runloop.config().clone(); + let mempool = Self::connect_mempool_db(&config); + let burn_db_path = config.get_burn_db_file_path(); + let stacks_chainstate_path = config.get_chainstate_path_str(); + + let sortdb = + SortitionDB::open(&burn_db_path, false).expect("FATAL: could not open sortition DB"); + let (chainstate, _) = StacksChainState::open( + config.is_mainnet(), + config.burnchain.chain_id, + &stacks_chainstate_path, + Some(config.node.get_marf_opts()), + ) + .expect("FATAL: could not open chainstate DB"); + + let p2p_sock: SocketAddr = config.node.p2p_bind.parse().expect(&format!( + "Failed to parse socket: {}", + &config.node.p2p_bind + )); + let rpc_sock = config.node.rpc_bind.parse().expect(&format!( + "Failed to parse socket: {}", + &config.node.rpc_bind + )); + + net.bind(&p2p_sock, &rpc_sock) + .expect("BUG: PeerNetwork could not bind or is already bound"); + + let poll_timeout = cmp::min(5000, config.miner.first_attempt_time_ms / 2000); + + PeerThread { + config, + net: Some(net), + globals: runloop.get_globals(), + poll_timeout, + attachments_rx, + sortdb: Some(sortdb), + chainstate: Some(chainstate), + mempool: Some(mempool), + results_with_data: VecDeque::new(), + num_p2p_state_machine_passes: 0, + num_inv_sync_passes: 0, + num_download_passes: 0, + mblock_deadline: 0, + } + } + + /// Do something with mutable references to the mempool, sortdb, and chainstate + /// Fools the borrow checker. + /// NOT COMPOSIBLE + fn with_chainstate(&mut self, func: F) -> R + where + F: FnOnce(&mut PeerThread, &mut SortitionDB, &mut StacksChainState, &mut MemPoolDB) -> R, + { + let mut sortdb = self.sortdb.take().expect("BUG: sortdb already taken"); + let mut chainstate = self + .chainstate + .take() + .expect("BUG: chainstate already taken"); + let mut mempool = self.mempool.take().expect("BUG: mempool already taken"); + + let res = func(self, &mut sortdb, &mut chainstate, &mut mempool); + + self.sortdb = Some(sortdb); + self.chainstate = Some(chainstate); + self.mempool = Some(mempool); + + res + } + + /// Get an immutable ref to the inner network. + /// DO NOT USE WITHIN with_network() + fn get_network(&self) -> &PeerNetwork { + self.net.as_ref().expect("BUG: did not replace net") + } + + /// Do something with mutable references to the network. + /// Fools the borrow checker. + /// NOT COMPOSIBLE. DO NOT CALL THIS OR get_network() IN func + fn with_network(&mut self, func: F) -> R + where + F: FnOnce(&mut PeerThread, &mut PeerNetwork) -> R, + { + let mut net = self.net.take().expect("BUG: net already taken"); + + let res = func(self, &mut net); + + self.net = Some(net); + res + } + + /// Run one pass of the p2p/http state machine + /// Return true if we should continue running passes; false if not + pub fn run_one_pass( + &mut self, + dns_client_opt: Option<&mut DNSClient>, + event_dispatcher: &EventDispatcher, + cost_estimator: &Box, + cost_metric: &Box, + fee_estimator: Option<&Box>, + ) -> bool { + // initial block download? + let ibd = self.globals.sync_comms.get_ibd(); + let download_backpressure = self.results_with_data.len() > 0; + let poll_ms = if !download_backpressure && self.get_network().has_more_downloads() { + // keep getting those blocks -- drive the downloader state-machine + debug!( + "P2P: backpressure: {}, more downloads: {}", + download_backpressure, + self.get_network().has_more_downloads() + ); + 1 + } else { + cmp::min(self.poll_timeout, self.config.node.microblock_frequency) + }; + + let mut expected_attachments = match self.attachments_rx.try_recv() { + Ok(expected_attachments) => { + debug!("Atlas: received attachments: {:?}", &expected_attachments); + expected_attachments + } + _ => { + debug!("Atlas: attachment channel is empty"); + HashSet::new() + } + }; + + // move over unconfirmed state obtained from the relayer + self.with_chainstate(|p2p_thread, sortdb, chainstate, _mempool| { + let _ = Relayer::setup_unconfirmed_state_readonly(chainstate, sortdb); + p2p_thread.globals.recv_unconfirmed_txs(chainstate); + }); + + // do one pass + let p2p_res = self.with_chainstate(|p2p_thread, sortdb, chainstate, mempool| { + // NOTE: handler_args must be created such that it outlives the inner net.run() call and + // doesn't ref anything within p2p_thread. + let handler_args = RPCHandlerArgs { + exit_at_block_height: p2p_thread + .config + .burnchain + .process_exit_at_block_height + .clone(), + genesis_chainstate_hash: Sha256Sum::from_hex(stx_genesis::GENESIS_CHAINSTATE_HASH) + .unwrap(), + event_observer: Some(event_dispatcher), + cost_estimator: Some(cost_estimator.as_ref()), + cost_metric: Some(cost_metric.as_ref()), + fee_estimator: fee_estimator.map(|boxed_estimator| boxed_estimator.as_ref()), + ..RPCHandlerArgs::default() + }; + p2p_thread.with_network(|_, net| { + net.run( + sortdb, + chainstate, + mempool, + dns_client_opt, + download_backpressure, + ibd, + poll_ms, + &handler_args, + &mut expected_attachments, + ) + }) + }); + + match p2p_res { + Ok(network_result) => { + if self.num_p2p_state_machine_passes < network_result.num_state_machine_passes { + // p2p state-machine did a full pass. Notify anyone listening. + self.globals.sync_comms.notify_p2p_state_pass(); + self.num_p2p_state_machine_passes = network_result.num_state_machine_passes; } - Err(e) => { - warn!( - "Failed to load descendant microblock stream from {}/{}: {:?}", - &parent_consensus_hash, - &stacks_parent_header.anchored_header.block_hash(), - &e - ); - None + + if self.num_inv_sync_passes < network_result.num_inv_sync_passes { + // inv-sync state-machine did a full pass. Notify anyone listening. + self.globals.sync_comms.notify_inv_sync_pass(); + self.num_inv_sync_passes = network_result.num_inv_sync_passes; + } + + if self.num_download_passes < network_result.num_download_passes { + // download state-machine did a full pass. Notify anyone listening. + self.globals.sync_comms.notify_download_pass(); + self.num_download_passes = network_result.num_download_passes; + } + + if network_result.has_data_to_store() { + self.results_with_data + .push_back(RelayerDirective::HandleNetResult(network_result)); + } + + // only do this on the Ok() path, even if we're mining, because an error in + // network dispatching is likely due to resource exhaustion + if self.mblock_deadline < get_epoch_time_ms() && self.config.node.mine_microblocks { + debug!("P2P: schedule microblock tenure"); + self.results_with_data + .push_back(RelayerDirective::RunMicroblockTenure( + self.get_network().burnchain_tip.clone(), + get_epoch_time_ms(), + )); + self.mblock_deadline = + get_epoch_time_ms() + (self.config.node.microblock_frequency as u128); + } + } + Err(e) => { + // this is only reachable if the network is not instantiated correctly -- + // i.e. you didn't connect it + panic!("P2P: Failed to process network dispatch: {:?}", &e); + } + }; + + while let Some(next_result) = self.results_with_data.pop_front() { + // have blocks, microblocks, and/or transactions (don't care about anything else), + // or a directive to mine microblocks + if let Err(e) = self.globals.relay_send.try_send(next_result) { + debug!( + "P2P: {:?}: download backpressure detected", + &self.get_network().local_peer + ); + match e { + TrySendError::Full(directive) => { + if let RelayerDirective::RunMicroblockTenure(..) = directive { + // can drop this + } else if let RelayerDirective::RunTenure(..) = directive { + // can drop this + } else { + // don't lose this data -- just try it again + self.results_with_data.push_front(directive); + } + break; + } + TrySendError::Disconnected(_) => { + info!("P2P: Relayer hang up with p2p channel"); + self.globals.signal_stop(); + return false; + } + } + } else { + debug!("P2P: Dispatched result to Relayer!"); + } + } + + true + } +} + +impl StacksNode { + /// Create a StacksPrivateKey from a given seed buffer + pub fn make_node_private_key_from_seed(seed: &[u8]) -> StacksPrivateKey { + let node_privkey = { + let mut re_hashed_seed = seed.to_vec(); + let my_private_key = loop { + match Secp256k1PrivateKey::from_slice(&re_hashed_seed[..]) { + Ok(sk) => break sk, + Err(_) => { + re_hashed_seed = Sha256Sum::from_data(&re_hashed_seed[..]) + .as_bytes() + .to_vec() + } } }; + my_private_key + }; + node_privkey + } - if let Some((ref microblocks, ref poison_opt)) = µblock_info_opt { - if let Some(ref tail) = microblocks.last() { - debug!( - "Confirm microblock stream tailed at {} (seq {})", - &tail.block_hash(), - tail.header.sequence - ); - } - - // try and confirm as many microblocks as we can (but note that the stream itself may - // be too long; we'll try again if that happens). - stacks_parent_header.microblock_tail = - microblocks.last().clone().map(|blk| blk.header.clone()); - - if let Some(poison_payload) = poison_opt { - let poison_microblock_tx = inner_generate_poison_microblock_tx( - keychain, - coinbase_nonce + 1, - poison_payload.clone(), - config.is_mainnet(), - config.burnchain.chain_id, - ); - - let stacks_epoch = burn_db - .index_conn() - .get_stacks_epoch(burn_block.block_height as u32) - .expect("Could not find a stacks epoch."); - - // submit the poison payload, privately, so we'll mine it when building the - // anchored block. - if let Err(e) = mem_pool.submit( - chain_state, - &parent_consensus_hash, - &stacks_parent_header.anchored_header.block_hash(), - &poison_microblock_tx, - Some(event_dispatcher), - &stacks_epoch.block_limit, - &stacks_epoch.epoch_id, - ) { - warn!( - "Detected but failed to mine poison-microblock transaction: {:?}", - &e - ); - } - } + /// Set up the AST size-precheck height, if configured + fn setup_ast_size_precheck(config: &Config, sortdb: &mut SortitionDB) { + if let Some(ast_precheck_size_height) = config.burnchain.ast_precheck_size_height { + info!( + "Override burnchain height of {:?} to {}", + ASTRules::PrecheckSize, + ast_precheck_size_height + ); + let mut tx = sortdb + .tx_begin() + .expect("FATAL: failed to begin tx on sortition DB"); + SortitionDB::override_ast_rule_height( + &mut tx, + ASTRules::PrecheckSize, + ast_precheck_size_height, + ) + .expect("FATAL: failed to override AST PrecheckSize rule height"); + tx.commit() + .expect("FATAL: failed to commit sortition DB transaction"); } + } - let (anchored_block, _, _) = match StacksBlockBuilder::build_anchored_block( - chain_state, - &burn_db.index_conn(), - mem_pool, - &stacks_parent_header, - parent_block_total_burn, - vrf_proof.clone(), - mblock_pubkey_hash, - &coinbase_tx, - config.make_block_builder_settings((last_mined_blocks.len() + 1) as u64, false), - Some(event_dispatcher), - ) { - Ok(block) => block, - Err(ChainstateError::InvalidStacksMicroblock(msg, mblock_header_hash)) => { - // part of the parent microblock stream is invalid, so try again - info!("Parent microblock stream is invalid; trying again without the offender {} (msg: {})", &mblock_header_hash, &msg); + /// Set up the mempool DB by making sure it exists. + /// Panics on failure. + fn setup_mempool_db(config: &Config) -> MemPoolDB { + // force early mempool instantiation + let cost_estimator = config + .make_cost_estimator() + .unwrap_or_else(|| Box::new(UnitEstimator)); + let metric = config + .make_cost_metric() + .unwrap_or_else(|| Box::new(UnitMetric)); - // truncate the stream - stacks_parent_header.microblock_tail = match microblock_info_opt { - Some((microblocks, _)) => { - let mut tail = None; - for mblock in microblocks.into_iter() { - if mblock.block_hash() == mblock_header_hash { - break; - } - tail = Some(mblock); - } - if let Some(ref t) = &tail { - debug!( - "New parent microblock stream tail is {} (seq {})", - t.block_hash(), - t.header.sequence - ); - } - tail.map(|t| t.header) - } - None => None, - }; + let mempool = MemPoolDB::open( + config.is_mainnet(), + config.burnchain.chain_id, + &config.get_chainstate_path_str(), + cost_estimator, + metric, + ) + .expect("BUG: failed to instantiate mempool"); - // try again - match StacksBlockBuilder::build_anchored_block( - chain_state, - &burn_db.index_conn(), - mem_pool, - &stacks_parent_header, - parent_block_total_burn, - vrf_proof.clone(), - mblock_pubkey_hash, - &coinbase_tx, - config.make_block_builder_settings((last_mined_blocks.len() + 1) as u64, false), - Some(event_dispatcher), - ) { - Ok(block) => block, - Err(e) => { - error!("Failure mining anchor block even after removing offending microblock {}: {}", &mblock_header_hash, &e); - return None; - } - } - } - Err(e) => { - error!("Failure mining anchored block: {}", e); - return None; - } - }; - let block_height = anchored_block.header.total_work.work; - info!( - "Succeeded assembling {} block #{}: {}, with {} txs, attempt {}", - if parent_block_total_burn == 0 { - "Genesis" - } else { - "Stacks" - }, - block_height, - anchored_block.block_hash(), - anchored_block.txs.len(), - attempt - ); + mempool + } - // let's figure out the recipient set! - let recipients = match get_next_recipients( - &burn_block, - chain_state, - burn_db, - burnchain, - &OnChainRewardSetProvider(), - ) { - Ok(x) => x, - Err(e) => { - error!("Failure fetching recipient set: {:?}", e); - return None; - } - }; - - let sunset_burn = burnchain.expected_sunset_burn(burn_block.block_height + 1, burn_fee_cap); - let rest_commit = burn_fee_cap - sunset_burn; - - let commit_outs = if burn_block.block_height + 1 < burnchain.pox_constants.sunset_end - && !burnchain.is_in_prepare_phase(burn_block.block_height + 1) - { - RewardSetInfo::into_commit_outs(recipients, config.is_mainnet()) + /// Set up the Peer DB and update any soft state from the config file. This includes: + /// * blacklisted/whitelisted nodes + /// * node keys + /// * bootstrap nodes + /// Returns the instantiated PeerDB + /// Panics on failure. + fn setup_peer_db(config: &Config, burnchain: &Burnchain) -> PeerDB { + let data_url = UrlString::try_from(format!("{}", &config.node.data_url)).unwrap(); + let initial_neighbors = config.node.bootstrap_node.clone(); + if initial_neighbors.len() > 0 { + info!( + "Will bootstrap from peers {}", + VecDisplay(&initial_neighbors) + ); } else { - vec![StacksAddress::burn_address(config.is_mainnet())] + warn!("Without a peer to bootstrap from, the node will start mining a new chain"); + } + + let p2p_sock: SocketAddr = config.node.p2p_bind.parse().expect(&format!( + "Failed to parse socket: {}", + &config.node.p2p_bind + )); + let p2p_addr: SocketAddr = config.node.p2p_address.parse().expect(&format!( + "Failed to parse socket: {}", + &config.node.p2p_address + )); + let node_privkey = + StacksNode::make_node_private_key_from_seed(&config.node.local_peer_seed); + + let mut peerdb = PeerDB::connect( + &config.get_peer_db_file_path(), + true, + config.burnchain.chain_id, + burnchain.network_id, + Some(node_privkey), + config.connection_options.private_key_lifetime.clone(), + PeerAddress::from_socketaddr(&p2p_addr), + p2p_sock.port(), + data_url, + &vec![], + Some(&initial_neighbors), + ) + .map_err(|e| { + eprintln!( + "Failed to open {}: {:?}", + &config.get_peer_db_file_path(), + &e + ); + panic!(); + }) + .unwrap(); + + // allow all bootstrap nodes + { + let mut tx = peerdb.tx_begin().unwrap(); + for initial_neighbor in initial_neighbors.iter() { + // update peer in case public key changed + PeerDB::update_peer(&mut tx, &initial_neighbor).unwrap(); + PeerDB::set_allow_peer( + &mut tx, + initial_neighbor.addr.network_id, + &initial_neighbor.addr.addrbytes, + initial_neighbor.addr.port, + -1, + ) + .unwrap(); + } + tx.commit().unwrap(); + } + + if !config.node.deny_nodes.is_empty() { + warn!("Will ignore nodes {:?}", &config.node.deny_nodes); + } + + // deny all config-denied peers + { + let mut tx = peerdb.tx_begin().unwrap(); + for denied in config.node.deny_nodes.iter() { + PeerDB::set_deny_peer( + &mut tx, + denied.addr.network_id, + &denied.addr.addrbytes, + denied.addr.port, + get_epoch_time_secs() + 24 * 365 * 3600, + ) + .unwrap(); + } + tx.commit().unwrap(); + } + + // update services to indicate we can support mempool sync + { + let mut tx = peerdb.tx_begin().unwrap(); + PeerDB::set_local_services( + &mut tx, + (ServiceFlags::RPC as u16) | (ServiceFlags::RELAY as u16), + ) + .unwrap(); + tx.commit().unwrap(); + } + + peerdb + } + + /// Set up the PeerNetwork, but do not bind it. + pub fn setup_peer_network( + config: &Config, + atlas_config: &AtlasConfig, + burnchain: Burnchain, + ) -> PeerNetwork { + let sortdb = SortitionDB::open(&config.get_burn_db_file_path(), true) + .expect("Error while instantiating sor/tition db"); + + let epochs = SortitionDB::get_stacks_epochs(sortdb.conn()) + .expect("Error while loading stacks epochs"); + + let view = { + let sortition_tip = SortitionDB::get_canonical_burn_chain_tip(&sortdb.conn()) + .expect("Failed to get sortition tip"); + SortitionDB::get_burnchain_view(&sortdb.conn(), &burnchain, &sortition_tip).unwrap() }; - // let's commit - let op = inner_generate_block_commit_op( - keychain.get_burnchain_signer(), - anchored_block.block_hash(), - rest_commit, - ®istered_key, - parent_block_burn_height - .try_into() - .expect("Could not convert parent block height into u32"), - parent_winning_vtxindex, - VRFSeed::from_proof(&vrf_proof), - commit_outs, - sunset_burn, - burn_block.block_height, + let peerdb = Self::setup_peer_db(config, &burnchain); + + let atlasdb = + AtlasDB::connect(atlas_config.clone(), &config.get_atlas_db_file_path(), true).unwrap(); + + let local_peer = match PeerDB::get_local_peer(peerdb.conn()) { + Ok(local_peer) => local_peer, + _ => panic!("Unable to retrieve local peer"), + }; + + let p2p_net = PeerNetwork::new( + peerdb, + atlasdb, + local_peer, + config.burnchain.peer_version, + burnchain, + view, + config.connection_options.clone(), + epochs, ); - let cur_burn_chain_tip = SortitionDB::get_canonical_burn_chain_tip(burn_db.conn()) - .expect("FATAL: failed to query sortition DB for canonical burn chain tip"); + p2p_net + } - // last chance -- confirm that the stacks tip and burnchain tip are unchanged (since it could have taken long - // enough to build this block that another block could have arrived). - if let Some(stacks_tip) = chain_state - .get_stacks_chain_tip(burn_db) - .expect("FATAL: could not query chain tip") + /// Main loop of the relayer. + /// Runs in a separate thread. + /// Continuously receives + pub fn relayer_main(mut relayer_thread: RelayerThread, relay_recv: Receiver) { + while let Ok(directive) = relay_recv.recv() { + if !relayer_thread.globals.keep_running() { + break; + } + + if !relayer_thread.handle_directive(directive) { + break; + } + } + + // kill miner if it's running + signal_mining_blocked(relayer_thread.globals.get_miner_status()); + + // set termination flag so other threads die + relayer_thread.globals.signal_stop(); + + debug!("Relayer exit!"); + } + + /// Main loop of the p2p thread. + /// Runs in a separate thread. + /// Continuously receives, until told otherwise. + pub fn p2p_main(mut p2p_thread: PeerThread, event_dispatcher: EventDispatcher) { + let (mut dns_resolver, mut dns_client) = DNSResolver::new(10); + // spawn a daemon thread that runs the DNS resolver. + // It will die when the rest of the system dies. { - if stacks_tip.anchored_block_hash != anchored_block.header.parent_block - || parent_consensus_hash != stacks_tip.consensus_hash - || cur_burn_chain_tip.sortition_id != burn_block.sortition_id - { - debug!( - "Cancel block-commit; chain tip(s) have changed"; - "block_hash" => %anchored_block.block_hash(), - "tx_count" => anchored_block.txs.len(), - "target_height" => %anchored_block.header.total_work.work, - "parent_consensus_hash" => %parent_consensus_hash, - "parent_block_hash" => %anchored_block.header.parent_block, - "parent_microblock_hash" => %anchored_block.header.parent_microblock, - "parent_microblock_seq" => anchored_block.header.parent_microblock_sequence, - "old_tip_burn_block_hash" => %burn_block.burn_header_hash, - "old_tip_burn_block_height" => burn_block.block_height, - "old_tip_burn_block_sortition_id" => %burn_block.sortition_id, - "attempt" => attempt, - "new_stacks_tip_block_hash" => %stacks_tip.anchored_block_hash, - "new_stacks_tip_consensus_hash" => %stacks_tip.consensus_hash, - "new_tip_burn_block_height" => cur_burn_chain_tip.block_height, - "new_tip_burn_block_sortition_id" => %cur_burn_chain_tip.sortition_id, - "new_burn_block_sortition_id" => %cur_burn_chain_tip.sortition_id - ); - return None; + let _jh = thread::Builder::new() + .name("dns-resolver".to_string()) + .spawn(move || { + dns_resolver.thread_main(); + }) + .unwrap(); + } + + // NOTE: these must be instantiated in the thread context, since it can't be safely sent + // between threads + let fee_estimator_opt = p2p_thread.config.make_fee_estimator(); + let cost_estimator = p2p_thread + .config + .make_cost_estimator() + .unwrap_or_else(|| Box::new(UnitEstimator)); + let cost_metric = p2p_thread + .config + .make_cost_metric() + .unwrap_or_else(|| Box::new(UnitMetric)); + + // receive until we can't reach the receiver thread + loop { + if !p2p_thread.globals.keep_running() { + break; + } + if !p2p_thread.run_one_pass( + Some(&mut dns_client), + &event_dispatcher, + &cost_estimator, + &cost_metric, + fee_estimator_opt.as_ref(), + ) { + break; } } - let mut op_signer = keychain.generate_op_signer(); - debug!( - "Submit block-commit"; - "block_hash" => %anchored_block.block_hash(), - "tx_count" => anchored_block.txs.len(), - "target_height" => anchored_block.header.total_work.work, - "parent_consensus_hash" => %parent_consensus_hash, - "parent_block_hash" => %anchored_block.header.parent_block, - "parent_microblock_hash" => %anchored_block.header.parent_microblock, - "parent_microblock_seq" => anchored_block.header.parent_microblock_sequence, - "tip_burn_block_hash" => %burn_block.burn_header_hash, - "tip_burn_block_height" => burn_block.block_height, - "tip_burn_block_sortition_id" => %burn_block.sortition_id, - "attempt" => attempt - ); + // kill miner + signal_mining_blocked(p2p_thread.globals.get_miner_status()); - let res = bitcoin_controller.submit_operation(op, &mut op_signer, attempt); - if !res { - if !config.node.mock_mining { - warn!("Failed to submit Bitcoin transaction"); - return None; - } else { - debug!("Mock-mining enabled; not sending Bitcoin transaction"); + // set termination flag so other threads die + p2p_thread.globals.signal_stop(); + + // thread exited, so signal to the relayer thread to die. + while let Err(TrySendError::Full(_)) = p2p_thread + .globals + .relay_send + .try_send(RelayerDirective::Exit) + { + warn!("Failed to direct relayer thread to exit, sleeping and trying again"); + thread::sleep(Duration::from_secs(5)); + } + info!("P2P thread exit!"); + } + + pub fn spawn( + runloop: &RunLoop, + globals: Globals, + // relay receiver endpoint for the p2p thread, so the relayer can feed it data to push + relay_recv: Receiver, + // attachments receiver endpoint for the p2p thread, so the chains coordinator can feed it + // attachments it discovers + attachments_receiver: Receiver>, + ) -> StacksNode { + let config = runloop.config().clone(); + let is_miner = runloop.is_miner(); + let burnchain = runloop.get_burnchain(); + let atlas_config = AtlasConfig::default(config.is_mainnet()); + let mut keychain = Keychain::default(config.node.seed.clone()); + + // we can call _open_ here rather than _connect_, since connect is first called in + // make_genesis_block + let mut sortdb = SortitionDB::open(&config.get_burn_db_file_path(), true) + .expect("Error while instantiating sor/tition db"); + + Self::setup_ast_size_precheck(&config, &mut sortdb); + + let _ = Self::setup_mempool_db(&config); + + let mut p2p_net = Self::setup_peer_network(&config, &atlas_config, burnchain.clone()); + let relayer = Relayer::from_p2p(&mut p2p_net); + + let local_peer = p2p_net.local_peer.clone(); + + let burnchain_signer = keychain.get_burnchain_signer(); + match monitoring::set_burnchain_signer(burnchain_signer.clone()) { + Err(e) => { + warn!("Failed to set global burnchain signer: {:?}", &e); } + _ => {} } - Some(( - AssembledAnchorBlock { - parent_consensus_hash: parent_consensus_hash, - my_burn_hash: burn_block.burn_header_hash, - anchored_block, - attempt, - }, - microblock_secret_key, - )) + let leader_key_registration_state = if config.node.mock_mining { + // mock mining, pretend to have a registered key + let vrf_public_key = keychain.rotate_vrf_keypair(1); + LeaderKeyRegistrationState::Active(RegisteredKey { + block_height: 1, + op_vtxindex: 1, + vrf_public_key, + }) + } else { + LeaderKeyRegistrationState::Inactive + }; + + let relayer_thread = RelayerThread::new(runloop, local_peer.clone(), relayer); + let relayer_thread_handle = thread::Builder::new() + .name(format!("relayer-{}", &local_peer.data_url)) + .spawn(move || { + Self::relayer_main(relayer_thread, relay_recv); + }) + .expect("FATAL: failed to start relayer thread"); + + let p2p_event_dispatcher = runloop.get_event_dispatcher(); + let p2p_thread = PeerThread::new(runloop, p2p_net, attachments_receiver); + let p2p_thread_handle = thread::Builder::new() + .name(format!( + "p2p-({},{})", + &config.node.p2p_bind, &config.node.rpc_bind + )) + .spawn(move || { + Self::p2p_main(p2p_thread, p2p_event_dispatcher); + }) + .expect("FATAL: failed to start p2p thread"); + + info!("Start HTTP server on: {}", &config.node.rpc_bind); + info!("Start P2P server on: {}", &config.node.p2p_bind); + + StacksNode { + config, + atlas_config, + globals, + burnchain_signer, + is_miner, + leader_key_registration_state, + p2p_thread_handle, + relayer_thread_handle, + } + } + + /// Manage the VRF public key registration state machine. + /// Tell the relayer thread to fire off a tenure and a block commit op, + /// if it is time to do so. + /// Called from the main thread. + /// Return true if we succeeded in carrying out the next task of the operation. + pub fn relayer_issue_tenure(&mut self) -> bool { + if !self.is_miner { + // node is a follower, don't try to issue a tenure + return true; + } + + if let Some(burnchain_tip) = self.globals.get_last_sortition() { + match self.leader_key_registration_state { + LeaderKeyRegistrationState::Active(ref key) => { + debug!( + "Tenure: Using key {:?} off of {}", + &key.vrf_public_key, &burnchain_tip.burn_header_hash + ); + + self.globals + .relay_send + .send(RelayerDirective::RunTenure( + key.clone(), + burnchain_tip, + get_epoch_time_ms(), + )) + .is_ok() + } + LeaderKeyRegistrationState::Inactive => { + warn!( + "Tenure: skipped tenure because no active VRF key. Trying to register one." + ); + self.leader_key_registration_state = LeaderKeyRegistrationState::Pending; + self.globals + .relay_send + .send(RelayerDirective::RegisterKey(burnchain_tip)) + .is_ok() + } + LeaderKeyRegistrationState::Pending => true, + } + } else { + warn!("Tenure: Do not know the last burn block. As a miner, this is bad."); + true + } + } + + /// Notify the relayer of a sortition, telling it to process the block + /// and advertize it if it was mined by the node. + /// returns _false_ if the relayer hung up the channel. + /// Called from the main thread. + pub fn relayer_sortition_notify(&self) -> bool { + if !self.is_miner { + // node is a follower, don't try to process my own tenure. + return true; + } + + if let Some(snapshot) = self.globals.get_last_sortition() { + debug!( + "Tenure: Notify sortition!"; + "consensus_hash" => %snapshot.consensus_hash, + "burn_block_hash" => %snapshot.burn_header_hash, + "winning_stacks_block_hash" => %snapshot.winning_stacks_block_hash, + "burn_block_height" => &snapshot.block_height, + "sortition_id" => %snapshot.sortition_id + ); + if snapshot.sortition { + return self + .globals + .relay_send + .send(RelayerDirective::ProcessTenure( + snapshot.consensus_hash.clone(), + snapshot.parent_burn_header_hash.clone(), + snapshot.winning_stacks_block_hash.clone(), + )) + .is_ok(); + } + } else { + debug!("Tenure: Notify sortition! No last burn block"); + } + true } /// Process a state coming from the burnchain, by extracting the validated KeyRegisterOp /// and inspecting if a sortition was won. /// `ibd`: boolean indicating whether or not we are in the initial block download + /// Called from the main thread. pub fn process_burnchain_state( &mut self, sortdb: &SortitionDB, @@ -2390,11 +3679,11 @@ impl StacksNode { } // no-op on UserBurnSupport ops are not supported / produced at this point. - - set_last_sortition(&mut self.last_sortition, block_snapshot); + self.globals.set_last_sortition(block_snapshot); last_sortitioned_block.map(|x| x.0) } + /// Join all inner threads pub fn join(self) { self.relayer_thread_handle.join().unwrap(); self.p2p_thread_handle.join().unwrap(); From c1d2132281c51301006ca5d76f490d93102fd79b Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:20:31 -0400 Subject: [PATCH 040/178] fix: send a copy of the exit at block height --- testnet/stacks-node/src/node.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testnet/stacks-node/src/node.rs b/testnet/stacks-node/src/node.rs index 0af4e5c6e..9bca951e4 100644 --- a/testnet/stacks-node/src/node.rs +++ b/testnet/stacks-node/src/node.rs @@ -180,7 +180,7 @@ fn spawn_peer( let fee_estimator = config.make_fee_estimator(); let handler_args = RPCHandlerArgs { - exit_at_block_height: exit_at_block_height.as_ref(), + exit_at_block_height: exit_at_block_height.clone(), cost_estimator: Some(cost_estimator.as_ref()), cost_metric: Some(metric.as_ref()), fee_estimator: fee_estimator.as_ref().map(|x| x.as_ref()), From 69ce0ff43dc395255e3ad80d534edc35d3e574a8 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:20:45 -0400 Subject: [PATCH 041/178] feat: make it so the neon runloop sets up structured global thread state, and blocks/unblocks the miner in response to new burnchain data arriving. This prevents the miner from stalling the node in the event of a slow mempool walk. --- testnet/stacks-node/src/run_loop/neon.rs | 88 +++++++++++++++++++----- 1 file changed, 71 insertions(+), 17 deletions(-) diff --git a/testnet/stacks-node/src/run_loop/neon.rs b/testnet/stacks-node/src/run_loop/neon.rs index 68246da2d..1557a31ba 100644 --- a/testnet/stacks-node/src/run_loop/neon.rs +++ b/testnet/stacks-node/src/run_loop/neon.rs @@ -7,6 +7,7 @@ use std::sync::atomic::AtomicU64; use std::sync::mpsc::sync_channel; use std::sync::mpsc::Receiver; use std::sync::Arc; +use std::sync::Mutex; use std::thread; use std::thread::JoinHandle; @@ -31,13 +32,16 @@ use stacks::util_lib::db::Error as db_error; use stx_genesis::GenesisData; use crate::monitoring::start_serving_monitoring_metrics; +use crate::neon_node::Globals; use crate::neon_node::StacksNode; +use crate::neon_node::RELAYER_MAX_BUFFER; use crate::node::use_test_genesis_chainstate; use crate::syncctl::{PoxSyncWatchdog, PoxSyncWatchdogComms}; use crate::{ node::{get_account_balances, get_account_lockups, get_names, get_namespaces}, BitcoinRegtestController, BurnchainController, Config, EventDispatcher, Keychain, }; +use stacks::chainstate::stacks::miner::{signal_mining_blocked, signal_mining_ready, MinerStatus}; use super::RunLoopCallbacks; use libc; @@ -126,6 +130,7 @@ impl Counters { pub struct RunLoop { config: Config, pub callbacks: RunLoopCallbacks, + globals: Option, counters: Counters, coordinator_channels: Option<(CoordinatorReceivers, CoordinatorChannels)>, should_keep_running: Arc, @@ -134,6 +139,9 @@ pub struct RunLoop { is_miner: Option, // not known until .start() is called burnchain: Option, // not known until .start() is called pox_watchdog_comms: PoxSyncWatchdogComms, + /// NOTE: this is duplicated in self.globals, but it needs to be accessible before globals is + /// instantiated (namely, so the test framework can access it). + miner_status: Arc>, } /// Write to stderr in an async-safe manner. @@ -160,6 +168,7 @@ impl RunLoop { let channels = CoordinatorCommunication::instantiate(); let should_keep_running = Arc::new(AtomicBool::new(true)); let pox_watchdog_comms = PoxSyncWatchdogComms::new(should_keep_running.clone()); + let miner_status = Arc::new(Mutex::new(MinerStatus::make_ready())); let mut event_dispatcher = EventDispatcher::new(); for observer in config.events_observers.iter() { @@ -168,6 +177,7 @@ impl RunLoop { Self { config, + globals: None, coordinator_channels: Some(channels), callbacks: RunLoopCallbacks::new(), counters: Counters::new(), @@ -177,9 +187,20 @@ impl RunLoop { is_miner: None, burnchain: None, pox_watchdog_comms, + miner_status, } } + pub fn get_globals(&self) -> Globals { + self.globals + .clone() + .expect("FATAL: globals not instantiated") + } + + fn set_globals(&mut self, globals: Globals) { + self.globals = Some(globals); + } + pub fn get_coordinator_channel(&self) -> Option { self.coordinator_channels.as_ref().map(|x| x.1.clone()) } @@ -225,7 +246,7 @@ impl RunLoop { } pub fn get_termination_switch(&self) -> Arc { - self.should_keep_running.clone() + self.get_globals().should_keep_running.clone() } pub fn get_burnchain(&self) -> Burnchain { @@ -240,6 +261,10 @@ impl RunLoop { .expect("FATAL: tried to get PoX watchdog before calling .start()") } + pub fn get_miner_status(&self) -> Arc> { + self.miner_status.clone() + } + /// Set up termination handler. Have a signal set the `should_keep_running` atomic bool to /// false. Panics of called more than once. fn setup_termination_handler(&self) { @@ -393,6 +418,7 @@ impl RunLoop { &mut self, burnchain_config: &Burnchain, coordinator_receivers: CoordinatorReceivers, + miner_status: Arc>, ) -> (JoinHandle<()>, Receiver>) { let use_test_genesis_data = use_test_genesis_chainstate(&self.config); @@ -451,7 +477,10 @@ impl RunLoop { let (attachments_tx, attachments_rx) = sync_channel(ATTACHMENTS_CHANNEL_SIZE); let coordinator_thread_handle = thread::Builder::new() - .name("chains-coordinator".to_string()) + .name(format!( + "chains-coordinator-{}", + &moved_config.node.rpc_bind + )) .spawn(move || { let mut cost_estimator = moved_config.make_cost_estimator(); let mut fee_estimator = moved_config.make_fee_estimator(); @@ -465,6 +494,7 @@ impl RunLoop { moved_atlas_config, cost_estimator.as_deref_mut(), fee_estimator.as_deref_mut(), + miner_status, ); }) .expect("FATAL: failed to start chains coordinator thread"); @@ -543,21 +573,39 @@ impl RunLoop { let burnchain_config = burnchain.get_burnchain(); self.burnchain = Some(burnchain_config.clone()); + // can we mine? let is_miner = self.check_is_miner(&mut burnchain); self.is_miner = Some(is_miner); + // relayer linkup + let (relay_send, relay_recv) = sync_channel(RELAYER_MAX_BUFFER); + + // set up globals so other subsystems can instantiate off of the runloop state. + let globals = Globals::new( + coordinator_senders, + self.get_miner_status(), + relay_send, + self.counters.clone(), + self.pox_watchdog_comms.clone(), + self.should_keep_running.clone(), + ); + self.set_globals(globals.clone()); + // have headers; boot up the chains coordinator and instantiate the chain state - let (coordinator_thread_handle, attachments_rx) = - self.spawn_chains_coordinator(&burnchain_config, coordinator_receivers); + let (coordinator_thread_handle, attachments_rx) = self.spawn_chains_coordinator( + &burnchain_config, + coordinator_receivers, + globals.get_miner_status(), + ); self.instantiate_pox_watchdog(); self.start_prometheus(); // We announce a new burn block so that the chains coordinator // can resume prior work and handle eventual unprocessed sortitions // stored during a previous session. - coordinator_senders.announce_new_burn_block(); + globals.coord().announce_new_burn_block(); - // Make sure at least one sortition has happened + // Make sure at least one sortition has happened, and make sure it's globally available let sortdb = burnchain.sortdb_mut(); let (rc_aligned_height, sn) = RunLoop::get_reward_cycle_sortition_db_height(&sortdb, &burnchain_config); @@ -572,14 +620,11 @@ impl RunLoop { sn }; + globals.set_last_sortition(burnchain_tip_snapshot); + // Boot up the p2p network and relayer, and figure out how many sortitions we have so far // (it could be non-zero if the node is resuming from chainstate) - let mut node = StacksNode::spawn( - self, - Some(burnchain_tip_snapshot), - coordinator_senders.clone(), - attachments_rx, - ); + let mut node = StacksNode::spawn(self, globals.clone(), relay_recv, attachments_rx); // Wait for all pending sortitions to process let mut burnchain_tip = burnchain @@ -609,14 +654,14 @@ impl RunLoop { let mut last_tenure_sortition_height = 0; loop { - if !self.should_keep_running.load(Ordering::SeqCst) { + if !globals.keep_running() { // The p2p thread relies on the same atomic_bool, it will // discontinue its execution after completing its ongoing runloop epoch. info!("Terminating p2p process"); info!("Terminating relayer"); info!("Terminating chains-coordinator"); - coordinator_senders.stop_chains_coordinator(); + globals.coord().stop_chains_coordinator(); coordinator_thread_handle.join().unwrap(); node.join(); @@ -652,7 +697,7 @@ impl RunLoop { // runloop will cause the PoX sync watchdog to wait until it believes that the node has // obtained all the Stacks blocks it can. while burnchain_height <= target_burnchain_block_height { - if !self.should_keep_running.load(Ordering::SeqCst) { + if !globals.keep_running() { break; } @@ -686,9 +731,15 @@ impl RunLoop { ); let mut sort_count = 0; + signal_mining_blocked(globals.get_miner_status()); // first, let's process all blocks in (sortition_db_height, next_sortition_height] for block_to_process in (sortition_db_height + 1)..(next_sortition_height + 1) { + // stop mining so we can advance the sortition DB and so our + // ProcessTenure() directive (sent by relayer_sortition_notify() below) + // will be unblocked. + debug!("Runloop: disable miner to process sortitions"); + let block = { let ic = burnchain.sortdb_ref().index_conn(); SortitionDB::get_ancestor_snapshot(&ic, block_to_process, sortition_tip) @@ -718,6 +769,8 @@ impl RunLoop { } } + signal_mining_ready(globals.get_miner_status()); + num_sortitions_in_last_cycle = sort_count; debug!( "Synchronized burnchain up to block height {} from {} (chain tip height is {}); {} sortitions", @@ -730,7 +783,7 @@ impl RunLoop { // we may have downloaded all the blocks already, // so we can't rely on the relayer alone to // drive it. - coordinator_senders.announce_new_stacks_block(); + globals.coord().announce_new_stacks_block(); } if burnchain_height == target_burnchain_block_height @@ -771,10 +824,11 @@ impl RunLoop { ); last_tenure_sortition_height = sortition_db_height; } + if !node.relayer_issue_tenure() { // relayer hung up, exit. error!("Block relayer and miner hung up, exiting."); - continue; + break; } } } From 8da076e7c72f3d743caecfd12a9b32496f0a236c Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:21:27 -0400 Subject: [PATCH 042/178] docs: document that this is only for helium --- testnet/stacks-node/src/tenure.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/testnet/stacks-node/src/tenure.rs b/testnet/stacks-node/src/tenure.rs index 4aaa70ae2..a7a5c2cba 100644 --- a/testnet/stacks-node/src/tenure.rs +++ b/testnet/stacks-node/src/tenure.rs @@ -1,3 +1,4 @@ +/// Only used by the Helium (Mocknet) node use super::node::ChainTip; use super::{BurnchainTip, Config}; From dc58cfc7b426b222a2ea539cca0f8c1f92fe70ee Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 5 Oct 2022 23:21:45 -0400 Subject: [PATCH 043/178] feat: add a poison-microblock integration test to verify that stream forks are detected and the offending miner gets punished, and also start working out multi-miner tests for verifying that the chain continues to work well under load. --- .../src/tests/neon_integrations.rs | 579 +++++++++++++++++- 1 file changed, 574 insertions(+), 5 deletions(-) diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index d712fa142..e09aaf968 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -72,6 +72,8 @@ use crate::{ use crate::util::hash::{MerkleTree, Sha512Trunc256Sum}; use crate::util::secp256k1::MessageSignature; +use crate::neon_node::StacksNode; + use rand::Rng; use super::bitcoin_regtest::BitcoinCoreController; @@ -87,15 +89,20 @@ use clarity::vm::ast::ASTRules; use clarity::vm::MAX_CALL_STACK_DEPTH; use stacks::chainstate::burn::db::sortdb::SortitionDB; use stacks::chainstate::stacks::miner::{ - TransactionErrorEvent, TransactionEvent, TransactionSkippedEvent, TransactionSuccessEvent, + signal_mining_blocked, signal_mining_ready, TransactionErrorEvent, TransactionEvent, + TransactionSkippedEvent, TransactionSuccessEvent, }; use stacks::net::RPCFeeEstimateResponse; use stacks::vm::ClarityName; use stacks::vm::ContractName; use std::convert::TryFrom; -pub fn neon_integration_test_conf() -> (Config, StacksAddress) { +use crate::stacks_common::types::PrivateKey; + +fn inner_neon_integration_test_conf(seed: Option>) -> (Config, StacksAddress) { let mut conf = super::new_test_conf(); + let seed = seed.unwrap_or(conf.node.seed.clone()); + conf.node.seed = seed; let keychain = Keychain::default(conf.node.seed.clone()); @@ -125,11 +132,22 @@ pub fn neon_integration_test_conf() -> (Config, StacksAddress) { conf.miner.first_attempt_time_ms = i64::max_value() as u64; conf.miner.subsequent_attempt_time_ms = i64::max_value() as u64; + // if there's just one node, then this must be true for tests to pass + conf.miner.wait_for_block_download = false; + let miner_account = keychain.origin_address(conf.is_mainnet()).unwrap(); (conf, miner_account) } +pub fn neon_integration_test_conf() -> (Config, StacksAddress) { + inner_neon_integration_test_conf(None) +} + +pub fn neon_integration_test_conf_with_seed(seed: Vec) -> (Config, StacksAddress) { + inner_neon_integration_test_conf(Some(seed)) +} + pub mod test_observer { use std::convert::Infallible; use std::sync::Mutex; @@ -393,6 +411,36 @@ pub fn next_block_and_wait( true } +/// Returns `false` on a timeout, true otherwise. +pub fn next_block_and_iterate( + btc_controller: &mut BitcoinRegtestController, + blocks_processed: &Arc, + iteration_delay_ms: u64, +) -> bool { + let current = blocks_processed.load(Ordering::SeqCst); + eprintln!( + "Issuing block at {}, waiting for bump ({})", + get_epoch_time_secs(), + current + ); + btc_controller.build_next_block(1); + let start = Instant::now(); + while blocks_processed.load(Ordering::SeqCst) <= current { + if start.elapsed() > Duration::from_secs(PANIC_TIMEOUT_SECS) { + error!("Timed out waiting for block to process, trying to continue test"); + return false; + } + thread::sleep(Duration::from_millis(iteration_delay_ms)); + btc_controller.build_next_block(1); + } + eprintln!( + "Block bumped at {} ({})", + get_epoch_time_secs(), + blocks_processed.load(Ordering::SeqCst) + ); + true +} + /// This function will call `next_block_and_wait` until the burnchain height underlying `BitcoinRegtestController` /// reaches *exactly* `target_height`. /// @@ -508,6 +556,22 @@ pub fn get_chain_info(conf: &Config) -> RPCPeerInfoData { tip_info } +pub fn get_chain_info_opt(conf: &Config) -> Option { + let http_origin = format!("http://{}", &conf.node.rpc_bind); + let client = reqwest::blocking::Client::new(); + + // get the canonical chain tip + let path = format!("{}/v2/info", &http_origin); + let tip_info_opt = client + .get(&path) + .send() + .unwrap() + .json::() + .ok(); + + tip_info_opt +} + fn get_tip_anchored_block(conf: &Config) -> (ConsensusHash, StacksBlock) { let tip_info = get_chain_info(conf); @@ -1604,6 +1668,236 @@ fn make_signed_microblock( mblock } +#[test] +#[ignore] +fn microblock_fork_poison_integration_test() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let spender_sk = StacksPrivateKey::from_hex(SK_1).unwrap(); + let spender_addr: PrincipalData = to_addr(&spender_sk).into(); + let second_spender_sk = StacksPrivateKey::from_hex(SK_2).unwrap(); + let second_spender_addr: PrincipalData = to_addr(&second_spender_sk).into(); + + let (mut conf, miner_account) = neon_integration_test_conf(); + + conf.initial_balances.push(InitialBalance { + address: spender_addr.clone(), + amount: 100300, + }); + conf.initial_balances.push(InitialBalance { + address: second_spender_addr.clone(), + amount: 10000, + }); + + // we'll manually post a forked stream to the node + conf.node.mine_microblocks = false; + + test_observer::spawn(); + + conf.events_observers.push(EventObserverConfig { + endpoint: format!("localhost:{}", test_observer::EVENT_OBSERVER_PORT), + events_keys: vec![EventKeyType::AnyEvent], + }); + + let mut btcd_controller = BitcoinCoreController::new(conf.clone()); + btcd_controller + .start_bitcoind() + .map_err(|_e| ()) + .expect("Failed starting bitcoind"); + + let mut btc_regtest_controller = BitcoinRegtestController::new(conf.clone(), None); + let http_origin = format!("http://{}", &conf.node.rpc_bind); + + btc_regtest_controller.bootstrap_chain(201); + + eprintln!("Chain bootstrapped..."); + + let mut run_loop = neon::RunLoop::new(conf.clone()); + let blocks_processed = run_loop.get_blocks_processed_arc(); + let client = reqwest::blocking::Client::new(); + let miner_status = run_loop.get_miner_status(); + + let channel = run_loop.get_coordinator_channel().unwrap(); + + thread::spawn(move || run_loop.start(None, 0)); + + // give the run loop some time to start up! + wait_for_runloop(&blocks_processed); + + // first block wakes up the run loop + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // first block will hold our VRF registration + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // second block will be the first mined Stacks block + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // let's query the miner's account nonce: + info!("Miner account: {}", miner_account); + let account = get_account(&http_origin, &miner_account); + assert_eq!(account.balance, 0); + assert_eq!(account.nonce, 1); + + // and our first spender + let account = get_account(&http_origin, &spender_addr); + assert_eq!(account.balance, 100300); + assert_eq!(account.nonce, 0); + + // and our second spender + let account = get_account(&http_origin, &second_spender_addr); + assert_eq!(account.balance, 10000); + assert_eq!(account.nonce, 0); + + info!("Test microblock"); + + let recipient = StacksAddress::from_string(ADDR_4).unwrap(); + let unconfirmed_tx_bytes = + make_stacks_transfer_mblock_only(&spender_sk, 0, 1000, &recipient.into(), 1000); + let unconfirmed_tx = + StacksTransaction::consensus_deserialize(&mut &unconfirmed_tx_bytes[..]).unwrap(); + let second_unconfirmed_tx_bytes = + make_stacks_transfer_mblock_only(&second_spender_sk, 0, 1000, &recipient.into(), 1500); + let second_unconfirmed_tx = + StacksTransaction::consensus_deserialize(&mut &second_unconfirmed_tx_bytes[..]).unwrap(); + + // TODO (hack) instantiate the sortdb in the burnchain + let _ = btc_regtest_controller.sortdb_mut(); + + // turn off the miner for now, so we can ensure both of these get accepted and preprocessed + // before we try and mine an anchor block that confirms them + eprintln!("Disable miner"); + signal_mining_blocked(miner_status.clone()); + + // put each into a microblock + let (first_microblock, second_microblock) = { + let tip_info = get_chain_info(&conf); + let stacks_tip = tip_info.stacks_tip; + + let (consensus_hash, stacks_block) = get_tip_anchored_block(&conf); + let tip_hash = + StacksBlockHeader::make_index_block_hash(&consensus_hash, &stacks_block.block_hash()); + let privk = + find_microblock_privkey(&conf, &stacks_block.header.microblock_pubkey_hash, 1024) + .unwrap(); + let (mut chainstate, _) = StacksChainState::open( + false, + CHAIN_ID_TESTNET, + &conf.get_chainstate_path_str(), + None, + ) + .unwrap(); + + chainstate + .reload_unconfirmed_state(&btc_regtest_controller.sortdb_ref().index_conn(), tip_hash) + .unwrap(); + let first_microblock = make_microblock( + &privk, + &mut chainstate, + &btc_regtest_controller.sortdb_ref().index_conn(), + consensus_hash, + stacks_block.clone(), + vec![unconfirmed_tx], + ); + + eprintln!( + "Created first microblock: {}: {:?}", + &first_microblock.block_hash(), + &first_microblock + ); + + // NOTE: this microblock conflicts because it has the same parent as the first microblock, + // even though it's seq is different. + let second_microblock = + make_signed_microblock(&privk, vec![second_unconfirmed_tx], stacks_tip, 1); + + eprintln!( + "Created second conflicting microblock: {}: {:?}", + &second_microblock.block_hash(), + &second_microblock + ); + (first_microblock, second_microblock) + }; + + let mut microblock_bytes = vec![]; + first_microblock + .consensus_serialize(&mut microblock_bytes) + .unwrap(); + + // post the first microblock + let path = format!("{}/v2/microblocks", &http_origin); + let res: String = client + .post(&path) + .header("Content-Type", "application/octet-stream") + .body(microblock_bytes.clone()) + .send() + .unwrap() + .json() + .unwrap(); + + assert_eq!(res, format!("{}", &first_microblock.block_hash())); + + let mut second_microblock_bytes = vec![]; + second_microblock + .consensus_serialize(&mut second_microblock_bytes) + .unwrap(); + + // post the second microblock + let path = format!("{}/v2/microblocks", &http_origin); + let res: String = client + .post(&path) + .header("Content-Type", "application/octet-stream") + .body(second_microblock_bytes.clone()) + .send() + .unwrap() + .json() + .unwrap(); + + assert_eq!(res, format!("{}", &second_microblock.block_hash())); + + eprintln!("Wait 10s and re-nable miner"); + sleep_ms(10_000); + + // resume mining + eprintln!("Enable miner"); + signal_mining_ready(miner_status.clone()); + + let mut found = false; + for i in 0..10 { + if found { + break; + } + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + let blocks = test_observer::get_blocks(); + for block in blocks.iter() { + let transactions = block.get("transactions").unwrap().as_array().unwrap(); + for tx in transactions.iter() { + let raw_tx = tx.get("raw_tx").unwrap().as_str().unwrap(); + if raw_tx == "0x00" { + continue; + } + let tx_bytes = hex_bytes(&raw_tx[2..]).unwrap(); + let parsed = StacksTransaction::consensus_deserialize(&mut &tx_bytes[..]).unwrap(); + + if let TransactionPayload::PoisonMicroblock(..) = &parsed.payload { + found = true; + break; + } + } + } + } + + assert!( + found, + "Did not find poison microblock tx in any mined block" + ); + + test_observer::clear(); + channel.stop_chains_coordinator(); +} + #[test] #[ignore] fn microblock_integration_test() { @@ -1618,6 +1912,7 @@ fn microblock_integration_test() { let (mut conf, miner_account) = neon_integration_test_conf(); + conf.miner.wait_for_block_download = false; conf.initial_balances.push(InitialBalance { address: spender_addr.clone(), amount: 100300, @@ -1763,9 +2058,26 @@ fn microblock_integration_test() { vec![unconfirmed_tx], ); + eprintln!( + "Created first microblock: {}: {:?}", + &first_microblock.block_hash(), + &first_microblock + ); + /* let second_microblock = make_signed_microblock(&privk, vec![second_unconfirmed_tx], stacks_tip, 1); - + */ + let second_microblock = make_signed_microblock( + &privk, + vec![second_unconfirmed_tx], + first_microblock.block_hash(), + 1, + ); + eprintln!( + "Created second microblock: {}: {:?}", + &second_microblock.block_hash(), + &second_microblock + ); (first_microblock, second_microblock) }; @@ -2017,8 +2329,8 @@ fn microblock_integration_test() { for next_nonce in 2..5 { // verify that the microblock miner can automatically pick up transactions debug!( - "Try to send unconfirmed tx from {} to {}", - &spender_addr, &recipient + "Try to send unconfirmed tx from {} to {} nonce {}", + &spender_addr, &recipient, next_nonce ); let unconfirmed_tx_bytes = make_stacks_transfer_mblock_only( &spender_sk, @@ -2045,6 +2357,7 @@ fn microblock_integration_test() { .txid() .to_string() ); + eprintln!("Sent {}", &res); } else { eprintln!("{}", res.text().unwrap()); panic!(""); @@ -7365,3 +7678,259 @@ fn test_flash_block_skip_tenure() { channel.stop_chains_coordinator(); } + +fn make_expensive_tx_chain(privk: &StacksPrivateKey, fee_plus: u64) -> Vec> { + let addr = to_addr(&privk); + let mut chain = vec![]; + for nonce in 0..25 { + let mut addr_prefix = addr.to_string(); + addr_prefix.split_off(12); + let contract_name = format!("large-{}-{}", nonce, &addr_prefix); + eprintln!("Make tx {}", &contract_name); + let tx = make_contract_publish( + privk, + nonce, + 1049230 + nonce + fee_plus, + &contract_name, + &format!( + " + ;; a single one of these transactions consumes over half the runtime budget + (define-constant BUFF_TO_BYTE (list + 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0a 0x0b 0x0c 0x0d 0x0e 0x0f + 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f + 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2a 0x2b 0x2c 0x2d 0x2e 0x2f + 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3a 0x3b 0x3c 0x3d 0x3e 0x3f + 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4a 0x4b 0x4c 0x4d 0x4e 0x4f + 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5a 0x5b 0x5c 0x5d 0x5e 0x5f + 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6a 0x6b 0x6c 0x6d 0x6e 0x6f + 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7a 0x7b 0x7c 0x7d 0x7e 0x7f + 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8a 0x8b 0x8c 0x8d 0x8e 0x8f + 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9a 0x9b 0x9c 0x9d 0x9e 0x9f + 0xa0 0xa1 0xa2 0xa3 0xa4 0xa5 0xa6 0xa7 0xa8 0xa9 0xaa 0xab 0xac 0xad 0xae 0xaf + 0xb0 0xb1 0xb2 0xb3 0xb4 0xb5 0xb6 0xb7 0xb8 0xb9 0xba 0xbb 0xbc 0xbd 0xbe 0xbf + 0xc0 0xc1 0xc2 0xc3 0xc4 0xc5 0xc6 0xc7 0xc8 0xc9 0xca 0xcb 0xcc 0xcd 0xce 0xcf + 0xd0 0xd1 0xd2 0xd3 0xd4 0xd5 0xd6 0xd7 0xd8 0xd9 0xda 0xdb 0xdc 0xdd 0xde 0xdf + 0xe0 0xe1 0xe2 0xe3 0xe4 0xe5 0xe6 0xe7 0xe8 0xe9 0xea 0xeb 0xec 0xed 0xee 0xef + 0xf0 0xf1 0xf2 0xf3 0xf4 0xf5 0xf6 0xf7 0xf8 0xf9 0xfa 0xfb 0xfc 0xfd 0xfe 0xff + )) + (define-private (crash-me-folder (input (buff 1)) (ctr uint)) + (begin + (unwrap-panic (index-of BUFF_TO_BYTE input)) + ;; (unwrap-panic (index-of BUFF_TO_BYTE input)) + ;; (unwrap-panic (index-of BUFF_TO_BYTE input)) + ;; (unwrap-panic (index-of BUFF_TO_BYTE input)) + ;; (unwrap-panic (index-of BUFF_TO_BYTE input)) + ;; (unwrap-panic (index-of BUFF_TO_BYTE input)) + ;; (unwrap-panic (index-of BUFF_TO_BYTE input)) + ;; (unwrap-panic (index-of BUFF_TO_BYTE input)) + (+ u1 ctr) + ) + ) + (define-public (crash-me (name (string-ascii 128))) + (begin + (fold crash-me-folder BUFF_TO_BYTE u0) + (print name) + (ok u0) + ) + ) + (begin + (crash-me \"{}\")) + ", + &format!("large-contract-{}-{}", nonce, &addr_prefix) + ), + ); + chain.push(tx); + } + chain +} + +#[test] +#[ignore] +fn test_competing_miners_build_on_same_chain() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let privks: Vec<_> = (0..200) + .into_iter() + .map(|_| StacksPrivateKey::new()) + .collect(); + let mut balances: Vec<_> = privks + .iter() + .map(|privk| { + let addr = to_addr(privk); + InitialBalance { + address: addr.into(), + amount: 1_000_000_000, + } + }) + .collect(); + + let num_miners = 5; + let mut confs = vec![]; + let mut burnchain_configs = vec![]; + let mut blocks_processed = vec![]; + + for _i in 0..num_miners { + let seed = StacksPrivateKey::new().to_bytes(); + let (mut conf, _) = neon_integration_test_conf_with_seed(seed); + + conf.node.mine_microblocks = false; + conf.miner.microblock_attempt_time_ms = 5_000; + conf.node.wait_time_for_microblocks = 0; + conf.initial_balances.append(&mut balances.clone()); + conf.miner.first_attempt_time_ms = 2_000; + conf.miner.subsequent_attempt_time_ms = 5_000; + + // multiple nodes so they must download from each other + conf.miner.wait_for_block_download = true; + + // don't RBF + conf.burnchain.max_rbf = 0; + + confs.push(conf); + } + + let node_privkey_1 = + StacksNode::make_node_private_key_from_seed(&confs[0].node.local_peer_seed); + for i in 1..num_miners { + let chain_id = confs[0].burnchain.chain_id; + let peer_version = confs[0].burnchain.peer_version; + let p2p_bind = confs[0].node.p2p_bind.clone(); + + confs[i].node.set_bootstrap_nodes( + format!( + "{}@{}", + &StacksPublicKey::from_private(&node_privkey_1).to_hex(), + p2p_bind + ), + chain_id, + peer_version, + ); + } + + let mut btcd_controller = BitcoinCoreController::new(confs[0].clone()); + btcd_controller + .start_bitcoind() + .map_err(|_e| ()) + .expect("Failed starting bitcoind"); + + // use long reward cycles + for i in 0..num_miners { + let mut burnchain_config = Burnchain::regtest(&confs[i].get_burn_db_path()); + let reward_cycle_len = 100; + let prepare_phase_len = 20; + let pox_constants = PoxConstants::new( + reward_cycle_len, + prepare_phase_len, + 4 * prepare_phase_len / 5, + 5, + 15, + (16 * reward_cycle_len - 1).into(), + (17 * reward_cycle_len).into(), + ); + burnchain_config.pox_constants = pox_constants.clone(); + + burnchain_configs.push(burnchain_config); + } + + let mut btc_regtest_controller = BitcoinRegtestController::with_burnchain( + confs[0].clone(), + None, + Some(burnchain_configs[0].clone()), + None, + ); + + btc_regtest_controller.bootstrap_chain(1); + + // make sure all miners have BTC + for i in 1..num_miners { + let old_mining_pubkey = btc_regtest_controller.get_mining_pubkey().unwrap(); + btc_regtest_controller + .set_mining_pubkey(confs[i].burnchain.local_mining_public_key.clone().unwrap()); + btc_regtest_controller.bootstrap_chain(1); + btc_regtest_controller.set_mining_pubkey(old_mining_pubkey); + } + + btc_regtest_controller.bootstrap_chain((199 - num_miners) as u64); + + eprintln!("Chain bootstrapped..."); + + for (i, burnchain_config) in burnchain_configs.into_iter().enumerate() { + let mut run_loop = neon::RunLoop::new(confs[i].clone()); + let blocks_processed_arc = run_loop.get_blocks_processed_arc(); + + blocks_processed.push(blocks_processed_arc); + thread::spawn(move || run_loop.start(Some(burnchain_config), 0)); + } + + let http_origin = format!("http://{}", &confs[0].node.rpc_bind); + + // give the run loops some time to start up! + for i in 0..num_miners { + wait_for_runloop(&blocks_processed[i as usize]); + } + + // activate miners + eprintln!("\n\nBoot miner 0\n\n"); + loop { + let tip_info_opt = get_chain_info_opt(&confs[0]); + if let Some(tip_info) = tip_info_opt { + eprintln!("\n\nMiner 1: {:?}\n\n", &tip_info); + if tip_info.stacks_tip_height > 0 { + break; + } + } else { + eprintln!("\n\nWaiting for miner 0...\n\n"); + } + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed[0]); + } + + for i in 1..num_miners { + eprintln!("\n\nBoot miner {}\n\n", i); + loop { + let tip_info_opt = get_chain_info_opt(&confs[i]); + if let Some(tip_info) = tip_info_opt { + eprintln!("\n\nMiner 2: {:?}\n\n", &tip_info); + if tip_info.stacks_tip_height > 0 { + break; + } + } else { + eprintln!("\n\nWaiting for miner {}...\n\n", i); + } + next_block_and_iterate( + &mut btc_regtest_controller, + &blocks_processed[i as usize], + 5_000, + ); + } + } + + eprintln!("\n\nBegin transactions\n\n"); + + // blast out lots of expensive transactions. + // keeps the mempool full, and makes it so miners will spend a nontrivial amount of time + // building blocks + let all_txs: Vec<_> = privks + .iter() + .enumerate() + .map(|(i, pk)| make_expensive_tx_chain(pk, (25 * i) as u64)) + .collect(); + let mut cnt = 0; + for tx_chain in all_txs { + for tx in tx_chain { + eprintln!("\n\nSubmit tx {}\n\n", &cnt); + submit_tx(&http_origin, &tx); + cnt += 1; + } + } + + eprintln!("\n\nBegin mining\n\n"); + + // mine quickly -- see if we can induce flash blocks + for i in 0..1000 { + eprintln!("\n\nBuild block {}\n\n", i); + btc_regtest_controller.build_next_block(1); + sleep_ms(10_000); + } +} From b5e7ee553c1336061c4b30d39b3d0db5f890f07f Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 10 Oct 2022 15:36:57 -0400 Subject: [PATCH 044/178] feat: add Error::ChannelClosed() variant to detect thread hangups --- src/chainstate/stacks/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/chainstate/stacks/mod.rs b/src/chainstate/stacks/mod.rs index fe35b8192..1c5b205be 100644 --- a/src/chainstate/stacks/mod.rs +++ b/src/chainstate/stacks/mod.rs @@ -121,6 +121,7 @@ pub enum Error { PoxNoRewardCycle, ProblematicTransaction(Txid), MinerAborted, + ChannelClosed(String), } impl From for Error { @@ -197,6 +198,7 @@ impl fmt::Display for Error { txid ), Error::MinerAborted => write!(f, "Mining attempt aborted by signal"), + Error::ChannelClosed(ref s) => write!(f, "Channel '{}' closed", s), } } } @@ -232,6 +234,7 @@ impl error::Error for Error { Error::StacksTransactionSkipped(ref _r) => None, Error::ProblematicTransaction(ref _txid) => None, Error::MinerAborted => None, + Error::ChannelClosed(ref _s) => None, } } } @@ -267,6 +270,7 @@ impl Error { Error::StacksTransactionSkipped(ref _r) => "StacksTransactionSkipped", Error::ProblematicTransaction(ref _txid) => "ProblematicTransaction", Error::MinerAborted => "MinerAborted", + Error::ChannelClosed(ref _s) => "ChannelClosed", } } From d08cd0acfcfc165c97da38c07e760e81a43fde60 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 10 Oct 2022 15:37:32 -0400 Subject: [PATCH 045/178] feat: log when we store a block --- src/net/relay.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/net/relay.rs b/src/net/relay.rs index 7766df86b..0d8a6e253 100644 --- a/src/net/relay.rs +++ b/src/net/relay.rs @@ -581,13 +581,21 @@ impl Relayer { return Ok(false); } - chainstate.preprocess_anchored_block( + let res = chainstate.preprocess_anchored_block( sort_ic, consensus_hash, block, &parent_block_snapshot.consensus_hash, download_time, - ) + )?; + if res { + debug!( + "Stored incoming block {}/{}", + consensus_hash, + &block.block_hash() + ); + } + Ok(res) } /// Coalesce a set of microblocks into relayer hints and MicroblocksData messages, as calculated by From c19ad0f70b20fefca10b1f7707d0e3e3eec24bd5 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 10 Oct 2022 15:37:46 -0400 Subject: [PATCH 046/178] feat: add wait_time_for_blocks config option similar to wait_time_for_microblocks to have the node wait for a block to arrive before starting tenure --- testnet/stacks-node/src/config.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index 0e8e20104..f3cd21f41 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -437,6 +437,9 @@ impl Config { wait_time_for_microblocks: node .wait_time_for_microblocks .unwrap_or(default_node_config.wait_time_for_microblocks), + wait_time_for_blocks: node + .wait_time_for_blocks + .unwrap_or(default_node_config.wait_time_for_blocks), prometheus_bind: node.prometheus_bind, marf_cache_strategy: node.marf_cache_strategy, marf_defer_hashing: node @@ -1139,6 +1142,7 @@ pub struct NodeConfig { pub microblock_frequency: u64, pub max_microblocks: u64, pub wait_time_for_microblocks: u64, + pub wait_time_for_blocks: u64, pub prometheus_bind: Option, pub marf_cache_strategy: Option, pub marf_defer_hashing: bool, @@ -1412,6 +1416,7 @@ impl NodeConfig { microblock_frequency: 30_000, max_microblocks: u16::MAX as u64, wait_time_for_microblocks: 30_000, + wait_time_for_blocks: 30_000, prometheus_bind: None, marf_cache_strategy: None, marf_defer_hashing: true, @@ -1600,6 +1605,7 @@ pub struct NodeConfigFile { pub microblock_frequency: Option, pub max_microblocks: Option, pub wait_time_for_microblocks: Option, + pub wait_time_for_blocks: Option, pub prometheus_bind: Option, pub marf_cache_strategy: Option, pub marf_defer_hashing: Option, From b3b652571396f391d31561beb24b2449610fd6f1 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 10 Oct 2022 15:38:24 -0400 Subject: [PATCH 047/178] fix: wait for one inv and download pass before running tenure, and when we win tenure, broadcast the block _before_ processing it. Also, because blocks can get mined out-of-order relative to burnchain blocks, advance the miner tip only if the tip is higher in the stacks chain --- testnet/stacks-node/src/neon_node.rs | 390 ++++++++++++++++++++------- 1 file changed, 290 insertions(+), 100 deletions(-) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index cfa5afbf1..b2b0ec9ca 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -93,7 +93,10 @@ enum MinerThreadResult { Secp256k1PrivateKey, Option, ), - Microblock(Result, NetError>, MinerTip), + Microblock( + Result, NetError>, + MinerTip, + ), } /// Fully-assembled Stacks anchored, block as well as some extra metadata pertaining to how it was @@ -128,8 +131,6 @@ pub enum RelayerDirective { RunTenure(RegisteredKey, BlockSnapshot, u128), // (vrf key, chain tip, time of issuance in ms) /// Try to register a VRF public key RegisterKey(BlockSnapshot), - /// Try to mine a microblock - RunMicroblockTenure(BlockSnapshot, u128), // time of issuance in ms /// Stop the relayer thread Exit, } @@ -164,14 +165,26 @@ pub struct MinerTip { block_hash: BlockHeaderHash, /// Microblock private key to use to sign microblocks microblock_privkey: Secp256k1PrivateKey, + /// Stacks height + stacks_height: u64, + /// burnchain height + burn_height: u64, } impl MinerTip { - pub fn new(ch: ConsensusHash, bh: BlockHeaderHash, pk: Secp256k1PrivateKey) -> MinerTip { + pub fn new( + ch: ConsensusHash, + bh: BlockHeaderHash, + pk: Secp256k1PrivateKey, + stacks_height: u64, + burn_height: u64, + ) -> MinerTip { MinerTip { consensus_hash: ch, block_hash: bh, microblock_privkey: pk, + stacks_height, + burn_height, } } } @@ -343,6 +356,8 @@ enum Error { SnapshotNotFoundForChainTip, /// The burnchain tip changed while this operation was in progress BurnchainTipChanged, + /// The coordinator channel closed + CoordinatorClosed, } /// Metadata required for beginning a new tenure @@ -404,13 +419,22 @@ pub struct RelayerThread { last_tenure_issue_time: u128, /// last observed burnchain block height from the p2p thread (obtained from network results) last_network_block_height: u64, + /// time at which we observed a change in the network block height (epoch time in millis) + last_network_block_height_ts: u128, /// last observed number of downloader state-machine passes from the p2p thread (obtained from /// network results) last_network_download_passes: u64, + /// last observed number of inventory state-machine passes from the p2p thread (obtained from + /// network results) + last_network_inv_passes: u64, /// minimum number of downloader state-machine passes that must take place before mining (this /// is used to ensure that the p2p thread attempts to download new Stacks block data before /// this thread tries to mine a block) min_network_download_passes: u64, + /// minimum number of inventory state-machine passes that must take place before mining (this + /// is used to ensure that the p2p thread attempts to download new Stacks block data before + /// this thread tries to mine a block) + min_network_inv_passes: u64, /// consensus hash of the last sortition we saw, even if we weren't the winner last_tenure_consensus_hash: Option, @@ -418,6 +442,10 @@ pub struct RelayerThread { miner_tip: Option, /// last time we mined a microblock, in millis last_microblock_tenure_time: u128, + /// when should we run the next microblock tenure, in millis + microblock_deadline: u128, + /// cost of the last-produced microblock stream + microblock_stream_cost: ExecutionCost, /// Inner relayer instance for forwarding broadcasted data back to the p2p thread for dispatch /// to neighbors @@ -465,9 +493,9 @@ struct MicroblockMinerThread { mempool: Option, /// Handle to the node's event dispatcher event_dispatcher: EventDispatcher, - /// Stacks block's sortition's consensus hash + /// Parent Stacks block's sortition's consensus hash parent_consensus_hash: ConsensusHash, - /// Stacks block's hash + /// Parent Stacks block's hash parent_block_hash: BlockHeaderHash, /// Microblock signing key miner_key: Secp256k1PrivateKey, @@ -550,6 +578,7 @@ impl MicroblockMinerThread { consensus_hash: ch, block_hash: bhh, microblock_privkey: miner_key, + .. } = miner_tip; debug!( @@ -561,12 +590,18 @@ impl MicroblockMinerThread { match StacksChainState::get_anchored_block_header_info(chainstate.db(), &ch, &bhh) { Ok(Some(_)) => { let parent_index_hash = StacksBlockHeader::make_index_block_hash(&ch, &bhh); - let cost_so_far = StacksChainState::get_stacks_block_anchored_cost( - chainstate.db(), - &parent_index_hash, - ) - .expect("FATAL: failed to get anchored block cost") - .expect("FATAL: no anchored block cost stored for processed anchored block"); + let cost_so_far = if relayer_thread.microblock_stream_cost == ExecutionCost::zero() + { + // unknown cost, or this is idempotent. + StacksChainState::get_stacks_block_anchored_cost( + chainstate.db(), + &parent_index_hash, + ) + .expect("FATAL: failed to get anchored block cost") + .expect("FATAL: no anchored block cost stored for processed anchored block") + } else { + relayer_thread.microblock_stream_cost.clone() + }; let frequency = config.node.microblock_frequency; let settings = @@ -823,7 +858,7 @@ impl MicroblockMinerThread { sortdb: &SortitionDB, chainstate: &mut StacksChainState, mem_pool: &mut MemPoolDB, - ) -> Result, NetError> { + ) -> Result, NetError> { if !self.can_mine_on_tip(&self.parent_consensus_hash, &self.parent_block_hash) { // not configured to mine on this tip return Ok(None); @@ -838,7 +873,7 @@ impl MicroblockMinerThread { return Ok(None); } - let mut next_microblock = None; + let mut next_microblock_and_runtime = None; // opportunistically try and mine, but only if there are no attachable blocks in // recent history (i.e. in the last 10 minutes) @@ -851,7 +886,7 @@ impl MicroblockMinerThread { match self.inner_mine_one_microblock(sortdb, chainstate, mem_pool) { Ok(microblock) => { // will need to relay this - next_microblock = Some(microblock); + next_microblock_and_runtime = Some((microblock, self.cost_so_far.clone())); } Err(ChainstateError::NoTransactionsToMine) => { info!("Will keep polling mempool for transactions to include in a microblock"); @@ -866,7 +901,7 @@ impl MicroblockMinerThread { self.last_mined = get_epoch_time_ms(); - Ok(next_microblock) + Ok(next_microblock_and_runtime) } /// Try to mine one microblock, given the current chain tip and access to the chain state DBs. @@ -880,7 +915,7 @@ impl MicroblockMinerThread { pub fn try_mine_microblock( &mut self, cur_tip: MinerTip, - ) -> Result, NetError> { + ) -> Result, NetError> { self.with_chainstate(|mblock_miner, sortdb, chainstate, mempool| { mblock_miner.inner_try_mine_microblock(cur_tip, sortdb, chainstate, mempool) }) @@ -1728,12 +1763,17 @@ impl RelayerThread { last_tenure_issue_time: 0, last_network_block_height: 0, + last_network_block_height_ts: 0, last_network_download_passes: 0, min_network_download_passes: 0, + last_network_inv_passes: 0, + min_network_inv_passes: 0, last_tenure_consensus_hash: None, miner_tip: None, last_microblock_tenure_time: 0, + microblock_deadline: 0, + microblock_stream_cost: ExecutionCost::zero(), relayer, @@ -1782,6 +1822,34 @@ impl RelayerThread { res } + /// have we waited for the right conditions under which to start mining a block off of our + /// chain tip? + pub fn has_waited_for_latest_blocks(&self) -> bool { + // a network download pass took place + (self.min_network_download_passes <= self.last_network_download_passes + // a network inv pass took place + && self.min_network_download_passes <= self.last_network_download_passes) + // we waited long enough for a download pass, but timed out waiting + || self.last_network_block_height_ts + (self.config.node.wait_time_for_blocks as u128) < get_epoch_time_ms() + // we're not supposed to wait at all + || !self.config.miner.wait_for_block_download + } + + /// Return debug string for waiting for latest blocks + pub fn debug_waited_for_latest_blocks(&self) -> String { + format!( + "({} <= {} && {} <= {}) || {} + {} < {} || {}", + self.min_network_download_passes, + self.last_network_download_passes, + self.min_network_inv_passes, + self.last_network_inv_passes, + self.last_network_block_height_ts, + self.config.node.wait_time_for_blocks, + get_epoch_time_ms(), + self.config.miner.wait_for_block_download + ) + } + /// Handle a NetworkResult from the p2p/http state machine. Usually this is the act of /// * preprocessing and storing new blocks and microblocks /// * relaying blocks, microblocks, and transacctions @@ -1801,10 +1869,11 @@ impl RelayerThread { } if self.last_network_block_height != net_result.burn_height { - // burnchain advanced; disable mining until we also do a download pass + // burnchain advanced; disable mining until we also do a download pass. self.last_network_block_height = net_result.burn_height; self.min_network_download_passes = net_result.num_download_passes + 1; - + self.min_network_inv_passes = net_result.num_inv_sync_passes + 1; + self.last_network_block_height_ts = get_epoch_time_ms(); debug!( "Relayer: block mining until the next download pass {}", self.min_network_download_passes @@ -1862,9 +1931,8 @@ impl RelayerThread { // resume mining if we blocked it, and if we've done the requisite download // passes self.last_network_download_passes = net_result.num_download_passes; - if self.min_network_download_passes <= self.last_network_download_passes - || !self.config.miner.wait_for_block_download - { + self.last_network_inv_passes = net_result.num_inv_sync_passes; + if self.has_waited_for_latest_blocks() { debug!("Relayer: did a download pass, so unblocking mining"); signal_mining_ready(self.globals.get_miner_status()); } @@ -1872,15 +1940,15 @@ impl RelayerThread { /// Process the block and microblocks from a sortition that we won. /// At this point, we're modifying the chainstate, and merging the artifacts from the previous tenure. - /// Blocks until the given stacks block is processed + /// Blocks until the given stacks block is processed. + /// Returns true if we accepted this block as new. + /// Returns false if we already processed this block. fn accept_winning_tenure( &mut self, anchored_block: &StacksBlock, consensus_hash: &ConsensusHash, parent_consensus_hash: &ConsensusHash, ) -> Result { - let stacks_blocks_processed = self.globals.coord_comms.get_stacks_blocks_processed(); - if StacksChainState::has_stored_block( self.chainstate_ref().db(), &self.chainstate_ref().blocks_path, @@ -1888,7 +1956,7 @@ impl RelayerThread { &anchored_block.block_hash(), )? { // already processed my tenure - return Ok(true); + return Ok(false); } let burn_height = SortitionDB::get_block_snapshot_consensus(self.sortdb_ref().conn(), consensus_hash) @@ -1975,20 +2043,54 @@ impl RelayerThread { ) })?; + Ok(true) + } + + /// Process a new block we mined + /// Return true if we processed it + /// Return false if we timed out waiting for it + /// Return Err(..) if we couldn't reach the chains coordiantor thread + fn process_new_block(&self) -> Result { + // process the block if !self.globals.coord_comms.announce_new_stacks_block() { - return Ok(false); + return Err(Error::CoordinatorClosed); } + let stacks_blocks_processed = self.globals.coord_comms.get_stacks_blocks_processed(); if !self .globals .coord_comms .wait_for_stacks_blocks_processed(stacks_blocks_processed, u64::MAX) { + // basically unreachable warn!("ChainsCoordinator timed out while waiting for new stacks block to be processed"); + return Ok(false); } - Ok(true) } + /// Given the two miner tips, return the newer tip. + fn pick_higher_tip(cur: Option, new: Option) -> Option { + match (cur, new) { + (Some(cur), None) => Some(cur), + (None, Some(new)) => Some(new), + (None, None) => None, + (Some(cur), Some(new)) => { + if cur.stacks_height < new.stacks_height { + Some(new) + } else if cur.stacks_height > new.stacks_height { + Some(cur) + } else if cur.burn_height < new.burn_height { + Some(new) + } else if cur.burn_height > new.burn_height { + Some(cur) + } else { + assert_eq!(cur, new); + Some(cur) + } + } + } + } + /// Given the pointer to a recently-discovered tenure, see if we won the sortition and if so, /// store it, preprocess it, and forward it to our neighbors. All the while, keep track of the /// latest Stacks mining tip we have produced so far. @@ -2006,7 +2108,7 @@ impl RelayerThread { block_header_hash: BlockHeaderHash, burn_hash: BurnchainHeaderHash, ) -> (bool, Option) { - let miner_tip; + let mut miner_tip = None; let sn = SortitionDB::get_block_snapshot_consensus(self.sortdb_ref().conn(), &consensus_hash) .expect("FATAL: failed to query sortition DB") @@ -2040,13 +2142,15 @@ impl RelayerThread { ); increment_stx_blocks_mined_counter(); - match self.accept_winning_tenure(&mined_block, &consensus_hash, &parent_consensus_hash) - { - Ok(coordinator_running) => { - if !coordinator_running { - warn!("Coordinator stopped, stopping relayer thread..."); - return (false, None); - } + let has_new_data = match self.accept_winning_tenure( + &mined_block, + &consensus_hash, + &parent_consensus_hash, + ) { + Ok(accepted) => accepted, + Err(ChainstateError::ChannelClosed(_)) => { + warn!("Coordinator stopped, stopping relayer thread..."); + return (false, None); } Err(e) => { warn!("Error processing my tenure, bad block produced: {}", e); @@ -2089,10 +2193,11 @@ impl RelayerThread { &consensus_hash, &mined_block.block_hash() ); - miner_tip = None; + miner_tip = Self::pick_higher_tip(miner_tip, None); } else { let ch = snapshot.consensus_hash.clone(); let bh = mined_block.block_hash(); + let height = mined_block.header.total_work.work; if let Err(e) = self .relayer @@ -2102,14 +2207,31 @@ impl RelayerThread { } // proceed to mine microblocks - miner_tip = Some(MinerTip::new(ch, bh, microblock_privkey)); + miner_tip = Self::pick_higher_tip( + miner_tip, + Some(MinerTip::new( + ch, + bh, + microblock_privkey, + height, + snapshot.block_height, + )), + ); + } + + if has_new_data { + // process the block, now that we've advertized it + if let Err(Error::CoordinatorClosed) = self.process_new_block() { + // coordiantor stopped + return (false, None); + } } } else { debug!( "Relayer: Did not win sortition in {}, winning block was {}/{}", &burn_hash, &consensus_hash, &block_header_hash ); - miner_tip = None; + miner_tip = Self::pick_higher_tip(miner_tip, None); } (true, miner_tip) @@ -2200,7 +2322,7 @@ impl RelayerThread { // coordinator thread hang-up return false; } - miner_tip = new_miner_tip; + miner_tip = Self::pick_higher_tip(miner_tip, new_miner_tip); // clear all blocks up to this consensus hash let this_burn_tip = SortitionDB::get_block_snapshot_consensus( @@ -2238,15 +2360,32 @@ impl RelayerThread { // resume mining if we blocked it if num_tenures > 0 { + self.mined_stacks_block = false; signal_mining_ready(self.globals.get_miner_status()); } - // update state - self.miner_tip = miner_tip; - + // update state for microblock mining + self.setup_microblock_mining_state(miner_tip); true } + /// Update the miner tip with a new tip. If it's changed, then clear out the microblock stream + /// cost since we won't be mining it anymore. + fn setup_microblock_mining_state(&mut self, new_miner_tip: Option) { + // update state + let my_miner_tip = std::mem::replace(&mut self.miner_tip, None); + let best_tip = Self::pick_higher_tip(my_miner_tip.clone(), new_miner_tip.clone()); + if best_tip == new_miner_tip && best_tip != my_miner_tip { + // tip has changed + debug!( + "Relayer: Best miner tip went from {:?} to {:?}", + &my_miner_tip, &new_miner_tip + ); + self.microblock_stream_cost = ExecutionCost::zero(); + } + self.miner_tip = best_tip; + } + /// Constructs and returns a LeaderKeyRegisterOp out of the provided params fn inner_generate_leader_key_register_op( address: StacksAddress, @@ -2303,7 +2442,16 @@ impl RelayerThread { ret } - /// Create the block miner thread state + /// Create the block miner thread state. + /// Only proceeds if all of the following are true: + /// * the miner is not blocked + /// * last_burn_block corresponds to the canonical sortition DB's chain tip + /// * the time of issuance is sufficiently recent + /// * there are no unprocessed stacks blocks in the staging DB + /// * the relayer has already tried a download scan that included this sortition (which, if a + /// block was found, would have placed it into the staging DB and marked it as + /// unprocessed) + /// * a miner thread is not running already fn create_block_miner( &mut self, registered_key: RegisteredKey, @@ -2370,12 +2518,11 @@ impl RelayerThread { } } - if burn_chain_sn.block_height == self.last_network_block_height - && (self.last_network_download_passes < self.min_network_download_passes - && self.config.miner.wait_for_block_download) + if burn_chain_sn.block_height != self.last_network_block_height + || !self.has_waited_for_latest_blocks() { - debug!("Relayer: network has not had a chance to process in-flight blocks ({} == {} && {} < {})", - burn_chain_sn.block_height, self.last_network_block_height, self.last_network_download_passes, self.min_network_download_passes); + debug!("Relayer: network has not had a chance to process in-flight blocks ({} != {} || !({}))", + burn_chain_sn.block_height, self.last_network_block_height, self.debug_waited_for_latest_blocks()); return None; } @@ -2402,8 +2549,7 @@ impl RelayerThread { debug!( "Relayer: Spawn tenure thread"; "height" => last_burn_block.block_height, - "burn_header_hash" => %burn_chain_tip, - "last_burn_header_hash" => %burn_header_hash + "burn_header_hash" => %burn_header_hash, ); let miner_thread_state = @@ -2423,6 +2569,11 @@ impl RelayerThread { return false; } + if self.mined_stacks_block && self.config.node.mine_microblocks { + debug!("Relayer: mined a Stacks block already; waiting for microblock miner"); + return false; + } + let mut miner_thread_state = match self.create_block_miner(registered_key, last_burn_block, issue_timestamp_ms) { Some(state) => state, @@ -2445,35 +2596,52 @@ impl RelayerThread { true } - /// Start up a microblock miner thread if we can: - /// * no miner thread must be running already - /// * the miner must not be blocked - /// * we must have won the sortition on the stacks chain tip - /// Returns true if the thread was started; false if not. - pub fn microblock_miner_thread_try_start( - &mut self, - burnchain_tip: BlockSnapshot, - tenure_issue_ms: u128, - ) -> bool { + /// See if we should run a microblock tenure now. + /// Return true if so; false if not + fn can_run_microblock_tenure(&mut self) -> bool { if !self.config.node.mine_microblocks { - // node will not mine microblocks + // not enabled + test_debug!("Relayer: not configured to mine microblocks"); return false; } - if self.last_microblock_tenure_time > tenure_issue_ms { - // stale request + if !self.miner_thread_try_join() { + // already running (for an anchored block or microblock) + test_debug!("Relayer: miner thread already running so cannot mine microblock"); + return false; + } + if self.microblock_deadline > get_epoch_time_ms() { + debug!( + "Relayer: Too soon to start a microblock tenure ({} > {})", + self.microblock_deadline, + get_epoch_time_ms() + ); + return false; + } + if self.miner_tip.is_none() { + debug!("Relayer: did not win last block, so cannot mine microblocks"); return false; } if !self.mined_stacks_block { // have not tried to mine a stacks block yet that confirms previously-mined unconfirmed // state (or have not tried to mine a new Stacks block yet for this active tenure); + debug!("Relayer: Did not mine a block yet, so will not mine a microblock"); return false; } - if let Some(cur_sortition) = self.globals.get_last_sortition() { - if burnchain_tip.sortition_id != cur_sortition.sortition_id { - debug!("Relayer: Drop stale RunMicroblockTenure for {}/{}: current sortition is for {} ({})", &burnchain_tip.consensus_hash, &burnchain_tip.winning_stacks_block_hash, &cur_sortition.consensus_hash, &cur_sortition.burn_header_hash); - return false; - } + if self.globals.get_last_sortition().is_none() { + debug!("Relayer: no first sortition yet"); + return false; } + + // go ahead + true + } + + /// Start up a microblock miner thread if we can: + /// * no miner thread must be running already + /// * the miner must not be blocked + /// * we must have won the sortition on the stacks chain tip + /// Returns true if the thread was started; false if not. + pub fn microblock_miner_thread_try_start(&mut self) -> bool { let miner_tip = match self.miner_tip.as_ref() { Some(tip) => tip.clone(), None => { @@ -2481,8 +2649,23 @@ impl RelayerThread { return false; } }; + + let burnchain_tip = match self.globals.get_last_sortition() { + Some(sn) => sn, + None => { + debug!("Relayer: no first sortition yet"); + return false; + } + }; + + debug!( + "Relayer: mined Stacks block {}/{} so can mine microblocks", + &miner_tip.consensus_hash, &miner_tip.block_hash + ); + if !self.miner_thread_try_join() { // already running (for an anchored block or microblock) + debug!("Relayer: miner thread already running so cannot mine microblock"); return false; } if self @@ -2528,7 +2711,10 @@ impl RelayerThread { e }) { + // thread started! self.miner_thread = Some(miner_handle); + self.microblock_deadline = + get_epoch_time_ms() + (self.config.node.microblock_frequency as u128); } true @@ -2596,7 +2782,7 @@ impl RelayerThread { MinerThreadResult::Microblock(microblock_result, miner_tip) => { // finished mining a microblock match microblock_result { - Ok(Some(next_microblock)) => { + Ok(Some((next_microblock, new_cost))) => { // apply it let microblock_hash = next_microblock.block_hash(); @@ -2644,13 +2830,15 @@ impl RelayerThread { ); } + self.last_microblock_tenure_time = get_epoch_time_ms(); + self.microblock_stream_cost = new_cost; + // synchronise state self.with_chainstate( |relayer_thread, _sortdb, chainstate, _mempool| { relayer_thread.globals.send_unconfirmed_txs(chainstate); }, ); - self.last_microblock_tenure_time = get_epoch_time_ms(); // have not yet mined a stacks block that confirms this microblock, so // do that on the next run @@ -2709,12 +2897,16 @@ impl RelayerThread { ); true } - RelayerDirective::RunMicroblockTenure(burnchain_tip, tenure_issue_ms) => { - self.microblock_miner_thread_try_start(burnchain_tip, tenure_issue_ms); - true - } RelayerDirective::Exit => false, }; + if !continue_running { + return false; + } + + // see if we need to run a microblock tenure + if self.can_run_microblock_tenure() { + self.microblock_miner_thread_try_start(); + } continue_running } } @@ -2854,8 +3046,8 @@ pub struct PeerThread { /// total number of download state-machine passes so far. Used to signal when to download the /// next reward cycle of blocks. num_download_passes: u64, - /// when should we queue up the next request to run a microblock tenure? (epoch time in millis) - mblock_deadline: u128, + /// last burnchain block seen in the PeerNetwork's chain view since the last run + last_burn_block_height: u64, } impl PeerThread { @@ -2932,7 +3124,7 @@ impl PeerThread { num_p2p_state_machine_passes: 0, num_inv_sync_passes: 0, num_download_passes: 0, - mblock_deadline: 0, + last_burn_block_height: 0, } } @@ -3002,7 +3194,7 @@ impl PeerThread { ); 1 } else { - cmp::min(self.poll_timeout, self.config.node.microblock_frequency) + self.poll_timeout }; let mut expected_attachments = match self.attachments_rx.try_recv() { @@ -3057,6 +3249,7 @@ impl PeerThread { match p2p_res { Ok(network_result) => { + let mut have_update = false; if self.num_p2p_state_machine_passes < network_result.num_state_machine_passes { // p2p state-machine did a full pass. Notify anyone listening. self.globals.sync_comms.notify_p2p_state_pass(); @@ -3067,31 +3260,30 @@ impl PeerThread { // inv-sync state-machine did a full pass. Notify anyone listening. self.globals.sync_comms.notify_inv_sync_pass(); self.num_inv_sync_passes = network_result.num_inv_sync_passes; + + // the relayer cares about the number of inventory passes, so pass this along + have_update = true; } if self.num_download_passes < network_result.num_download_passes { // download state-machine did a full pass. Notify anyone listening. self.globals.sync_comms.notify_download_pass(); self.num_download_passes = network_result.num_download_passes; + + // the relayer cares about the number of download passes, so pass this along + have_update = true; } - if network_result.has_data_to_store() { + if network_result.has_data_to_store() + || self.last_burn_block_height != network_result.burn_height + || have_update + { + // pass along if we have blocks, microblocks, or transactions, or a status + // update on the network's view of the burnchain + self.last_burn_block_height = network_result.burn_height; self.results_with_data .push_back(RelayerDirective::HandleNetResult(network_result)); } - - // only do this on the Ok() path, even if we're mining, because an error in - // network dispatching is likely due to resource exhaustion - if self.mblock_deadline < get_epoch_time_ms() && self.config.node.mine_microblocks { - debug!("P2P: schedule microblock tenure"); - self.results_with_data - .push_back(RelayerDirective::RunMicroblockTenure( - self.get_network().burnchain_tip.clone(), - get_epoch_time_ms(), - )); - self.mblock_deadline = - get_epoch_time_ms() + (self.config.node.microblock_frequency as u128); - } } Err(e) => { // this is only reachable if the network is not instantiated correctly -- @@ -3105,14 +3297,13 @@ impl PeerThread { // or a directive to mine microblocks if let Err(e) = self.globals.relay_send.try_send(next_result) { debug!( - "P2P: {:?}: download backpressure detected", - &self.get_network().local_peer + "P2P: {:?}: download backpressure detected (bufferred {})", + &self.get_network().local_peer, + self.results_with_data.len() ); match e { TrySendError::Full(directive) => { - if let RelayerDirective::RunMicroblockTenure(..) = directive { - // can drop this - } else if let RelayerDirective::RunTenure(..) = directive { + if let RelayerDirective::RunTenure(..) = directive { // can drop this } else { // don't lose this data -- just try it again @@ -3479,7 +3670,7 @@ impl StacksNode { } else { LeaderKeyRegistrationState::Inactive }; - + let relayer_thread = RelayerThread::new(runloop, local_peer.clone(), relayer); let relayer_thread_handle = thread::Builder::new() .name(format!("relayer-{}", &local_peer.data_url)) @@ -3678,7 +3869,6 @@ impl StacksNode { } } - // no-op on UserBurnSupport ops are not supported / produced at this point. self.globals.set_last_sortition(block_snapshot); last_sortitioned_block.map(|x| x.0) } From c3656a379d189e27f1f31e5c5a6820c15aa0e579 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 10 Oct 2022 15:40:18 -0400 Subject: [PATCH 048/178] feat: add integration tests for chain quality when there are multiple miners --- .../src/tests/neon_integrations.rs | 228 +++++++++++++----- 1 file changed, 163 insertions(+), 65 deletions(-) diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index e09aaf968..703be8479 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -7679,7 +7679,11 @@ fn test_flash_block_skip_tenure() { channel.stop_chains_coordinator(); } -fn make_expensive_tx_chain(privk: &StacksPrivateKey, fee_plus: u64) -> Vec> { +fn make_expensive_tx_chain( + privk: &StacksPrivateKey, + fee_plus: u64, + mblock_only: bool, +) -> Vec> { let addr = to_addr(&privk); let mut chain = vec![]; for nonce in 0..25 { @@ -7687,71 +7691,129 @@ fn make_expensive_tx_chain(privk: &StacksPrivateKey, fee_plus: u64) -> Vec = (0..200) + let privks: Vec<_> = (0..100) .into_iter() .map(|_| StacksPrivateKey::new()) .collect(); @@ -7766,7 +7828,6 @@ fn test_competing_miners_build_on_same_chain() { }) .collect(); - let num_miners = 5; let mut confs = vec![]; let mut burnchain_configs = vec![]; let mut blocks_processed = vec![]; @@ -7775,19 +7836,20 @@ fn test_competing_miners_build_on_same_chain() { let seed = StacksPrivateKey::new().to_bytes(); let (mut conf, _) = neon_integration_test_conf_with_seed(seed); - conf.node.mine_microblocks = false; - conf.miner.microblock_attempt_time_ms = 5_000; - conf.node.wait_time_for_microblocks = 0; conf.initial_balances.append(&mut balances.clone()); - conf.miner.first_attempt_time_ms = 2_000; - conf.miner.subsequent_attempt_time_ms = 5_000; + + conf.node.mine_microblocks = conf_template.node.mine_microblocks; + conf.miner.microblock_attempt_time_ms = conf_template.miner.microblock_attempt_time_ms; + conf.node.wait_time_for_microblocks = conf_template.node.wait_time_for_microblocks; + conf.node.microblock_frequency = conf_template.node.microblock_frequency; + conf.miner.first_attempt_time_ms = conf_template.miner.first_attempt_time_ms; + conf.miner.subsequent_attempt_time_ms = conf_template.miner.subsequent_attempt_time_ms; + conf.node.wait_time_for_blocks = conf_template.node.wait_time_for_blocks; + conf.burnchain.max_rbf = conf_template.burnchain.max_rbf; // multiple nodes so they must download from each other conf.miner.wait_for_block_download = true; - // don't RBF - conf.burnchain.max_rbf = 0; - confs.push(conf); } @@ -7914,7 +7976,7 @@ fn test_competing_miners_build_on_same_chain() { let all_txs: Vec<_> = privks .iter() .enumerate() - .map(|(i, pk)| make_expensive_tx_chain(pk, (25 * i) as u64)) + .map(|(i, pk)| make_expensive_tx_chain(pk, (25 * i) as u64, mblock_only)) .collect(); let mut cnt = 0; for tx_chain in all_txs { @@ -7931,6 +7993,42 @@ fn test_competing_miners_build_on_same_chain() { for i in 0..1000 { eprintln!("\n\nBuild block {}\n\n", i); btc_regtest_controller.build_next_block(1); - sleep_ms(10_000); + sleep_ms(block_time_ms); } } + +// TODO: this needs to run as a smoke test, since they take too long to run in CI +#[test] +#[ignore] +fn test_competing_miners_build_anchor_blocks_on_same_chain_without_rbf() { + let (mut conf, _) = neon_integration_test_conf(); + + conf.node.mine_microblocks = false; + conf.miner.microblock_attempt_time_ms = 5_000; + conf.node.wait_time_for_microblocks = 0; + conf.node.microblock_frequency = 10_000; + conf.miner.first_attempt_time_ms = 2_000; + conf.miner.subsequent_attempt_time_ms = 5_000; + conf.burnchain.max_rbf = 0; + conf.node.wait_time_for_blocks = 1_000; + + test_competing_miners_build_on_same_chain(5, conf, false, 10_000) +} + +// TODO: this needs to run as a smoke test, since they take too long to run in CI +#[test] +#[ignore] +fn test_competing_miners_build_anchor_blocks_and_microblocks_on_same_chain() { + let (mut conf, _) = neon_integration_test_conf(); + + conf.node.mine_microblocks = true; + conf.miner.microblock_attempt_time_ms = 2_000; + conf.node.wait_time_for_microblocks = 0; + conf.node.microblock_frequency = 0; + conf.miner.first_attempt_time_ms = 1; + conf.miner.subsequent_attempt_time_ms = 1; + conf.burnchain.max_rbf = 1000000; + conf.node.wait_time_for_blocks = 1_000; + + test_competing_miners_build_on_same_chain(5, conf, true, 15_000) +} From c3148e790c9ea5fb73d710ef69bc2ef06afa9d22 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 10 Oct 2022 15:45:16 -0400 Subject: [PATCH 049/178] fix: avoid gratuitous call to pick_higher_tip --- testnet/stacks-node/src/neon_node.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index b2b0ec9ca..366c74c15 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -2207,16 +2207,13 @@ impl RelayerThread { } // proceed to mine microblocks - miner_tip = Self::pick_higher_tip( - miner_tip, - Some(MinerTip::new( - ch, - bh, - microblock_privkey, - height, - snapshot.block_height, - )), - ); + miner_tip = Some(MinerTip::new( + ch, + bh, + microblock_privkey, + height, + snapshot.block_height, + )); } if has_new_data { @@ -2231,7 +2228,7 @@ impl RelayerThread { "Relayer: Did not win sortition in {}, winning block was {}/{}", &burn_hash, &consensus_hash, &block_header_hash ); - miner_tip = Self::pick_higher_tip(miner_tip, None); + miner_tip = None; } (true, miner_tip) From b7b68790c5b67b1dd21138f24542bdcb6ca19e00 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Mon, 10 Oct 2022 20:38:14 -0400 Subject: [PATCH 050/178] feat: document the structure of the Stacks node --- testnet/stacks-node/src/neon_node.rs | 138 +++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index 366c74c15..f87974cfb 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -1,3 +1,141 @@ +// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation +// Copyright (C) 2020 Stacks Open Internet Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/// Main body of code for the Stacks node and miner. +/// +/// System schematic. +/// Legend: +/// |------| Thread +/// /------\ Shared memory +/// @------@ Database +/// .------. Code module +/// +/// +/// |------------------| +/// | RunLoop thread | [1,7] +/// | .----------. |--------------------------------------. +/// | .StacksNode. | | +/// |---.----------.---| | +/// [1] | | | [1] | +/// .----------------* | *---------------. | +/// | [3] | | | +/// V | V V +/// |----------------| | [9,10] |---------------| [11] |--------------------------| +/// .--- | Relayer thread | <-----------|-----------> | P2P Thread | <--- | ChainsCoordinator thread | <--. +/// | |----------------| V |---------------| |--------------------------| | +/// | | | /-------------\ [2,3] | | | | | +/// | [1] | *--------> / Globals \ <-----------*----|--------------* | [4] | +/// | | [2,3,7] /-------------\ | | | +/// | V V [5] V | +/// | |----------------| @--------------@ @------------------@ | +/// | | Miner thread | <------------------------------ @ Mempool DB @ @ Chainstate DBs @ | +/// | |----------------| [6] @--------------@ @------------------@ | +/// | ^ | +/// | [8] | | +/// *----------------------------------------------------------------------------------------* | +/// | [7] | +/// *--------------------------------------------------------------------------------------------------------* +/// +/// [1] Spawns +/// [2] Synchronize unconfirmed state +/// [3] Enable/disable miner +/// [4] Processes block data +/// [5] Stores unconfirmed transactions +/// [6] Reads unconfirmed transactions +/// [7] Signals block arrival +/// [8] Store blocks and microblocks +/// [9] Pushes retrieved blocks and microblocks +/// [10] Broadcasts new blocks, microblocks, and transactions +/// [11] Notifies about new transaction attachment events +/// +/// When the node is running, there are 4-5 active threads at once. They are: +/// +/// * **RunLoop Thread**: This is the main thread, whose code body lives in src/run_loop/neon.rs. +/// This thread is responsible for: +/// * Bootup +/// * Running the burnchain indexer +/// * Notifying the ChainsCoordinator thread when there are new burnchain blocks to process +/// +/// * **Relayer Thread**: This is the thread that stores and relays blocks and microblocks. Both +/// it and the ChainsCoordinator thread are very I/O-heavy threads, and care has been taken to +/// ensure that neither one attempts to acquire a write-lock in the underlying databases. +/// Specifically, this thread directs the ChainsCoordinator thread when to process new Stacks +/// blocks, and it directs the miner thread (if running) to stop when either it or the +/// ChainsCoordinator thread needs to acquire the write-lock. +/// This thread is responsible for: +/// * Receiving new blocks and microblocks from the P2P thread via a shared channel +/// * (Sychronously) requesting the CoordinatorThread to process newly-stored Stacks blocks and +/// microblocks +/// * Building up the node's unconfirmed microblock stream state, and sharing it with the P2P +/// thread so it can answer queries about the unconfirmed microblock chain +/// * Pushing newly-discovered blocks and microblocks to the P2P thread for broadcast +/// * Registering the VRF public key for the miner +/// * Spawning the block and microblock miner threads, and stopping them if their continued +/// execution would inhibit block or microblock storage or processing. +/// * Submitting the burnchain operation to commit to a freshly-mined block +/// +/// * **Miner thread**: This is the thread that actually produces new blocks and microblocks. It +/// is spawned only by the Relayer thread to carry out mining activity when the underlying +/// chainstate is not needed by either the Relayer or ChainsCoordinator threeads. +/// This thread does the following: +/// * Walk the mempool DB to build a new block or microblock +/// * Return the block or microblock to the Relayer thread +/// +/// * **P2P Thread**: This is the thread that communicates with the rest of the p2p network, and +/// handles RPC requests. It is meant to do as little storage-write I/O as possible to avoid lock +/// contention with the Miner, Relayer, and ChainsCoordinator threads. In particular, it forwards +/// data it receives from the p2p thread to the Relayer thread for I/O-bound processing. At the +/// time of this writing, it still requires holding a write-lock to handle some RPC request, but +/// future work will remove this so that this thread's execution will not interfere with the +/// others. This is the only thread that does socket I/O. +/// This thread runs the PeerNetwork state machines, which include the following: +/// * Learning the node's public IP address +/// * Discovering neighbor nodes +/// * Forwarding newly-discovered blocks, microblocks, and transactions from the Relayer thread to +/// other neighbors +/// * Synchronizing block and microblock inventory state with other neighbors +/// * Downloading blocks and microblocks, and passing them to the Relayer for storage and processing +/// * Downloading transaction attachments as their hashes are discovered during block processing +/// * Synchronizing the local mempool database with other neighbors +/// (notifications for new attachments come from a shared channel in the ChainsCoordinator thread) +/// * Handling HTTP requests +/// +/// * **ChainsCoordinator Thread**: This thread process sortitions and Stacks blocks and +/// microblocks, and handles PoX reorgs should they occur (this mainly happens in boot-up). It, +/// like the Relayer thread, is a very I/O-heavy thread, and it will hold a write-lock on the +/// chainstate DBs while it works. Its actions are controlled by a CoordinatorComms structure in +/// the Globals shared state, which the Relayer thread and RunLoop thread both drive (the former +/// drives Stacks blocks processing, the latter sortitions). +/// This thread is responsible for: +/// * Responding to requests from other threads to process sortitions +/// * Responding to requests from other threads to process Stacks blocks and microblocks +/// * Processing PoX chain reorgs, should they ever happen +/// * Detecting attachment creation events, and informing the P2P thread of them so it can go +/// and download them +/// +/// In addition to the mempool and chainstate databases, these threads share access to a Globals +/// singleton that contains soft state shared between threads. Mainly, the Globals struct is meant +/// to store inter-thread shared singleton communication media all in one convenient struct. Each +/// thread has a handle to the struct's shared state handles. Global state includes: +/// * The global flag as to whether or not the miner thread can be running +/// * The global shutdown flag that, when set, causes all threads to terminate +/// * Sender channel endpoints that can be shared between threads +/// * Metrics about the node's behavior (e.g. number of blocks processed, etc.) +/// +/// This file may be refactored in the future into a full-fledged module. use std::cmp; use std::collections::HashMap; use std::collections::{HashSet, VecDeque}; From e5e39b1b04f497e4b703d81a0875842d5e3f6c69 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 11 Oct 2022 11:01:38 -0400 Subject: [PATCH 051/178] fix: fix compile-time issue with unit tests --- src/net/rpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/rpc.rs b/src/net/rpc.rs index cce01f265..133b30aa6 100644 --- a/src/net/rpc.rs +++ b/src/net/rpc.rs @@ -4088,7 +4088,7 @@ mod test { let peer_info = RPCPeerInfoData::from_network( &peer_server.network, &peer_server.stacks_node.as_ref().unwrap().chainstate, - &None, + None, &Sha256Sum::zero(), ); From 9a62096894b1d8de157e811384b7f44653a46a03 Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Tue, 11 Oct 2022 20:52:03 -0400 Subject: [PATCH 052/178] feat: improvements to mempool iteration These changes take lessons learned from experiments by @gregorycoppola and myself, as well as feedback on various related PRs. It uses the following techniques to improve the speed and scalability of the mempool walk. * Uses rusqlite's `Rows` iterator to read one row at a time * Caches the nonces in memory to avoid repeated lookups * Restarts search from the highest fee-rate transactions after every executed transaction * Caches potential transactions in memory to retry on next pass With this implementation, miners can reliably fill a block in <30s, regardless of how large the mempool gets. --- src/chainstate/stacks/db/accounts.rs | 4 + src/chainstate/stacks/miner.rs | 3 +- src/core/mempool.rs | 415 ++++++++++++++++++--------- 3 files changed, 282 insertions(+), 140 deletions(-) diff --git a/src/chainstate/stacks/db/accounts.rs b/src/chainstate/stacks/db/accounts.rs index 5d2b19355..6829c6d5b 100644 --- a/src/chainstate/stacks/db/accounts.rs +++ b/src/chainstate/stacks/db/accounts.rs @@ -153,6 +153,10 @@ impl StacksChainState { }) } + pub fn get_nonce(clarity_tx: &mut T, principal: &PrincipalData) -> u64 { + clarity_tx.with_clarity_db_readonly(|ref mut db| db.get_account_nonce(principal)) + } + pub fn get_account_ft( clarity_tx: &mut ClarityTx, contract_id: &QualifiedContractIdentifier, diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index 87026e8b4..a12dd431b 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -2425,7 +2425,7 @@ impl StacksBlockBuilder { ); } - debug!( + info!( "Miner: mined anchored block"; "block_hash" => %block.block_hash(), "height" => block.header.total_work.work, @@ -2435,6 +2435,7 @@ impl StacksBlockBuilder { "parent_stacks_microblock_seq" => block.header.parent_microblock_sequence, "block_size" => size, "execution_consumed" => %consumed, + "%-full" => block_limit.proportion_largest_dimension(&consumed), "assembly_time_ms" => ts_end.saturating_sub(ts_start), "tx_fees_microstacks" => block.txs.iter().fold(0, |agg: u64, tx| { agg.saturating_add(tx.get_tx_fee()) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index db9b95f32..546ef00f4 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -14,8 +14,8 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::cmp; -use std::collections::HashSet; +use std::cmp::{self, Ordering}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::fs; use std::hash::Hasher; use std::io::{Read, Write}; @@ -31,6 +31,7 @@ use rusqlite::Error as SqliteError; use rusqlite::OpenFlags; use rusqlite::OptionalExtension; use rusqlite::Row; +use rusqlite::Rows; use rusqlite::Transaction; use rusqlite::NO_PARAMS; @@ -241,10 +242,16 @@ pub struct MemPoolTxInfo { pub metadata: MemPoolTxMetadata, } -#[derive(Debug, PartialEq, Clone)] -pub enum MemPoolTxInfoPartial { - NeedsNonces { addrs_needed: Vec }, - HasNonces(MemPoolTxInfo), +/// This class is a minimal version of `MemPoolTxInfo`. It contains +/// just enough information to 1) filter by nonce readiness, 2) sort by fee rate. +#[derive(Debug, Clone)] +pub struct MemPoolTxInfoPartial { + pub txid: Txid, + pub fee_rate: Option, + pub origin_address: StacksAddress, + pub origin_nonce: u64, + pub sponsor_address: StacksAddress, + pub sponsor_nonce: u64, } #[derive(Debug, PartialEq, Clone)] @@ -367,25 +374,24 @@ impl FromRow for MemPoolTxInfo { impl FromRow for MemPoolTxInfoPartial { fn from_row<'a>(row: &'a Row) -> Result { - let md = MemPoolTxMetadata::from_row(row)?; - let needs_nonces = md.get_unknown_nonces(); - let consider = if !needs_nonces.is_empty() { - MemPoolTxInfoPartial::NeedsNonces { - addrs_needed: needs_nonces, - } - } else { - let tx_bytes: Vec = row.get_unwrap("tx"); - let tx = StacksTransaction::consensus_deserialize(&mut &tx_bytes[..]) - .map_err(|_e| db_error::ParseError)?; - - if tx.txid() != md.txid { - return Err(db_error::ParseError); - } - - MemPoolTxInfoPartial::HasNonces(MemPoolTxInfo { tx, metadata: md }) + let txid = Txid::from_column(row, "txid")?; + let fee_rate: Option = match row.get("fee_rate") { + Ok(rate) => Some(rate), + Err(_) => None, }; + let origin_address = StacksAddress::from_column(row, "origin_address")?; + let origin_nonce = u64::from_column(row, "origin_nonce")?; + let sponsor_address = StacksAddress::from_column(row, "sponsor_address")?; + let sponsor_nonce = u64::from_column(row, "sponsor_nonce")?; - Ok(consider) + Ok(MemPoolTxInfoPartial { + txid, + fee_rate, + origin_address, + origin_nonce, + sponsor_address, + sponsor_nonce, + }) } } @@ -750,6 +756,110 @@ impl MemPoolTxInfo { } } +/// Used to locally cache nonces to avoid repeatedly looking them up in the nonce. +struct NonceCache { + cache: HashMap, + size: usize, +} + +impl NonceCache { + const MAX_SIZE: usize = 1024 * 1024; + + fn new() -> Self { + Self { + cache: HashMap::new(), + size: 0, + } + } + + fn get(&mut self, address: &StacksAddress, clarity_tx: &mut C) -> u64 + where + C: ClarityConnection, + { + match self.cache.get(address) { + Some(nonce) => *nonce, + None => { + let nonce = StacksChainState::get_nonce(clarity_tx, &address.clone().into()); + // Simple size cap to the cache -- once it's full, all nonces + // will be looked up every time. This is bad for performance + // but is unlikely to occur due to the typical number of + // transactions processed before filling a block. + if self.size < Self::MAX_SIZE { + self.cache.insert(address.clone(), nonce); + self.size += 1; + } + nonce + } + } + } + + fn increment(&mut self, address: StacksAddress) { + let nonce = self.cache.entry(address).or_insert(0); + *nonce += 1; + } +} + +struct CandidateCache { + cache: VecDeque, + next: VecDeque, + size: usize, +} + +impl CandidateCache { + const MAX_SIZE: usize = 64 * 1024; + + fn new() -> Self { + Self { + cache: VecDeque::new(), + next: VecDeque::new(), + size: 0, + } + } + + fn next(&mut self) -> Option { + self.cache.pop_back() + } + + fn push(&mut self, tx: MemPoolTxInfoPartial) { + if self.size < Self::MAX_SIZE { + self.next.push_back(tx); + } + } + + fn reset(&mut self) { + self.cache = std::mem::take(&mut self.next); + } +} + +/// Evaluates the pair of nonces, to determine an order +/// +/// Returns: +/// `Equal` if both origin and sponsor nonces match expected +/// `Less` if the origin nonce is less than expected, or the origin matches expected and the +/// sponsor nonce is less than expected +/// `Greater` if the origin nonce is greater than expected, or the origin matches expected +/// and the sponsor nonce is greater than expected +fn order_nonces( + origin_actual: u64, + origin_expected: u64, + sponsor_actual: u64, + sponsor_expected: u64, +) -> Ordering { + if origin_actual < origin_expected { + return Ordering::Less; + } else if origin_actual > origin_expected { + return Ordering::Greater; + } + + if sponsor_actual < sponsor_expected { + return Ordering::Less; + } else if sponsor_actual > sponsor_expected { + return Ordering::Greater; + } + + Ordering::Equal +} + impl MemPoolDB { fn instantiate_mempool_db(conn: &mut DBConn) -> Result<(), db_error> { let mut tx = tx_begin_immediate(conn)?; @@ -991,73 +1101,6 @@ impl MemPoolDB { Ok(()) } - /// Select the next TX to consider from the pool of transactions without cost estimates. - /// If a transaction is found, returns Some object containing the transaction and a boolean indicating - /// whether or not the miner should propagate transaction receipts back to the estimator. - fn get_next_tx_to_consider_no_estimate( - &self, - ) -> Result, db_error> { - let select_no_estimate = "SELECT * FROM mempool WHERE - ((origin_nonce = last_known_origin_nonce AND - sponsor_nonce = last_known_sponsor_nonce) OR (last_known_origin_nonce is NULL) OR (last_known_sponsor_nonce is NULL)) - AND fee_rate IS NULL ORDER BY tx_fee DESC LIMIT 1"; - query_row(&self.db, select_no_estimate, rusqlite::NO_PARAMS) - .map(|opt_tx| opt_tx.map(|tx| (tx, true))) - } - - /// Select the next TX to consider from the pool of transactions with cost estimates. - /// If a transaction is found, returns Some object containing the transaction and a boolean indicating - /// whether or not the miner should propagate transaction receipts back to the estimator. - fn get_next_tx_to_consider_with_estimate( - &self, - ) -> Result, db_error> { - let select_estimate = "SELECT * FROM mempool WHERE - ((origin_nonce = last_known_origin_nonce AND - sponsor_nonce = last_known_sponsor_nonce) OR (last_known_origin_nonce is NULL) OR (last_known_sponsor_nonce is NULL)) - AND fee_rate IS NOT NULL ORDER BY fee_rate DESC LIMIT 1"; - query_row(&self.db, select_estimate, rusqlite::NO_PARAMS) - .map(|opt_tx| opt_tx.map(|tx| (tx, false))) - } - - /// * `start_with_no_estimate` - Pass `true` to make this function - /// start by considering transactions without a cost - /// estimate, and if none are found, use transactions with a cost estimate. - /// Pass `false` for the opposite behavior. - fn get_next_tx_to_consider( - &self, - start_with_no_estimate: bool, - ) -> Result { - let (next_tx, update_estimate): (MemPoolTxInfoPartial, bool) = if start_with_no_estimate { - match self.get_next_tx_to_consider_no_estimate()? { - Some(result) => result, - None => match self.get_next_tx_to_consider_with_estimate()? { - Some(result) => result, - None => return Ok(ConsiderTransactionResult::NoTransactions), - }, - } - } else { - match self.get_next_tx_to_consider_with_estimate()? { - Some(result) => result, - None => match self.get_next_tx_to_consider_no_estimate()? { - Some(result) => result, - None => return Ok(ConsiderTransactionResult::NoTransactions), - }, - } - }; - - match next_tx { - MemPoolTxInfoPartial::NeedsNonces { addrs_needed } => { - Ok(ConsiderTransactionResult::UpdateNonces(addrs_needed)) - } - MemPoolTxInfoPartial::HasNonces(tx) => { - Ok(ConsiderTransactionResult::Consider(ConsiderTransaction { - tx, - update_estimate, - })) - } - } - } - /// Find the origin addresses who have sent the highest-fee transactions fn find_origin_addresses_by_descending_fees( &self, @@ -1166,7 +1209,35 @@ impl MemPoolDB { let tx_consideration_sampler = Uniform::new(0, 100); let mut rng = rand::thread_rng(); - let mut remember_start_with_estimate = None; + let mut candidate_cache = CandidateCache::new(); + let mut nonce_cache = NonceCache::new(); + + let sql = " + SELECT txid, origin_nonce, origin_address, sponsor_nonce, sponsor_address, fee_rate + FROM mempool + WHERE fee_rate IS NULL + "; + let mut query_stmt = self + .db + .prepare(&sql) + .map_err(|err| Error::SqliteError(err))?; + let mut null_iterator = query_stmt + .query(NO_PARAMS) + .map_err(|err| Error::SqliteError(err))?; + + let sql = " + SELECT txid, origin_nonce, origin_address, sponsor_nonce, sponsor_address, fee_rate + FROM mempool + WHERE fee_rate IS NOT NULL + ORDER BY fee_rate DESC + "; + let mut query_stmt = self + .db + .prepare(&sql) + .map_err(|err| Error::SqliteError(err))?; + let mut fee_iterator = query_stmt + .query(NO_PARAMS) + .map_err(|err| Error::SqliteError(err))?; loop { if start_time.elapsed().as_millis() > settings.max_walk_time_ms as u128 { @@ -1175,71 +1246,137 @@ impl MemPoolDB { break; } - let start_with_no_estimate = remember_start_with_estimate.unwrap_or_else(|| { - tx_consideration_sampler.sample(&mut rng) < settings.consider_no_estimate_tx_prob - }); + let start_with_no_estimate = + tx_consideration_sampler.sample(&mut rng) < settings.consider_no_estimate_tx_prob; - match self.get_next_tx_to_consider(start_with_no_estimate)? { - ConsiderTransactionResult::NoTransactions => { - debug!("No more transactions to consider in mempool"); - break; + // First, try to read from the retry list + let (candidate, update_estimate) = match candidate_cache.next() { + Some(tx) => { + let update_estimate = tx.fee_rate.is_none(); + (tx, update_estimate) } - ConsiderTransactionResult::UpdateNonces(addresses) => { - // if we need to update the nonce for the considered transaction, - // use the last value of start_with_no_estimate on the next loop - remember_start_with_estimate = Some(start_with_no_estimate); - let mut last_addr = None; - for address in addresses.into_iter() { - debug!("Update nonce"; "address" => %address); - // do not recheck nonces if the sponsor == origin - if last_addr.as_ref() == Some(&address) { - continue; + None => { + // When the retry list is empty, read from the mempool db, + // randomly selecting from either the null fee-rate transactions + // or those with fee-rate estimates. + let opt_tx = if start_with_no_estimate { + null_iterator + .next() + .map_err(|err| Error::SqliteError(err))? + } else { + fee_iterator.next().map_err(|err| Error::SqliteError(err))? + }; + match opt_tx { + Some(row) => (MemPoolTxInfoPartial::from_row(row)?, start_with_no_estimate), + None => { + // If the selected iterator is empty, check the other + match if start_with_no_estimate { + fee_iterator.next().map_err(|err| Error::SqliteError(err))? + } else { + null_iterator + .next() + .map_err(|err| Error::SqliteError(err))? + } { + Some(row) => ( + MemPoolTxInfoPartial::from_row(row)?, + !start_with_no_estimate, + ), + None => { + debug!("No more transactions to consider in mempool"); + break; + } + } } - let min_nonce = - StacksChainState::get_account(clarity_tx, &address.clone().into()) - .nonce; - - self.update_last_known_nonces(&address, min_nonce)?; - last_addr = Some(address) } } - ConsiderTransactionResult::Consider(consider) => { - // if we actually consider the chosen transaction, - // compute a new start_with_no_estimate on the next loop - remember_start_with_estimate = None; - debug!("Consider mempool transaction"; + }; + + // Check the nonces. + let expected_origin_nonce = nonce_cache.get(&candidate.origin_address, clarity_tx); + let expected_sponsor_nonce = nonce_cache.get(&candidate.sponsor_address, clarity_tx); + match order_nonces( + candidate.origin_nonce, + expected_origin_nonce, + candidate.sponsor_nonce, + expected_sponsor_nonce, + ) { + Ordering::Less => { + debug!( + "Mempool: unexecutable: drop tx ({})", + candidate.fee_rate.unwrap_or_default() + ); + // This transaction cannot execute in this pass, just drop it + continue; + } + Ordering::Greater => { + debug!( + "Mempool: nonces too high, cached for later ({})", + candidate.fee_rate.unwrap_or_default() + ); + // This transaction could become runnable in this pass, save it for later + candidate_cache.push(candidate); + continue; + } + Ordering::Equal => { + // Candidate transaction: fall through + } + }; + + // Read in and deserialize the transaction. + let tx_info_option = MemPoolDB::get_tx(&self.conn(), &candidate.txid)?; + let tx_info = match tx_info_option { + Some(tx) => tx, + None => { + // Note: Don't panic here because maybe the state has changed from garbage collection. + warn!("Miner: could not find a tx for id {:?}", &candidate.txid); + continue; + } + }; + + let consider = ConsiderTransaction { + tx: tx_info, + update_estimate, + }; + debug!("Consider mempool transaction"; "txid" => %consider.tx.tx.txid(), "origin_addr" => %consider.tx.metadata.origin_address, "sponsor_addr" => %consider.tx.metadata.sponsor_address, "accept_time" => consider.tx.metadata.accept_time, "tx_fee" => consider.tx.metadata.tx_fee, + "fee_rate" => candidate.fee_rate, "size" => consider.tx.metadata.len); - total_considered += 1; + total_considered += 1; - // Run `todo` on the transaction. - match todo(clarity_tx, &consider, self.cost_estimator.as_mut())? { - Some(tx_event) => { - match tx_event { - TransactionEvent::Skipped(_) => { - // don't push `Skipped` events to the observer - } - _ => { - output_events.push(tx_event); - } - } + // Run `todo` on the transaction. + match todo(clarity_tx, &consider, self.cost_estimator.as_mut())? { + Some(tx_event) => { + match tx_event { + TransactionEvent::Skipped(_) => { + // don't push `Skipped` events to the observer } - None => { - debug!("Mempool iteration early exit from iterator"); - break; + _ => { + output_events.push(tx_event); } } - - self.bump_last_known_nonces(&consider.tx.metadata.origin_address)?; - if consider.tx.tx.auth.is_sponsored() { - self.bump_last_known_nonces(&consider.tx.metadata.sponsor_address)?; - } + } + None => { + debug!("Mempool iteration early exit from iterator"); + break; } } + + // Bump nonces in the cache for the executed transaction + nonce_cache.increment(consider.tx.metadata.origin_address); + if consider.tx.tx.auth.is_sponsored() { + nonce_cache.increment(consider.tx.metadata.sponsor_address); + } + + // Reset for finding the next transaction to process + debug!( + "Mempool: reset: retry list has {} entries", + candidate_cache.size + ); + candidate_cache.reset(); } debug!( From 91900935725d7bfe757f3afc2a1e220f44bbc526 Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Wed, 12 Oct 2022 12:40:46 -0400 Subject: [PATCH 053/178] fix: candidate cache should pop from the front This was a typo when refactoring, and caused worse ordering for the transactions. --- src/core/mempool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 546ef00f4..2766b180e 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -817,7 +817,7 @@ impl CandidateCache { } fn next(&mut self) -> Option { - self.cache.pop_back() + self.cache.pop_front() } fn push(&mut self, tx: MemPoolTxInfoPartial) { From 1586e86d20d8931c3e51804e337539f3b3c1d33a Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Wed, 12 Oct 2022 13:11:19 -0400 Subject: [PATCH 054/178] fix: resolve problems with candidate cache --- src/core/mempool.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 2766b180e..de164d231 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -823,11 +823,14 @@ impl CandidateCache { fn push(&mut self, tx: MemPoolTxInfoPartial) { if self.size < Self::MAX_SIZE { self.next.push_back(tx); + self.size += 1; } } fn reset(&mut self) { + self.next.append(&mut self.cache); self.cache = std::mem::take(&mut self.next); + self.size = 0; } } From 6fe44078f68518e77c357cb3c80bb0d08f631d5f Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 12 Oct 2022 13:49:47 -0400 Subject: [PATCH 055/178] fix: fix failing unit tests by *not* returning a microblock who has a sibling in a fork whereby it shares a parent with another microblock --- src/chainstate/stacks/db/blocks.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/chainstate/stacks/db/blocks.rs b/src/chainstate/stacks/db/blocks.rs index 81b87cd51..544e2da4c 100644 --- a/src/chainstate/stacks/db/blocks.rs +++ b/src/chainstate/stacks/db/blocks.rs @@ -1680,6 +1680,7 @@ impl StacksChainState { mblock.header, conflict.header, )); + ret.pop(); // last microblock pushed (i.e. the tip) conflicts with mblock break; } From 2ac20824f527a18f302f96111916e327e66c4c47 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 12 Oct 2022 13:50:37 -0400 Subject: [PATCH 056/178] fix: mine a microblock before mining another stacks block when we win sortition --- testnet/stacks-node/src/neon_node.rs | 45 +++++++++++++++++++++------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index f87974cfb..fb9d3bf5e 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -2493,14 +2493,26 @@ impl RelayerThread { }); } - // resume mining if we blocked it - if num_tenures > 0 { - self.mined_stacks_block = false; - signal_mining_ready(self.globals.get_miner_status()); - } - // update state for microblock mining self.setup_microblock_mining_state(miner_tip); + + // resume mining if we blocked it + if num_tenures > 0 { + if self.miner_tip.is_some() { + // we won the highest tenure + if self.config.node.mine_microblocks { + // mine a microblock first + self.mined_stacks_block = true; + } else { + // mine a Stacks block first -- we won't build microblocks + self.mined_stacks_block = false; + } + } else { + // mine a Stacks block first -- we didn't win + self.mined_stacks_block = false; + } + signal_mining_ready(self.globals.get_miner_status()); + } true } @@ -2718,7 +2730,7 @@ impl RelayerThread { }; if let Ok(miner_handle) = thread::Builder::new() - .name(format!("miner-{}", self.local_peer.data_url)) + .name(format!("miner-block-{}", self.local_peer.data_url)) .spawn(move || miner_thread_state.run_tenure()) .map_err(|e| { error!("Relayer: Failed to start tenure thread: {:?}", &e); @@ -2834,7 +2846,7 @@ impl RelayerThread { }; if let Ok(miner_handle) = thread::Builder::new() - .name(format!("miner-{}", self.local_peer.data_url)) + .name(format!("miner-microblock-{}", self.local_peer.data_url)) .spawn(move || { Some(MinerThreadResult::Microblock( microblock_thread_state.try_mine_microblock(miner_tip.clone()), @@ -2877,7 +2889,14 @@ impl RelayerThread { ongoing_commit_opt, ) => { // finished mining a block - if self.last_mined_blocks.len() == 0 { + if BlockMinerThread::find_inflight_mined_blocks( + last_mined_block.my_block_height, + &self.last_mined_blocks, + ) + .len() + == 0 + { + // first time we've mined a block in this burnchain block self.globals.counters.bump_blocks_processed(); } @@ -2980,10 +2999,16 @@ impl RelayerThread { self.mined_stacks_block = false; } Ok(None) => { - debug!("Relayer: did not mine microblock in this tenure") + debug!("Relayer: did not mine microblock in this tenure"); + + // switch back to block mining + self.mined_stacks_block = false; } Err(e) => { warn!("Relayer: Failed to mine next microblock: {:?}", &e); + + // switch back to block mining + self.mined_stacks_block = false; } } } From 365215284b2db1fee78e921d86306774a6e8675f Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Wed, 12 Oct 2022 13:50:57 -0400 Subject: [PATCH 057/178] fix: shorter microblock deadline for microblocks_event_test --- testnet/stacks-node/src/tests/neon_integrations.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index 703be8479..02f3fde12 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -4312,7 +4312,7 @@ fn mining_events_integration_test() { }); conf.node.mine_microblocks = true; - conf.node.wait_time_for_microblocks = 30000; + conf.node.wait_time_for_microblocks = 1000; conf.node.microblock_frequency = 1000; conf.miner.min_tx_fee = 1; From 022b75373fe27b964a9c03afc8f6aeeb6e27372a Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Wed, 12 Oct 2022 17:48:36 -0400 Subject: [PATCH 058/178] docs: add details for corner cases of `pow` Fixes: #3295 --- clarity/src/vm/docs/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/clarity/src/vm/docs/mod.rs b/clarity/src/vm/docs/mod.rs index 7523b04c1..dd9eec0df 100644 --- a/clarity/src/vm/docs/mod.rs +++ b/clarity/src/vm/docs/mod.rs @@ -216,7 +216,13 @@ const MOD_API: SimpleFunctionAPI = SimpleFunctionAPI { const POW_API: SimpleFunctionAPI = SimpleFunctionAPI { name: None, signature: "(pow i1 i2)", - description: "Returns the result of raising `i1` to the power of `i2`. In the event of an _overflow_, throws a runtime error.", + description: "Returns the result of raising `i1` to the power of `i2`. In the event of an _overflow_, throws a runtime error. +Note: Corner cases are handled with the following rules: + * if both `i1` and `i2` are `0`, return `1` + * if `i1` is `1`, return `1` + * if `i1` is `0`, return `0` + * if `i2` is `1`, return `i1` + * if `i2` is negative or greater than `u32::MAX`, throw a runtime error", example: "(pow 2 3) ;; Returns 8 (pow 2 2) ;; Returns 4 (pow 7 1) ;; Returns 7 From 0d8d205eba52cb780c23a23ee6709a2749388f78 Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Thu, 13 Oct 2022 13:04:39 -0400 Subject: [PATCH 059/178] fix: candidate cache size limitation --- src/core/mempool.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index de164d231..71c7c9bec 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -802,7 +802,6 @@ impl NonceCache { struct CandidateCache { cache: VecDeque, next: VecDeque, - size: usize, } impl CandidateCache { @@ -812,7 +811,6 @@ impl CandidateCache { Self { cache: VecDeque::new(), next: VecDeque::new(), - size: 0, } } @@ -821,16 +819,18 @@ impl CandidateCache { } fn push(&mut self, tx: MemPoolTxInfoPartial) { - if self.size < Self::MAX_SIZE { + if self.next.len() < Self::MAX_SIZE { self.next.push_back(tx); - self.size += 1; } } fn reset(&mut self) { self.next.append(&mut self.cache); self.cache = std::mem::take(&mut self.next); - self.size = 0; + } + + fn len(&self) -> usize { + self.cache.len() + self.next.len() } } @@ -1377,7 +1377,7 @@ impl MemPoolDB { // Reset for finding the next transaction to process debug!( "Mempool: reset: retry list has {} entries", - candidate_cache.size + candidate_cache.len() ); candidate_cache.reset(); } From 7043e5477d6846ac9b70d332d2a8343569ac86c0 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Mon, 17 Oct 2022 14:35:50 -0500 Subject: [PATCH 060/178] add one unit test: test_fee_order_mismatch_nonce_order --- src/chainstate/stacks/miner.rs | 137 +++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index a12dd431b..55f9673d1 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -10989,6 +10989,143 @@ pub mod test { } } + #[test] + /// Test the situation in which the nonce order of transactions from a user. That is, + /// nonce 1 has a higher fee than nonce 0. + /// Want to see that both transactions can go into the same block, because the miner + /// should make multiple passes. + fn test_fee_order_mismatch_nonce_order() { + let privk = StacksPrivateKey::from_hex( + "42faca653724860da7a41bfcef7e6ba78db55146f6900de8cb2a9f760ffac70c01", + ) + .unwrap(); + let addr = StacksAddress::from_public_keys( + C32_ADDRESS_VERSION_TESTNET_SINGLESIG, + &AddressHashMode::SerializeP2PKH, + 1, + &vec![StacksPublicKey::from_private(&privk)], + ) + .unwrap(); + + let mut peer_config = TestPeerConfig::new( + "test_build_anchored_blocks_stx_transfers_single", + 2002, + 2003, + ); + peer_config.initial_balances = vec![(addr.to_account_principal(), 1000000000)]; + + let mut peer = TestPeer::new(peer_config); + + let chainstate_path = peer.chainstate_path.clone(); + + let num_blocks = 10; + let first_stacks_block_height = { + let sn = + SortitionDB::get_canonical_burn_chain_tip(&peer.sortdb.as_ref().unwrap().conn()) + .unwrap(); + sn.block_height + }; + + let recipient_addr_str = "ST1RFD5Q2QPK3E0F08HG9XDX7SSC7CNRS0QR0SGEV"; + let recipient = StacksAddress::from_string(recipient_addr_str).unwrap(); + let mut sender_nonce = 0; + + let mut last_block = None; + // send transactions to the mempool + let tip = SortitionDB::get_canonical_burn_chain_tip(&peer.sortdb.as_ref().unwrap().conn()) + .unwrap(); + + let (burn_ops, stacks_block, microblocks) = peer.make_tenure( + |ref mut miner, + ref mut sortdb, + ref mut chainstate, + vrf_proof, + ref parent_opt, + ref parent_microblock_header_opt| { + let parent_tip = match parent_opt { + None => StacksChainState::get_genesis_header_info(chainstate.db()).unwrap(), + Some(block) => { + let ic = sortdb.index_conn(); + let snapshot = SortitionDB::get_block_snapshot_for_winning_stacks_block( + &ic, + &tip.sortition_id, + &block.block_hash(), + ) + .unwrap() + .unwrap(); // succeeds because we don't fork + StacksChainState::get_anchored_block_header_info( + chainstate.db(), + &snapshot.consensus_hash, + &snapshot.winning_stacks_block_hash, + ) + .unwrap() + .unwrap() + } + }; + + let parent_header_hash = parent_tip.anchored_header.block_hash(); + let parent_consensus_hash = parent_tip.consensus_hash.clone(); + + let mut mempool = + MemPoolDB::open_test(false, 0x80000000, &chainstate_path).unwrap(); + + let coinbase_tx = make_coinbase(miner, 0); + + let stx_transfer0 = + make_user_stacks_transfer(&privk, 0, 200, &recipient.to_account_principal(), 1); + let stx_transfer1 = + make_user_stacks_transfer(&privk, 1, 400, &recipient.to_account_principal(), 1); + + mempool + .submit( + chainstate, + &parent_consensus_hash, + &parent_header_hash, + &stx_transfer0, + None, + &ExecutionCost::max_value(), + &StacksEpochId::Epoch20, + ) + .unwrap(); + + mempool + .submit( + chainstate, + &parent_consensus_hash, + &parent_header_hash, + &stx_transfer1, + None, + &ExecutionCost::max_value(), + &StacksEpochId::Epoch20, + ) + .unwrap(); + + let anchored_block = StacksBlockBuilder::build_anchored_block( + chainstate, + &sortdb.index_conn(), + &mut mempool, + &parent_tip, + tip.total_burn, + vrf_proof, + Hash160([0 as u8; 20]), + &coinbase_tx, + BlockBuilderSettings::max_value(), + None, + ) + .unwrap(); + (anchored_block.0, vec![]) + }, + ); + + last_block = Some(stacks_block.clone()); + + peer.next_burnchain_block(burn_ops.clone()); + peer.process_stacks_epoch_at_tip(&stacks_block, µblocks); + + // Both user transactions and the coinbase should have been mined. + assert_eq!(stacks_block.txs.len(), 3); + } + static CONTRACT: &'static str = " (define-map my-map int int) (define-private (do (input bool)) From 27ee2f88c0a4639547f7d8670393182b6377ea21 Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Mon, 17 Oct 2022 17:17:22 -0400 Subject: [PATCH 061/178] docs: document the `CandidateCache` --- src/core/mempool.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 71c7c9bec..1b966585f 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -799,6 +799,11 @@ impl NonceCache { } } +/// Cache potential candidate transactions for subsequent iterations. +/// While walking the mempool, transactions that have nonces that are too high +/// to process yet (but could be processed in the future) are added to `next`. +/// In the next pass, `next` is moved to `cache` and these transactions are +/// checked before reading more from the mempool DB. struct CandidateCache { cache: VecDeque, next: VecDeque, @@ -814,21 +819,31 @@ impl CandidateCache { } } + /// Retrieve the next candidate transaction from the cache. fn next(&mut self) -> Option { self.cache.pop_front() } + /// Push a candidate to the cache for the next iteration. fn push(&mut self, tx: MemPoolTxInfoPartial) { if self.next.len() < Self::MAX_SIZE { self.next.push_back(tx); } } + /// Prepare for the next iteration, transferring transactions from `next` to `cache`. fn reset(&mut self) { + // We do not need a size check here, because the cache can only grow in size + // after `cache` is empty. New transactions are not walked until the entire + // cache has been walked, so whenever we are adding brand new transactions to + // the cache, `cache` must, by definition, be empty. The size of `next` + // can grow beyond the previous iteration's cache, and that is limited inside + // the `push` method. self.next.append(&mut self.cache); self.cache = std::mem::take(&mut self.next); } + /// Total length of the cache. fn len(&self) -> usize { self.cache.len() + self.next.len() } From 6cff1385924c86f45af3e27ecfecaea57db587af Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Mon, 17 Oct 2022 16:30:03 -0500 Subject: [PATCH 062/178] make mempool walk cache sizes configurable --- src/chainstate/stacks/miner.rs | 1 - src/core/mempool.rs | 9 +++++++++ testnet/stacks-node/src/config.rs | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index 55f9673d1..bfe29e8da 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -11018,7 +11018,6 @@ pub mod test { let chainstate_path = peer.chainstate_path.clone(); - let num_blocks = 10; let first_stacks_block_height = { let sn = SortitionDB::get_canonical_burn_chain_tip(&peer.sortdb.as_ref().unwrap().conn()) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 71c7c9bec..1bf8ede0c 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -295,6 +295,11 @@ pub struct MemPoolWalkSettings { /// That is, with x%, when picking the next transaction to include a block, select one that /// either failed to get a cost estimate or has not been estimated yet. pub consider_no_estimate_tx_prob: u8, + /// Size of the nonce cache. This avoids MARF look-ups. + pub nonce_cache_size: u64, + /// Size of the candidate cache. These are the candidates that will be retried after each + /// transaction is mined. + pub candidate_retry_cache_size: u64, } impl MemPoolWalkSettings { @@ -303,6 +308,8 @@ impl MemPoolWalkSettings { min_tx_fee: 1, max_walk_time_ms: u64::max_value(), consider_no_estimate_tx_prob: 5, + nonce_cache_size: 10_000, + candidate_retry_cache_size: 10_000, } } pub fn zero() -> MemPoolWalkSettings { @@ -310,6 +317,8 @@ impl MemPoolWalkSettings { min_tx_fee: 0, max_walk_time_ms: u64::max_value(), consider_no_estimate_tx_prob: 5, + nonce_cache_size: 10_000, + candidate_retry_cache_size: 10_000, } } } diff --git a/testnet/stacks-node/src/config.rs b/testnet/stacks-node/src/config.rs index 7e449f9bb..b101cacd1 100644 --- a/testnet/stacks-node/src/config.rs +++ b/testnet/stacks-node/src/config.rs @@ -595,6 +595,12 @@ impl Config { probability_pick_no_estimate_tx: miner .probability_pick_no_estimate_tx .unwrap_or(miner_default_config.probability_pick_no_estimate_tx), + nonce_cache_size: miner + .nonce_cache_size + .unwrap_or(miner_default_config.nonce_cache_size), + candidate_retry_cache_size: miner + .candidate_retry_cache_size + .unwrap_or(miner_default_config.candidate_retry_cache_size), }, None => miner_default_config, }; @@ -970,6 +976,8 @@ impl Config { self.miner.subsequent_attempt_time_ms }, consider_no_estimate_tx_prob: self.miner.probability_pick_no_estimate_tx, + nonce_cache_size: self.miner.nonce_cache_size, + candidate_retry_cache_size: self.miner.candidate_retry_cache_size, }, } } @@ -1514,6 +1522,8 @@ pub struct MinerConfig { pub subsequent_attempt_time_ms: u64, pub microblock_attempt_time_ms: u64, pub probability_pick_no_estimate_tx: u8, + pub nonce_cache_size: u64, + pub candidate_retry_cache_size: u64, } impl MinerConfig { @@ -1524,6 +1534,8 @@ impl MinerConfig { subsequent_attempt_time_ms: 30_000, microblock_attempt_time_ms: 30_000, probability_pick_no_estimate_tx: 5, + nonce_cache_size: 10_000, + candidate_retry_cache_size: 10_000, } } } @@ -1628,6 +1640,8 @@ pub struct MinerConfigFile { pub subsequent_attempt_time_ms: Option, pub microblock_attempt_time_ms: Option, pub probability_pick_no_estimate_tx: Option, + pub nonce_cache_size: Option, + pub candidate_retry_cache_size: Option, } #[derive(Clone, Deserialize, Default, Debug)] From 3a491e6a0ac7c02178d5c379cba81851227b3c1d Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Mon, 17 Oct 2022 16:54:40 -0500 Subject: [PATCH 063/178] replace MAX_SIZE hard-coded limits with config options --- src/core/mempool.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 5c4be586e..632b89019 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -769,15 +769,17 @@ impl MemPoolTxInfo { struct NonceCache { cache: HashMap, size: usize, + /// The maximum size that this cache can be. + max_size: usize, } impl NonceCache { - const MAX_SIZE: usize = 1024 * 1024; - - fn new() -> Self { + fn new(nonce_cache_size:u64) -> Self { + let max_size:usize = nonce_cache_size.try_into().expect("Could not cast `nonce_cache_size` as `usize`."); Self { cache: HashMap::new(), size: 0, + max_size, } } @@ -793,7 +795,7 @@ impl NonceCache { // will be looked up every time. This is bad for performance // but is unlikely to occur due to the typical number of // transactions processed before filling a block. - if self.size < Self::MAX_SIZE { + if self.size < self.max_size { self.cache.insert(address.clone(), nonce); self.size += 1; } @@ -816,15 +818,17 @@ impl NonceCache { struct CandidateCache { cache: VecDeque, next: VecDeque, + /// The maximum size that this cache can be. + max_size: usize, } impl CandidateCache { - const MAX_SIZE: usize = 64 * 1024; - - fn new() -> Self { + fn new(candidate_retry_cache_size:u64) -> Self { + let max_size:usize = candidate_retry_cache_size.try_into().expect("Could not cast `candidate_retry_cache_size` as usize."); Self { cache: VecDeque::new(), next: VecDeque::new(), + max_size, } } @@ -835,7 +839,7 @@ impl CandidateCache { /// Push a candidate to the cache for the next iteration. fn push(&mut self, tx: MemPoolTxInfoPartial) { - if self.next.len() < Self::MAX_SIZE { + if self.next.len() < self.max_size { self.next.push_back(tx); } } @@ -1236,8 +1240,8 @@ impl MemPoolDB { let tx_consideration_sampler = Uniform::new(0, 100); let mut rng = rand::thread_rng(); - let mut candidate_cache = CandidateCache::new(); - let mut nonce_cache = NonceCache::new(); + let mut candidate_cache = CandidateCache::new(settings.candidate_retry_cache_size); + let mut nonce_cache = NonceCache::new(settings.nonce_cache_size); let sql = " SELECT txid, origin_nonce, origin_address, sponsor_nonce, sponsor_address, fee_rate From 209d751878a24cecde98e068dcf2f14d563e34f1 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Mon, 17 Oct 2022 20:50:08 -0500 Subject: [PATCH 064/178] add a test with one user, many more transactions than cache size --- src/chainstate/stacks/miner.rs | 131 +++++++++++++++++++++++++++++++++ src/core/mempool.rs | 12 ++- 2 files changed, 139 insertions(+), 4 deletions(-) diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index bfe29e8da..97208280c 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -11125,6 +11125,137 @@ pub mod test { assert_eq!(stacks_block.txs.len(), 3); } + #[test] + /// Test with a number of transactions (10) that is "much greater" than the cache size (2). + /// Test that all transactions get into the block. + fn test_1_user_10_transactions_with_cache_size_2() { + let privk = StacksPrivateKey::from_hex( + "42faca653724860da7a41bfcef7e6ba78db55146f6900de8cb2a9f760ffac70c01", + ) + .unwrap(); + let addr = StacksAddress::from_public_keys( + C32_ADDRESS_VERSION_TESTNET_SINGLESIG, + &AddressHashMode::SerializeP2PKH, + 1, + &vec![StacksPublicKey::from_private(&privk)], + ) + .unwrap(); + + let mut peer_config = TestPeerConfig::new( + "test_build_anchored_blocks_stx_transfers_single", + 2002, + 2003, + ); + peer_config.initial_balances = vec![(addr.to_account_principal(), 1000000000)]; + + let mut peer = TestPeer::new(peer_config); + + let chainstate_path = peer.chainstate_path.clone(); + + let first_stacks_block_height = { + let sn = + SortitionDB::get_canonical_burn_chain_tip(&peer.sortdb.as_ref().unwrap().conn()) + .unwrap(); + sn.block_height + }; + + let recipient_addr_str = "ST1RFD5Q2QPK3E0F08HG9XDX7SSC7CNRS0QR0SGEV"; + let recipient = StacksAddress::from_string(recipient_addr_str).unwrap(); + let mut sender_nonce = 0; + + let mut last_block = None; + // send transactions to the mempool + let tip = SortitionDB::get_canonical_burn_chain_tip(&peer.sortdb.as_ref().unwrap().conn()) + .unwrap(); + + let (burn_ops, stacks_block, microblocks) = peer.make_tenure( + |ref mut miner, + ref mut sortdb, + ref mut chainstate, + vrf_proof, + ref parent_opt, + ref parent_microblock_header_opt| { + let parent_tip = match parent_opt { + None => StacksChainState::get_genesis_header_info(chainstate.db()).unwrap(), + Some(block) => { + let ic = sortdb.index_conn(); + let snapshot = SortitionDB::get_block_snapshot_for_winning_stacks_block( + &ic, + &tip.sortition_id, + &block.block_hash(), + ) + .unwrap() + .unwrap(); // succeeds because we don't fork + StacksChainState::get_anchored_block_header_info( + chainstate.db(), + &snapshot.consensus_hash, + &snapshot.winning_stacks_block_hash, + ) + .unwrap() + .unwrap() + } + }; + + let parent_header_hash = parent_tip.anchored_header.block_hash(); + let parent_consensus_hash = parent_tip.consensus_hash.clone(); + + let mut mempool = + MemPoolDB::open_test(false, 0x80000000, &chainstate_path).unwrap(); + + let coinbase_tx = make_coinbase(miner, 0); + + for i in 0..10 { + let stx_transfer = make_user_stacks_transfer( + &privk, + i, + 200, + &recipient.to_account_principal(), + 1, + ); + + mempool + .submit( + chainstate, + &parent_consensus_hash, + &parent_header_hash, + &stx_transfer, + None, + &ExecutionCost::max_value(), + &StacksEpochId::Epoch20, + ) + .unwrap(); + } + + let mut settings = BlockBuilderSettings::max_value(); + settings.mempool_settings.nonce_cache_size = 2; + settings.mempool_settings.consider_no_estimate_tx_prob = 50; + settings.mempool_settings.candidate_retry_cache_size = 2; + let anchored_block = StacksBlockBuilder::build_anchored_block( + chainstate, + &sortdb.index_conn(), + &mut mempool, + &parent_tip, + tip.total_burn, + vrf_proof, + Hash160([0 as u8; 20]), + &coinbase_tx, + settings, + None, + ) + .unwrap(); + (anchored_block.0, vec![]) + }, + ); + + last_block = Some(stacks_block.clone()); + + peer.next_burnchain_block(burn_ops.clone()); + peer.process_stacks_epoch_at_tip(&stacks_block, µblocks); + + // Both user transactions and the coinbase should have been mined. + assert_eq!(stacks_block.txs.len(), 11); + } + static CONTRACT: &'static str = " (define-map my-map int int) (define-private (do (input bool)) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 632b89019..ea21e67fd 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -774,8 +774,10 @@ struct NonceCache { } impl NonceCache { - fn new(nonce_cache_size:u64) -> Self { - let max_size:usize = nonce_cache_size.try_into().expect("Could not cast `nonce_cache_size` as `usize`."); + fn new(nonce_cache_size: u64) -> Self { + let max_size: usize = nonce_cache_size + .try_into() + .expect("Could not cast `nonce_cache_size` as `usize`."); Self { cache: HashMap::new(), size: 0, @@ -823,8 +825,10 @@ struct CandidateCache { } impl CandidateCache { - fn new(candidate_retry_cache_size:u64) -> Self { - let max_size:usize = candidate_retry_cache_size.try_into().expect("Could not cast `candidate_retry_cache_size` as usize."); + fn new(candidate_retry_cache_size: u64) -> Self { + let max_size: usize = candidate_retry_cache_size + .try_into() + .expect("Could not cast `candidate_retry_cache_size` as usize."); Self { cache: VecDeque::new(), next: VecDeque::new(), From fa95c06d43cec7a3c69a04df558c104756da3ac7 Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Mon, 17 Oct 2022 23:37:36 -0400 Subject: [PATCH 065/178] test: add test for `consider_no_estimate_tx_prob` --- src/core/tests/mod.rs | 177 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/src/core/tests/mod.rs b/src/core/tests/mod.rs index da5e7c929..0ebf7f326 100644 --- a/src/core/tests/mod.rs +++ b/src/core/tests/mod.rs @@ -553,6 +553,183 @@ fn mempool_walk_over_fork() { ); } +#[test] +/// This test verifies that all transactions are visited, regardless of the +/// setting for `consider_no_estimate_tx_prob`. +fn test_iterate_candidates_consider_no_estimate_tx_prob() { + let mut chainstate = instantiate_chainstate_with_balances( + false, + 0x80000000, + "test_iterate_candidates_consider_no_estimate_tx_prob", + vec![], + ); + let chainstate_path = chainstate_path("test_iterate_candidates_consider_no_estimate_tx_prob"); + let mut mempool = MemPoolDB::open_test(false, 0x80000000, &chainstate_path).unwrap(); + let b_1 = make_block( + &mut chainstate, + ConsensusHash([0x1; 20]), + &( + FIRST_BURNCHAIN_CONSENSUS_HASH.clone(), + FIRST_STACKS_BLOCK_HASH.clone(), + ), + 1, + 1, + ); + let b_2 = make_block(&mut chainstate, ConsensusHash([0x2; 20]), &b_1, 2, 2); + + let mut mempool_settings = MemPoolWalkSettings::default(); + mempool_settings.min_tx_fee = 10; + let mut tx_events = Vec::new(); + + let mut txs = codec_all_transactions( + &TransactionVersion::Testnet, + 0x80000000, + &TransactionAnchorMode::Any, + &TransactionPostConditionMode::Allow, + ); + + // Load 24 transactions into the mempool, alternating whether or not they have a fee-rate. + for nonce in 0..24 { + let mut tx = txs.pop().unwrap(); + let mut mempool_tx = mempool.tx_begin().unwrap(); + + let origin_address = tx.origin_address(); + let origin_nonce = tx.get_origin_nonce(); + let sponsor_address = tx.sponsor_address().unwrap_or(origin_address); + let sponsor_nonce = tx.get_sponsor_nonce().unwrap_or(origin_nonce); + + tx.set_tx_fee(100); + let txid = tx.txid(); + let tx_bytes = tx.serialize_to_vec(); + let tx_fee = tx.get_tx_fee(); + let height = 100; + + MemPoolDB::try_add_tx( + &mut mempool_tx, + &mut chainstate, + &b_1.0, + &b_1.1, + txid, + tx_bytes, + tx_fee, + height, + &origin_address, + nonce, + &sponsor_address, + nonce, + None, + ) + .unwrap(); + + if nonce & 1 == 0 { + mempool_tx + .execute( + "UPDATE mempool SET fee_rate = ? WHERE txid = ?", + rusqlite::params![Some(123.0), &txid], + ) + .unwrap(); + } else { + let none: Option = None; + mempool_tx + .execute( + "UPDATE mempool SET fee_rate = ? WHERE txid = ?", + rusqlite::params![none, &txid], + ) + .unwrap(); + } + + mempool_tx.commit().unwrap(); + } + + // First, with default (5%) + chainstate.with_read_only_clarity_tx( + &TEST_BURN_STATE_DB, + &StacksBlockHeader::make_index_block_hash(&b_2.0, &b_2.1), + |clarity_conn| { + let mut count_txs = 0; + mempool + .iterate_candidates::<_, ChainstateError, _>( + clarity_conn, + &mut tx_events, + 2, + mempool_settings.clone(), + |_, available_tx, _| { + count_txs += 1; + Ok(Some( + TransactionResult::skipped( + &available_tx.tx.tx, + "event not relevant to test".to_string(), + ) + .convert_to_event(), + )) + }, + ) + .unwrap(); + assert_eq!(count_txs, 24, "Mempool should find all 24 transactions"); + }, + ); + + // Next with 0% + mempool_settings.consider_no_estimate_tx_prob = 0; + + chainstate.with_read_only_clarity_tx( + &TEST_BURN_STATE_DB, + &StacksBlockHeader::make_index_block_hash(&b_2.0, &b_2.1), + |clarity_conn| { + let mut count_txs = 0; + mempool + .iterate_candidates::<_, ChainstateError, _>( + clarity_conn, + &mut tx_events, + 2, + mempool_settings.clone(), + |_, available_tx, _| { + count_txs += 1; + Ok(Some( + TransactionResult::skipped( + &available_tx.tx.tx, + "event not relevant to test".to_string(), + ) + .convert_to_event(), + )) + }, + ) + .unwrap(); + assert_eq!(count_txs, 24, "Mempool should find all 24 transactions"); + }, + ); + + // Then with with 100% + mempool_settings.consider_no_estimate_tx_prob = 100; + + chainstate.with_read_only_clarity_tx( + &TEST_BURN_STATE_DB, + &StacksBlockHeader::make_index_block_hash(&b_2.0, &b_2.1), + |clarity_conn| { + let mut count_txs = 0; + mempool + .iterate_candidates::<_, ChainstateError, _>( + clarity_conn, + &mut tx_events, + 2, + mempool_settings.clone(), + |_, available_tx, _| { + count_txs += 1; + Ok(Some( + TransactionResult::skipped( + &available_tx.tx.tx, + "event not relevant to test".to_string(), + ) + .convert_to_event(), + )) + }, + ) + .unwrap(); + assert_eq!(count_txs, 24, "Mempool should find all 24 transactions"); + }, + ); +} + #[test] fn mempool_do_not_replace_tx() { let mut chainstate = instantiate_chainstate_with_balances( From 5cd1c6194812e29ea5f3ed37a28f4baac10aa277 Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Mon, 17 Oct 2022 23:51:37 -0400 Subject: [PATCH 066/178] refactor: remove `size` from `NonceCache` --- src/core/mempool.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index ea21e67fd..f0c5ee814 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -768,7 +768,6 @@ impl MemPoolTxInfo { /// Used to locally cache nonces to avoid repeatedly looking them up in the nonce. struct NonceCache { cache: HashMap, - size: usize, /// The maximum size that this cache can be. max_size: usize, } @@ -780,7 +779,6 @@ impl NonceCache { .expect("Could not cast `nonce_cache_size` as `usize`."); Self { cache: HashMap::new(), - size: 0, max_size, } } @@ -797,9 +795,8 @@ impl NonceCache { // will be looked up every time. This is bad for performance // but is unlikely to occur due to the typical number of // transactions processed before filling a block. - if self.size < self.max_size { + if self.cache.len() < self.max_size { self.cache.insert(address.clone(), nonce); - self.size += 1; } nonce } From 3f70fb0519598d11684f7a770f03b1c9a81ab252 Mon Sep 17 00:00:00 2001 From: Greg Coppola Date: Tue, 18 Oct 2022 08:03:27 -0500 Subject: [PATCH 067/178] parameterized mempool walk test --- src/chainstate/stacks/miner.rs | 184 +++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) diff --git a/src/chainstate/stacks/miner.rs b/src/chainstate/stacks/miner.rs index 97208280c..c7656db67 100644 --- a/src/chainstate/stacks/miner.rs +++ b/src/chainstate/stacks/miner.rs @@ -11256,6 +11256,190 @@ pub mod test { assert_eq!(stacks_block.txs.len(), 11); } + #[test] + fn mempool_walk_test_users_1_rounds_10_cache_size_2_null_prob_0() { + paramaterized_mempool_walk_test(1, 10, 2, 0) + } + + #[test] + fn mempool_walk_test_users_10_rounds_3_cache_size_2_null_prob_0() { + paramaterized_mempool_walk_test(10, 3, 2, 0) + } + + #[test] + fn mempool_walk_test_users_1_rounds_10_cache_size_2_null_prob_50() { + paramaterized_mempool_walk_test(1, 10, 2, 50) + } + + #[test] + fn mempool_walk_test_users_10_rounds_3_cache_size_2_null_prob_50() { + paramaterized_mempool_walk_test(10, 3, 2, 50) + } + + #[test] + fn mempool_walk_test_users_1_rounds_10_cache_size_2_null_prob_100() { + paramaterized_mempool_walk_test(1, 10, 2, 100) + } + + #[test] + fn mempool_walk_test_users_10_rounds_3_cache_size_2_null_prob_100() { + paramaterized_mempool_walk_test(10, 3, 2, 100) + } + + /// With the parameters given, create `num_rounds` transactions per each user in `num_users`. + /// `nonce_and_candidate_cache_size` is the cache size used for both of the nonce cache + /// and the candidate cache. + fn paramaterized_mempool_walk_test( + num_users: usize, + num_rounds: usize, + nonce_and_candidate_cache_size: u64, + consider_no_estimate_tx_prob: u8, + ) { + let key_address_pairs: Vec<(Secp256k1PrivateKey, StacksAddress)> = (0..num_users) + .map(|_user_index| { + let privk = StacksPrivateKey::new(); + let addr = StacksAddress::from_public_keys( + C32_ADDRESS_VERSION_TESTNET_SINGLESIG, + &AddressHashMode::SerializeP2PKH, + 1, + &vec![StacksPublicKey::from_private(&privk)], + ) + .unwrap(); + (privk, addr) + }) + .collect(); + + let mut peer_config = TestPeerConfig::new( + &format!( + "mempool_walk_test_users_{}_rounds_{}_cache_size_{}_null_prob_{}", + num_users, num_rounds, nonce_and_candidate_cache_size, consider_no_estimate_tx_prob + ), + 2002, + 2003, + ); + + peer_config.initial_balances = vec![]; + for (privk, addr) in &key_address_pairs { + peer_config + .initial_balances + .push((addr.to_account_principal(), 1000000000)); + } + + let mut peer = TestPeer::new(peer_config); + + let chainstate_path = peer.chainstate_path.clone(); + + let first_stacks_block_height = { + let sn = + SortitionDB::get_canonical_burn_chain_tip(&peer.sortdb.as_ref().unwrap().conn()) + .unwrap(); + sn.block_height + }; + + let recipient_addr_str = "ST1RFD5Q2QPK3E0F08HG9XDX7SSC7CNRS0QR0SGEV"; + let recipient = StacksAddress::from_string(recipient_addr_str).unwrap(); + let mut sender_nonce = 0; + + let mut last_block = None; + // send transactions to the mempool + let tip = SortitionDB::get_canonical_burn_chain_tip(&peer.sortdb.as_ref().unwrap().conn()) + .unwrap(); + + let (burn_ops, stacks_block, microblocks) = peer.make_tenure( + |ref mut miner, + ref mut sortdb, + ref mut chainstate, + vrf_proof, + ref parent_opt, + ref parent_microblock_header_opt| { + let parent_tip = match parent_opt { + None => StacksChainState::get_genesis_header_info(chainstate.db()).unwrap(), + Some(block) => { + let ic = sortdb.index_conn(); + let snapshot = SortitionDB::get_block_snapshot_for_winning_stacks_block( + &ic, + &tip.sortition_id, + &block.block_hash(), + ) + .unwrap() + .unwrap(); // succeeds because we don't fork + StacksChainState::get_anchored_block_header_info( + chainstate.db(), + &snapshot.consensus_hash, + &snapshot.winning_stacks_block_hash, + ) + .unwrap() + .unwrap() + } + }; + + let parent_header_hash = parent_tip.anchored_header.block_hash(); + let parent_consensus_hash = parent_tip.consensus_hash.clone(); + + let mut mempool = + MemPoolDB::open_test(false, 0x80000000, &chainstate_path).unwrap(); + + let coinbase_tx = make_coinbase(miner, 0); + + for round_index in 0..num_rounds { + for user_index in 0..num_users { + let stx_transfer = make_user_stacks_transfer( + &key_address_pairs[user_index].0, + round_index as u64, + 200, + &recipient.to_account_principal(), + 1, + ); + + mempool + .submit( + chainstate, + &parent_consensus_hash, + &parent_header_hash, + &stx_transfer, + None, + &ExecutionCost::max_value(), + &StacksEpochId::Epoch20, + ) + .unwrap(); + } + } + + let mut settings = BlockBuilderSettings::max_value(); + settings.mempool_settings.nonce_cache_size = nonce_and_candidate_cache_size; + settings.mempool_settings.candidate_retry_cache_size = + nonce_and_candidate_cache_size; + settings.mempool_settings.consider_no_estimate_tx_prob = + consider_no_estimate_tx_prob; + let anchored_block = StacksBlockBuilder::build_anchored_block( + chainstate, + &sortdb.index_conn(), + &mut mempool, + &parent_tip, + tip.total_burn, + vrf_proof, + Hash160([0 as u8; 20]), + &coinbase_tx, + settings, + None, + ) + .unwrap(); + (anchored_block.0, vec![]) + }, + ); + + last_block = Some(stacks_block.clone()); + + peer.next_burnchain_block(burn_ops.clone()); + peer.process_stacks_epoch_at_tip(&stacks_block, µblocks); + + // Both user transactions and the coinbase should have been mined. + assert_eq!( + stacks_block.txs.len() as u64, + (1 + num_rounds * num_users) as u64 + ); + } + static CONTRACT: &'static str = " (define-map my-map int int) (define-private (do (input bool)) From 21ab472c670d83b63e98c7bd831bcc79e8ab133d Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 09:44:24 -0400 Subject: [PATCH 068/178] chore: run sunset-gate integration test --- .github/workflows/bitcoin-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index e5eaf030f..906f97c01 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -78,6 +78,7 @@ jobs: - tests::epoch_21::transition_fixes_bitcoin_rigidity - tests::epoch_21::transition_adds_pay_to_contract - tests::epoch_21::transition_adds_get_pox_addr_recipients + - tests::epoch_21::transition_removes_pox_sunset steps: - uses: actions/checkout@v2 - name: Download docker image From aa94a8f0860913ae1e4a7a4d948b24d82a56abfd Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 09:44:38 -0400 Subject: [PATCH 069/178] chore: ignore .orig and .rej files from patch --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 49aea8d27..e0a0092ca 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,7 @@ net-test/mnt/archive/* *.profraw *.profdata lcov.info + +# patch +*.orig +*.rej From 16f2f1210457149bff11e7da624c00ac0496f858 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 09:45:54 -0400 Subject: [PATCH 070/178] feat: gate sunset burn logic based on what epoch we're in, for the purposes of calculating the sunset burn and deciding whether or not to validate PoX block-commits and on-Bitcoin transactions --- src/burnchains/burnchain.rs | 156 +++++++++++++++++++++++++++++++----- 1 file changed, 134 insertions(+), 22 deletions(-) diff --git a/src/burnchains/burnchain.rs b/src/burnchains/burnchain.rs index 10bcc9631..9b3ed012d 100644 --- a/src/burnchains/burnchain.rs +++ b/src/burnchains/burnchain.rs @@ -57,12 +57,12 @@ use crate::chainstate::burn::{BlockSnapshot, Opcodes}; use crate::chainstate::coordinator::comm::CoordinatorChannels; use crate::chainstate::stacks::address::PoxAddress; use crate::chainstate::stacks::StacksPublicKey; -use crate::core::StacksEpoch; use crate::core::MINING_COMMITMENT_WINDOW; use crate::core::NETWORK_ID_MAINNET; use crate::core::NETWORK_ID_TESTNET; use crate::core::PEER_VERSION_MAINNET; use crate::core::PEER_VERSION_TESTNET; +use crate::core::{StacksEpoch, StacksEpochId}; use crate::deps; use crate::monitoring::update_burnchain_height; use crate::types::chainstate::StacksAddress; @@ -168,7 +168,19 @@ impl BurnchainStateTransition { let mut windowed_block_commits = vec![block_commits]; let mut windowed_missed_commits = vec![]; - if !burnchain.is_in_prepare_phase(parent_snapshot.block_height + 1) { + // what epoch are we in? + let epoch_id = SortitionDB::get_stacks_epoch(sort_tx, parent_snapshot.block_height + 1)? + .expect(&format!( + "FATAL: no epoch defined at burn height {}", + parent_snapshot.block_height + 1 + )) + .epoch_id; + + if !burnchain.is_in_prepare_phase(parent_snapshot.block_height + 1) + && !burnchain + .pox_constants + .is_after_pox_sunset_end(parent_snapshot.block_height + 1, epoch_id) + { // PoX reward-phase is active! // build a map of intended sortition -> missed commit for the missed commits // discovered in this block. @@ -211,7 +223,7 @@ impl BurnchainStateTransition { } else { // PoX reward-phase is not active debug!( - "Block {} is in a prepare phase, so no windowing will take place", + "Block {} is in a prepare phase or post-PoX sunset, so no windowing will take place", parent_snapshot.block_height + 1 ); @@ -223,14 +235,22 @@ impl BurnchainStateTransition { windowed_block_commits.reverse(); windowed_missed_commits.reverse(); - // figure out which sortitions must be PoB due to them falling in a prepare phase. + // figure out if the PoX sunset finished during the window, + // and/or which sortitions must be PoB due to them falling in a prepare phase. let window_end_height = parent_snapshot.block_height + 1; let window_start_height = window_end_height + 1 - (windowed_block_commits.len() as u64); let mut burn_blocks = vec![false; windowed_block_commits.len()]; - // set burn_blocks flags to accomodate prepare phases + // set burn_blocks flags to accomodate prepare phases and PoX sunset for (i, b) in burn_blocks.iter_mut().enumerate() { - if burnchain.is_in_prepare_phase(window_start_height + (i as u64)) { + if PoxConstants::has_pox_sunset(epoch_id) + && burnchain + .pox_constants + .is_after_pox_sunset_end(window_start_height + (i as u64), epoch_id) + { + // past PoX sunset, so must burn + *b = true; + } else if burnchain.is_in_prepare_phase(window_start_height + (i as u64)) { // must burn *b = true; } else { @@ -454,10 +474,63 @@ impl Burnchain { }) } + /// BROKEN; DO NOT USE IN NEW CODE pub fn is_mainnet(&self) -> bool { self.network_id == NETWORK_ID_MAINNET } + /// the expected sunset burn is: + /// total_commit * (progress through sunset phase) / (sunset phase duration) + pub fn expected_sunset_burn( + &self, + burn_height: u64, + total_commit: u64, + epoch_id: StacksEpochId, + ) -> u64 { + if !PoxConstants::has_pox_sunset(epoch_id) { + // sunset is disabled + return 0; + } + if !self + .pox_constants + .is_after_pox_sunset_start(burn_height, epoch_id) + { + // too soon to do this + return 0; + } + if self + .pox_constants + .is_after_pox_sunset_end(burn_height, epoch_id) + { + // no need to do an extra burn; PoX is already disabled + return 0; + } + + // no sunset burn needed in prepare phase -- it's already getting burnt + if self.is_in_prepare_phase(burn_height) { + return 0; + } + + let reward_cycle_height = self.reward_cycle_to_block_height( + self.block_height_to_reward_cycle(burn_height) + .expect("BUG: Sunset start is less than first_block_height"), + ); + + if reward_cycle_height <= self.pox_constants.sunset_start { + return 0; + } + + let sunset_duration = + (self.pox_constants.sunset_end - self.pox_constants.sunset_start) as u128; + let sunset_progress = (reward_cycle_height - self.pox_constants.sunset_start) as u128; + + // use u128 to avoid any possibilities of overflowing in the calculation here. + let expected_u128 = (total_commit as u128) * (sunset_progress) / sunset_duration; + u64::try_from(expected_u128) + // should never be possible, because sunset_burn is <= total_commit, which is a u64 + .expect("Overflowed u64 in calculating expected sunset_burn") + } + pub fn is_reward_cycle_start(&self, burn_height: u64) -> bool { let effective_height = burn_height - self.first_block_height; // first block of the new reward cycle @@ -688,6 +761,7 @@ impl Burnchain { burnchain: &Burnchain, burnchain_db: &BurnchainDB, block_header: &BurnchainBlockHeader, + epoch_id: StacksEpochId, burn_tx: &BurnchainTransaction, pre_stx_op_map: &HashMap, ) -> Option { @@ -707,7 +781,7 @@ impl Burnchain { } } x if x == Opcodes::LeaderBlockCommit as u8 => { - match LeaderBlockCommitOp::from_tx(burnchain, block_header, burn_tx) { + match LeaderBlockCommitOp::from_tx(burnchain, block_header, epoch_id, burn_tx) { Ok(op) => Some(BlockstackOperationType::LeaderBlockCommit(op)), Err(e) => { warn!( @@ -734,18 +808,25 @@ impl Burnchain { } } } - x if x == Opcodes::PreStx as u8 => match PreStxOp::from_tx(block_header, burn_tx) { - Ok(op) => Some(BlockstackOperationType::PreStx(op)), - Err(e) => { - warn!( - "Failed to parse pre stack stx tx"; - "txid" => %burn_tx.txid(), - "data" => %to_hex(&burn_tx.data()), - "error" => ?e, - ); - None + x if x == Opcodes::PreStx as u8 => { + match PreStxOp::from_tx( + block_header, + epoch_id, + burn_tx, + burnchain.pox_constants.sunset_end, + ) { + Ok(op) => Some(BlockstackOperationType::PreStx(op)), + Err(e) => { + warn!( + "Failed to parse pre stack stx tx"; + "txid" => %burn_tx.txid(), + "data" => %to_hex(&burn_tx.data()), + "error" => ?e, + ); + None + } } - }, + } x if x == Opcodes::TransferStx as u8 => { let pre_stx_txid = TransferStxOp::get_sender_txid(burn_tx).ok()?; let pre_stx_tx = match pre_stx_op_map.get(&pre_stx_txid) { @@ -783,7 +864,13 @@ impl Burnchain { }; if let Some(BlockstackOperationType::PreStx(pre_stack_stx)) = pre_stx_tx { let sender = &pre_stack_stx.output; - match StackStxOp::from_tx(block_header, burn_tx, sender) { + match StackStxOp::from_tx( + block_header, + epoch_id, + burn_tx, + sender, + burnchain.pox_constants.sunset_end, + ) { Ok(op) => Some(BlockstackOperationType::StackStx(op)), Err(e) => { warn!( @@ -859,6 +946,7 @@ impl Burnchain { burnchain: &Burnchain, burnchain_db: &mut BurnchainDB, block: &BurnchainBlock, + epoch_id: StacksEpochId, ) -> Result { debug!( "Process block {} {}", @@ -866,7 +954,8 @@ impl Burnchain { &block.block_hash() ); - let _blockstack_txs = burnchain_db.store_new_burnchain_block(burnchain, &block)?; + let _blockstack_txs = + burnchain_db.store_new_burnchain_block(burnchain, &block, epoch_id)?; let header = block.header(); @@ -887,8 +976,15 @@ impl Burnchain { &block.block_hash() ); + let cur_epoch = + SortitionDB::get_stacks_epoch(db.conn(), block.block_height())?.expect(&format!( + "FATAL: no epoch for burn block height {}", + block.block_height() + )); + let header = block.header(); - let blockstack_txs = burnchain_db.store_new_burnchain_block(burnchain, &block)?; + let blockstack_txs = + burnchain_db.store_new_burnchain_block(burnchain, &block, cur_epoch.epoch_id)?; let sortition_tip = SortitionDB::get_canonical_sortition_tip(db.conn())?; @@ -1325,6 +1421,11 @@ impl Burnchain { let myself = self.clone(); + let epochs = { + let (sortdb, _) = self.open_db(false)?; + SortitionDB::get_stacks_epochs(sortdb.conn())? + }; + // TODO: don't re-process blocks. See if the block hash is already present in the burn db, // and if so, do nothing. let download_thread: thread::JoinHandle> = @@ -1406,6 +1507,8 @@ impl Burnchain { } if is_mainnet { + // NOTE: This code block is unreachable due to a bug in + // self.is_mainnet() if last_processed.block_height == STACKS_2_0_LAST_BLOCK_TO_PROCESS { info!("Reached Stacks 2.0 last block to processed, ignoring subsequent burn blocks"; "block_height" => last_processed.block_height); @@ -1418,9 +1521,14 @@ impl Burnchain { } } + let epoch_index = StacksEpoch::find_epoch(&epochs, block_height) + .expect(&format!("FATAL: no epoch defined for height {}", block_height)); + + let epoch_id = epochs[epoch_index].epoch_id; + let insert_start = get_epoch_time_ms(); last_processed = - Burnchain::process_block(&myself, &mut burnchain_db, &burnchain_block)?; + Burnchain::process_block(&myself, &mut burnchain_db, &burnchain_block, epoch_id)?; if !coord_comm.announce_new_burn_block() { return Err(burnchain_error::CoordinatorClosed); } @@ -1845,6 +1953,7 @@ pub mod tests { }; let block_commit_1 = LeaderBlockCommitOp { + sunset_burn: 0, commit_outs: vec![], block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222222") @@ -1885,6 +1994,7 @@ pub mod tests { }; let block_commit_2 = LeaderBlockCommitOp { + sunset_burn: 0, commit_outs: vec![], block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222223") @@ -1925,6 +2035,7 @@ pub mod tests { }; let block_commit_3 = LeaderBlockCommitOp { + sunset_burn: 0, commit_outs: vec![], block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222224") @@ -2487,6 +2598,7 @@ pub mod tests { // insert block commit paired to previous round's leader key, as well as a user burn if i > 0 { let next_block_commit = LeaderBlockCommitOp { + sunset_burn: 0, commit_outs: vec![], block_header_hash: BlockHeaderHash::from_bytes(&vec![ i, i, i, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, From 899497b6e01db2e7477b0d1b9395b99345353e3b Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 09:46:42 -0400 Subject: [PATCH 071/178] chore: op-checking is now gated on epoch ID --- src/burnchains/db.rs | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/burnchains/db.rs b/src/burnchains/db.rs index 2b7a6d55e..c8540265c 100644 --- a/src/burnchains/db.rs +++ b/src/burnchains/db.rs @@ -34,6 +34,8 @@ use crate::util_lib::db::{ use crate::chainstate::stacks::index::ClarityMarfTrieId; use stacks_common::types::chainstate::BurnchainHeaderHash; +use crate::core::StacksEpochId; + pub struct BurnchainDB { conn: Connection, } @@ -320,6 +322,7 @@ impl BurnchainDB { burnchain: &Burnchain, block: &BurnchainBlock, block_header: &BurnchainBlockHeader, + epoch_id: StacksEpochId, ) -> Vec { debug!( "Extract Blockstack transactions from block {} {}", @@ -331,8 +334,14 @@ impl BurnchainDB { let mut pre_stx_ops = HashMap::new(); for tx in block.txs().iter() { - let result = - Burnchain::classify_transaction(burnchain, self, block_header, &tx, &pre_stx_ops); + let result = Burnchain::classify_transaction( + burnchain, + self, + block_header, + epoch_id, + &tx, + &pre_stx_ops, + ); if let Some(classified_tx) = result { if let BlockstackOperationType::PreStx(pre_stx_op) = classified_tx { pre_stx_ops.insert(pre_stx_op.txid.clone(), pre_stx_op); @@ -357,11 +366,13 @@ impl BurnchainDB { &mut self, burnchain: &Burnchain, block: &BurnchainBlock, + epoch_id: StacksEpochId, ) -> Result, BurnchainError> { let header = block.header(); debug!("Storing new burnchain block"; "burn_header_hash" => %header.block_hash.to_string()); - let mut blockstack_ops = self.get_blockstack_transactions(burnchain, block, &header); + let mut blockstack_ops = + self.get_blockstack_transactions(burnchain, block, &header, epoch_id); apply_blockstack_txs_safety_checks(header.block_height, &mut blockstack_ops); let db_tx = self.tx_begin()?; @@ -432,6 +443,8 @@ mod tests { let mut burnchain = Burnchain::regtest(":memory:"); burnchain.pox_constants = PoxConstants::test_default(); + burnchain.pox_constants.sunset_start = 999; + burnchain.pox_constants.sunset_end = 1000; let first_block_header = burnchain_db.get_canonical_chain_tip().unwrap(); assert_eq!(&first_block_header.block_hash, &first_bhh); @@ -452,7 +465,7 @@ mod tests { 485, )); let ops = burnchain_db - .store_new_burnchain_block(&burnchain, &canonical_block) + .store_new_burnchain_block(&burnchain, &canonical_block, StacksEpochId::Epoch21) .unwrap(); assert_eq!(ops.len(), 0); @@ -490,7 +503,7 @@ mod tests { )); let ops = burnchain_db - .store_new_burnchain_block(&burnchain, &non_canonical_block) + .store_new_burnchain_block(&burnchain, &non_canonical_block, StacksEpochId::Epoch21) .unwrap(); assert_eq!(ops.len(), expected_ops.len()); for op in ops.iter() { @@ -542,6 +555,8 @@ mod tests { let mut burnchain = Burnchain::regtest(":memory:"); burnchain.pox_constants = PoxConstants::test_default(); + burnchain.pox_constants.sunset_start = 999; + burnchain.pox_constants.sunset_end = 1000; let first_block_header = burnchain_db.get_canonical_chain_tip().unwrap(); assert_eq!(&first_block_header.block_hash, &first_bhh); @@ -562,7 +577,7 @@ mod tests { 485, )); let ops = burnchain_db - .store_new_burnchain_block(&burnchain, &canonical_block) + .store_new_burnchain_block(&burnchain, &canonical_block, StacksEpochId::Epoch21) .unwrap(); assert_eq!(ops.len(), 0); @@ -713,7 +728,7 @@ mod tests { )); let processed_ops_0 = burnchain_db - .store_new_burnchain_block(&burnchain, &block_0) + .store_new_burnchain_block(&burnchain, &block_0, StacksEpochId::Epoch21) .unwrap(); assert_eq!( @@ -723,7 +738,7 @@ mod tests { ); let processed_ops_1 = burnchain_db - .store_new_burnchain_block(&burnchain, &block_1) + .store_new_burnchain_block(&burnchain, &block_1, StacksEpochId::Epoch21) .unwrap(); assert_eq!( From 46abd7426362191245f37ce57baf1797233d3489 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 09:47:01 -0400 Subject: [PATCH 072/178] feat: confine StacksEpochId checks for sunset burn to wrapper methods in PoxConstants, so we don't pepper the codebase with this check --- src/burnchains/mod.rs | 57 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/src/burnchains/mod.rs b/src/burnchains/mod.rs index a00a5bdc1..265391dad 100644 --- a/src/burnchains/mod.rs +++ b/src/burnchains/mod.rs @@ -302,6 +302,10 @@ pub struct PoxConstants { /// percentage of liquid STX that must participate for PoX /// to occur pub pox_participation_threshold_pct: u64, + /// last+1 block height of sunset phase + pub sunset_end: u64, + /// first block height of sunset phase + pub sunset_start: u64, /// The auto unlock height for PoX v1 lockups before transition to PoX v2. This /// also defines the burn height at which PoX reward sets are calculated using /// PoX v2 rather than v1 @@ -316,10 +320,13 @@ impl PoxConstants { anchor_threshold: u32, pox_rejection_fraction: u64, pox_participation_threshold_pct: u64, + sunset_start: u64, + sunset_end: u64, v1_unlock_height: u32, ) -> PoxConstants { assert!(anchor_threshold > (prepare_length / 2)); assert!(prepare_length < reward_cycle_length); + assert!(sunset_start <= sunset_end); PoxConstants { reward_cycle_length, @@ -327,6 +334,8 @@ impl PoxConstants { anchor_threshold, pox_rejection_fraction, pox_participation_threshold_pct, + sunset_start, + sunset_end, v1_unlock_height, _shadow: PhantomData, } @@ -334,7 +343,7 @@ impl PoxConstants { #[cfg(test)] pub fn test_default() -> PoxConstants { // 20 reward slots; 10 prepare-phase slots - PoxConstants::new(10, 5, 3, 25, 5, u32::max_value()) + PoxConstants::new(10, 5, 3, 25, 5, 5000, 10000, u32::max_value()) } /// Returns the PoX contract that is "active" at the given burn block height @@ -367,6 +376,8 @@ impl PoxConstants { 80, 25, 5, + BITCOIN_MAINNET_FIRST_BLOCK_HEIGHT + POX_SUNSET_START, + BITCOIN_MAINNET_FIRST_BLOCK_HEIGHT + POX_SUNSET_END, POX_V1_MAINNET_EARLY_UNLOCK_HEIGHT, ) } @@ -378,12 +389,54 @@ impl PoxConstants { 40, 12, 2, + BITCOIN_TESTNET_FIRST_BLOCK_HEIGHT + POX_SUNSET_START, + BITCOIN_TESTNET_FIRST_BLOCK_HEIGHT + POX_SUNSET_END, POX_V1_TESTNET_EARLY_UNLOCK_HEIGHT, ) // total liquid supply is 40000000000000000 µSTX } pub fn regtest_default() -> PoxConstants { - PoxConstants::new(5, 1, 1, 3333333333333333, 1, 1_000_000) + PoxConstants::new( + 5, + 1, + 1, + 3333333333333333, + 1, + BITCOIN_REGTEST_FIRST_BLOCK_HEIGHT + POX_SUNSET_START, + BITCOIN_REGTEST_FIRST_BLOCK_HEIGHT + POX_SUNSET_END, + 1_000_000, + ) + } + + /// Return true if PoX should sunset at all + /// return false if not. + pub fn has_pox_sunset(epoch_id: StacksEpochId) -> bool { + epoch_id < StacksEpochId::Epoch21 + } + + /// Returns true if PoX has been fully disabled by the PoX sunset. + /// Behavior is epoch-specific + pub fn is_after_pox_sunset_end(&self, burn_block_height: u64, epoch_id: StacksEpochId) -> bool { + if !Self::has_pox_sunset(epoch_id) { + false + } else { + burn_block_height > self.sunset_end + } + } + + /// Returns true if the burn height falls into the PoX sunset period. + /// Returns false if not, or if the sunset isn't active in this epoch + /// (Note that this is true if burn_block_height is beyond the sunset height) + pub fn is_after_pox_sunset_start( + &self, + burn_block_height: u64, + epoch_id: StacksEpochId, + ) -> bool { + if !Self::has_pox_sunset(epoch_id) { + false + } else { + self.sunset_start <= burn_block_height + } } } From 8921179199e1c90464ccffc2081e2cc2c4c615f8 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 09:47:33 -0400 Subject: [PATCH 073/178] fix: restore sunset_burn --- src/chainstate/burn/db/processing.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/chainstate/burn/db/processing.rs b/src/chainstate/burn/db/processing.rs index aba2e7fb0..e0d1e1acb 100644 --- a/src/chainstate/burn/db/processing.rs +++ b/src/chainstate/burn/db/processing.rs @@ -418,6 +418,7 @@ mod tests { }; let block_commit = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash([0x22; 32]), new_seed: VRFSeed::from_hex( "3333333333333333333333333333333333333333333333333333333333333333", From 2999558f33555a821322d6c1ceaa634b87b6b95b Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 09:47:51 -0400 Subject: [PATCH 074/178] fix: restore sunset_burn --- src/chainstate/burn/db/sortdb.rs | 78 +++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/src/chainstate/burn/db/sortdb.rs b/src/chainstate/burn/db/sortdb.rs index e326fea72..0ef74e78e 100644 --- a/src/chainstate/burn/db/sortdb.rs +++ b/src/chainstate/burn/db/sortdb.rs @@ -252,6 +252,7 @@ impl FromRow for LeaderBlockCommitOp { let burn_fee_str: String = row.get_unwrap("burn_fee"); let input_json: String = row.get_unwrap("input"); let apparent_sender_json: String = row.get_unwrap("apparent_sender"); + let sunset_burn_str: String = row.get_unwrap("sunset_burn"); let commit_outs = serde_json::from_value(row.get_unwrap("commit_outs")) .expect("Unparseable value stored to database"); @@ -268,7 +269,11 @@ impl FromRow for LeaderBlockCommitOp { let burn_fee = burn_fee_str .parse::() - .expect("DB Corruption: burn is not parseable as u64"); + .expect("DB Corruption: Sunset burn is not parseable as u64"); + + let sunset_burn = sunset_burn_str + .parse::() + .expect("DB Corruption: Sunset burn is not parseable as u64"); let burn_parent_modulus: u8 = row.get_unwrap("burn_parent_modulus"); @@ -286,6 +291,7 @@ impl FromRow for LeaderBlockCommitOp { input, apparent_sender, commit_outs, + sunset_burn, txid, vtxindex, block_height, @@ -523,7 +529,7 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[ memo TEXT, commit_outs TEXT, burn_fee TEXT NOT NULL, -- use text to encode really big numbers - sunset_burn TEXT NOT NULL, -- use text to encode really big numbers (OBSOLETE; IGNORED) + sunset_burn TEXT NOT NULL, -- use text to encode really big numbers input TEXT NOT NULL, apparent_sender TEXT NOT NULL, burn_parent_modulus INTEGER NOT NULL, @@ -1540,6 +1546,9 @@ impl<'a> SortitionHandleTx<'a> { // This block was mined on this fork, but it's acceptance doesn't overtake // the current stacks chain tip. Remember it so that we can process its children, // which might do so later. + // + // TODO: flip a coin here to decide the tie (or use the burn block hash or + // whatever) debug!("Accepted Stacks block {}/{} builds on a non-canonical Stacks tip in this burnchain fork ({})", consensus_hash, stacks_block_hash, &burn_tip.burn_header_hash); } SortitionDB::insert_accepted_stacks_block_pointer( @@ -3052,6 +3061,12 @@ impl SortitionDB { BurnchainError::MissingParentBlock })?; + let cur_epoch = SortitionDB::get_stacks_epoch(self.conn(), burn_header.block_height)? + .expect(&format!( + "FATAL: no epoch defined for burn height {}", + burn_header.block_height + )); + let mut sortition_db_handle = SortitionHandleTx::begin(self, &parent_sort_id)?; let parent_snapshot = sortition_db_handle .get_block_snapshot(&burn_header.parent_block_hash, &parent_sort_id)? @@ -3066,12 +3081,24 @@ impl SortitionDB { .sortition_hash .mix_burn_header(&parent_snapshot.burn_header_hash); - let reward_set_info = sortition_db_handle.pick_recipients( - burnchain, - burn_header.block_height, - &reward_set_vrf_hash, - next_pox_info.as_ref(), - )?; + let reward_set_info = if cur_epoch.epoch_id < StacksEpochId::Epoch21 + && burn_header.block_height >= burnchain.pox_constants.sunset_end + { + test_debug!( + "No recipients for burn height {}: epoch = {}, sunset_end = {}", + burnchain.pox_constants.sunset_end, + cur_epoch.epoch_id, + burn_header.block_height + ); + None + } else { + sortition_db_handle.pick_recipients( + burnchain, + burn_header.block_height, + &reward_set_vrf_hash, + next_pox_info.as_ref(), + )? + }; // Get any initial mining bonus which would be due to the winner of this block. let bonus_remaining = @@ -3128,15 +3155,28 @@ impl SortitionDB { .sortition_hash .mix_burn_header(&parent_snapshot.burn_header_hash); + let cur_epoch = + SortitionDB::get_stacks_epoch(self.conn(), parent_snapshot.block_height + 1)?.expect( + &format!( + "FATAL: no epoch defined for burn height {}", + parent_snapshot.block_height + 1 + ), + ); + let mut sortition_db_handle = SortitionHandleTx::begin(self, &parent_snapshot.sortition_id)?; - - sortition_db_handle.pick_recipients( - burnchain, - parent_snapshot.block_height + 1, - &reward_set_vrf_hash, - next_pox_info, - ) + if cur_epoch.epoch_id < StacksEpochId::Epoch21 + && parent_snapshot.block_height + 1 >= burnchain.pox_constants.sunset_end + { + Ok(None) + } else { + sortition_db_handle.pick_recipients( + burnchain, + parent_snapshot.block_height + 1, + &reward_set_vrf_hash, + next_pox_info, + ) + } } pub fn is_stacks_block_in_sortition_set( @@ -4190,7 +4230,7 @@ impl<'a> SortitionHandleTx<'a> { &tx_input_str, sort_id, &serde_json::to_value(&block_commit.commit_outs).unwrap(), - &0i64, + &block_commit.sunset_burn.to_string(), &apparent_sender_str, &block_commit.burn_parent_modulus, ]; @@ -5040,6 +5080,7 @@ pub mod tests { }; let block_commit = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222222") .unwrap(), @@ -5861,6 +5902,7 @@ pub mod tests { }; let block_commit = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222222") .unwrap(), @@ -7988,6 +8030,7 @@ pub mod tests { }; let genesis_block_commit = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222221") .unwrap(), @@ -8030,6 +8073,7 @@ pub mod tests { // descends from genesis let block_commit_1 = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222222") .unwrap(), @@ -8071,6 +8115,7 @@ pub mod tests { // descends from block_commit_1 let block_commit_1_1 = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222224") .unwrap(), @@ -8112,6 +8157,7 @@ pub mod tests { // descends from genesis_block_commit let block_commit_2 = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222223") .unwrap(), From 243f719018ba48508226d3dca88b371dd75da419 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 09:48:07 -0400 Subject: [PATCH 075/178] fix: restore sunset burn --- src/chainstate/burn/distribution.rs | 115 +++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/src/chainstate/burn/distribution.rs b/src/chainstate/burn/distribution.rs index 2ea8f91a3..0c99173c6 100644 --- a/src/chainstate/burn/distribution.rs +++ b/src/chainstate/burn/distribution.rs @@ -126,8 +126,8 @@ impl BurnSamplePoint { /// /// All operations need to be supplied in an ordered Vec of Vecs containing /// the ops at each block height in a mining commit window. Normally, this window - /// is the constant `MINING_COMMITMENT_WINDOW`, except during prepare-phases. - /// In this particular case, the window is only one block. The code does not + /// is the constant `MINING_COMMITMENT_WINDOW`, except during prepare-phases and post-PoX + /// sunset. In either of these two cases, the window is only one block. The code does not /// consider which window is active; it merely deduces it by inspecting the length of the /// given `block_commits` argument. /// @@ -150,7 +150,7 @@ impl BurnSamplePoint { /// `missed_commits.len() = block_commits.len() - 1` /// * `burn_blocks`: this is a vector of booleans that indicate whether or not a block-commit /// occurred during a PoB-only sortition or a possibly-PoX sortition. The former occurs - /// during a prepare phase, and must have only one (burn) output. + /// during either a prepare phase or after PoX sunset, and must have only one (burn) output. /// The latter occurs everywhere else, and must have `OUTPUTS_PER_COMMIT` outputs after the /// `OP_RETURN` payload. The length of this vector must be equal to the length of the /// `block_commits` vector. `burn_blocks[i]` is `true` if the `ith` block-commit must be PoB. @@ -519,6 +519,7 @@ mod tests { input: (input_txid, 3), apparent_sender: BurnchainSigner::new_p2pkh(&StacksPublicKey::new()), commit_outs: vec![], + sunset_burn: 0, txid, vtxindex: 0, block_height: block_ht, @@ -531,6 +532,111 @@ mod tests { } } + #[test] + fn make_mean_min_median_sunset_in_window() { + // miner 1: 3 4 5 4 5 4 + // ub : 1 0 0 0 0 0 + // | sunset end + // miner 2: 1 3 3 3 3 3 + // ub : 1 0 0 0 0 0 + // 0 1 0 0 0 0 + // .. + + // miner 1 => min = 1, median = 1, last_burn = 4 + // miner 2 => min = 1, median = 1, last_burn = 3 + + let mut commits = vec![ + vec![ + make_block_commit(3, 1, 1, 1, None, 1), + make_block_commit(1, 2, 2, 2, None, 1), + ], + vec![ + make_block_commit(4, 3, 3, 3, Some(1), 2), + make_block_commit(3, 4, 4, 4, Some(2), 2), + ], + vec![ + make_block_commit(5, 5, 5, 5, Some(3), 3), + make_block_commit(3, 6, 6, 6, Some(4), 3), + ], + vec![ + make_block_commit(4, 7, 7, 7, Some(5), 4), + make_block_commit(3, 8, 8, 8, Some(6), 4), + ], + vec![ + make_block_commit(5, 9, 9, 9, Some(7), 5), + make_block_commit(3, 10, 10, 10, Some(8), 5), + ], + vec![ + make_block_commit(4, 11, 11, 11, Some(9), 6), + make_block_commit(3, 12, 12, 12, Some(10), 6), + ], + ]; + let user_burns = vec![ + vec![make_user_burn(1, 1, 1, 1, 1), make_user_burn(1, 2, 2, 2, 1)], + vec![make_user_burn(1, 4, 4, 4, 2)], + vec![make_user_burn(1, 6, 6, 6, 3)], + vec![make_user_burn(1, 8, 8, 8, 4)], + vec![make_user_burn(1, 10, 10, 10, 5)], + vec![make_user_burn(1, 12, 12, 12, 6)], + ]; + + let mut result = BurnSamplePoint::make_min_median_distribution( + commits.clone(), + vec![vec![]; (MINING_COMMITMENT_WINDOW - 1) as usize], + vec![false, false, false, true, true, true], + ); + + assert_eq!(result.len(), 2, "Should be two miners"); + + result.sort_by_key(|sample| sample.candidate.txid); + + // block-commits are currently malformed -- the post-sunset commits spend the wrong UTXO. + assert_eq!(result[0].burns, 1); + assert_eq!(result[1].burns, 1); + + // make sure that we're associating with the last commit in the window. + assert_eq!(result[0].candidate.txid, commits[5][0].txid); + assert_eq!(result[1].candidate.txid, commits[5][1].txid); + + assert_eq!(result[0].user_burns.len(), 0); + assert_eq!(result[1].user_burns.len(), 0); + + // now correct the back pointers so that they point + // at the correct UTXO position *post-sunset* + for (ix, window_slice) in commits.iter_mut().enumerate() { + if ix >= 4 { + for commit in window_slice.iter_mut() { + commit.input.1 = 2; + } + } + } + + // miner 1: 3 4 5 4 5 4 + // miner 2: 1 3 3 3 3 3 + // miner 1 => min = 3, median = 4, last_burn = 4 + // miner 2 => min = 1, median = 3, last_burn = 3 + + let mut result = BurnSamplePoint::make_min_median_distribution( + commits.clone(), + vec![vec![]; (MINING_COMMITMENT_WINDOW - 1) as usize], + vec![false, false, false, true, true, true], + ); + + assert_eq!(result.len(), 2, "Should be two miners"); + + result.sort_by_key(|sample| sample.candidate.txid); + + assert_eq!(result[0].burns, 4); + assert_eq!(result[1].burns, 3); + + // make sure that we're associating with the last commit in the window. + assert_eq!(result[0].candidate.txid, commits[5][0].txid); + assert_eq!(result[1].candidate.txid, commits[5][1].txid); + + assert_eq!(result[0].user_burns.len(), 0); + assert_eq!(result[1].user_burns.len(), 0); + } + #[test] fn make_mean_min_median() { // test case 1: @@ -1018,6 +1124,7 @@ mod tests { }; let block_commit_1 = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222222") .unwrap(), @@ -1062,6 +1169,7 @@ mod tests { }; let block_commit_2 = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222223") .unwrap(), @@ -1106,6 +1214,7 @@ mod tests { }; let block_commit_3 = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222224") .unwrap(), From 038b03071a86a7f560c24dd5083a38245f6880dc Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:03:59 -0400 Subject: [PATCH 076/178] fix: >=, not >, for sunset end --- src/burnchains/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/burnchains/mod.rs b/src/burnchains/mod.rs index 265391dad..d18fa011d 100644 --- a/src/burnchains/mod.rs +++ b/src/burnchains/mod.rs @@ -420,7 +420,7 @@ impl PoxConstants { if !Self::has_pox_sunset(epoch_id) { false } else { - burn_block_height > self.sunset_end + burn_block_height >= self.sunset_end } } From ae693f1c1664089454db65e95c8f4e4cd818fe40 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:04:43 -0400 Subject: [PATCH 077/178] fix: restore PoX sunset, and add epoch-2.1-specific tests to verify that the sunset burn requirement disappears at the epoch boundary --- .../burn/operations/leader_block_commit.rs | 314 ++++++++++++++++-- 1 file changed, 286 insertions(+), 28 deletions(-) diff --git a/src/chainstate/burn/operations/leader_block_commit.rs b/src/chainstate/burn/operations/leader_block_commit.rs index 130d46a5a..052887112 100644 --- a/src/chainstate/burn/operations/leader_block_commit.rs +++ b/src/chainstate/burn/operations/leader_block_commit.rs @@ -20,6 +20,7 @@ use crate::burnchains::bitcoin::BitcoinNetworkType; use crate::burnchains::Address; use crate::burnchains::Burnchain; use crate::burnchains::BurnchainBlockHeader; +use crate::burnchains::PoxConstants; use crate::burnchains::Txid; use crate::burnchains::{BurnchainRecipient, BurnchainSigner}; use crate::burnchains::{BurnchainTransaction, PublicKey}; @@ -74,6 +75,7 @@ impl LeaderBlockCommitOp { apparent_sender: &BurnchainSigner, ) -> LeaderBlockCommitOp { LeaderBlockCommitOp { + sunset_burn: 0, block_height: block_height, burn_parent_modulus: if block_height > 0 { ((block_height - 1) % BURN_BLOCK_MINED_AT_MODULUS) as u8 @@ -112,6 +114,7 @@ impl LeaderBlockCommitOp { apparent_sender: &BurnchainSigner, ) -> LeaderBlockCommitOp { LeaderBlockCommitOp { + sunset_burn: 0, new_seed: new_seed.clone(), key_block_ptr: key_block_ptr, key_vtxindex: key_vtxindex, @@ -146,7 +149,7 @@ impl LeaderBlockCommitOp { pub fn expected_chained_utxo(burn_only: bool) -> u32 { if burn_only { - 2 // if we're in the prepare phase, then chained commits should spend the output after the burn commit + 2 // if sunset has occurred, or we're in the prepare phase, then chained commits should spend the output after the burn commit } else { // otherwise, it's the output after the last PoX output (OUTPUTS_PER_COMMIT as u32) + 1 @@ -209,12 +212,14 @@ impl LeaderBlockCommitOp { pub fn from_tx( burnchain: &Burnchain, block_header: &BurnchainBlockHeader, + epoch_id: StacksEpochId, tx: &BurnchainTransaction, ) -> Result { LeaderBlockCommitOp::parse_from_tx( burnchain, block_header.block_height, &block_header.block_hash, + epoch_id, tx, ) } @@ -228,6 +233,7 @@ impl LeaderBlockCommitOp { burnchain: &Burnchain, block_height: u64, block_hash: &BurnchainHeaderHash, + epoch_id: StacksEpochId, tx: &BurnchainTransaction, ) -> Result { // can't be too careful... @@ -292,15 +298,32 @@ impl LeaderBlockCommitOp { return Err(op_error::ParseError); } - let (commit_outs, burn_fee) = if burnchain.is_in_prepare_phase(block_height) { - // check if we're in a prepare phase - // should be only one burn output + // check if we've reached PoX disable + let (commit_outs, sunset_burn, burn_fee) = if burnchain + .pox_constants + .is_after_pox_sunset_end(block_height, epoch_id) + { + // PoX is disabled by sunset (not possible in epoch 2.1 or later). + // should be only one burn output. if !outputs[0].address.is_burn() { return Err(op_error::BlockCommitBadOutputs); } let BurnchainRecipient { address, amount } = outputs.remove(0); - (vec![address], amount) + (vec![address], 0, amount) + } else if burnchain.is_in_prepare_phase(block_height) { + // we're in a prepare phase. + // should be only one burn output. + if !outputs[0].address.is_burn() { + return Err(op_error::BlockCommitBadOutputs); + } + let BurnchainRecipient { address, amount } = outputs.remove(0); + (vec![address], 0, amount) } else { + // we're in a reward phase, which may or may not be PoX. + // check if this transaction provided a sunset burn (which is still allowed in epoch + // 2.1; it's just not doing anything for the miner). + let sunset_burn = tx.get_burn_amount(); + let mut commit_outs = vec![]; let mut pox_fee = None; for (ix, output) in outputs.into_iter().enumerate() { @@ -311,7 +334,7 @@ impl LeaderBlockCommitOp { // all pox outputs must have the same fee if let Some(pox_fee) = pox_fee { if output.amount != pox_fee { - warn!("Invalid commit tx: different output amounts for different PoX reward addresses"); + warn!("Invalid commit tx: different output amounts for different PoX reward addresses ({} != {})", pox_fee, output.amount); return Err(op_error::ParseError); } } else { @@ -337,7 +360,7 @@ impl LeaderBlockCommitOp { return Err(op_error::ParseError); } - (commit_outs, burn_fee) + (commit_outs, sunset_burn, burn_fee) }; let input = tx @@ -360,6 +383,7 @@ impl LeaderBlockCommitOp { burn_parent_modulus: data.burn_parent_modulus, commit_outs, + sunset_burn, burn_fee, input, apparent_sender, @@ -472,12 +496,33 @@ impl RewardSetInfo { impl LeaderBlockCommitOp { fn check_pox( &self, + epoch_id: StacksEpochId, burnchain: &Burnchain, tx: &mut SortitionHandleTx, reward_set_info: Option<&RewardSetInfo>, ) -> Result<(), op_error> { let parent_block_height = self.parent_block_ptr as u64; + if PoxConstants::has_pox_sunset(epoch_id) { + // sunset only applies in epochs prior to 2.1. After 2.1, miners can put whatever they + // want into the sunset burn but it won't be checked, nor will it count towards their + // sortition + let total_committed = self + .burn_fee + .checked_add(self.sunset_burn) + .expect("BUG: Overflow in total committed calculation"); + + let expected_sunset_burn = + burnchain.expected_sunset_burn(self.block_height, total_committed, epoch_id); + if self.sunset_burn < expected_sunset_burn { + warn!( + "Invalid block commit: should have included sunset burn amount of {}, found {}", + expected_sunset_burn, self.sunset_burn + ); + return Err(op_error::BlockCommitBadOutputs); + } + } + ///////////////////////////////////////////////////////////////////////////////////// // This tx must have the expected commit or burn outputs: // * if there is a known anchor block for the current reward cycle, and this @@ -594,16 +639,20 @@ impl LeaderBlockCommitOp { fn check_single_burn_output(&self) -> Result<(), op_error> { if self.commit_outs.len() != 1 { - warn!("Invalid block commit, should have 1 commit out"); + warn!("Invalid post-sunset block commit, should have 1 commit out"); return Err(op_error::BlockCommitBadOutputs); } if !self.commit_outs[0].is_burn() { - warn!("Invalid block commit, should have burn address output"); + warn!("Invalid post-sunset block commit, should have burn address output"); return Err(op_error::BlockCommitBadOutputs); } Ok(()) } + fn check_after_pox_sunset(&self) -> Result<(), op_error> { + self.check_single_burn_output() + } + fn check_prepare_commit_burn(&self) -> Result<(), op_error> { self.check_single_burn_output() } @@ -633,6 +682,10 @@ impl LeaderBlockCommitOp { ); return Err(op_error::BlockCommitBadInput); } + let epoch = SortitionDB::get_stacks_epoch(tx, self.block_height)?.expect(&format!( + "FATAL: impossible block height: no epoch defined for {}", + self.block_height + )); let intended_modulus = (self.burn_block_mined_at() + 1) % BURN_BLOCK_MINED_AT_MODULUS; let actual_modulus = self.block_height % BURN_BLOCK_MINED_AT_MODULUS; @@ -647,10 +700,6 @@ impl LeaderBlockCommitOp { // is not valid, but we should allow this UTXO to "chain" to valid // UTXOs to allow the miner windowing to work in the face of missed // blocks. - let epoch = SortitionDB::get_stacks_epoch(tx, self.block_height)?.expect(&format!( - "FATAL: impossible block height: no epoch defined for {}", - self.block_height - )); let miss_distance = if actual_modulus > intended_modulus { actual_modulus - intended_modulus } else { @@ -692,12 +741,25 @@ impl LeaderBlockCommitOp { return Err(op_error::MissedBlockCommit(missed_data)); } - self.check_pox(burnchain, tx, reward_set_info) - .map_err(|e| { - warn!("Invalid block-commit: bad PoX: {:?}", &e; - "apparent_sender" => %apparent_sender_address); + if burnchain + .pox_constants + .is_after_pox_sunset_end(self.block_height, epoch.epoch_id) + { + // sunset has begun and we're not in epoch 2.1 or later, so apply sunset check + self.check_after_pox_sunset().map_err(|e| { + warn!("Invalid block-commit: bad PoX after sunset: {:?}", &e; + "apparent_sender" => %apparent_sender_address); e })?; + } else { + // either in epoch 2.1, or the PoX sunset hasn't completed yet + self.check_pox(epoch.epoch_id, burnchain, tx, reward_set_info) + .map_err(|e| { + warn!("Invalid block-commit: bad PoX: {:?}", &e; + "apparent_sender" => %apparent_sender_address); + e + })?; + } ///////////////////////////////////////////////////////////////////////////////////// // This tx must occur after the start of the network @@ -913,6 +975,151 @@ mod tests { } } + #[test] + fn test_parse_sunset_end() { + let tx = BurnchainTransaction::Bitcoin(BitcoinTransaction { + data_amt: 0, + txid: Txid([0; 32]), + vtxindex: 0, + opcode: Opcodes::LeaderBlockCommit as u8, + data: vec![1; 80], + inputs: vec![BitcoinTxInput { + keys: vec![], + num_required: 0, + in_type: BitcoinInputType::Standard, + tx_ref: (Txid([0; 32]), 0), + }], + outputs: vec![ + BitcoinTxOutput { + units: 10, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([1; 20]), + }, + }, + BitcoinTxOutput { + units: 10, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([2; 20]), + }, + }, + BitcoinTxOutput { + units: 30, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([0; 20]), + }, + }, + ], + }); + + let mut burnchain = Burnchain::regtest("nope"); + burnchain.pox_constants.sunset_start = 16843021; + burnchain.pox_constants.sunset_end = 16843022; + + // should fail in epoch 2.05 -- not a burn output + let err = LeaderBlockCommitOp::parse_from_tx( + &burnchain, + 16843022, + &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, + &tx, + ) + .unwrap_err(); + + assert!(if let op_error::BlockCommitBadOutputs = err { + true + } else { + false + }); + + // should succeed in epoch 2.1 -- can be PoX in 2.1 + let _op = LeaderBlockCommitOp::parse_from_tx( + &burnchain, + 16843022, + &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch21, + &tx, + ) + .unwrap(); + + let tx = BurnchainTransaction::Bitcoin(BitcoinTransaction { + data_amt: 0, + txid: Txid([0; 32]), + vtxindex: 0, + opcode: Opcodes::LeaderBlockCommit as u8, + data: vec![1; 80], + inputs: vec![BitcoinTxInput { + keys: vec![], + num_required: 0, + in_type: BitcoinInputType::Standard, + tx_ref: (Txid([0; 32]), 0), + }], + outputs: vec![ + BitcoinTxOutput { + units: 10, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([0; 20]), + }, + }, + BitcoinTxOutput { + units: 10, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([2; 20]), + }, + }, + BitcoinTxOutput { + units: 30, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([0; 20]), + }, + }, + ], + }); + + let mut burnchain = Burnchain::regtest("nope"); + burnchain.pox_constants.sunset_start = 16843021; + burnchain.pox_constants.sunset_end = 16843022; + + // succeeds in epoch 2.05 because of the burn output + let op = LeaderBlockCommitOp::parse_from_tx( + &burnchain, + 16843022, + &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, + &tx, + ) + .unwrap(); + + assert_eq!(op.commit_outs.len(), 1); + assert!(op.commit_outs[0].is_burn()); + assert_eq!(op.burn_fee, 10); + + // succeeds in epoch 2.1 because the sunset doesn't apply + let op = LeaderBlockCommitOp::parse_from_tx( + &burnchain, + 16843022, + &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch21, + &tx, + ) + .unwrap(); + + assert_eq!(op.commit_outs.len(), 2); + assert!(op.commit_outs[0].is_burn()); + assert_eq!(op.burn_fee, 20); + } + #[test] fn test_parse_pox_commits() { let tx = BurnchainTransaction::Bitcoin(BitcoinTransaction { @@ -955,12 +1162,15 @@ mod tests { ], }); - let burnchain = Burnchain::regtest("nope"); + let mut burnchain = Burnchain::regtest("nope"); + burnchain.pox_constants.sunset_start = 16843019; + burnchain.pox_constants.sunset_end = 16843020; let op = LeaderBlockCommitOp::parse_from_tx( &burnchain, 16843019, &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, &tx, ) .unwrap(); @@ -968,6 +1178,8 @@ mod tests { // should have 2 commit outputs, summing to 20 burned units assert_eq!(op.commit_outs.len(), 2); assert_eq!(op.burn_fee, 20); + // the third output, because it's a burn, should have counted as a sunset_burn + assert_eq!(op.sunset_burn, 30); let tx = BurnchainTransaction::Bitcoin(BitcoinTransaction { data_amt: 0, @@ -1001,13 +1213,16 @@ mod tests { ], }); - let burnchain = Burnchain::regtest("nope"); + let mut burnchain = Burnchain::regtest("nope"); + burnchain.pox_constants.sunset_start = 16843019; + burnchain.pox_constants.sunset_end = 16843020; // burn amount should have been 10, not 9 match LeaderBlockCommitOp::parse_from_tx( &burnchain, 16843019, &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, &tx, ) .unwrap_err() @@ -1072,12 +1287,15 @@ mod tests { ], }); - let burnchain = Burnchain::regtest("nope"); + let mut burnchain = Burnchain::regtest("nope"); + burnchain.pox_constants.sunset_start = 16843019; + burnchain.pox_constants.sunset_end = 16843020; let op = LeaderBlockCommitOp::parse_from_tx( &burnchain, 16843019, &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, &tx, ) .unwrap(); @@ -1085,6 +1303,8 @@ mod tests { // should have 2 commit outputs assert_eq!(op.commit_outs.len(), 2); assert_eq!(op.burn_fee, 26); + // the third output, because it's not a burn, should not have counted as a sunset_burn + assert_eq!(op.sunset_burn, 0); let tx = BurnchainTransaction::Bitcoin(BitcoinTransaction { data_amt: 0, @@ -1108,13 +1328,16 @@ mod tests { }], }); - let burnchain = Burnchain::regtest("nope"); + let mut burnchain = Burnchain::regtest("nope"); + burnchain.pox_constants.sunset_start = 16843019; + burnchain.pox_constants.sunset_end = 16843020; // not enough PoX outputs match LeaderBlockCommitOp::parse_from_tx( &burnchain, 16843019, &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, &tx, ) .unwrap_err() @@ -1155,13 +1378,16 @@ mod tests { ], }); - let burnchain = Burnchain::regtest("nope"); + let mut burnchain = Burnchain::regtest("nope"); + burnchain.pox_constants.sunset_start = 16843019; + burnchain.pox_constants.sunset_end = 16843020; // unequal PoX outputs match LeaderBlockCommitOp::parse_from_tx( &burnchain, 16843019, &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, &tx, ) .unwrap_err() @@ -1226,13 +1452,16 @@ mod tests { ], }); - let burnchain = Burnchain::regtest("nope"); + let mut burnchain = Burnchain::regtest("nope"); + burnchain.pox_constants.sunset_start = 16843019; + burnchain.pox_constants.sunset_end = 16843020; // 0 total burn match LeaderBlockCommitOp::parse_from_tx( &burnchain, 16843019, &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, &tx, ) .unwrap_err() @@ -1257,6 +1486,7 @@ mod tests { txstr: "01000000011111111111111111111111111111111111111111111111111111111111111111000000006b483045022100eba8c0a57c1eb71cdfba0874de63cf37b3aace1e56dcbd61701548194a79af34022041dd191256f3f8a45562e5d60956bb871421ba69db605716250554b23b08277b012102d8015134d9db8178ac93acbc43170a2f20febba5087a5b0437058765ad5133d000000000040000000000000000536a4c5069645b22222222222222222222222222222222222222222222222222222222222222223333333333333333333333333333333333333333333333333333333333333333404142435051606162637071fa39300000000000001976a914000000000000000000000000000000000000000088ac39300000000000001976a914000000000000000000000000000000000000000088aca05b0000000000001976a9140be3e286a15ea85882761618e366586b5574100d88ac00000000".into(), opstr: "69645b22222222222222222222222222222222222222222222222222222222222222223333333333333333333333333333333333333333333333333333333333333333404142435051606162637071fa".to_string(), result: Some(LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes(&hex_bytes("2222222222222222222222222222222222222222222222222222222222222222").unwrap()).unwrap(), new_seed: VRFSeed::from_bytes(&hex_bytes("3333333333333333333333333333333333333333333333333333333333333333").unwrap()).unwrap(), parent_block_ptr: 0x40414243, @@ -1342,9 +1572,16 @@ mod tests { let burnchain_tx = BurnchainTransaction::Bitcoin(parser.parse_tx(&tx, vtxindex as usize).unwrap()); - let burnchain = Burnchain::regtest("nope"); + let mut burnchain = Burnchain::regtest("nope"); + burnchain.pox_constants.sunset_start = block_height; + burnchain.pox_constants.sunset_end = block_height + 1; - let op = LeaderBlockCommitOp::from_tx(&burnchain, &header, &burnchain_tx); + let op = LeaderBlockCommitOp::from_tx( + &burnchain, + &header, + StacksEpochId::Epoch2_05, + &burnchain_tx, + ); match (op, tx_fixture.result) { (Ok(parsed_tx), Some(result)) => { @@ -1412,7 +1649,7 @@ mod tests { ]; let burnchain = Burnchain { - pox_constants: PoxConstants::new(6, 2, 2, 25, 5, u32::max_value()), + pox_constants: PoxConstants::new(6, 2, 2, 25, 5, 5000, 10000, u32::max_value()), peer_version: 0x012345678, network_id: 0x9abcdef0, chain_name: "bitcoin".to_string(), @@ -1486,6 +1723,7 @@ mod tests { // consumes leader_key_1 let block_commit_1 = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222222") .unwrap(), @@ -1636,6 +1874,7 @@ mod tests { CheckFixture { // accept -- consumes leader_key_2 op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -1685,6 +1924,7 @@ mod tests { CheckFixture { // accept -- builds directly off of genesis block and consumes leader_key_2 op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -1734,6 +1974,7 @@ mod tests { CheckFixture { // accept -- also consumes leader_key_1 op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -1783,6 +2024,7 @@ mod tests { CheckFixture { // reject -- bad burn parent modulus op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -1844,6 +2086,7 @@ mod tests { CheckFixture { // reject -- bad burn parent modulus op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -1963,7 +2206,7 @@ mod tests { ]; let burnchain = Burnchain { - pox_constants: PoxConstants::new(6, 2, 2, 25, 5, u32::max_value()), + pox_constants: PoxConstants::new(6, 2, 2, 25, 5, 5000, 10000, u32::max_value()), peer_version: 0x012345678, network_id: 0x9abcdef0, chain_name: "bitcoin".to_string(), @@ -2037,6 +2280,7 @@ mod tests { // consumes leader_key_1 let block_commit_1 = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes("2222222222222222222222222222222222222222222222222222222222222222") .unwrap(), @@ -2184,6 +2428,7 @@ mod tests { CheckFixture { // reject -- predates start block op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -2233,6 +2478,7 @@ mod tests { CheckFixture { // reject -- no such leader key op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -2282,6 +2528,7 @@ mod tests { CheckFixture { // reject -- previous block must exist op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -2331,6 +2578,7 @@ mod tests { CheckFixture { // reject -- previous block must exist in a different block op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -2380,6 +2628,7 @@ mod tests { CheckFixture { // reject -- tx input does not match any leader keys op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -2429,6 +2678,7 @@ mod tests { CheckFixture { // reject -- fee is 0 op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -2478,6 +2728,7 @@ mod tests { CheckFixture { // accept -- consumes leader_key_2 op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -2527,6 +2778,7 @@ mod tests { CheckFixture { // accept -- builds directly off of genesis block and consumes leader_key_2 op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -2576,6 +2828,7 @@ mod tests { CheckFixture { // accept -- also consumes leader_key_1 op: LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash::from_bytes( &hex_bytes( "2222222222222222222222222222222222222222222222222222222222222222", @@ -2654,7 +2907,7 @@ mod tests { .unwrap(); let burnchain = Burnchain { - pox_constants: PoxConstants::new(6, 2, 2, 25, 5, u32::max_value()), + pox_constants: PoxConstants::new(6, 2, 2, 25, 5, 5000, 10000, u32::max_value()), peer_version: 0x012345678, network_id: 0x9abcdef0, chain_name: "bitcoin".to_string(), @@ -2728,6 +2981,7 @@ mod tests { }; let block_commit_pre_2_05 = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash([0x02; 32]), new_seed: VRFSeed([0x03; 32]), parent_block_ptr: 0, @@ -2756,6 +3010,7 @@ mod tests { }; let block_commit_post_2_05_valid = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash([0x03; 32]), new_seed: VRFSeed([0x04; 32]), parent_block_ptr: 0, @@ -2784,6 +3039,7 @@ mod tests { }; let block_commit_post_2_05_valid_bigger_epoch = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash([0x03; 32]), new_seed: VRFSeed([0x04; 32]), parent_block_ptr: 0, @@ -2812,6 +3068,7 @@ mod tests { }; let block_commit_post_2_05_invalid_bad_memo = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash([0x04; 32]), new_seed: VRFSeed([0x05; 32]), parent_block_ptr: 0, @@ -2840,6 +3097,7 @@ mod tests { }; let block_commit_post_2_05_invalid_no_memo = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: BlockHeaderHash([0x05; 32]), new_seed: VRFSeed([0x06; 32]), parent_block_ptr: 0, From 2c749850ddf95d715cab34af6b1e1cf54250787f Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:05:09 -0400 Subject: [PATCH 078/178] fix: restore sunset burn --- src/chainstate/burn/operations/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/chainstate/burn/operations/mod.rs b/src/chainstate/burn/operations/mod.rs index fcf5a83b3..36333b9fc 100644 --- a/src/chainstate/burn/operations/mod.rs +++ b/src/chainstate/burn/operations/mod.rs @@ -233,6 +233,8 @@ pub struct LeaderBlockCommitOp { /// PoX/Burn outputs pub commit_outs: Vec, + // PoX sunset burn + pub sunset_burn: u64, // common to all transactions pub txid: Txid, // transaction ID From 2cbd0602cf372da3edd1accce6968e0da4055832 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:05:27 -0400 Subject: [PATCH 079/178] fix: gate StackStxOp and PreStxOp's expiration on the epoch so they keep working in 2.1, and add test coverage --- src/chainstate/burn/operations/stack_stx.rs | 209 +++++++++++++++++++- 1 file changed, 208 insertions(+), 1 deletion(-) diff --git a/src/chainstate/burn/operations/stack_stx.rs b/src/chainstate/burn/operations/stack_stx.rs index ebf7cb459..3880ca61b 100644 --- a/src/chainstate/burn/operations/stack_stx.rs +++ b/src/chainstate/burn/operations/stack_stx.rs @@ -19,6 +19,7 @@ use std::io::{Read, Write}; use crate::burnchains::Address; use crate::burnchains::Burnchain; use crate::burnchains::BurnchainBlockHeader; +use crate::burnchains::PoxConstants; use crate::burnchains::Txid; use crate::burnchains::{BurnchainRecipient, BurnchainSigner}; use crate::burnchains::{BurnchainTransaction, PublicKey}; @@ -33,6 +34,7 @@ use crate::chainstate::stacks::address::PoxAddress; use crate::chainstate::stacks::index::storage::TrieFileStorage; use crate::chainstate::stacks::{StacksPrivateKey, StacksPublicKey}; use crate::codec::{write_next, Error as codec_error, StacksMessageCodec}; +use crate::core::StacksEpochId; use crate::core::POX_MAX_NUM_CYCLES; use crate::net::Error as net_error; use crate::types::chainstate::TrieHash; @@ -67,16 +69,27 @@ impl PreStxOp { pub fn from_tx( block_header: &BurnchainBlockHeader, + epoch_id: StacksEpochId, tx: &BurnchainTransaction, + pox_sunset_ht: u64, ) -> Result { - PreStxOp::parse_from_tx(block_header.block_height, &block_header.block_hash, tx) + PreStxOp::parse_from_tx( + block_header.block_height, + &block_header.block_hash, + epoch_id, + tx, + pox_sunset_ht, + ) } /// parse a PreStxOp + /// `pox_sunset_ht` is the height at which PoX *disables* pub fn parse_from_tx( block_height: u64, block_hash: &BurnchainHeaderHash, + epoch_id: StacksEpochId, tx: &BurnchainTransaction, + pox_sunset_ht: u64, ) -> Result { // can't be too careful... let inputs = tx.get_signers(); @@ -114,6 +127,15 @@ impl PreStxOp { op_error::InvalidInput })?; + // check if we've reached PoX disable + if PoxConstants::has_pox_sunset(epoch_id) && block_height >= pox_sunset_ht { + debug!( + "PreStxOp broadcasted after sunset. Ignoring. txid={}", + tx.txid() + ); + return Err(op_error::InvalidInput); + } + Ok(PreStxOp { output: output, txid: tx.txid(), @@ -197,23 +219,30 @@ impl StackStxOp { pub fn from_tx( block_header: &BurnchainBlockHeader, + epoch_id: StacksEpochId, tx: &BurnchainTransaction, sender: &StacksAddress, + pox_sunset_ht: u64, ) -> Result { StackStxOp::parse_from_tx( block_header.block_height, &block_header.block_hash, + epoch_id, tx, sender, + pox_sunset_ht, ) } /// parse a StackStxOp + /// `pox_sunset_ht` is the height at which PoX *disables* pub fn parse_from_tx( block_height: u64, block_hash: &BurnchainHeaderHash, + epoch_id: StacksEpochId, tx: &BurnchainTransaction, sender: &StacksAddress, + pox_sunset_ht: u64, ) -> Result { // can't be too careful... let outputs = tx.get_recipients(); @@ -250,6 +279,15 @@ impl StackStxOp { // address into the .pox contract let reward_addr = outputs[0].address.clone().coerce_hash_mode(); + // check if we've reached PoX disable + if PoxConstants::has_pox_sunset(epoch_id) && block_height >= pox_sunset_ht { + debug!( + "StackStxOp broadcasted after sunset. Ignoring. txid={}", + tx.txid() + ); + return Err(op_error::InvalidInput); + } + Ok(StackStxOp { sender: sender.clone(), reward_addr: reward_addr, @@ -328,6 +366,7 @@ mod tests { use crate::chainstate::stacks::address::PoxAddress; use crate::chainstate::stacks::address::StacksAddressExtensions; use crate::chainstate::stacks::StacksPublicKey; + use crate::core::StacksEpochId; use stacks_common::address::AddressHashMode; use stacks_common::deps_common::bitcoin::blockdata::transaction::Transaction; use stacks_common::deps_common::bitcoin::network::serialize::{deserialize, serialize_hex}; @@ -406,7 +445,87 @@ mod tests { let op = PreStxOp::parse_from_tx( 16843022, &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, &BurnchainTransaction::Bitcoin(tx.clone()), + 16843023, + ) + .unwrap(); + + assert_eq!( + &op.output, + &StacksAddress::from_bitcoin_address(&tx.outputs[0].address) + ); + } + + #[test] + fn test_parse_pre_stack_stx_sunset() { + let tx = BitcoinTransaction { + txid: Txid([0; 32]), + vtxindex: 0, + opcode: Opcodes::PreStx as u8, + data: vec![1; 80], + data_amt: 0, + inputs: vec![BitcoinTxInput { + keys: vec![], + num_required: 0, + in_type: BitcoinInputType::Standard, + tx_ref: (Txid([0; 32]), 0), + }], + outputs: vec![ + BitcoinTxOutput { + units: 10, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([1; 20]), + }, + }, + BitcoinTxOutput { + units: 10, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([2; 20]), + }, + }, + BitcoinTxOutput { + units: 30, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([0; 20]), + }, + }, + ], + }; + + let sender = StacksAddress { + version: 0, + bytes: Hash160([0; 20]), + }; + + // pre-2.1 this fails + let op_err = PreStxOp::parse_from_tx( + 16843022, + &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, + &BurnchainTransaction::Bitcoin(tx.clone()), + 16843022, + ) + .unwrap_err(); + + if let op_error::InvalidInput = op_err { + } else { + panic!("Parsed post-sunset prestx"); + } + + // post-2.1 this succeeds + let op = PreStxOp::parse_from_tx( + 16843022, + &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch21, + &BurnchainTransaction::Bitcoin(tx.clone()), + 16843022, ) .unwrap(); @@ -465,8 +584,96 @@ mod tests { let op = StackStxOp::parse_from_tx( 16843022, &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, &BurnchainTransaction::Bitcoin(tx.clone()), &sender, + 16843023, + ) + .unwrap(); + + assert_eq!(&op.sender, &sender); + assert_eq!( + &op.reward_addr, + &PoxAddress::Standard( + StacksAddress::from_bitcoin_address(&tx.outputs[0].address), + Some(AddressHashMode::SerializeP2PKH) + ) + ); + assert_eq!(op.stacked_ustx, u128::from_be_bytes([1; 16])); + assert_eq!(op.num_cycles, 1); + } + + #[test] + fn test_parse_stack_stx_sunset() { + let tx = BitcoinTransaction { + txid: Txid([0; 32]), + vtxindex: 0, + opcode: Opcodes::StackStx as u8, + data: vec![1; 80], + data_amt: 0, + inputs: vec![BitcoinTxInput { + keys: vec![], + num_required: 0, + in_type: BitcoinInputType::Standard, + tx_ref: (Txid([0; 32]), 0), + }], + outputs: vec![ + BitcoinTxOutput { + units: 10, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([1; 20]), + }, + }, + BitcoinTxOutput { + units: 10, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([2; 20]), + }, + }, + BitcoinTxOutput { + units: 30, + address: BitcoinAddress { + addrtype: BitcoinAddressType::PublicKeyHash, + network_id: BitcoinNetworkType::Mainnet, + bytes: Hash160([0; 20]), + }, + }, + ], + }; + + let sender = StacksAddress { + version: 0, + bytes: Hash160([0; 20]), + }; + + // pre-2.1: this fails + let op_err = StackStxOp::parse_from_tx( + 16843022, + &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch2_05, + &BurnchainTransaction::Bitcoin(tx.clone()), + &sender, + 16843022, + ) + .unwrap_err(); + + if let op_error::InvalidInput = op_err { + } else { + panic!("Parsed post-sunset epoch 2.05"); + } + + // post-2.1: this succeeds + let op = StackStxOp::parse_from_tx( + 16843022, + &BurnchainHeaderHash([0; 32]), + StacksEpochId::Epoch21, + &BurnchainTransaction::Bitcoin(tx.clone()), + &sender, + 16843022, ) .unwrap(); From af7d04613057225771c53038356499df549eafc9 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:05:55 -0400 Subject: [PATCH 080/178] fix: gate PoX reward set calculation by epoch if we're in the sunset period --- src/chainstate/coordinator/mod.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/chainstate/coordinator/mod.rs b/src/chainstate/coordinator/mod.rs index b22b3a664..235f344e6 100644 --- a/src/chainstate/coordinator/mod.rs +++ b/src/chainstate/coordinator/mod.rs @@ -445,7 +445,20 @@ pub fn get_reward_cycle_info( sort_db: &SortitionDB, provider: &U, ) -> Result, Error> { + let epoch_at_height = SortitionDB::get_stacks_epoch(sort_db.conn(), burn_height)?.expect( + &format!("FATAL: no epoch defined for burn height {}", burn_height), + ); + if burnchain.is_reward_cycle_start(burn_height) { + if burnchain + .pox_constants + .is_after_pox_sunset_end(burn_height, epoch_at_height.epoch_id) + { + return Ok(Some(RewardCycleInfo { + anchor_status: PoxAnchorBlockStatus::NotSelected, + })); + } + debug!("Beginning reward cycle"; "burn_height" => burn_height, "reward_cycle_length" => burnchain.pox_constants.reward_cycle_length, From df8b6e86c276594875ceaf6aead3b07e2e7425c9 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:06:21 -0400 Subject: [PATCH 081/178] feat: restore sunset burn testing and add coverage to verify that the sunset burn gets interrupted by the epoch 2.1 transition --- src/chainstate/coordinator/tests.rs | 848 +++++++++++++++++++++++++++- 1 file changed, 820 insertions(+), 28 deletions(-) diff --git a/src/chainstate/coordinator/tests.rs b/src/chainstate/coordinator/tests.rs index 0cb4b6303..be4932330 100644 --- a/src/chainstate/coordinator/tests.rs +++ b/src/chainstate/coordinator/tests.rs @@ -191,13 +191,36 @@ pub fn setup_states( pox_consts: Option, initial_balances: Option>, stacks_epoch_id: StacksEpochId, +) { + setup_states_with_epochs( + paths, + vrf_keys, + committers, + pox_consts, + initial_balances, + stacks_epoch_id, + None, + ); +} + +pub fn setup_states_with_epochs( + paths: &[&str], + vrf_keys: &[VRFPrivateKey], + committers: &[StacksPrivateKey], + pox_consts: Option, + initial_balances: Option>, + stacks_epoch_id: StacksEpochId, + epochs_opt: Option>, ) { let mut burn_block = None; let mut others = vec![]; for path in paths.iter() { let burnchain = get_burnchain(path, pox_consts.clone()); - let epochs = StacksEpoch::unit_test(stacks_epoch_id, burnchain.first_block_height); + let epochs = epochs_opt.clone().unwrap_or(StacksEpoch::unit_test( + stacks_epoch_id, + burnchain.first_block_height, + )); let sortition_db = SortitionDB::connect( &burnchain.get_db_path(), burnchain.first_block_height, @@ -394,8 +417,18 @@ fn make_reward_set_coordinator<'a>( pub fn get_burnchain(path: &str, pox_consts: Option) -> Burnchain { let mut b = Burnchain::regtest(&format!("{}/burnchain/db/", path)); - b.pox_constants = - pox_consts.unwrap_or_else(|| PoxConstants::new(5, 3, 3, 25, 5, u32::max_value())); + b.pox_constants = pox_consts.unwrap_or_else(|| { + PoxConstants::new( + 5, + 3, + 3, + 25, + 5, + u64::max_value(), + u64::max_value(), + u32::max_value(), + ) + }); b } @@ -522,6 +555,7 @@ fn make_genesis_block_with_recipients( }; let commit_op = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: block.block_hash(), burn_fee: my_burn, input: (Txid([0; 32]), 0), @@ -589,6 +623,39 @@ fn make_stacks_block_with_recipients( vrf_key: &VRFPrivateKey, key_index: u32, recipients: Option<&RewardSetInfo>, +) -> (BlockstackOperationType, StacksBlock) { + make_stacks_block_with_recipients_and_sunset_burn( + sort_db, + state, + burnchain, + parent_block, + parent_height, + miner, + my_burn, + vrf_key, + key_index, + recipients, + 0, + false, + ) +} + +/// build a stacks block with just the coinbase off of +/// parent_block, in the canonical sortition fork of SortitionDB. +/// parent_block _must_ be included in the StacksChainState +fn make_stacks_block_with_recipients_and_sunset_burn( + sort_db: &SortitionDB, + state: &mut StacksChainState, + burnchain: &Burnchain, + parent_block: &BlockHeaderHash, + parent_height: u64, + miner: &StacksPrivateKey, + my_burn: u64, + vrf_key: &VRFPrivateKey, + key_index: u32, + recipients: Option<&RewardSetInfo>, + sunset_burn: u64, + post_sunset_burn: bool, ) -> (BlockstackOperationType, StacksBlock) { make_stacks_block_with_input( sort_db, @@ -601,6 +668,8 @@ fn make_stacks_block_with_recipients( vrf_key, key_index, recipients, + sunset_burn, + post_sunset_burn, (Txid([0; 32]), 0), ) } @@ -619,6 +688,8 @@ fn make_stacks_block_with_input( vrf_key: &VRFPrivateKey, key_index: u32, recipients: Option<&RewardSetInfo>, + sunset_burn: u64, + post_sunset_burn: bool, input: (Txid, u32), ) -> (BlockstackOperationType, StacksBlock) { let tx_auth = TransactionAuth::from_p2pkh(miner).unwrap(); @@ -695,7 +766,7 @@ fn make_stacks_block_with_input( commit_outs.push(PoxAddress::standard_burn_address(false)); } commit_outs - } else if burnchain.is_in_prepare_phase(parent_height + 1) { + } else if post_sunset_burn || burnchain.is_in_prepare_phase(parent_height + 1) { test_debug!("block-commit in {} will burn", parent_height + 1); vec![PoxAddress::standard_burn_address(false)] } else { @@ -703,6 +774,7 @@ fn make_stacks_block_with_input( }; let commit_op = LeaderBlockCommitOp { + sunset_burn, block_header_hash: block.block_hash(), burn_fee: my_burn, input, @@ -735,7 +807,17 @@ fn missed_block_commits() { let path = "/tmp/stacks-blockchain-missed_block_commits"; let _r = std::fs::remove_dir_all(path); - let pox_consts = Some(PoxConstants::new(5, 3, 3, 25, 5, u32::max_value())); + let sunset_ht = 8000; + let pox_consts = Some(PoxConstants::new( + 5, + 3, + 3, + 25, + 5, + 7010, + sunset_ht, + u32::max_value(), + )); let burnchain_conf = get_burnchain(path, pox_consts.clone()); let vrf_keys: Vec<_> = (0..50).map(|_| VRFPrivateKey::new()).collect(); @@ -753,7 +835,7 @@ fn missed_block_commits() { &committers, pox_consts.clone(), Some(initial_balances), - StacksEpochId::Epoch20, + StacksEpochId::Epoch2_05, ); let mut coord = make_coordinator(path, Some(burnchain_conf)); @@ -821,6 +903,8 @@ fn missed_block_commits() { vrf_key, ix as u32, next_block_recipients.as_ref(), + 0, + false, last_input.as_ref().unwrap().clone(), ); // NOTE: intended for block block_height - 2 @@ -871,6 +955,8 @@ fn missed_block_commits() { vrf_key, ix as u32, next_block_recipients.as_ref(), + 0, + false, last_input.as_ref().unwrap().clone(), ) }; @@ -1035,7 +1121,7 @@ fn test_simple_setup() { &committers, None, None, - StacksEpochId::Epoch20, + StacksEpochId::Epoch2_05, ); let mut coord = make_coordinator(path, None); @@ -1249,7 +1335,7 @@ fn test_sortition_with_reward_set() { &committers, None, None, - StacksEpochId::Epoch20, + StacksEpochId::Epoch2_05, ); let mut coord = make_reward_set_coordinator(path, reward_set, None); @@ -1518,7 +1604,7 @@ fn test_sortition_with_burner_reward_set() { &committers, None, None, - StacksEpochId::Epoch20, + StacksEpochId::Epoch2_05, ); let mut coord = make_reward_set_coordinator(path, reward_set, None); @@ -1745,8 +1831,18 @@ fn test_pox_btc_ops() { let path = "/tmp/stacks-blockchain-pox-btc-ops"; let _r = std::fs::remove_dir_all(path); + let sunset_ht = 8000; let pox_v1_unlock_ht = u32::max_value(); - let pox_consts = Some(PoxConstants::new(5, 3, 3, 25, 5, pox_v1_unlock_ht)); + let pox_consts = Some(PoxConstants::new( + 5, + 3, + 3, + 25, + 5, + 7010, + sunset_ht, + pox_v1_unlock_ht, + )); let burnchain_conf = get_burnchain(path, pox_consts.clone()); let vrf_keys: Vec<_> = (0..50).map(|_| VRFPrivateKey::new()).collect(); @@ -1764,7 +1860,7 @@ fn test_pox_btc_ops() { &committers, pox_consts.clone(), Some(initial_balances), - StacksEpochId::Epoch20, + StacksEpochId::Epoch2_05, ); let mut coord = make_coordinator(path, Some(burnchain_conf.clone())); @@ -1830,6 +1926,9 @@ fn test_pox_btc_ops() { let next_block_recipients = get_rw_sortdb(path, pox_consts.clone()) .test_get_next_block_recipients(&burnchain_conf, reward_cycle_info.as_ref()) .unwrap(); + if next_mock_header.block_height >= sunset_ht { + assert!(next_block_recipients.is_none()); + } if let Some(ref next_block_recipients) = next_block_recipients { for (addr, _) in next_block_recipients.recipients.iter() { @@ -1940,11 +2039,20 @@ fn test_pox_btc_ops() { let new_burnchain_tip = burnchain.get_canonical_chain_tip().unwrap(); if b.is_reward_cycle_start(new_burnchain_tip.block_height) { - started_first_reward_cycle = true; - // store the anchor block for this sortition for later checking - let ic = sort_db.index_handle_at_tip(); - let bhh = ic.get_last_anchor_block_hash().unwrap().unwrap(); - anchor_blocks.push(bhh); + if new_burnchain_tip.block_height < sunset_ht { + started_first_reward_cycle = true; + // store the anchor block for this sortition for later checking + let ic = sort_db.index_handle_at_tip(); + let bhh = ic.get_last_anchor_block_hash().unwrap().unwrap(); + anchor_blocks.push(bhh); + } else { + // store the anchor block for this sortition for later checking + let ic = sort_db.index_handle_at_tip(); + assert!( + ic.get_last_anchor_block_hash().unwrap().is_none(), + "No PoX anchor block should be chosen after PoX sunset" + ); + } } let tip = SortitionDB::get_canonical_burn_chain_tip(sort_db.conn()).unwrap(); @@ -2000,7 +2108,17 @@ fn test_stx_transfer_btc_ops() { let _r = std::fs::remove_dir_all(path); let pox_v1_unlock_ht = u32::max_value(); - let pox_consts = Some(PoxConstants::new(5, 3, 3, 25, 5, pox_v1_unlock_ht)); + let sunset_ht = 8000; + let pox_consts = Some(PoxConstants::new( + 5, + 3, + 3, + 25, + 5, + 7010, + sunset_ht, + pox_v1_unlock_ht, + )); let burnchain_conf = get_burnchain(path, pox_consts.clone()); let vrf_keys: Vec<_> = (0..50).map(|_| VRFPrivateKey::new()).collect(); @@ -2018,7 +2136,7 @@ fn test_stx_transfer_btc_ops() { &committers, pox_consts.clone(), Some(initial_balances), - StacksEpochId::Epoch20, + StacksEpochId::Epoch2_05, ); let mut coord = make_coordinator(path, Some(burnchain_conf.clone())); @@ -2080,6 +2198,9 @@ fn test_stx_transfer_btc_ops() { let next_block_recipients = get_rw_sortdb(path, pox_consts.clone()) .test_get_next_block_recipients(&burnchain_conf, reward_cycle_info.as_ref()) .unwrap(); + if next_mock_header.block_height >= sunset_ht { + assert!(next_block_recipients.is_none()); + } if let Some(ref next_block_recipients) = next_block_recipients { for (addr, _) in next_block_recipients.recipients.iter() { @@ -2233,11 +2354,20 @@ fn test_stx_transfer_btc_ops() { let new_burnchain_tip = burnchain.get_canonical_chain_tip().unwrap(); if b.is_reward_cycle_start(new_burnchain_tip.block_height) { - started_first_reward_cycle = true; - // store the anchor block for this sortition for later checking - let ic = sort_db.index_handle_at_tip(); - let bhh = ic.get_last_anchor_block_hash().unwrap().unwrap(); - anchor_blocks.push(bhh); + if new_burnchain_tip.block_height < sunset_ht { + started_first_reward_cycle = true; + // store the anchor block for this sortition for later checking + let ic = sort_db.index_handle_at_tip(); + let bhh = ic.get_last_anchor_block_hash().unwrap().unwrap(); + anchor_blocks.push(bhh); + } else { + // store the anchor block for this sortition for later checking + let ic = sort_db.index_handle_at_tip(); + assert!( + ic.get_last_anchor_block_hash().unwrap().is_none(), + "No PoX anchor block should be chosen after PoX sunset" + ); + } } let tip = SortitionDB::get_canonical_burn_chain_tip(sort_db.conn()).unwrap(); @@ -2292,7 +2422,17 @@ fn test_initial_coinbase_reward_distributions() { let path = "/tmp/initial_coinbase_reward_distributions"; let _r = std::fs::remove_dir_all(path); - let pox_consts = Some(PoxConstants::new(5, 3, 3, 25, 5, u32::max_value())); + let sunset_ht = 8000; + let pox_consts = Some(PoxConstants::new( + 5, + 3, + 3, + 25, + 5, + 7010, + sunset_ht, + u32::max_value(), + )); let burnchain_conf = get_burnchain(path, pox_consts.clone()); let vrf_keys: Vec<_> = (0..50).map(|_| VRFPrivateKey::new()).collect(); @@ -2516,7 +2656,17 @@ fn test_epoch_switch_cost_contract_instantiation() { let path = "/tmp/stacks-blockchain-epoch-switch-cost-contract-instantiation"; let _r = std::fs::remove_dir_all(path); - let pox_consts = Some(PoxConstants::new(6, 3, 3, 25, 5, u32::max_value())); + let sunset_ht = 8000; + let pox_consts = Some(PoxConstants::new( + 6, + 3, + 3, + 25, + 5, + 10, + sunset_ht, + u32::max_value(), + )); let burnchain_conf = get_burnchain(path, pox_consts.clone()); let vrf_keys: Vec<_> = (0..10).map(|_| VRFPrivateKey::new()).collect(); @@ -2696,6 +2846,648 @@ fn test_epoch_switch_cost_contract_instantiation() { } } +#[test] +fn test_sortition_with_sunset() { + let path = "/tmp/stacks-blockchain-sortition-with-sunset"; + let _r = std::fs::remove_dir_all(path); + + let sunset_ht = 80; + let pox_consts = Some(PoxConstants::new( + 6, + 3, + 3, + 25, + 5, + 10, + sunset_ht, + u32::max_value(), + )); + let burnchain_conf = get_burnchain(path, pox_consts.clone()); + + let mut vrf_keys: Vec<_> = (0..200).map(|_| VRFPrivateKey::new()).collect(); + let mut committers: Vec<_> = (0..200).map(|_| StacksPrivateKey::new()).collect(); + + let reward_set_size = pox_consts.as_ref().unwrap().reward_slots() as usize; + assert_eq!(reward_set_size, 6); + let reward_set: Vec<_> = (0..reward_set_size) + .map(|_| pox_addr_from(&StacksPrivateKey::new())) + .collect(); + + setup_states( + &[path], + &vrf_keys, + &committers, + pox_consts.clone(), + None, + StacksEpochId::Epoch2_05, + ); + + let mut coord = make_reward_set_coordinator(path, reward_set, pox_consts.clone()); + + coord.handle_new_burnchain_block().unwrap(); + + let sort_db = get_sortition_db(path, pox_consts.clone()); + + let tip = SortitionDB::get_canonical_burn_chain_tip(sort_db.conn()).unwrap(); + assert_eq!(tip.block_height, 1); + assert_eq!(tip.sortition, false); + let (_, ops) = sort_db + .get_sortition_result(&tip.sortition_id) + .unwrap() + .unwrap(); + + // we should have all the VRF registrations accepted + assert_eq!(ops.accepted_ops.len(), vrf_keys.len()); + assert_eq!(ops.consumed_leader_keys.len(), 0); + + let mut started_first_reward_cycle = false; + // process sequential blocks, and their sortitions... + let mut stacks_blocks: Vec<(SortitionId, StacksBlock)> = vec![]; + let mut anchor_blocks = vec![]; + + // split up the vrf keys and committers so that we have some that will be mining "correctly" + // and some that will be producing bad outputs + + let WRONG_OUTS_OFFSET = 100; + let vrf_key_wrong_outs = vrf_keys.split_off(WRONG_OUTS_OFFSET); + let miner_wrong_outs = committers.split_off(WRONG_OUTS_OFFSET); + + // track the reward set consumption + let mut reward_recipients = HashSet::new(); + for ix in 0..vrf_keys.len() { + let vrf_key = &vrf_keys[ix]; + let miner = &committers[ix]; + + let vrf_wrong_out = &vrf_key_wrong_outs[ix]; + let miner_wrong_out = &miner_wrong_outs[ix]; + + let mut burnchain = get_burnchain_db(path, pox_consts.clone()); + let mut chainstate = get_chainstate(path); + + let parent = if ix == 0 { + BlockHeaderHash([0; 32]) + } else { + stacks_blocks[ix - 1].1.header.block_hash() + }; + + let burnchain_tip = burnchain.get_canonical_chain_tip().unwrap(); + let next_mock_header = BurnchainBlockHeader { + block_height: burnchain_tip.block_height + 1, + block_hash: BurnchainHeaderHash([0; 32]), + parent_block_hash: burnchain_tip.block_hash, + num_txs: 0, + timestamp: 1, + }; + + let reward_cycle_info = coord.get_reward_cycle_info(&next_mock_header).unwrap(); + let cur_epoch = + SortitionDB::get_stacks_epoch(sort_db.conn(), next_mock_header.block_height) + .unwrap() + .unwrap(); + + if reward_cycle_info.is_some() { + // did we process a reward set last cycle? check if the + // recipient set size matches our expectation + if started_first_reward_cycle { + let last_reward_cycle_block = (sunset_ht + / (pox_consts.as_ref().unwrap().reward_cycle_length as u64)) + * (pox_consts.as_ref().unwrap().reward_cycle_length as u64); + if burnchain_tip.block_height == last_reward_cycle_block { + eprintln!( + "End of PoX (at sunset height {}): reward set size is {}", + burnchain_tip.block_height, + reward_recipients.len() + ); + assert_eq!(reward_recipients.len(), 6); // still hasn't cleared yet, so still 6 + } else if burnchain_tip.block_height + > last_reward_cycle_block + + (pox_consts.as_ref().unwrap().reward_cycle_length as u64) + { + eprintln!("End of PoX (beyond sunset height {} and in next reward cycle): reward set size is {}", burnchain_tip.block_height, reward_recipients.len()); + assert_eq!(reward_recipients.len(), 0); + } else if burnchain_tip.block_height > last_reward_cycle_block { + eprintln!( + "End of PoX (beyond sunset height {}): reward set size is {}", + burnchain_tip.block_height, + reward_recipients.len() + ); + assert_eq!(reward_recipients.len(), 2); // still haven't cleared this yet, so still 2 + } else { + eprintln!( + "End of PoX (before sunset height {}): reward set size is {}", + burnchain_tip.block_height, + reward_recipients.len() + ); + assert_eq!(reward_recipients.len(), reward_set_size); + } + } + // clear the reward recipients tracker, since those + // recipients are now eligible again in the new reward cycle + reward_recipients.clear(); + } + let next_block_recipients = get_rw_sortdb(path, pox_consts.clone()) + .test_get_next_block_recipients(&burnchain_conf, reward_cycle_info.as_ref()) + .unwrap(); + if next_mock_header.block_height >= sunset_ht { + assert!(next_block_recipients.is_none()); + } + + if let Some(ref next_block_recipients) = next_block_recipients { + // this is only Some(..) if we're pre-sunset + assert!(burnchain_tip.block_height <= sunset_ht); + for (addr, _) in next_block_recipients.recipients.iter() { + if !addr.is_burn() { + assert!( + !reward_recipients.contains(addr), + "Reward set should not already contain address {}", + addr + ); + } + reward_recipients.insert(addr.clone()); + } + eprintln!( + "at {}: reward_recipients ({}) = {:?}", + burnchain_tip.block_height, + reward_recipients.len(), + reward_recipients + ); + } + + let sunset_burn = burnchain_conf.expected_sunset_burn( + next_mock_header.block_height, + 10000, + cur_epoch.epoch_id, + ); + let rest_commit = 10000 - sunset_burn; + let b = get_burnchain(path, pox_consts.clone()); + + let (good_op, block) = if ix == 0 { + make_genesis_block_with_recipients( + &sort_db, + &mut chainstate, + &parent, + miner, + 10000, + vrf_key, + ix as u32, + next_block_recipients.as_ref(), + ) + } else { + make_stacks_block_with_recipients_and_sunset_burn( + &sort_db, + &mut chainstate, + &b, + &parent, + burnchain_tip.block_height, + miner, + rest_commit, + vrf_key, + ix as u32, + next_block_recipients.as_ref(), + sunset_burn + (rand::random::() as u64), + next_mock_header.block_height >= sunset_ht, + ) + }; + + eprintln!("good op: {:?}", &good_op); + let expected_winner = good_op.txid(); + let mut ops = vec![good_op]; + + if sunset_burn > 0 { + let (bad_outs_op, _) = make_stacks_block_with_recipients_and_sunset_burn( + &sort_db, + &mut chainstate, + &b, + &parent, + burnchain_tip.block_height, + miner, + 10000, + vrf_wrong_out, + (ix + WRONG_OUTS_OFFSET) as u32, + next_block_recipients.as_ref(), + sunset_burn - 1, + false, + ); + ops.push(bad_outs_op); + } + + let burnchain_tip = burnchain.get_canonical_chain_tip().unwrap(); + produce_burn_block( + &mut burnchain, + &burnchain_tip.block_hash, + ops, + vec![].iter_mut(), + ); + // handle the sortition + coord.handle_new_burnchain_block().unwrap(); + + let new_burnchain_tip = burnchain.get_canonical_chain_tip().unwrap(); + if b.is_reward_cycle_start(new_burnchain_tip.block_height) { + if new_burnchain_tip.block_height < sunset_ht { + started_first_reward_cycle = true; + // store the anchor block for this sortition for later checking + let ic = sort_db.index_handle_at_tip(); + let bhh = ic.get_last_anchor_block_hash().unwrap().unwrap(); + anchor_blocks.push(bhh); + } else { + // store the anchor block for this sortition for later checking + let ic = sort_db.index_handle_at_tip(); + assert!( + ic.get_last_anchor_block_hash().unwrap().is_none(), + "No PoX anchor block should be chosen after PoX sunset" + ); + } + } + + let tip = SortitionDB::get_canonical_burn_chain_tip(sort_db.conn()).unwrap(); + assert_eq!(&tip.winning_block_txid, &expected_winner); + + // load the block into staging + let block_hash = block.header.block_hash(); + + assert_eq!(&tip.winning_stacks_block_hash, &block_hash); + stacks_blocks.push((tip.sortition_id.clone(), block.clone())); + + preprocess_block(&mut chainstate, &sort_db, &tip, block); + + // handle the stacks block + coord.handle_new_stacks_block().unwrap(); + } + + let stacks_tip = SortitionDB::get_canonical_stacks_chain_tip_hash(sort_db.conn()).unwrap(); + let mut chainstate = get_chainstate(path); + assert_eq!( + chainstate + .with_read_only_clarity_tx( + &sort_db.index_conn(), + &StacksBlockId::new(&stacks_tip.0, &stacks_tip.1), + |conn| conn + .with_readonly_clarity_env( + false, + CHAIN_ID_TESTNET, + ClarityVersion::Clarity1, + PrincipalData::parse("SP3Q4A5WWZ80REGBN0ZXNE540ECJ9JZ4A765Q5K2Q").unwrap(), + None, + LimitedCostTracker::new_free(), + |env| env.eval_raw("block-height") + ) + .unwrap() + ) + .unwrap(), + Value::UInt(100) + ); + { + let ic = sort_db.index_handle_at_tip(); + let pox_id = ic.get_pox_id().unwrap(); + assert_eq!(&pox_id.to_string(), + "111111111111111111", + "PoX ID should reflect the 10 reward cycles _with_ a known anchor block, plus the 'initial' known reward cycle at genesis"); + } +} + +/// Verify that the PoX sunset is stopped at the 2.1 epoch switch. +/// Runs a mocked blockchain for 100 sortitions. +/// PoX sunset begins at block 10, and completes at block 80. +/// Epoch 2.1 activates at block 50 (n.b. reward cycles are 6 blocks long) +#[test] +fn test_sortition_with_sunset_and_epoch_switch() { + let path = "/tmp/stacks-blockchain-sortition-with-sunset-and-epoch-switch"; + let _r = std::fs::remove_dir_all(path); + + let rc_len = 6; + let sunset_ht = 80; + let epoch_switch_ht = 50; + let v1_unlock_ht = 56; + let pox_consts = Some(PoxConstants::new( + rc_len, + 3, + 3, + 25, + 5, + 10, + sunset_ht, + v1_unlock_ht, + )); + + let burnchain_conf = get_burnchain(path, pox_consts.clone()); + + let mut vrf_keys: Vec<_> = (0..200).map(|_| VRFPrivateKey::new()).collect(); + let mut committers: Vec<_> = (0..200).map(|_| StacksPrivateKey::new()).collect(); + + let reward_set_size = pox_consts.as_ref().unwrap().reward_slots() as usize; + assert_eq!(reward_set_size, 6); + let reward_set: Vec<_> = (0..reward_set_size) + .map(|_| pox_addr_from(&StacksPrivateKey::new())) + .collect(); + + setup_states_with_epochs( + &[path], + &vrf_keys, + &committers, + pox_consts.clone(), + None, + StacksEpochId::Epoch20, + Some(StacksEpoch::all(0, 5, epoch_switch_ht)), + ); + + let mut coord = make_reward_set_coordinator(path, reward_set, pox_consts.clone()); + + coord.handle_new_burnchain_block().unwrap(); + + let sort_db = get_sortition_db(path, pox_consts.clone()); + + let tip = SortitionDB::get_canonical_burn_chain_tip(sort_db.conn()).unwrap(); + assert_eq!(tip.block_height, 1); + assert_eq!(tip.sortition, false); + let (_, ops) = sort_db + .get_sortition_result(&tip.sortition_id) + .unwrap() + .unwrap(); + + // we should have all the VRF registrations accepted + assert_eq!(ops.accepted_ops.len(), vrf_keys.len()); + assert_eq!(ops.consumed_leader_keys.len(), 0); + + let mut started_first_reward_cycle = false; + // process sequential blocks, and their sortitions... + let mut stacks_blocks: Vec<(SortitionId, StacksBlock)> = vec![]; + let mut anchor_blocks = vec![]; + + // split up the vrf keys and committers so that we have some that will be mining "correctly" + // and some that will be producing bad outputs + + let WRONG_OUTS_OFFSET = 100; + let vrf_key_wrong_outs = vrf_keys.split_off(WRONG_OUTS_OFFSET); + let miner_wrong_outs = committers.split_off(WRONG_OUTS_OFFSET); + + // track the reward set consumption. + // epoch switch to 2.1 disables the in-progress sunset + let mut reward_recipients = HashSet::new(); + for ix in 0..vrf_keys.len() { + let vrf_key = &vrf_keys[ix]; + let miner = &committers[ix]; + + let vrf_wrong_out = &vrf_key_wrong_outs[ix]; + let miner_wrong_out = &miner_wrong_outs[ix]; + + let mut burnchain = get_burnchain_db(path, pox_consts.clone()); + let mut chainstate = get_chainstate(path); + + let parent = if ix == 0 { + BlockHeaderHash([0; 32]) + } else { + stacks_blocks[ix - 1].1.header.block_hash() + }; + + let burnchain_tip = burnchain.get_canonical_chain_tip().unwrap(); + let next_mock_header = BurnchainBlockHeader { + block_height: burnchain_tip.block_height + 1, + block_hash: BurnchainHeaderHash([0; 32]), + parent_block_hash: burnchain_tip.block_hash, + num_txs: 0, + timestamp: 1, + }; + + let reward_cycle_info = coord.get_reward_cycle_info(&next_mock_header).unwrap(); + let cur_epoch = + SortitionDB::get_stacks_epoch(sort_db.conn(), next_mock_header.block_height) + .unwrap() + .unwrap(); + + if reward_cycle_info.is_some() { + // did we process a reward set last cycle? check if the + // recipient set size matches our expectation + if started_first_reward_cycle { + let last_reward_cycle_block = (sunset_ht + / (pox_consts.as_ref().unwrap().reward_cycle_length as u64)) + * (pox_consts.as_ref().unwrap().reward_cycle_length as u64); + + if cur_epoch.epoch_id < StacksEpochId::Epoch21 { + if burnchain_tip.block_height == last_reward_cycle_block { + eprintln!( + "End of PoX (at sunset height {}): reward set size is {}", + burnchain_tip.block_height, + reward_recipients.len() + ); + assert_eq!(reward_recipients.len(), 6); // still hasn't cleared yet, so still 6 + } else if burnchain_tip.block_height + > last_reward_cycle_block + + (pox_consts.as_ref().unwrap().reward_cycle_length as u64) + { + eprintln!("End of PoX (beyond sunset height {} and in next reward cycle): reward set size is {}", burnchain_tip.block_height, reward_recipients.len()); + assert_eq!(reward_recipients.len(), 0); + } else if burnchain_tip.block_height > last_reward_cycle_block { + eprintln!( + "End of PoX (beyond sunset height {}): reward set size is {}", + burnchain_tip.block_height, + reward_recipients.len() + ); + assert_eq!(reward_recipients.len(), 2); // still haven't cleared this yet, so still 2 + } else { + eprintln!( + "End of PoX (before sunset height {}): reward set size is {}", + burnchain_tip.block_height, + reward_recipients.len() + ); + assert_eq!(reward_recipients.len(), reward_set_size); + } + } else { + assert!(burnchain_tip.block_height > epoch_switch_ht); + eprintln!( + "In epoch 2.1 at height {}: reward set size is {}", + burnchain_tip.block_height, + reward_recipients.len() + ); + assert_eq!(reward_recipients.len(), reward_set_size); + } + } + // clear the reward recipients tracker, since those + // recipients are now eligible again in the new reward cycle + reward_recipients.clear(); + } else if started_first_reward_cycle + && burnchain_conf.is_reward_cycle_start(next_mock_header.block_height) + { + // unreachable -- Epoch 2.1 activates at block 50, so we never reach the PoX sunset. + // So, we should always have a reward set once we pass the first reward cycle. + panic!("FATAL: Epoch 2.1 switch did not prevent PoX from disabling"); + } + + let next_block_recipients = get_rw_sortdb(path, pox_consts.clone()) + .test_get_next_block_recipients(&burnchain_conf, reward_cycle_info.as_ref()) + .unwrap(); + if cur_epoch.epoch_id < StacksEpochId::Epoch21 && next_mock_header.block_height >= sunset_ht + { + assert!(next_block_recipients.is_none()); + } + + if let Some(ref next_block_recipients) = next_block_recipients { + // this is only Some(..) if we're pre-sunset or in epoch 2.1 + assert!( + burnchain_tip.block_height <= sunset_ht + || cur_epoch.epoch_id >= StacksEpochId::Epoch21 + ); + for (addr, _) in next_block_recipients.recipients.iter() { + if !addr.is_burn() { + assert!( + !reward_recipients.contains(addr), + "Reward set should not already contain address {}", + addr + ); + } + reward_recipients.insert(addr.clone()); + } + eprintln!( + "at {}: reward_recipients ({}) = {:?}", + burnchain_tip.block_height, + reward_recipients.len(), + reward_recipients + ); + } + + let sunset_burn = burnchain_conf.expected_sunset_burn( + next_mock_header.block_height, + 10000, + cur_epoch.epoch_id, + ); + if cur_epoch.epoch_id >= StacksEpochId::Epoch21 { + assert_eq!(sunset_burn, 0); + } + let rest_commit = 10000 - sunset_burn; + let b = get_burnchain(path, pox_consts.clone()); + + let (good_op, block) = if ix == 0 { + make_genesis_block_with_recipients( + &sort_db, + &mut chainstate, + &parent, + miner, + 10000, + vrf_key, + ix as u32, + next_block_recipients.as_ref(), + ) + } else { + make_stacks_block_with_recipients_and_sunset_burn( + &sort_db, + &mut chainstate, + &b, + &parent, + burnchain_tip.block_height, + miner, + rest_commit, + vrf_key, + ix as u32, + next_block_recipients.as_ref(), + sunset_burn + (rand::random::() as u64), + next_mock_header.block_height >= sunset_ht + && cur_epoch.epoch_id < StacksEpochId::Epoch21, + ) + }; + + eprintln!("good op in {}: {:?}", cur_epoch.epoch_id, &good_op); + let expected_winner = good_op.txid(); + let mut ops = vec![good_op]; + + if sunset_burn > 0 && cur_epoch.epoch_id < StacksEpochId::Epoch21 { + // this is a "good op" post-2.1, since the sunset is disabled + let (bad_outs_op, _) = make_stacks_block_with_recipients_and_sunset_burn( + &sort_db, + &mut chainstate, + &b, + &parent, + burnchain_tip.block_height, + miner, + 10000, + vrf_wrong_out, + (ix + WRONG_OUTS_OFFSET) as u32, + next_block_recipients.as_ref(), + sunset_burn - 1, + false, + ); + ops.push(bad_outs_op); + } + + let burnchain_tip = burnchain.get_canonical_chain_tip().unwrap(); + produce_burn_block( + &mut burnchain, + &burnchain_tip.block_hash, + ops, + vec![].iter_mut(), + ); + // handle the sortition + coord.handle_new_burnchain_block().unwrap(); + + let new_burnchain_tip = burnchain.get_canonical_chain_tip().unwrap(); + if b.is_reward_cycle_start(new_burnchain_tip.block_height) { + if new_burnchain_tip.block_height < sunset_ht { + started_first_reward_cycle = true; + // store the anchor block for this sortition for later checking + let ic = sort_db.index_handle_at_tip(); + let bhh = ic.get_last_anchor_block_hash().unwrap().unwrap(); + anchor_blocks.push(bhh); + } else { + // store the anchor block for this sortition for later checking + let ic = sort_db.index_handle_at_tip(); + if cur_epoch.epoch_id < StacksEpochId::Epoch21 { + assert!( + ic.get_last_anchor_block_hash().unwrap().is_none(), + "No PoX anchor block should be chosen after PoX sunset" + ); + } else { + assert!( + ic.get_last_anchor_block_hash().unwrap().is_some(), + "PoX anchor block should be chosen after Epoch 2.1" + ); + } + } + } + + let tip = SortitionDB::get_canonical_burn_chain_tip(sort_db.conn()).unwrap(); + assert_eq!(&tip.winning_block_txid, &expected_winner); + + // load the block into staging + let block_hash = block.header.block_hash(); + + assert_eq!(&tip.winning_stacks_block_hash, &block_hash); + stacks_blocks.push((tip.sortition_id.clone(), block.clone())); + + preprocess_block(&mut chainstate, &sort_db, &tip, block); + + // handle the stacks block + coord.handle_new_stacks_block().unwrap(); + } + + let stacks_tip = SortitionDB::get_canonical_stacks_chain_tip_hash(sort_db.conn()).unwrap(); + let mut chainstate = get_chainstate(path); + assert_eq!( + chainstate + .with_read_only_clarity_tx( + &sort_db.index_conn(), + &StacksBlockId::new(&stacks_tip.0, &stacks_tip.1), + |conn| conn + .with_readonly_clarity_env( + false, + CHAIN_ID_TESTNET, + ClarityVersion::Clarity1, + PrincipalData::parse("SP3Q4A5WWZ80REGBN0ZXNE540ECJ9JZ4A765Q5K2Q").unwrap(), + None, + LimitedCostTracker::new_free(), + |env| env.eval_raw("block-height") + ) + .unwrap() + ) + .unwrap(), + Value::UInt(100) + ); + { + let ic = sort_db.index_handle_at_tip(); + let pox_id = ic.get_pox_id().unwrap(); + assert_eq!(&pox_id.to_string(), + "111111111111111111", + "PoX ID should reflect the 10 reward cycles _with_ a known anchor block, plus the 'initial' known reward cycle at genesis"); + } +} + #[test] // This test should panic until the MARF stability issue // https://github.com/blockstack/stacks-blockchain/issues/1805 @@ -2722,7 +3514,7 @@ fn test_pox_processable_block_in_different_pox_forks() { &committers, None, None, - StacksEpochId::Epoch20, + StacksEpochId::Epoch2_05, ); let mut coord = make_coordinator(path, None); @@ -2973,7 +3765,7 @@ fn test_pox_no_anchor_selected() { &committers, None, None, - StacksEpochId::Epoch20, + StacksEpochId::Epoch2_05, ); let mut coord = make_coordinator(path, None); @@ -3186,7 +3978,7 @@ fn test_pox_fork_out_of_order() { &committers, None, None, - StacksEpochId::Epoch20, + StacksEpochId::Epoch2_05, ); let mut coord = make_coordinator(path, None); From 41450fae72e9fc6e4aac6a181bca2f95085bcc7a Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:06:42 -0400 Subject: [PATCH 082/178] fix: restore sunset burn --- src/chainstate/stacks/block.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/chainstate/stacks/block.rs b/src/chainstate/stacks/block.rs index 4ec78b945..608ba6192 100644 --- a/src/chainstate/stacks/block.rs +++ b/src/chainstate/stacks/block.rs @@ -1295,6 +1295,7 @@ mod test { }; let mut block_commit = LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: header.block_hash(), new_seed: VRFSeed::from_proof(&header.proof), parent_block_ptr: 0, From 11c92ae9661f6f607b0f6b291ca81ca9a8aec5bf Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:06:58 -0400 Subject: [PATCH 083/178] fix: log PoX participation --- src/chainstate/stacks/boot/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/chainstate/stacks/boot/mod.rs b/src/chainstate/stacks/boot/mod.rs index 2012d0afd..9de48035c 100644 --- a/src/chainstate/stacks/boot/mod.rs +++ b/src/chainstate/stacks/boot/mod.rs @@ -535,8 +535,8 @@ impl StacksChainState { }; let threshold = threshold_precise + ceil_amount; info!( - "PoX participation threshold is {}, from {} + {} ({})", - threshold, threshold_precise, ceil_amount, scale_by, + "PoX participation threshold is {}, from {} + {} ({}), participation is {}", + threshold, threshold_precise, ceil_amount, scale_by, participation ); (threshold, participation) } @@ -853,7 +853,7 @@ pub mod test { #[test] fn get_reward_threshold_units() { - let test_pox_constants = PoxConstants::new(501, 1, 1, 1, 5, u32::max_value()); + let test_pox_constants = PoxConstants::new(501, 1, 1, 1, 5, 5000, 10000, u32::max_value()); // when the liquid amount = the threshold step, // the threshold should always be the step size. let liquid = POX_THRESHOLD_STEPS_USTX; From 5f13e5c514721a8fda7436800abcb7e154c61ef9 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:07:15 -0400 Subject: [PATCH 084/178] feat: log epoch initialization --- src/clarity_vm/clarity.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/clarity_vm/clarity.rs b/src/clarity_vm/clarity.rs index 3e50d536a..c9089e5b5 100644 --- a/src/clarity_vm/clarity.rs +++ b/src/clarity_vm/clarity.rs @@ -760,6 +760,7 @@ impl<'a, 'b> ClarityBlockConnection<'a, 'b> { // C'est la vie. // initialize with a synthetic transaction + debug!("Instantiate .costs-2 contract"); let receipt = StacksChainState::process_transaction_payload( tx_conn, &costs_2_contract_tx, @@ -782,6 +783,7 @@ impl<'a, 'b> ClarityBlockConnection<'a, 'b> { // NOTE: we don't set self.epoch to Epoch2_05 here, even though we probably // should, because doing so risks a chain split. + debug!("Epoch 2.05 initialized"); (old_cost_tracker, Ok(initialization_receipt)) }) } @@ -877,6 +879,7 @@ impl<'a, 'b> ClarityBlockConnection<'a, 'b> { tx_conn.epoch = StacksEpochId::Epoch21; // initialize with a synthetic transaction + debug!("Instantiate {} contract", &pox_2_contract_id); let receipt = StacksChainState::process_transaction_payload( tx_conn, &pox_2_contract_tx, @@ -917,6 +920,7 @@ impl<'a, 'b> ClarityBlockConnection<'a, 'b> { ); } + debug!("Epoch 2.1 initialized"); (old_cost_tracker, Ok(initialization_receipt)) }) } From d4397773941f42c30851ea3091d0f82ba8ce8fbb Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:07:28 -0400 Subject: [PATCH 085/178] fix: permit calls to read-only PoX 1 functions if we're in PoX 2 (this might fix #3261) --- src/clarity_vm/special.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/clarity_vm/special.rs b/src/clarity_vm/special.rs index 75927c09f..3299de724 100644 --- a/src/clarity_vm/special.rs +++ b/src/clarity_vm/special.rs @@ -372,6 +372,21 @@ fn handle_pox_v2_api_contract_call( Ok(()) } +/// Is a PoX-1 function read-only? +/// i.e. can we call it without incurring an error? +fn is_pox_v1_read_only(func_name: &str) -> bool { + func_name == "get-pox-rejection" + || func_name == "is-pox-active" + || func_name == "get-stacker-info" + || func_name == "get-reward-set-size" + || func_name == "get-total-ustx-stacked" + || func_name == "get-reward-set-pox-address" + || func_name == "get-stacking-minimum" + || func_name == "can-stack-stx" + || func_name == "minimal-can-stack-stx" + || func_name == "get-pox-info" +} + /// Handle special cases of contract-calls -- namely, those into PoX that should lock up STX pub fn handle_contract_call_special_cases( global_context: &mut GlobalContext, @@ -382,12 +397,16 @@ pub fn handle_contract_call_special_cases( result: &Value, ) -> Result<()> { if *contract_id == boot_code_id(POX_1_NAME, global_context.mainnet) { - if global_context.database.get_v1_unlock_height() - <= global_context.database.get_current_burnchain_block_height() + if !is_pox_v1_read_only(function_name) + && global_context.database.get_v1_unlock_height() + <= global_context.database.get_current_burnchain_block_height() { + // NOTE: get-pox-info is read-only, so it can call old pox v1 stuff warn!("PoX-1 Lock attempted on an account after v1 unlock height"; "v1_unlock_ht" => global_context.database.get_v1_unlock_height(), "current_burn_ht" => global_context.database.get_current_burnchain_block_height(), + "function_name" => function_name, + "contract_id" => %contract_id ); return Err(Error::Runtime(RuntimeErrorType::DefunctPoxContract, None)); } From 2e945a26b0a0cbb35c9d99f9476b36ccfc33318e Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:08:06 -0400 Subject: [PATCH 086/178] fix: API sync --- src/net/inv.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/net/inv.rs b/src/net/inv.rs index 932b6e75d..7901bb1cc 100644 --- a/src/net/inv.rs +++ b/src/net/inv.rs @@ -3091,7 +3091,16 @@ mod test { #[test] fn test_inv_merge_pox_inv() { let mut burnchain = Burnchain::regtest("unused"); - burnchain.pox_constants = PoxConstants::new(5, 3, 3, 25, 5, u32::max_value()); + burnchain.pox_constants = PoxConstants::new( + 5, + 3, + 3, + 25, + 5, + u64::max_value(), + u64::max_value(), + u32::max_value(), + ); let mut peer_inv = PeerBlocksInv::new(vec![0x01], vec![0x01], vec![0x01], 1, 1, 0); for i in 0..32 { @@ -3109,7 +3118,16 @@ mod test { #[test] fn test_inv_truncate_pox_inv() { let mut burnchain = Burnchain::regtest("unused"); - burnchain.pox_constants = PoxConstants::new(5, 3, 3, 25, 5, u32::max_value()); + burnchain.pox_constants = PoxConstants::new( + 5, + 3, + 3, + 25, + 5, + u64::max_value(), + u64::max_value(), + u32::max_value(), + ); let mut peer_inv = PeerBlocksInv::new(vec![0x01], vec![0x01], vec![0x01], 1, 1, 0); for i in 0..5 { From 0f33ad2bf0a3f6923535349ee0e3c418627ca28e Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:08:14 -0400 Subject: [PATCH 087/178] fix API sync --- src/net/mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/net/mod.rs b/src/net/mod.rs index b1cbbd352..391075e52 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -2431,7 +2431,16 @@ pub mod test { .unwrap(), ); - burnchain.pox_constants = PoxConstants::new(5, 3, 3, 25, 5, u32::max_value()); + burnchain.pox_constants = PoxConstants::new( + 5, + 3, + 3, + 25, + 5, + u64::max_value(), + u64::max_value(), + u32::max_value(), + ); let mut spending_account = TestMinerFactory::new().next_miner( &burnchain, From 1444c8cbce20a7bada7d8d21966fcdff7e9d69ac Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:08:26 -0400 Subject: [PATCH 088/178] fix: the current burn height is the height of the latest sortition, not what we've seen in the Clarity DB --- src/net/rpc.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/net/rpc.rs b/src/net/rpc.rs index d9ede602e..13fa1438d 100644 --- a/src/net/rpc.rs +++ b/src/net/rpc.rs @@ -269,19 +269,18 @@ impl RPCPoxInfoData { ) -> Result { let mainnet = chainstate.mainnet; let chain_id = chainstate.chain_id; - - let current_burn_height = chainstate - .with_read_only_clarity_tx(&sortdb.index_conn(), tip, |clarity_tx| { - clarity_tx.with_clarity_db_readonly(|clarity_db| { - clarity_db.get_current_burnchain_block_height() as u64 - }) - }) - .ok_or(net_error::NotFoundError)?; + let current_burn_height = + SortitionDB::get_canonical_burn_chain_tip(sortdb.conn())?.block_height; let pox_contract_name = burnchain .pox_constants .active_pox_contract(current_burn_height); + debug!( + "Active PoX contract is '{}' (current_burn_height = {}, v1_unlock_height = {}", + &pox_contract_name, current_burn_height, burnchain.pox_constants.v1_unlock_height + ); + let contract_identifier = boot_code_id(pox_contract_name, mainnet); let function = "get-pox-info"; let cost_track = LimitedCostTracker::new_free(); @@ -306,7 +305,7 @@ impl RPCPoxInfoData { clarity_tx.with_readonly_clarity_env( mainnet, chain_id, - ClarityVersion::Clarity1, + ClarityVersion::Clarity2, sender, None, cost_track, From f9d231c9ae43f4b9d2f58f668950eef0793aeeb4 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:08:53 -0400 Subject: [PATCH 089/178] fix: restore sunset burn --- .../burnchains/bitcoin_regtest_controller.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs b/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs index a6e1f7ca5..7bb614bd1 100644 --- a/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs +++ b/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs @@ -96,6 +96,7 @@ impl OngoingBlockCommit { #[derive(Clone)] struct LeaderBlockCommitFees { + sunset_fee: u64, fee_rate: u64, sortition_fee: u64, outputs_len: u64, @@ -123,6 +124,12 @@ impl LeaderBlockCommitFees { payload: &LeaderBlockCommitOp, config: &Config, ) -> LeaderBlockCommitFees { + let sunset_fee = if payload.sunset_burn > 0 { + cmp::max(payload.sunset_burn, DUST_UTXO_LIMIT) + } else { + 0 + }; + let number_of_transfers = payload.commit_outs.len() as u64; let value_per_transfer = payload.burn_fee / number_of_transfers; let sortition_fee = value_per_transfer * number_of_transfers; @@ -131,6 +138,7 @@ impl LeaderBlockCommitFees { let default_tx_size = config.burnchain.block_commit_tx_estimated_size; LeaderBlockCommitFees { + sunset_fee, fee_rate, sortition_fee, outputs_len: number_of_transfers, @@ -154,11 +162,14 @@ impl LeaderBlockCommitFees { } pub fn estimated_amount_required(&self) -> u64 { - self.estimated_miner_fee() + self.rbf_fee() + self.sortition_fee + self.estimated_miner_fee() + self.rbf_fee() + self.sunset_fee + self.sortition_fee } pub fn total_spent(&self) -> u64 { - self.fee_rate * self.final_size + self.spent_in_attempts + self.sortition_fee + self.fee_rate * self.final_size + + self.spent_in_attempts + + self.sunset_fee + + self.sortition_fee } pub fn amount_per_output(&self) -> u64 { @@ -166,7 +177,7 @@ impl LeaderBlockCommitFees { } pub fn total_spent_in_outputs(&self) -> u64 { - self.sortition_fee + self.sunset_fee + self.sortition_fee } pub fn min_tx_size(&self) -> u64 { @@ -955,7 +966,7 @@ impl BitcoinRegtestController { }; let consensus_output = TxOut { - value: 0, + value: estimated_fees.sunset_fee, script_pubkey: Builder::new() .push_opcode(opcodes::All::OP_RETURN) .push_slice(&op_bytes) From 35d44661f6a48820a498e9bd4f768bb5894edac0 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:09:05 -0400 Subject: [PATCH 090/178] fix: restore sunset burn --- testnet/stacks-node/src/burnchains/mocknet_controller.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/testnet/stacks-node/src/burnchains/mocknet_controller.rs b/testnet/stacks-node/src/burnchains/mocknet_controller.rs index 7dc0e5c2e..ae2f28f73 100644 --- a/testnet/stacks-node/src/burnchains/mocknet_controller.rs +++ b/testnet/stacks-node/src/burnchains/mocknet_controller.rs @@ -184,6 +184,7 @@ impl BurnchainController for MocknetController { } BlockstackOperationType::LeaderBlockCommit(payload) => { BlockstackOperationType::LeaderBlockCommit(LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash: payload.block_header_hash, new_seed: payload.new_seed, parent_block_ptr: payload.parent_block_ptr, From ba9e08b9ca9f80a725b334efe3ff491e826a13e4 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:09:19 -0400 Subject: [PATCH 091/178] fix: restore sunset burn --- testnet/stacks-node/src/neon_node.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/testnet/stacks-node/src/neon_node.rs b/testnet/stacks-node/src/neon_node.rs index e3058a0b0..4c9b32fec 100644 --- a/testnet/stacks-node/src/neon_node.rs +++ b/testnet/stacks-node/src/neon_node.rs @@ -10,6 +10,7 @@ use std::time::Duration; use std::{thread, thread::JoinHandle}; use stacks::burnchains::BurnchainSigner; +use stacks::burnchains::PoxConstants; use stacks::burnchains::{Burnchain, BurnchainParameters, Txid}; use stacks::chainstate::burn::db::sortdb::SortitionDB; use stacks::chainstate::burn::operations::{ @@ -414,6 +415,7 @@ fn inner_generate_block_commit_op( parent_winning_vtx: u16, vrf_seed: VRFSeed, commit_outs: Vec, + sunset_burn: u64, current_burn_height: u64, ) -> BlockstackOperationType { let (parent_block_ptr, parent_vtxindex) = (parent_burnchain_height, parent_winning_vtx); @@ -421,6 +423,7 @@ fn inner_generate_block_commit_op( let burn_parent_modulus = (current_burn_height % BURN_BLOCK_MINED_AT_MODULUS) as u8; BlockstackOperationType::LeaderBlockCommit(LeaderBlockCommitOp { + sunset_burn, block_header_hash, burn_fee, input: (Txid([0; 32]), 0), @@ -2167,9 +2170,26 @@ impl StacksNode { } }; - let commit_outs = if !burnchain.is_in_prepare_phase(burn_block.block_height + 1) { + let epoch_id = SortitionDB::get_stacks_epoch(burn_db.conn(), burn_block.block_height + 1) + .expect("FATAL: unable to read sortition DB") + .expect("FATAL: no epoch defined for current block height") + .epoch_id; + + let sunset_burn = + burnchain.expected_sunset_burn(burn_block.block_height + 1, burn_fee_cap, epoch_id); + let rest_commit = burn_fee_cap - sunset_burn; + + let commit_outs = if !burnchain.is_in_prepare_phase(burn_block.block_height + 1) + && (!PoxConstants::has_pox_sunset(epoch_id) + || !burnchain + .pox_constants + .is_after_pox_sunset_end(burn_block.block_height + 1, epoch_id)) + { + // not in prepare phase, and if we have a sunset, it's not yet completed RewardSetInfo::into_commit_outs(recipients, config.is_mainnet()) } else { + // in prepare phase, or either there's no sunset at all in this epoch, or if there is, + // it has completed. vec![PoxAddress::standard_burn_address(config.is_mainnet())] }; @@ -2177,7 +2197,7 @@ impl StacksNode { let op = inner_generate_block_commit_op( keychain.get_burnchain_signer(), anchored_block.block_hash(), - burn_fee_cap, + rest_commit, ®istered_key, parent_block_burn_height .try_into() @@ -2185,9 +2205,12 @@ impl StacksNode { parent_winning_vtxindex, VRFSeed::from_proof(&vrf_proof), commit_outs, + sunset_burn, burn_block.block_height, ); + debug!("Leader block commit: {:?}", &op); + let cur_burn_chain_tip = SortitionDB::get_canonical_burn_chain_tip(burn_db.conn()) .expect("FATAL: failed to query sortition DB for canonical burn chain tip"); From fd24b5e79cd2bf93203b84c7e599c015c95149f8 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:09:32 -0400 Subject: [PATCH 092/178] fix: restore sunset burn --- testnet/stacks-node/src/node.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/testnet/stacks-node/src/node.rs b/testnet/stacks-node/src/node.rs index 8ac2ef54c..b325ec956 100644 --- a/testnet/stacks-node/src/node.rs +++ b/testnet/stacks-node/src/node.rs @@ -1010,16 +1010,20 @@ impl Node { self.config .update_pox_constants(&mut burnchain.pox_constants); - let commit_outs = - if !burnchain.is_in_prepare_phase(burnchain_tip.block_snapshot.block_height + 1) { - RewardSetInfo::into_commit_outs(None, self.config.is_mainnet()) - } else { - vec![PoxAddress::standard_burn_address(self.config.is_mainnet())] - }; + let commit_outs = if burnchain_tip.block_snapshot.block_height + 1 + < burnchain.pox_constants.sunset_end + && !burnchain.is_in_prepare_phase(burnchain_tip.block_snapshot.block_height + 1) + { + RewardSetInfo::into_commit_outs(None, self.config.is_mainnet()) + } else { + vec![PoxAddress::standard_burn_address(self.config.is_mainnet())] + }; + let burn_parent_modulus = (burnchain_tip.block_snapshot.block_height % BURN_BLOCK_MINED_AT_MODULUS) as u8; BlockstackOperationType::LeaderBlockCommit(LeaderBlockCommitOp { + sunset_burn: 0, block_header_hash, burn_fee, input: (Txid([0; 32]), 0), From 7798b9addc03aff18f20a9d59087f1e2ba870422 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:09:46 -0400 Subject: [PATCH 093/178] fix: restore sunset burn --- testnet/stacks-node/src/tests/epoch_205.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/testnet/stacks-node/src/tests/epoch_205.rs b/testnet/stacks-node/src/tests/epoch_205.rs index d7b90b077..233d1c637 100644 --- a/testnet/stacks-node/src/tests/epoch_205.rs +++ b/testnet/stacks-node/src/tests/epoch_205.rs @@ -589,8 +589,16 @@ fn transition_empty_blocks() { if tip_info.burn_block_height + 1 >= epoch_2_05 { let burn_fee_cap = 100000000; // 1 BTC + let sunset_burn = burnchain.expected_sunset_burn( + tip_info.burn_block_height + 1, + burn_fee_cap, + StacksEpochId::Epoch2_05, + ); + let rest_commit = burn_fee_cap - sunset_burn; - let commit_outs = if !burnchain.is_in_prepare_phase(tip_info.burn_block_height + 1) { + let commit_outs = if tip_info.burn_block_height + 1 < burnchain.pox_constants.sunset_end + && !burnchain.is_in_prepare_phase(tip_info.burn_block_height + 1) + { vec![ PoxAddress::standard_burn_address(conf.is_mainnet()), PoxAddress::standard_burn_address(conf.is_mainnet()), @@ -603,8 +611,9 @@ fn transition_empty_blocks() { let burn_parent_modulus = (tip_info.burn_block_height % BURN_BLOCK_MINED_AT_MODULUS) as u8; let op = BlockstackOperationType::LeaderBlockCommit(LeaderBlockCommitOp { + sunset_burn, block_header_hash: BlockHeaderHash([0xff; 32]), - burn_fee: burn_fee_cap, + burn_fee: rest_commit, input: (Txid([0; 32]), 0), apparent_sender: keychain.get_burnchain_signer(), key_block_ptr, From 09ccc3381b6349359e74d3bb0e521ff2b4ac4184 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:09:58 -0400 Subject: [PATCH 094/178] feat: integration test verifying that the sunset burn gets interrupted by the epoch 2.1 transition --- testnet/stacks-node/src/tests/epoch_21.rs | 305 +++++++++++++++++++++- 1 file changed, 304 insertions(+), 1 deletion(-) diff --git a/testnet/stacks-node/src/tests/epoch_21.rs b/testnet/stacks-node/src/tests/epoch_21.rs index 696672817..2e5bb0fc1 100644 --- a/testnet/stacks-node/src/tests/epoch_21.rs +++ b/testnet/stacks-node/src/tests/epoch_21.rs @@ -53,7 +53,6 @@ use stacks::core::BURNCHAIN_TX_SEARCH_WINDOW; use crate::burnchains::bitcoin_regtest_controller::UTXO; use crate::operations::BurnchainOpSigner; -use crate::tests::neon_integrations::get_balance; use crate::Keychain; fn advance_to_2_1( @@ -101,6 +100,8 @@ fn advance_to_2_1( 4 * prepare_phase_len / 5, 5, 15, + u64::max_value() - 2, + u64::max_value() - 1, u32::max_value(), )); burnchain_config.pox_constants = pox_constants.clone(); @@ -609,6 +610,8 @@ fn transition_fixes_bitcoin_rigidity() { 4 * prepare_phase_len / 5, 5, 15, + (16 * reward_cycle_len - 1).into(), + (17 * reward_cycle_len).into(), u32::max_value(), ); burnchain_config.pox_constants = pox_constants.clone(); @@ -1014,6 +1017,8 @@ fn transition_adds_get_pox_addr_recipients() { 4 * prepare_phase_len / 5, 1, 1, + u64::max_value() - 2, + u64::max_value() - 1, v1_unlock_height, ); @@ -1245,3 +1250,301 @@ fn transition_adds_get_pox_addr_recipients() { eprintln!("found pox addrs: {:?}", &found_pox_addrs); assert_eq!(found_pox_addrs.len(), 4); } + +/// Verify that a sunset-in-progress will be halted by the epoch 2.1 transition +#[test] +#[ignore] +fn transition_removes_pox_sunset() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let spender_sk = StacksPrivateKey::new(); + let spender_addr: PrincipalData = to_addr(&spender_sk).into(); + let first_bal = 6_000_000_000 * (core::MICROSTACKS_PER_STACKS as u64); + + let pox_pubkey = Secp256k1PublicKey::from_hex( + "02f006a09b59979e2cb8449f58076152af6b124aa29b948a3714b8d5f15aa94ede", + ) + .unwrap(); + let pox_pubkey_hash = bytes_to_hex( + &Hash160::from_node_public_key(&pox_pubkey) + .to_bytes() + .to_vec(), + ); + + let (mut conf, miner_account) = neon_integration_test_conf(); + + test_observer::spawn(); + + conf.events_observers.push(EventObserverConfig { + endpoint: format!("localhost:{}", test_observer::EVENT_OBSERVER_PORT), + events_keys: vec![EventKeyType::AnyEvent], + }); + + conf.initial_balances.push(InitialBalance { + address: spender_addr.clone(), + amount: first_bal, + }); + + // reward cycle length = 15, so 10 reward cycle slots + 5 prepare-phase burns + let first_sortition_height = 201; + let reward_cycle_len = 15; + let prepare_phase_len = 5; + let sunset_start_rc = 16; + let sunset_end_rc = 20; + let epoch_21_rc = 18; + + let epoch_21 = epoch_21_rc * reward_cycle_len + 1; + + let mut epochs = core::STACKS_EPOCHS_REGTEST.to_vec(); + epochs[1].end_height = 1; + epochs[2].start_height = 1; + epochs[2].end_height = epoch_21; + epochs[3].start_height = epoch_21; + + conf.burnchain.epochs = Some(epochs); + + let mut btcd_controller = BitcoinCoreController::new(conf.clone()); + btcd_controller + .start_bitcoind() + .map_err(|_e| ()) + .expect("Failed starting bitcoind"); + + let mut burnchain_config = Burnchain::regtest(&conf.get_burn_db_path()); + + let pox_constants = PoxConstants::new( + reward_cycle_len as u32, + prepare_phase_len, + 4 * prepare_phase_len / 5, + 5, + 15, + (sunset_start_rc * reward_cycle_len - 1).into(), + (sunset_end_rc * reward_cycle_len).into(), + (epoch_21 as u32) + 1, + ); + burnchain_config.pox_constants = pox_constants.clone(); + + let mut btc_regtest_controller = BitcoinRegtestController::with_burnchain( + conf.clone(), + None, + Some(burnchain_config.clone()), + None, + ); + let http_origin = format!("http://{}", &conf.node.rpc_bind); + + btc_regtest_controller.bootstrap_chain(first_sortition_height); + + eprintln!("Chain bootstrapped..."); + + let mut run_loop = neon::RunLoop::new(conf.clone()); + let blocks_processed = run_loop.get_blocks_processed_arc(); + let channel = run_loop.get_coordinator_channel().unwrap(); + let thread_burnchain = burnchain_config.clone(); + + thread::spawn(move || run_loop.start(Some(thread_burnchain), 0)); + + // give the run loop some time to start up! + wait_for_runloop(&blocks_processed); + + // first block wakes up the run loop + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // first block will hold our VRF registration + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + // second block will be the first mined Stacks block + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + + let sort_height = channel.get_sortitions_processed(); + + // let's query the miner's account nonce: + let account = get_account(&http_origin, &miner_account); + assert_eq!(account.balance, 0); + assert_eq!(account.nonce, 1); + + // and our potential spenders: + let account = get_account(&http_origin, &spender_addr); + assert_eq!(account.balance, first_bal as u128); + assert_eq!(account.nonce, 0); + + let pox_info = get_pox_info(&http_origin); + + assert_eq!( + &pox_info.contract_id, + &format!("ST000000000000000000002AMW42H.pox") + ); + assert_eq!(pox_info.current_cycle.is_pox_active, false); + assert_eq!(pox_info.next_cycle.stacked_ustx, 0); + + let tx = make_contract_call( + &spender_sk, + 0, + 260, + &StacksAddress::from_string("ST000000000000000000002AMW42H").unwrap(), + "pox", + "stack-stx", + &[ + Value::UInt(first_bal as u128 - 260 * 3), + execute( + &format!("{{ hashbytes: 0x{}, version: 0x00 }}", pox_pubkey_hash), + ClarityVersion::Clarity1, + ) + .unwrap() + .unwrap(), + Value::UInt(sort_height as u128), + Value::UInt(12), + ], + ); + + // okay, let's push that stacking transaction! + submit_tx(&http_origin, &tx); + + let mut sort_height = channel.get_sortitions_processed(); + eprintln!("Sort height pox-1: {}", sort_height); + + // advance to next reward cycle + for _i in 0..(reward_cycle_len * 2 + 2) { + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + sort_height = channel.get_sortitions_processed(); + eprintln!("Sort height pox-1: {} <= {}", sort_height, epoch_21); + } + + // pox must activate + let pox_info = get_pox_info(&http_origin); + eprintln!("pox_info in pox-1 = {:?}", &pox_info); + assert_eq!(pox_info.current_cycle.is_pox_active, true); + assert_eq!( + &pox_info.contract_id, + &format!("ST000000000000000000002AMW42H.pox") + ); + + // advance to 2.1 + while sort_height <= epoch_21 + 1 { + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + sort_height = channel.get_sortitions_processed(); + eprintln!("Sort height pox-1: {} <= {}", sort_height, epoch_21); + } + + let pox_info = get_pox_info(&http_origin); + + // pox is still "active" despite unlock, because there's enough participation, and also even + // though the v1 blocok height has passed, the pox-2 contract won't be managing reward sets + // until the next reward cycle + eprintln!("pox_info in pox-2 = {:?}", &pox_info); + assert_eq!(pox_info.current_cycle.is_pox_active, true); + assert_eq!( + &pox_info.contract_id, + &format!("ST000000000000000000002AMW42H.pox") + ); + + // re-stack + let tx = make_contract_call( + &spender_sk, + 1, + 260 * 2, + &StacksAddress::from_string("ST000000000000000000002AMW42H").unwrap(), + "pox-2", + "stack-stx", + &[ + Value::UInt(first_bal as u128 - 260 * 3), + execute( + &format!("{{ hashbytes: 0x{}, version: 0x00 }}", pox_pubkey_hash), + ClarityVersion::Clarity2, + ) + .unwrap() + .unwrap(), + Value::UInt(sort_height as u128), + Value::UInt(12), + ], + ); + + // okay, let's push that stacking transaction! + submit_tx(&http_origin, &tx); + + eprintln!("Try and confirm pox-2 stack-stx"); + + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + sort_height = channel.get_sortitions_processed(); + eprintln!( + "Sort height pox-1 to pox-2 with stack-stx to pox-2: {}", + sort_height + ); + + let pox_info = get_pox_info(&http_origin); + assert_eq!(pox_info.current_cycle.is_pox_active, true); + + // get pox back online + while sort_height <= epoch_21 + reward_cycle_len { + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + sort_height = channel.get_sortitions_processed(); + eprintln!("Sort height pox-2: {}", sort_height); + } + + let pox_info = get_pox_info(&http_origin); + eprintln!("pox_info = {:?}", &pox_info); + assert_eq!(pox_info.current_cycle.is_pox_active, true); + + // first full reward cycle with pox-2 + assert_eq!( + &pox_info.contract_id, + &format!("ST000000000000000000002AMW42H.pox-2") + ); + + let burn_blocks = test_observer::get_burn_blocks(); + let mut pox_out_opt = None; + for (i, block) in burn_blocks.into_iter().enumerate() { + let recipients: Vec<(String, u64)> = block + .get("reward_recipients") + .unwrap() + .as_array() + .unwrap() + .iter() + .map(|value| { + let recipient: String = value + .get("recipient") + .unwrap() + .as_str() + .unwrap() + .to_string(); + let amount = value.get("amt").unwrap().as_u64().unwrap(); + (recipient, amount) + }) + .collect(); + + if (i as u64) < (sunset_start_rc * reward_cycle_len) { + // before sunset + if recipients.len() >= 1 { + for (_, amt) in recipients.into_iter() { + pox_out_opt = if let Some(pox_out) = pox_out_opt.clone() { + Some(std::cmp::max(amt, pox_out)) + } else { + Some(amt) + }; + } + } + } else if (i as u64) >= (sunset_start_rc * reward_cycle_len) && (i as u64) + 1 < epoch_21 { + // some sunset burn happened + let pox_out = pox_out_opt.clone().unwrap(); + if recipients.len() >= 1 { + for (_, amt) in recipients.into_iter() { + assert!(amt < pox_out); + } + } + } else if (i as u64) + 1 >= epoch_21 { + // no sunset burn happened + let pox_out = pox_out_opt.clone().unwrap(); + if recipients.len() >= 1 { + for (_, amt) in recipients.into_iter() { + // NOTE: odd number of reward cycles + if !burnchain_config.is_in_prepare_phase((i + 2) as u64) { + assert_eq!(amt, pox_out); + } + } + } + } + } + + test_observer::clear(); + channel.stop_chains_coordinator(); +} From b309777ecdecb1fa6fb7c9530ec1c89779bf516f Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:10:17 -0400 Subject: [PATCH 095/178] fix: remove dead comment --- testnet/stacks-node/src/tests/integrations.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/testnet/stacks-node/src/tests/integrations.rs b/testnet/stacks-node/src/tests/integrations.rs index d41fe6af5..295a5bec4 100644 --- a/testnet/stacks-node/src/tests/integrations.rs +++ b/testnet/stacks-node/src/tests/integrations.rs @@ -58,9 +58,6 @@ const CALL_READ_CONTRACT: &'static str = " (ok (contract-call? .other f2 u5))) "; -// TODO(gregorycoppola) Add "burn-block-info" section to this test, once a way to configure the -// Mocknet default Stacks epoch has been decided. Verify that we get (none) for any block height -// prior to the first burnchain block height. const GET_INFO_CONTRACT: &'static str = " (define-map block-data { height: uint } From 6ae4ef1834fc4f88549be7fe3a40d22c64a1aada Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:10:31 -0400 Subject: [PATCH 096/178] fix: restore PoX sunset test --- .../src/tests/neon_integrations.rs | 58 +++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/testnet/stacks-node/src/tests/neon_integrations.rs b/testnet/stacks-node/src/tests/neon_integrations.rs index b6f299254..3cd54f834 100644 --- a/testnet/stacks-node/src/tests/neon_integrations.rs +++ b/testnet/stacks-node/src/tests/neon_integrations.rs @@ -759,7 +759,7 @@ pub fn get_account(http_origin: &str, account: &F) -> Acco } } -fn get_pox_info(http_origin: &str) -> RPCPoxInfoData { +pub fn get_pox_info(http_origin: &str) -> RPCPoxInfoData { let client = reqwest::blocking::Client::new(); let path = format!("{}/v2/pox", http_origin); client @@ -4925,6 +4925,8 @@ fn pox_integration_test() { 4 * prepare_phase_len / 5, 5, 15, + (16 * reward_cycle_len - 1).into(), + (17 * reward_cycle_len).into(), u32::max_value(), ); burnchain_config.pox_constants = pox_constants.clone(); @@ -5016,7 +5018,7 @@ fn pox_integration_test() { Value::UInt(stacked_bal), execute( &format!("{{ hashbytes: 0x{}, version: 0x00 }}", pox_pubkey_hash), - ClarityVersion::Clarity2, + ClarityVersion::Clarity1, ) .unwrap() .unwrap(), @@ -5129,7 +5131,7 @@ fn pox_integration_test() { Value::UInt(stacked_bal / 2), execute( &format!("{{ hashbytes: 0x{}, version: 0x00 }}", pox_2_pubkey_hash), - ClarityVersion::Clarity2, + ClarityVersion::Clarity1, ) .unwrap() .unwrap(), @@ -5152,7 +5154,7 @@ fn pox_integration_test() { Value::UInt(stacked_bal / 2), execute( &format!("{{ hashbytes: 0x{}, version: 0x00 }}", pox_2_pubkey_hash), - ClarityVersion::Clarity2, + ClarityVersion::Clarity1, ) .unwrap() .unwrap(), @@ -5217,6 +5219,7 @@ fn pox_integration_test() { // let's test the reward information in the observer test_observer::clear(); + // before sunset // mine until the end of the next reward cycle, // the participation threshold now should be met. while sort_height < ((16 * pox_constants.reward_cycle_length) - 1).into() { @@ -5305,6 +5308,53 @@ fn pox_integration_test() { eprintln!("Stacks tip is now {}", tip_info.stacks_tip_height); assert_eq!(tip_info.stacks_tip_height, 36); + // now let's mine into the sunset + while sort_height < ((17 * pox_constants.reward_cycle_length) - 1).into() { + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + sort_height = channel.get_sortitions_processed(); + eprintln!("Sort height: {}", sort_height); + } + + // get the canonical chain tip + let tip_info = get_chain_info(&conf); + + eprintln!("Stacks tip is now {}", tip_info.stacks_tip_height); + assert_eq!(tip_info.stacks_tip_height, 51); + + let utxos = btc_regtest_controller.get_all_utxos(&pox_2_pubkey); + + // should receive more rewards during this cycle... + eprintln!("Got UTXOs: {}", utxos.len()); + assert_eq!( + utxos.len(), + 14, + "Should have received more outputs during the sunsetting PoX reward cycle" + ); + + // and after sunset + while sort_height < ((18 * pox_constants.reward_cycle_length) - 1).into() { + next_block_and_wait(&mut btc_regtest_controller, &blocks_processed); + sort_height = channel.get_sortitions_processed(); + eprintln!("Sort height: {}", sort_height); + } + + let utxos = btc_regtest_controller.get_all_utxos(&pox_2_pubkey); + + // should *not* receive more rewards during the after sunset cycle... + eprintln!("Got UTXOs: {}", utxos.len()); + assert_eq!( + utxos.len(), + 14, + "Should have received no more outputs after sunset PoX reward cycle" + ); + + // should have progressed the chain, though! + // get the canonical chain tip + let tip_info = get_chain_info(&conf); + + eprintln!("Stacks tip is now {}", tip_info.stacks_tip_height); + assert_eq!(tip_info.stacks_tip_height, 66); + test_observer::clear(); channel.stop_chains_coordinator(); } From 32694115f08a9b65bb0b198c31c3d1e57cbb59a2 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:45:19 -0400 Subject: [PATCH 097/178] chore: run new integration tests --- .github/workflows/bitcoin-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 3476219c3..8695d655b 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -73,6 +73,11 @@ jobs: - tests::epoch_205::test_cost_limit_switch_version205 - tests::epoch_205::test_exact_block_costs - tests::epoch_205::bigger_microblock_streams_in_2_05 + - tests::neon_integrations::test_problematic_txs_are_not_stored + - tests::neon_integrations::test_problematic_blocks_are_not_mined + - tests::neon_integrations::test_problematic_blocks_are_not_relayed_or_stored + - tests::neon_integrations::test_problematic_microblocks_are_not_mined + - tests::neon_integrations::test_problematic_microblocks_are_not_relayed_or_stored steps: - uses: actions/checkout@v2 - name: Download docker image From ef1ecb71f3be4e44b3b231ee8b1a1e9125cb84cb Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:45:34 -0400 Subject: [PATCH 098/178] fix: backport unit tests from 2.05.0.3.0 --- clarity/src/vm/ast/mod.rs | 137 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/clarity/src/vm/ast/mod.rs b/clarity/src/vm/ast/mod.rs index 4688a5b96..7f10d6218 100644 --- a/clarity/src/vm/ast/mod.rs +++ b/clarity/src/vm/ast/mod.rs @@ -151,6 +151,143 @@ mod test { use crate::vm::MAX_CALL_STACK_DEPTH; use std::collections::HashMap; + #[derive(PartialEq, Debug)] + struct UnitTestTracker { + invoked_functions: Vec<(ClarityCostFunction, Vec)>, + invocation_count: u64, + cost_addition_count: u64, + } + impl UnitTestTracker { + pub fn new() -> Self { + UnitTestTracker { + invoked_functions: vec![], + invocation_count: 0, + cost_addition_count: 0, + } + } + } + impl CostTracker for UnitTestTracker { + fn compute_cost( + &mut self, + cost_f: ClarityCostFunction, + input: &[u64], + ) -> std::result::Result { + self.invoked_functions.push((cost_f, input.to_vec())); + self.invocation_count += 1; + Ok(ExecutionCost::zero()) + } + fn add_cost(&mut self, _cost: ExecutionCost) -> std::result::Result<(), CostErrors> { + self.cost_addition_count += 1; + Ok(()) + } + fn add_memory(&mut self, _memory: u64) -> std::result::Result<(), CostErrors> { + Ok(()) + } + fn drop_memory(&mut self, _memory: u64) {} + fn reset_memory(&mut self) {} + fn short_circuit_contract_call( + &mut self, + _contract: &QualifiedContractIdentifier, + _function: &ClarityName, + _input: &[u64], + ) -> Result { + Ok(false) + } + } + + #[test] + fn test_cost_tracking_deep_contracts() { + let stack_limit = + (AST_CALL_STACK_DEPTH_BUFFER + (MAX_CALL_STACK_DEPTH as u64) + 1) as usize; + let exceeds_stack_depth_tuple = format!( + "{}u1 {}", + "{ a : ".repeat(stack_limit + 1), + "} ".repeat(stack_limit + 1) + ); + + // for deep lists, a test like this works: + // it can assert a limit, that you can also verify + // by disabling `VaryStackDepthChecker` and arbitrarily bumping up the parser lexer limits + // and see that it produces the same result + let exceeds_stack_depth_list = format!( + "{}u1 {}", + "(list ".repeat(stack_limit + 1), + ")".repeat(stack_limit + 1) + ); + + // with old rules, this is just ExpressionStackDepthTooDeep + let mut cost_track = UnitTestTracker::new(); + let err = build_ast_with_rules( + &QualifiedContractIdentifier::transient(), + &exceeds_stack_depth_list, + &mut cost_track, + ASTRules::Typical, + ) + .expect_err("Contract should error in parsing"); + + let expected_err = ParseErrors::ExpressionStackDepthTooDeep; + let expected_list_cost_state = UnitTestTracker { + invoked_functions: vec![(ClarityCostFunction::AstParse, vec![500])], + invocation_count: 1, + cost_addition_count: 1, + }; + + assert_eq!(&expected_err, &err.err); + assert_eq!(expected_list_cost_state, cost_track); + + // with new rules, this is now VaryExpressionStackDepthTooDeep + let mut cost_track = UnitTestTracker::new(); + let err = build_ast_with_rules( + &QualifiedContractIdentifier::transient(), + &exceeds_stack_depth_list, + &mut cost_track, + ASTRules::PrecheckSize, + ) + .expect_err("Contract should error in parsing"); + + let expected_err = ParseErrors::VaryExpressionStackDepthTooDeep; + let expected_list_cost_state = UnitTestTracker { + invoked_functions: vec![(ClarityCostFunction::AstParse, vec![500])], + invocation_count: 1, + cost_addition_count: 1, + }; + + assert_eq!(&expected_err, &err.err); + assert_eq!(expected_list_cost_state, cost_track); + + // you cannot do the same for tuples! + // in ASTRules::Typical, this passes + let mut cost_track = UnitTestTracker::new(); + let _ = build_ast_with_rules( + &QualifiedContractIdentifier::transient(), + &exceeds_stack_depth_tuple, + &mut cost_track, + ASTRules::Typical, + ) + .expect("Contract should aprse with ASTRules::Typical"); + + // this actually won't even error without + // the VaryStackDepthChecker changes. + let mut cost_track = UnitTestTracker::new(); + let err = build_ast_with_rules( + &QualifiedContractIdentifier::transient(), + &exceeds_stack_depth_tuple, + &mut cost_track, + ASTRules::PrecheckSize, + ) + .expect_err("Contract should error in parsing with ASTRules::PrecheckSize"); + + let expected_err = ParseErrors::VaryExpressionStackDepthTooDeep; + let expected_list_cost_state = UnitTestTracker { + invoked_functions: vec![(ClarityCostFunction::AstParse, vec![571])], + invocation_count: 1, + cost_addition_count: 1, + }; + + assert_eq!(&expected_err, &err.err); + assert_eq!(expected_list_cost_state, cost_track); + } + #[test] fn test_expression_identification_tuples() { let progn = "{ a: (+ 1 2 3), From 620b72efe978eae59f66ea27a3cb04a122a93dfe Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:45:51 -0400 Subject: [PATCH 099/178] chore: backport unit tests from 2.05.0.3.0 --- clarity/src/vm/ast/parser/mod.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/clarity/src/vm/ast/parser/mod.rs b/clarity/src/vm/ast/parser/mod.rs index e0c0605f5..f9d0a4ecc 100644 --- a/clarity/src/vm/ast/parser/mod.rs +++ b/clarity/src/vm/ast/parser/mod.rs @@ -1059,6 +1059,18 @@ mod test { let string_with_multiple_slashes = r#" "hello\\\"world" "#; + let stack_limit = + (AST_CALL_STACK_DEPTH_BUFFER + (MAX_CALL_STACK_DEPTH as u64) + 1) as usize; + let exceeds_stack_depth_tuple = format!( + "{}u1 {}", + "{ a : ".repeat(stack_limit + 1), + "} ".repeat(stack_limit + 1) + ); + let exceeds_stack_depth_list = format!( + "{}u1 {}", + "(list ".repeat(stack_limit + 1), + ")".repeat(stack_limit + 1) + ); assert!(match ast::parser::parse(&split_tokens).unwrap_err().err { ParseErrors::SeparatorExpected(_) => true, @@ -1270,5 +1282,25 @@ mod test { _ => false, } ); + + assert!(match ast::parser::parse(&exceeds_stack_depth_tuple) + .unwrap_err() + .err + { + ParseErrors::VaryExpressionStackDepthTooDeep => true, + x => { + panic!("Got {:?}", &x); + } + }); + + assert!(match ast::parser::parse(&exceeds_stack_depth_list) + .unwrap_err() + .err + { + ParseErrors::VaryExpressionStackDepthTooDeep => true, + x => { + panic!("Got {:?}", &x); + } + }); } } From ea81b42ecff20fd75755e77d8236f1aa8da23e45 Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 18 Oct 2022 10:46:11 -0400 Subject: [PATCH 100/178] chore: backport unit tests from 2.05.0.3.0 --- src/burnchains/bitcoin/spv.rs | 155 +++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 2 deletions(-) diff --git a/src/burnchains/bitcoin/spv.rs b/src/burnchains/bitcoin/spv.rs index fa21c5559..40ac7f6af 100644 --- a/src/burnchains/bitcoin/spv.rs +++ b/src/burnchains/bitcoin/spv.rs @@ -771,7 +771,7 @@ impl SpvClient { // check work let chain_tip = self.get_headers_height()?; self.validate_header_work( - (chain_tip - 1) / BLOCK_DIFFICULTY_CHUNK_SIZE, + (insert_height.saturating_sub(1)) / BLOCK_DIFFICULTY_CHUNK_SIZE, chain_tip / BLOCK_DIFFICULTY_CHUNK_SIZE + 1, ) .map_err(|e| { @@ -784,6 +784,7 @@ impl SpvClient { } else { // fetching headers in descending order, so verify that the last item in // `block_headers` connects to a child in the DB (if it has one) + let headers_len = block_headers.len() as u64; self.insert_block_headers_before(insert_height, block_headers) .map_err(|e| { error!("Failed to insert block headers: {:?}", &e); @@ -797,7 +798,9 @@ impl SpvClient { insert_height / BLOCK_DIFFICULTY_CHUNK_SIZE + 1 }; - self.validate_header_work(interval_start, interval_start + 1) + let interval_end = (insert_height + 1 + headers_len) / BLOCK_DIFFICULTY_CHUNK_SIZE + 1; + + self.validate_header_work(interval_start, interval_end) .map_err(|e| { error!( "Received headers with bad target, difficulty, or continuity: {:?}", @@ -1598,4 +1601,152 @@ mod test { spv_client.validate_header_work(i, i + 1).unwrap(); } } + + #[test] + fn test_spv_check_work_bad_blocks_rejected() { + if !env::var("BLOCKSTACK_SPV_HEADERS_DB").is_ok() { + eprintln!("Skipping test_spv_check_work_reorg_accepted -- no BLOCKSTACK_SPV_HEADERS_DB envar set"); + return; + } + let db_path_source = env::var("BLOCKSTACK_SPV_HEADERS_DB").unwrap(); + let db_path = "/tmp/test_spv_check_work_reorg_accepted.dat".to_string(); + + if fs::metadata(&db_path).is_ok() { + fs::remove_file(&db_path).unwrap(); + } + + fs::copy(&db_path_source, &db_path).unwrap(); + + // set up SPV client so we don't have chain work at first + let mut spv_client = + SpvClient::new(&db_path, 0, None, BitcoinNetworkType::Mainnet, true, false).unwrap(); + + assert!( + spv_client.get_headers_height().unwrap() >= 40317, + "This test needs headers up to 40317" + ); + spv_client.drop_headers(40317).unwrap(); + + // update chain work + let total_work_before = spv_client.update_chain_work().unwrap(); + + // fake block headers for mainnet 40319-40320, which is on a difficulty adjustment boundary + let bad_headers = vec![ + LoneBlockHeader { + header: BlockHeader { + version: 1, + prev_blockhash: Sha256dHash::from_hex( + "000000003ae696c44274e40817d4acaf40c1ff1853411d4f0573421caf5faa07", + ) + .unwrap(), + merkle_root: Sha256dHash::from_hex( + "f1b2e16f74ee0e90cad3dc2e5be4806ff0581ed50a9cb3dfeee591dac76b17a7", + ) + .unwrap(), + time: 1654939798, + bits: 386485098, + nonce: 1, + }, + tx_count: VarInt(0), + }, + LoneBlockHeader { + header: BlockHeader { + version: 1, + prev_blockhash: Sha256dHash::from_hex( + "2c32da7fbdcef6ba34bcc5cc9707f2b351dbf094b4df5d3a2a38ea47b3e61f35", + ) + .unwrap(), + merkle_root: Sha256dHash::from_hex( + "b4d736ca74838036ebd19b085c3eeb9ffec2307f6452347cdd8ddaa249686f39", + ) + .unwrap(), + time: 1654939798, + bits: 486575299, + nonce: 1, + }, + tx_count: VarInt(0), + }, + LoneBlockHeader { + header: BlockHeader { + version: 1, + prev_blockhash: Sha256dHash::from_hex( + "7c1d4fd424f626c13bcd5442db080df9000d8f3dbfa1809ba1b05ddcdba58dd5", + ) + .unwrap(), + merkle_root: Sha256dHash::from_hex( + "cd71b0c247cfa777748c84d5376ad6a4a19e7802793dfd27c4d5834aa4393299", + ) + .unwrap(), + time: 1654996886, + bits: 0x1d00ffff, + nonce: 178162936, + }, + tx_count: VarInt(0), + }, + LoneBlockHeader { + header: BlockHeader { + version: 1, + prev_blockhash: Sha256dHash::from_hex( + "000000008a8cab438b9c98599c5363c7a4eff5b246871dd7c3f46886da375170", + ) + .unwrap(), + merkle_root: Sha256dHash::from_hex( + "cd71b0c247cfa777748c84d5376ad6a4a19e7802793dfd27c4d5834aa4393299", + ) + .unwrap(), + time: 1655018902, + bits: 0x1d00ffff, + nonce: 168408960, + }, + tx_count: VarInt(0), + }, + ]; + + // should fail + if let btc_error::InvalidPoW = spv_client + .handle_headers(40317, bad_headers.clone()) + .unwrap_err() + { + } else { + panic!("Bad PoW headers accepted"); + } + } + + #[test] + fn test_witness_size() { + use stacks_common::deps_common::bitcoin::blockdata::script::Script; + use stacks_common::deps_common::bitcoin::blockdata::transaction::OutPoint; + use stacks_common::deps_common::bitcoin::blockdata::transaction::TxIn; + use stacks_common::deps_common::bitcoin::blockdata::transaction::TxOut; + use std::mem; + + println!("OutPoint size in memory {}", mem::size_of::()); + println!("TxIn in memory {}", mem::size_of::()); + println!("TxOut size in memory {}", mem::size_of::()); + println!("Script size in memory {}", mem::size_of::