From fe45e9157bbbbe690260b306039a1adae0e7a86d Mon Sep 17 00:00:00 2001 From: Jeff Bencin Date: Fri, 1 Sep 2023 13:25:36 -0400 Subject: [PATCH 01/24] ci: Fix CI failing when we don't have DockerHub credentials --- .github/workflows/image-build-alpine-binary.yml | 5 ++++- .github/workflows/image-build-debian-binary.yml | 5 ++++- .github/workflows/image-build-debian-source.yml | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/image-build-alpine-binary.yml b/.github/workflows/image-build-alpine-binary.yml index f5dc99238..75a9144d6 100644 --- a/.github/workflows/image-build-alpine-binary.yml +++ b/.github/workflows/image-build-alpine-binary.yml @@ -39,6 +39,7 @@ jobs: run: | echo "GITHUB_SHA_SHORT=${GITHUB_SHA::7}" >> $GITHUB_ENV echo "GITHUB_REF_SHORT=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "DOCKER_PUSH=${{ (secrets.DOCKERHUB_USERNAME != '') && (secrets.DOCKERHUB_PASSWORD != '') }}" >> $GITHUB_ENV - name: Set up QEMU id: docker_qemu uses: docker/setup-qemu-action@v2 @@ -62,6 +63,8 @@ jobs: - name: Login to DockerHub id: docker_login uses: docker/login-action@v2 + # Only attempt login and push if we have credentials + if: env.DOCKER_PUSH == 'true' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -78,4 +81,4 @@ jobs: STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }} GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} - push: true + push: ${{ env.DOCKER_PUSH }} diff --git a/.github/workflows/image-build-debian-binary.yml b/.github/workflows/image-build-debian-binary.yml index e1584abbc..052a2f87f 100644 --- a/.github/workflows/image-build-debian-binary.yml +++ b/.github/workflows/image-build-debian-binary.yml @@ -49,6 +49,7 @@ jobs: run: | echo "GITHUB_SHA_SHORT=${GITHUB_SHA::7}" >> $GITHUB_ENV echo "GITHUB_REF_SHORT=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "DOCKER_PUSH=${{ (secrets.DOCKERHUB_USERNAME != '') && (secrets.DOCKERHUB_PASSWORD != '') }}" >> $GITHUB_ENV - name: Set up QEMU id: docker_qemu uses: docker/setup-qemu-action@v2 @@ -73,6 +74,8 @@ jobs: - name: Login to DockerHub id: docker_login uses: docker/login-action@v2 + # Only attempt login and push if we have credentials + if: env.DOCKER_PUSH == 'true' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -89,4 +92,4 @@ jobs: STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }} GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} - push: true + push: ${{ env.DOCKER_PUSH }} diff --git a/.github/workflows/image-build-debian-source.yml b/.github/workflows/image-build-debian-source.yml index d60166e26..0ba8b2bfb 100644 --- a/.github/workflows/image-build-debian-source.yml +++ b/.github/workflows/image-build-debian-source.yml @@ -46,6 +46,7 @@ jobs: run: | echo "GITHUB_SHA_SHORT=${GITHUB_SHA::7}" >> $GITHUB_ENV echo "GITHUB_REF_SHORT=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "DOCKER_PUSH=${{ (secrets.DOCKERHUB_USERNAME != '') && (secrets.DOCKERHUB_PASSWORD != '') }}" >> $GITHUB_ENV - name: Set up QEMU id: docker_qemu uses: docker/setup-qemu-action@v2 @@ -72,6 +73,8 @@ jobs: - name: Login to DockerHub id: docker_login uses: docker/login-action@v2 + # Only attempt login and push if we have credentials + if: env.DOCKER_PUSH == 'true' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -87,4 +90,4 @@ jobs: STACKS_NODE_VERSION=${{ env.GITHUB_SHA_SHORT }} GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} - push: true + push: ${{ env.DOCKER_PUSH }} From 6134cc877d383465770a3f803d14996994452f35 Mon Sep 17 00:00:00 2001 From: Jacinta Ferrant Date: Wed, 20 Sep 2023 09:00:38 -0700 Subject: [PATCH 02/24] Add 'sudo apt-get update' to the build image step Signed-off-by: Jacinta Ferrant --- .github/workflows/bitcoin-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 66adcf4f9..0d5613ff5 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -31,6 +31,7 @@ jobs: - name: Reclaim disk space id: cleanup run: | + sudo apt-get update sudo apt-get remove -y '^dotnet-.*' sudo apt-get remove -y '^llvm-.*' sudo apt-get remove -y 'php.*' From fec356804a90b1c36ad7ed6644cd06cbcafd4621 Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Thu, 21 Sep 2023 10:10:06 -0400 Subject: [PATCH 03/24] fix: lower warning about mempool nonce caching to `debug!` This case happens often in normal execution, so it is just spamming the logs as a `warn!`. --- src/core/mempool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/mempool.rs b/src/core/mempool.rs index 7f3c195d2..5efb76281 100644 --- a/src/core/mempool.rs +++ b/src/core/mempool.rs @@ -853,7 +853,7 @@ impl NonceCache { let should_store_again = match db_set_nonce(mempool_db, address, nonce) { Ok(_) => false, Err(e) => { - warn!("error caching nonce to sqlite: {}", e); + debug!("error caching nonce to sqlite: {}", e); true } }; From 34703d7b63135987c296014bff7377b4115ff57f Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 10 Jul 2023 11:02:45 -0500 Subject: [PATCH 04/24] feat: ignore logging errors rather than panicking --- stacks-common/src/util/log.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stacks-common/src/util/log.rs b/stacks-common/src/util/log.rs index 26de14e67..73463cc45 100644 --- a/stacks-common/src/util/log.rs +++ b/stacks-common/src/util/log.rs @@ -205,8 +205,8 @@ fn make_json_logger() -> Logger { }), ); - let drain = Mutex::new(slog_json::Json::default(std::io::stderr())).map(slog::Fuse); - let filtered_drain = slog::LevelFilter::new(drain, get_loglevel()).fuse(); + let drain = Mutex::new(slog_json::Json::default(std::io::stderr())); + let filtered_drain = slog::LevelFilter::new(drain, get_loglevel()).ignore_res(); slog::Logger::root(filtered_drain, def_keys) } @@ -225,7 +225,7 @@ fn make_logger() -> Logger { let decorator = slog_term::PlainSyncDecorator::new(std::io::stderr()); let atty = isatty(Stream::Stderr); let drain = TermFormat::new(decorator, pretty_print, debug, atty); - let logger = Logger::root(drain.fuse(), o!()); + let logger = Logger::root(drain.ignore_res(), o!()); logger } } @@ -239,7 +239,7 @@ fn make_logger() -> Logger { let plain = slog_term::PlainSyncDecorator::new(slog_term::TestStdoutWriter); let isatty = isatty(Stream::Stdout); let drain = TermFormat::new(plain, false, debug, isatty); - let logger = Logger::root(drain.fuse(), o!()); + let logger = Logger::root(drain.ignore_res(), o!()); logger } } From fb5e2f592fc56907752a0d12d3e9f8f69a57c9ba Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Wed, 20 Sep 2023 13:24:21 -0500 Subject: [PATCH 05/24] chore: add changelog entry for 3784 hotfix --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40ed70618..66f936c5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to the versioning scheme outlined in the [README.md](README.md). +## [2.4.0.0.2] + +This is a hotfix that changes the logging failure behavior from panicking to dropping +the log message (PR #3784). + ## [2.4.0.0.1] This is a minor change to add `txid` fields into the log messages from failing From 96872fb657b0292873a94ff3f3733e94911c705a Mon Sep 17 00:00:00 2001 From: Jude Nelson Date: Tue, 24 Oct 2023 17:50:21 -0400 Subject: [PATCH 06/24] fix: fix trait handler and cut release --- CHANGELOG.md | 5 +++++ clarity/src/vm/mod.rs | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f936c5e..eb9a75b0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to the versioning scheme outlined in the [README.md](README.md). +## [2.4.0.0.3] + +This is a high-priority hotfix that addresses a bug in transaction processing which +could impact miner availability. + ## [2.4.0.0.2] This is a hotfix that changes the logging failure behavior from panicking to dropping diff --git a/clarity/src/vm/mod.rs b/clarity/src/vm/mod.rs index db54d5b24..9943f0038 100644 --- a/clarity/src/vm/mod.rs +++ b/clarity/src/vm/mod.rs @@ -344,7 +344,12 @@ pub fn eval<'a>( let f = lookup_function(&function_name, env)?; apply(&f, &rest, env, context) } - TraitReference(_, _) | Field(_) => unreachable!("can't be evaluated"), + TraitReference(_, _) | Field(_) => { + return Err(InterpreterError::BadSymbolicRepresentation( + "Unexpected trait reference".into(), + ) + .into()) + } }; if let Some(mut eval_hooks) = env.global_context.eval_hooks.take() { From f7be9aff24642c33781813bbc402f0d2c5d27ea5 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 6 Nov 2023 12:59:45 -0600 Subject: [PATCH 07/24] fix: address issue in print eval --- CHANGELOG.md | 5 +++++ clarity/src/vm/functions/mod.rs | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb9a75b0a..812eb18c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to the versioning scheme outlined in the [README.md](README.md). +## [2.4.0.0.4] + +This is a high-priority hotfix that addresses a bug in transaction processing which +could impact miner availability. + ## [2.4.0.0.3] This is a high-priority hotfix that addresses a bug in transaction processing which diff --git a/clarity/src/vm/functions/mod.rs b/clarity/src/vm/functions/mod.rs index 991e074ff..da8cbde4d 100644 --- a/clarity/src/vm/functions/mod.rs +++ b/clarity/src/vm/functions/mod.rs @@ -69,6 +69,8 @@ macro_rules! switch_on_global_epoch { use crate::vm::ClarityVersion; +use super::errors::InterpreterError; + mod arithmetic; mod assets; mod boolean; @@ -606,7 +608,10 @@ fn special_print( env: &mut Environment, context: &LocalContext, ) -> Result { - let input = eval(&args[0], env, context)?; + let arg = args.get(0).ok_or_else(|| { + InterpreterError::BadSymbolicRepresentation("Print should have an argument".into()) + })?; + let input = eval(arg, env, context)?; runtime_cost(ClarityCostFunction::Print, env, input.size())?; From 2547ddce4c27622c0d924d3a995b7c4531e625b0 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Tue, 7 Nov 2023 08:55:22 -0800 Subject: [PATCH 08/24] remove missing packags from ci --- .github/workflows/bitcoin-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 0d5613ff5..9aba4e625 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -37,7 +37,6 @@ jobs: sudo apt-get remove -y 'php.*' sudo apt-get remove -y '^mongodb-.*' sudo apt-get remove -y '^mysql-.*' - sudo apt-get remove -y azure-cli google-cloud-sdk google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri sudo apt-get autoremove -y sudo apt-get clean docker system prune --force From 4354a9d5fd25d4f0168d41d4227b739b9b339189 Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Thu, 9 Nov 2023 10:58:06 -0500 Subject: [PATCH 09/24] docs: fix invalid example in docs --- clarity/src/vm/docs/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarity/src/vm/docs/mod.rs b/clarity/src/vm/docs/mod.rs index 9a019dbbc..8a432481e 100644 --- a/clarity/src/vm/docs/mod.rs +++ b/clarity/src/vm/docs/mod.rs @@ -1417,7 +1417,7 @@ The function returns the result of evaluating `expr`. example: " (define-data-var data int 1) (at-block 0x0000000000000000000000000000000000000000000000000000000000000000 block-height) ;; Returns u0 -(at-block (get-block-info? id-header-hash 0) (var-get data)) ;; Throws NoSuchDataVariable because `data` wasn't initialized at block height 0" +(at-block (unwrap-panic (get-block-info? id-header-hash u0)) (var-get data)) ;; Throws NoSuchDataVariable because `data` wasn't initialized at block height 0" }; const AS_CONTRACT_API: SpecialAPI = SpecialAPI { From 049db7ec56b80812e9a40a051b0911a7cc2c308b Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Mon, 13 Nov 2023 07:42:57 -0800 Subject: [PATCH 10/24] Add new docker tag to address repo name change --- .github/workflows/image-build-alpine-binary.yml | 1 + .github/workflows/image-build-debian-binary.yml | 1 + .github/workflows/image-build-debian-source.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/image-build-alpine-binary.yml b/.github/workflows/image-build-alpine-binary.yml index 75a9144d6..5422baffd 100644 --- a/.github/workflows/image-build-alpine-binary.yml +++ b/.github/workflows/image-build-alpine-binary.yml @@ -55,6 +55,7 @@ jobs: uses: docker/metadata-action@v4 with: images: | + blockstack/stacks-blockchain blockstack/${{ github.event.repository.name }} tags: | type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) )}} diff --git a/.github/workflows/image-build-debian-binary.yml b/.github/workflows/image-build-debian-binary.yml index 052a2f87f..c7c30ff26 100644 --- a/.github/workflows/image-build-debian-binary.yml +++ b/.github/workflows/image-build-debian-binary.yml @@ -67,6 +67,7 @@ jobs: uses: docker/metadata-action@v4 with: images: | + blockstack/stacks-blockchain blockstack/${{ github.event.repository.name }} tags: | type=raw,value=latest-${{ inputs.linux_version }},enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) )}} diff --git a/.github/workflows/image-build-debian-source.yml b/.github/workflows/image-build-debian-source.yml index 0ba8b2bfb..3a8e37997 100644 --- a/.github/workflows/image-build-debian-source.yml +++ b/.github/workflows/image-build-debian-source.yml @@ -66,6 +66,7 @@ jobs: uses: docker/metadata-action@v4 with: images: | + blockstack/stacks-blockchain blockstack/${{ github.event.repository.name }} tags: | type=raw,value=${{ env.BRANCH_NAME }} From b70dd0033678ea266334ed02f274cdb5163130d2 Mon Sep 17 00:00:00 2001 From: Mitchell Cuevas <6131188+cuevasm@users.noreply.github.com> Date: Wed, 1 Nov 2023 09:53:01 -0400 Subject: [PATCH 11/24] Update README.md Updating L2 language in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 06fca3166..2f1be0887 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Reference implementation of the [Stacks blockchain](https://github.com/stacks-network/stacks) in Rust. -Stacks is a layer-1 blockchain that connects to Bitcoin for security and enables decentralized apps and predictable smart contracts using the [Clarity language](https://clarity-lang.org/). Stacks implements [Proof of Transfer (PoX)](https://community.stacks.org/pox) mining that anchors to Bitcoin security. Leader election happens at the Bitcoin blockchain and Stacks (STX) miners write new blocks on the separate Stacks blockchain. With PoX there is no need to modify Bitcoin to enable smart contracts and decentralized apps. +Stacks is a layer-2 blockchain that uses Bitcoin as a base layer for security and enables decentralized apps and predictable smart contracts using the [Clarity language](https://clarity-lang.org/). Stacks implements [Proof of Transfer (PoX)](https://community.stacks.org/pox) mining that anchors to Bitcoin security. Leader election happens at the Bitcoin blockchain and Stacks (STX) miners write new blocks on the separate Stacks blockchain. With PoX there is no need to modify Bitcoin to enable smart contracts and decentralized apps. [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg?style=flat)](https://www.gnu.org/licenses/gpl-3.0) [![Release](https://img.shields.io/github/v/release/stacks-network/stacks-blockchain?style=flat)](https://github.com/stacks-network/stacks-blockchain/releases/latest) From b3de84a78f507d92b274ddccf8c1c91f45645281 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Sun, 19 Nov 2023 22:31:01 -0800 Subject: [PATCH 12/24] (3847) - Adding/updating Dockerfiles for ARM arch --- .../dockerfiles/Dockerfile.alpine-binary | 3 ++- .../dockerfiles/Dockerfile.debian-binary | 3 ++- build-scripts/Dockerfile.linux-glibc-arm64 | 1 + build-scripts/Dockerfile.linux-glibc-armv7 | 26 +++++++++++++++++++ build-scripts/Dockerfile.linux-glibc-x64 | 1 + build-scripts/Dockerfile.linux-musl-armv7 | 21 +++++++++++++++ build-scripts/build-dist.sh | 4 +++ 7 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 build-scripts/Dockerfile.linux-glibc-armv7 create mode 100644 build-scripts/Dockerfile.linux-musl-armv7 diff --git a/.github/actions/dockerfiles/Dockerfile.alpine-binary b/.github/actions/dockerfiles/Dockerfile.alpine-binary index 8c450a67f..2388ffa03 100644 --- a/.github/actions/dockerfiles/Dockerfile.alpine-binary +++ b/.github/actions/dockerfiles/Dockerfile.alpine-binary @@ -7,11 +7,12 @@ ARG TARGETPLATFORM ARG BUILDPLATFORM ARG TARGETARCH ARG TARGETVARIANT -ARG REPO=stacks-network/stacks-blockchain +ARG REPO=stacks-network/stacks-core RUN case ${TARGETARCH} in \ "amd64") BIN_ARCH=linux-musl-x64 ;; \ "arm64") BIN_ARCH=linux-musl-arm64 ;; \ + "arm") BIN_ARCH=linux-musl-armv7 ;; \ "*") exit 1 ;; \ esac \ && echo "wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \ diff --git a/.github/actions/dockerfiles/Dockerfile.debian-binary b/.github/actions/dockerfiles/Dockerfile.debian-binary index cf1380361..1cc5341d2 100644 --- a/.github/actions/dockerfiles/Dockerfile.debian-binary +++ b/.github/actions/dockerfiles/Dockerfile.debian-binary @@ -7,11 +7,12 @@ ARG TARGETPLATFORM ARG BUILDPLATFORM ARG TARGETARCH ARG TARGETVARIANT -ARG REPO=stacks-network/stacks-blockchain +ARG REPO=stacks-network/stacks-core RUN case ${TARGETARCH} in \ "amd64") BIN_ARCH=linux-musl-x64 ;; \ "arm64") BIN_ARCH=linux-musl-arm64 ;; \ + "arm") BIN_ARCH=linux-musl-armv7 ;; \ "*") exit 1 ;; \ esac \ && echo "wget -q https://github.com/${REPO}/releases/download/${TAG}/${BIN_ARCH}.zip -O /${BIN_ARCH}.zip" \ diff --git a/build-scripts/Dockerfile.linux-glibc-arm64 b/build-scripts/Dockerfile.linux-glibc-arm64 index 7ce50b6a6..61ff5f4a0 100644 --- a/build-scripts/Dockerfile.linux-glibc-arm64 +++ b/build-scripts/Dockerfile.linux-glibc-arm64 @@ -24,3 +24,4 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \ FROM scratch AS export-stage COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node / +## comment diff --git a/build-scripts/Dockerfile.linux-glibc-armv7 b/build-scripts/Dockerfile.linux-glibc-armv7 new file mode 100644 index 000000000..eb893baeb --- /dev/null +++ b/build-scripts/Dockerfile.linux-glibc-armv7 @@ -0,0 +1,26 @@ +FROM rust:bullseye as build + +ARG STACKS_NODE_VERSION="No Version Info" +ARG GIT_BRANCH='No Branch Info' +ARG GIT_COMMIT='No Commit Info' +ARG BUILD_DIR=/build +ARG TARGET=armv7-unknown-linux-gnueabihf +WORKDIR /src + +COPY . . + +RUN apt-get update && apt-get install -y git gcc-arm-linux-gnueabihf + +# Run all the build steps in ramdisk in an attempt to speed things up +RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \ + && cd ${BUILD_DIR} \ + && rustup target add ${TARGET} \ + && CC=arm-linux-gnueabihf-gcc \ + CC_armv7_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc \ + CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc \ + cargo build --features monitoring_prom,slog_json --release --workspace --target ${TARGET} \ + && mkdir -p /out \ + && cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out + +FROM scratch AS export-stage +COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node / \ No newline at end of file diff --git a/build-scripts/Dockerfile.linux-glibc-x64 b/build-scripts/Dockerfile.linux-glibc-x64 index 2db13cb51..9c68c58af 100644 --- a/build-scripts/Dockerfile.linux-glibc-x64 +++ b/build-scripts/Dockerfile.linux-glibc-x64 @@ -21,3 +21,4 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \ FROM scratch AS export-stage COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node / +## comment diff --git a/build-scripts/Dockerfile.linux-musl-armv7 b/build-scripts/Dockerfile.linux-musl-armv7 new file mode 100644 index 000000000..57b93b47e --- /dev/null +++ b/build-scripts/Dockerfile.linux-musl-armv7 @@ -0,0 +1,21 @@ +FROM messense/rust-musl-cross:armv7-musleabihf as build + +ARG STACKS_NODE_VERSION="No Version Info" +ARG GIT_BRANCH='No Branch Info' +ARG GIT_COMMIT='No Commit Info' +ARG BUILD_DIR=/build +ARG TARGET=armv7-unknown-linux-musleabihf +WORKDIR /src + +COPY . . + +# Run all the build steps in ramdisk in an attempt to speed things up +RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \ + && cd ${BUILD_DIR} \ + && rustup target add ${TARGET} \ + && cargo build --features monitoring_prom,slog_json --release --workspace --target ${TARGET} \ + && mkdir -p /out \ + && cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out + +FROM scratch AS export-stage +COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node / \ No newline at end of file diff --git a/build-scripts/build-dist.sh b/build-scripts/build-dist.sh index ac2c8bcd5..8be8f4f8a 100755 --- a/build-scripts/build-dist.sh +++ b/build-scripts/build-dist.sh @@ -17,8 +17,10 @@ case $DIST_TARGET_FILTER in case $DIST_TARGET_FILTER in linux-glibc-x64) build_platform linux-glibc-x64 ;; linux-glibc-arm64) build_platform linux-glibc-arm64 ;; + linux-glibc-armv7) build_platform linux-glibc-armv7 ;; linux-musl-x64) build_platform linux-musl-x64 ;; linux-musl-arm64) build_platform linux-musl-arm64 ;; + linux-musl-armv7) build_platform linux-musl-armv7 ;; windows-x64) build_platform windows-x64 ;; macos-x64) build_platform macos-x64 ;; macos-arm64) build_platform macos-arm64 ;; @@ -32,8 +34,10 @@ case $DIST_TARGET_FILTER in echo "Building distrubtions for all targets." build_platform linux-glibc-x64 build_platform linux-glibc-arm64 + build_platform linux-glibc-armv7 build_platform linux-musl-x64 build_platform linux-musl-arm64 + build_platform linux-musl-armv7 build_platform windows-x64 build_platform macos-x64 build_platform macos-arm64 From 99b54fd2d79c4a2469d44b7ad1fcdd5669d26597 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Sun, 19 Nov 2023 22:34:22 -0800 Subject: [PATCH 13/24] (3847) - Removing files --- .github/workflows/audit.yml | 35 ------- .github/workflows/build-source-binary.yml | 65 ------------- .github/workflows/clippy.yml | 44 --------- .../workflows/image-build-alpine-binary.yml | 85 ---------------- .../workflows/image-build-debian-binary.yml | 96 ------------------- .../workflows/image-build-debian-source.yml | 94 ------------------ 6 files changed, 419 deletions(-) delete mode 100644 .github/workflows/audit.yml delete mode 100644 .github/workflows/build-source-binary.yml delete mode 100644 .github/workflows/clippy.yml delete mode 100644 .github/workflows/image-build-alpine-binary.yml delete mode 100644 .github/workflows/image-build-debian-binary.yml delete mode 100644 .github/workflows/image-build-debian-source.yml diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml deleted file mode 100644 index c3864d565..000000000 --- a/.github/workflows/audit.yml +++ /dev/null @@ -1,35 +0,0 @@ -## -## Performs an audit for crate advisories against cargo dependencies -## - -name: Security Audit - -# Only run when: -# - workflow is manually triggered -# - Cargo.toml/lock is changed -# - Daily at 0330 UTC -# Note: this will create issues for any crate advisories unless they already exist - -on: - workflow_dispatch: - push: - paths: - - "**/Cargo.toml" - - "**/Cargo.lock" - schedule: - - cron: 30 03 * * * - -jobs: - security_audit: - if: ${{ false }} - name: Crate Vulnerability Check - runs-on: ubuntu-latest - steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Rust Dependency Check - id: rust_dep_check - uses: actions-rs/audit-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-source-binary.yml b/.github/workflows/build-source-binary.yml deleted file mode 100644 index 284171d67..000000000 --- a/.github/workflows/build-source-binary.yml +++ /dev/null @@ -1,65 +0,0 @@ -## -## Builds binary assets of stacks-blockchain and creates a named tag github (draft) release -## - -name: Build Distributable Assets - -# Only run when: -# - manually triggered via the ci.yml workflow with a provided input tag - -on: - workflow_call: - inputs: - tag: - description: "Tag name of this release (x.y.z)" - required: true - type: string - parallel_jobs: - description: "Number of parallel binary builds" - required: false - type: number - default: 4 - arch: - description: "Stringified JSON object listing of platform matrix" - required: true - type: string - -jobs: - artifact: - if: ${{ inputs.tag != '' }} - name: Create Artifacts - runs-on: ubuntu-latest - strategy: - max-parallel: ${{ inputs.parallel_jobs }} - matrix: - platform: ${{ fromJson(inputs.arch) }} - steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Set Vars - id: set_vars - run: | - echo "GITHUB_SHA_SHORT=${GITHUB_SHA::7}" >> $GITHUB_ENV - echo "GITHUB_REF_SHORT=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Set up Docker Buildx - id: setup_buildx - uses: docker/setup-buildx-action@v2 - - name: Build Binaries - id: build_binaries - uses: docker/build-push-action@v3 - with: - file: build-scripts/Dockerfile.${{ matrix.platform }} - outputs: type=local,dest=./release/${{ matrix.platform }} - build-args: | - OS_ARCH=${{ matrix.platform }} - GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} - GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} - - name: Compress artifact - id: compress_artifact - run: zip --junk-paths ${{ matrix.platform }} ./release/${{ matrix.platform }}/* - - name: Upload artifact - id: upload_artifact - uses: actions/upload-artifact@v3 - with: - path: ${{ matrix.platform }}.zip diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml deleted file mode 100644 index 1e6872bd6..000000000 --- a/.github/workflows/clippy.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Disabled - this workflow needs more work so it's not incredibly chatty -## -## Perform Clippy checks - currently set to defaults -## https://github.com/rust-lang/rust-clippy#usage -## https://rust-lang.github.io/rust-clippy/master/index.html -## - -name: Clippy Checks - -# Only run when: -# - PRs are (re)opened against develop branch - -on: - pull_request: - branches: - - develop - types: - - opened - - reopened - -jobs: - clippy_check: - if: ${{ false }} - name: Clippy Check - runs-on: ubuntu-latest - steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Define Rust Toolchain - id: define_rust_toolchain - run: echo "RUST_TOOLCHAIN=$(cat ./rust-toolchain)" >> $GITHUB_ENV - - name: Setup Rust Toolchain - id: setup_rust_toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - components: clippy - - name: Clippy - id: clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features diff --git a/.github/workflows/image-build-alpine-binary.yml b/.github/workflows/image-build-alpine-binary.yml deleted file mode 100644 index 5422baffd..000000000 --- a/.github/workflows/image-build-alpine-binary.yml +++ /dev/null @@ -1,85 +0,0 @@ -## -## Build the Docker Alpine image from the pre-built downloaded binary asset -## - -name: Build Alpine Binary Image - -# Only run when: -# - manually triggered via the ci.yml workflow - -on: - workflow_call: - inputs: - tag: - required: true - type: string - description: "semver tag for alpine images" - docker_platforms: - required: true - description: "Arch to buid alpine images" - type: string - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_PASSWORD: - required: true - -jobs: - image: - # Only run if a tag is provided manually - if: ${{ inputs.tag != '' }} - name: Build Image - runs-on: ubuntu-latest - steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Set Vars - id: set_vars - run: | - echo "GITHUB_SHA_SHORT=${GITHUB_SHA::7}" >> $GITHUB_ENV - echo "GITHUB_REF_SHORT=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "DOCKER_PUSH=${{ (secrets.DOCKERHUB_USERNAME != '') && (secrets.DOCKERHUB_PASSWORD != '') }}" >> $GITHUB_ENV - - name: Set up QEMU - id: docker_qemu - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - id: docker_buildx - uses: docker/setup-buildx-action@v2 - # tag image with: - # latest: `latest` - # input tag: `` - # git tag: `1234` - - name: Docker Metadata - id: docker_metadata - uses: docker/metadata-action@v4 - with: - images: | - blockstack/stacks-blockchain - blockstack/${{ github.event.repository.name }} - tags: | - type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) )}} - type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} - type=ref,event=tag,enable=true - - name: Login to DockerHub - id: docker_login - uses: docker/login-action@v2 - # Only attempt login and push if we have credentials - if: env.DOCKER_PUSH == 'true' - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build and Push - id: docker_build - uses: docker/build-push-action@v3 - with: - file: ./.github/actions/dockerfiles/Dockerfile.alpine-binary - platforms: ${{ inputs.docker_platforms }} - tags: ${{ steps.docker_metadata.outputs.tags }} - labels: ${{ steps.docker_metadata.outputs.labels }} - build-args: | - TAG=${{ inputs.tag}} - STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }} - GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} - GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} - push: ${{ env.DOCKER_PUSH }} diff --git a/.github/workflows/image-build-debian-binary.yml b/.github/workflows/image-build-debian-binary.yml deleted file mode 100644 index c7c30ff26..000000000 --- a/.github/workflows/image-build-debian-binary.yml +++ /dev/null @@ -1,96 +0,0 @@ -## -## Build the Docker Debian image from the pre-built downloaded binary asset -## - -name: Build Linux Binary Image - -# Only run when: -# - manually triggered via the ci.yml workflow - -on: - workflow_call: - inputs: - tag: - required: true - type: string - description: "semver tag for linux images" - docker_platforms: - required: true - description: "Arch to buid linux images" - type: string - linux_version: - required: true - description: "Linux image to build" - type: string - default: debian - build_type: - required: true - description: Build type (source/binary) - type: string - default: binary - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_PASSWORD: - required: true - -jobs: - image: - # Only run if a tag is provided manually - if: ${{ inputs.tag != '' }} - name: Build Image - runs-on: ubuntu-latest - steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Set Vars - id: set_vars - run: | - echo "GITHUB_SHA_SHORT=${GITHUB_SHA::7}" >> $GITHUB_ENV - echo "GITHUB_REF_SHORT=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "DOCKER_PUSH=${{ (secrets.DOCKERHUB_USERNAME != '') && (secrets.DOCKERHUB_PASSWORD != '') }}" >> $GITHUB_ENV - - name: Set up QEMU - id: docker_qemu - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - id: docker_buildx - uses: docker/setup-buildx-action@v2 - - name: Extract branch name - id: extract_branch - run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - # tag image with: - # branch name: `latest-` - # input tag: `-` - - name: Docker Metadata - id: docker_metadata - uses: docker/metadata-action@v4 - with: - images: | - blockstack/stacks-blockchain - blockstack/${{ github.event.repository.name }} - tags: | - type=raw,value=latest-${{ inputs.linux_version }},enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) )}} - type=raw,value=${{ inputs.tag }}-${{ inputs.linux_version }},enable=${{ inputs.tag != '' }} - - name: Login to DockerHub - id: docker_login - uses: docker/login-action@v2 - # Only attempt login and push if we have credentials - if: env.DOCKER_PUSH == 'true' - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build and Push - id: docker_build - uses: docker/build-push-action@v3 - with: - file: ./.github/actions/dockerfiles/Dockerfile.${{ inputs.linux_version }}-${{ inputs.build_type }} - platforms: ${{ inputs.docker_platforms }} - tags: ${{ steps.docker_metadata.outputs.tags }} - labels: ${{ steps.docker_metadata.outputs.labels }} - build-args: | - TAG=${{ inputs.tag}} - STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }} - GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} - GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} - push: ${{ env.DOCKER_PUSH }} diff --git a/.github/workflows/image-build-debian-source.yml b/.github/workflows/image-build-debian-source.yml deleted file mode 100644 index 3a8e37997..000000000 --- a/.github/workflows/image-build-debian-source.yml +++ /dev/null @@ -1,94 +0,0 @@ -## -## Build the Docker Debian image from source -## - -name: Build Linux Source Image - -# Only run when: -# - workflow is manually triggered -# - manually triggered via the ci.yml workflow - -on: - workflow_dispatch: - workflow_call: - inputs: - docker_platforms: - required: true - description: "Arch to buid images" - type: string - default: linux/amd64 - linux_version: - required: true - description: "Linux image to build" - type: string - default: debian - build_type: - required: true - description: Build type (source/binary) - type: string - default: source - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_PASSWORD: - required: true - -jobs: - image: - name: Build Image - runs-on: ubuntu-latest - steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Set Vars - id: set_vars - run: | - echo "GITHUB_SHA_SHORT=${GITHUB_SHA::7}" >> $GITHUB_ENV - echo "GITHUB_REF_SHORT=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "DOCKER_PUSH=${{ (secrets.DOCKERHUB_USERNAME != '') && (secrets.DOCKERHUB_PASSWORD != '') }}" >> $GITHUB_ENV - - name: Set up QEMU - id: docker_qemu - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - id: docker_buildx - uses: docker/setup-buildx-action@v2 - - name: Extract branch name - id: extract_branch - if: ${{ github.event_name != 'pull_request' }} - run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - - name: Extract branch name (PR) - id: extract_branch_pr - if: ${{ github.event_name == 'pull_request' }} - run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF})" >> $GITHUB_ENV - - name: Docker Metadata - id: docker_metadata - uses: docker/metadata-action@v4 - with: - images: | - blockstack/stacks-blockchain - blockstack/${{ github.event.repository.name }} - tags: | - type=raw,value=${{ env.BRANCH_NAME }} - type=ref,event=pr - - name: Login to DockerHub - id: docker_login - uses: docker/login-action@v2 - # Only attempt login and push if we have credentials - if: env.DOCKER_PUSH == 'true' - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build and Push - id: docker_build - uses: docker/build-push-action@v3 - with: - file: ./.github/actions/dockerfiles/Dockerfile.${{ inputs.linux_version }}-${{ inputs.build_type }} - platforms: ${{ inputs.docker_platforms }} - tags: ${{ steps.docker_metadata.outputs.tags }} - labels: ${{ steps.docker_metadata.outputs.labels }} - build-args: | - STACKS_NODE_VERSION=${{ env.GITHUB_SHA_SHORT }} - GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} - GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} - push: ${{ env.DOCKER_PUSH }} From f07190e65a507e9fc9733668bf86d069f1a2e4e7 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Sun, 19 Nov 2023 22:37:19 -0800 Subject: [PATCH 14/24] (3847) - Adding updated files --- .github/workflows/bitcoin-tests.yml | 251 ++++--------- .github/workflows/ci.yml | 333 ++++++++++-------- .github/workflows/clarity-js-sdk-pr.yml | 11 +- .github/workflows/docs-pr.yml | 19 +- .github/workflows/github-release.yml | 75 +++- .github/workflows/stacks-blockchain-tests.yml | 187 ++++++---- docs/ci-release.md | 248 +++++++------ 7 files changed, 602 insertions(+), 522 deletions(-) diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 9aba4e625..48e53915c 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -1,205 +1,100 @@ -## -## Bitcoin Integration Tests -## +## Github workflow to run bitcoin tests -name: Bitcoin Integration Tests - -# Only run when: -# - PRs are (re)opened against master branch +name: Tests::Bitcoin on: - pull_request: - types: - - opened - - reopened + workflow_call: + +defaults: + run: + shell: bash + +## env vars are transferred to composite action steps +env: + BITCOIND_TEST: 1 + RUST_BACKTRACE: full + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 + RETRIES: 3 + RETRY_DELAY: 10000 + TEST_TIMEOUT: 30 + TEST_RETRIES: 2 concurrency: - group: stacks-bitcoin-integration-tests-${{ github.ref }} - # Only cancel in progress if this is for a PR + group: bitcoin-tests-${{ github.head_ref || github.ref || github.run_id}} + ## Only cancel in progress if this is for a PR cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - # Create bitcoin image used for later tests - build-integration-image: - name: Build Image + # Bitcoin integration tests with code coverage + integration-tests: + name: Integration Tests runs-on: ubuntu-latest - steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - - name: Reclaim disk space - id: cleanup - run: | - sudo apt-get update - sudo apt-get remove -y '^dotnet-.*' - sudo apt-get remove -y '^llvm-.*' - sudo apt-get remove -y 'php.*' - sudo apt-get remove -y '^mongodb-.*' - sudo apt-get remove -y '^mysql-.*' - sudo apt-get autoremove -y - sudo apt-get clean - docker system prune --force - - - name: Build bitcoin integration testing image - id: build_docker_image - env: - DOCKER_BUILDKIT: 1 - # Remove .dockerignore file so codecov has access to git info and build the image - run: | - rm .dockerignore - docker build -f ./.github/actions/bitcoin-int-tests/Dockerfile.generic.bitcoin-tests -t stacks-blockchain:integrations . - - - name: Export docker image as tarball - id: export_docker_image - run: docker save stacks-blockchain:integrations | gzip > integration-image.tar.gz - - - name: Upload built docker image - id: upload_docker_image - uses: actions/upload-artifact@v3 - with: - name: integration-image.tar.gz - path: integration-image.tar.gz - - # Run integration tests using sampled genesis block - sampled-genesis: - name: Sampled Genesis - runs-on: ubuntu-latest - needs: - - build-integration-image strategy: + ## Continue with the test matrix even if we've had a failure fail-fast: false + ## Run a maximum of 32 concurrent tests from the test matrix + max-parallel: 32 matrix: test-name: - - tests::neon_integrations::miner_submit_twice - - tests::neon_integrations::microblock_integration_test - - tests::neon_integrations::microblock_fork_poison_integration_test - - tests::neon_integrations::size_check_integration_test - - tests::neon_integrations::cost_voting_integration - - tests::integrations::integration_test_get_info - - tests::neon_integrations::bitcoind_integration_test - - tests::neon_integrations::liquid_ustx_integration - - tests::neon_integrations::stx_transfer_btc_integration_test - - tests::neon_integrations::stx_delegate_btc_integration_test - - tests::neon_integrations::bitcoind_forking_test - - tests::neon_integrations::should_fix_2771 - - tests::neon_integrations::pox_integration_test - - tests::neon_integrations::mining_events_integration_test - tests::bitcoin_regtest::bitcoind_integration_test - - tests::should_succeed_handling_malformed_and_valid_txs + - tests::integrations::integration_test_get_info + - tests::neon_integrations::antientropy_integration_test ## forced failure + - tests::neon_integrations::bad_microblock_pubkey + - tests::neon_integrations::bitcoind_forking_test + - tests::neon_integrations::bitcoind_integration_test + - tests::neon_integrations::block_large_tx_integration_test + - tests::neon_integrations::block_limit_hit_integration_test + - tests::neon_integrations::cost_voting_integration + - tests::neon_integrations::filter_long_runtime_tx_integration_test + - tests::neon_integrations::filter_low_fee_tx_integration_test + - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window10 + - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window5 + - tests::neon_integrations::liquid_ustx_integration + - tests::neon_integrations::microblock_fork_poison_integration_test + - tests::neon_integrations::microblock_integration_test + - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY + - tests::neon_integrations::microblock_limit_hit_integration_test + - tests::neon_integrations::miner_submit_twice + - tests::neon_integrations::mining_events_integration_test + - tests::neon_integrations::pox_integration_test + - tests::neon_integrations::push_boot_receipts + - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test + - tests::neon_integrations::should_fix_2771 + - tests::neon_integrations::size_check_integration_test + - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test - tests::neon_integrations::size_overflow_unconfirmed_microblocks_integration_test - tests::neon_integrations::size_overflow_unconfirmed_stream_microblocks_integration_test - - tests::neon_integrations::size_overflow_unconfirmed_invalid_stream_microblocks_integration_test - - tests::neon_integrations::runtime_overflow_unconfirmed_microblocks_integration_test - - tests::neon_integrations::antientropy_integration_test - - tests::neon_integrations::filter_low_fee_tx_integration_test - - tests::neon_integrations::filter_long_runtime_tx_integration_test - - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY - - tests::neon_integrations::block_large_tx_integration_test - - tests::neon_integrations::microblock_limit_hit_integration_test - - tests::neon_integrations::block_limit_hit_integration_test - - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window5 - - tests::neon_integrations::fuzzed_median_fee_rate_estimation_test_window10 - - tests::neon_integrations::use_latest_tip_integration_test - - tests::neon_integrations::test_flash_block_skip_tenure + - tests::neon_integrations::stx_delegate_btc_integration_test + - tests::neon_integrations::stx_transfer_btc_integration_test - tests::neon_integrations::test_chainwork_first_intervals - tests::neon_integrations::test_chainwork_partial_interval - - tests::neon_integrations::test_problematic_txs_are_not_stored + - tests::neon_integrations::test_flash_block_skip_tenure - tests::neon_integrations::test_problematic_blocks_are_not_mined - tests::neon_integrations::test_problematic_blocks_are_not_relayed_or_stored - tests::neon_integrations::test_problematic_microblocks_are_not_mined - tests::neon_integrations::test_problematic_microblocks_are_not_relayed_or_stored - - tests::neon_integrations::push_boot_receipts - - tests::epoch_205::test_dynamic_db_method_costs - - tests::epoch_205::transition_empty_blocks - - tests::epoch_205::test_cost_limit_switch_version205 - - tests::epoch_205::test_exact_block_costs - - tests::epoch_205::bigger_microblock_streams_in_2_05 - - tests::epoch_21::transition_adds_burn_block_height - - tests::epoch_21::transition_fixes_bitcoin_rigidity - - tests::epoch_21::transition_adds_pay_to_contract - - tests::epoch_21::transition_adds_get_pox_addr_recipients - - tests::epoch_21::transition_adds_mining_from_segwit - - tests::epoch_21::transition_removes_pox_sunset - - tests::epoch_21::transition_empty_blocks - - tests::epoch_21::test_pox_reorgs_three_flaps - - tests::epoch_21::test_pox_reorg_one_flap - - tests::epoch_21::test_pox_reorg_flap_duel - - tests::epoch_21::test_pox_reorg_flap_reward_cycles - - tests::epoch_21::test_pox_missing_five_anchor_blocks - - tests::epoch_21::test_sortition_divergence_pre_21 - - tests::epoch_21::test_v1_unlock_height_with_current_stackers - - tests::epoch_21::test_v1_unlock_height_with_delay_and_current_stackers - - tests::epoch_21::trait_invocation_cross_epoch - - tests::epoch_22::pox_2_unlock_all - - tests::epoch_22::disable_pox - - tests::epoch_22::test_pox_reorg_one_flap - - tests::epoch_23::trait_invocation_behavior - - tests::neon_integrations::bad_microblock_pubkey - - tests::epoch_24::fix_to_pox_contract - - tests::epoch_24::verify_auto_unlock_behavior + - tests::neon_integrations::test_problematic_txs_are_not_stored + - tests::neon_integrations::use_latest_tip_integration_test + - tests::should_succeed_handling_malformed_and_valid_txs steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Download docker image - id: download_docker_image - uses: actions/download-artifact@v3 - with: - name: integration-image.tar.gz - - name: Load docker image - id: load_docker_image - run: docker load -i integration-image.tar.gz && rm integration-image.tar.gz - - name: All integration tests with sampled genesis - id: bitcoin_integration_tests - timeout-minutes: 30 - env: - DOCKER_BUILDKIT: 1 - TEST_NAME: ${{ matrix.test-name }} - run: docker build -o coverage-output --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests . - - name: Code Coverage - id: code_coverage - uses: codecov/codecov-action@v3 - with: - files: ./coverage-output/lcov.info - name: ${{ matrix.test-name }} - fail_ci_if_error: false + ## Setup test environment + - name: Setup Test Environment + id: setup_tests + uses: stacks-network/actions/stacks-core/testenv@main - # Run atlas integration tests - atlas-test: - name: Atlas Test - runs-on: ubuntu-latest - needs: - - build-integration-image - strategy: - fail-fast: false - matrix: - test-name: - - tests::neon_integrations::atlas_integration_test - - tests::neon_integrations::atlas_stress_integration_test - steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Download docker image - id: download_docker_image - uses: actions/download-artifact@v3 + ## Run test matrix using restored cache of archive file + ## - Test will timeout after env.TEST_TIMEOUT minutes + - name: Run Tests + id: run_tests + timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT) }} + uses: stacks-network/actions/stacks-core/run-tests@main with: - name: integration-image.tar.gz - - name: Load docker image - id: load_docker_image - run: docker load -i integration-image.tar.gz && rm integration-image.tar.gz - - name: Atlas integration tests - id: atlas_integration_tests - timeout-minutes: 40 - env: - DOCKER_BUILDKIT: 1 - TEST_NAME: ${{ matrix.test-name }} - run: docker build -o coverage-output --build-arg test_name=${{ matrix.test-name }} -f ./.github/actions/bitcoin-int-tests/Dockerfile.bitcoin-tests . + test-name: ${{ matrix.test-name }} + threads: 1 + + ## Create and upload code coverage file - name: Code Coverage - id: code_coverage - uses: codecov/codecov-action@v3 + id: codecov + uses: stacks-network/actions/codecov@main with: - files: ./coverage-output/lcov.info - name: ${{ matrix.test-name }} - fail_ci_if_error: false + test-name: ${{ matrix.test-name }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b72c9faf..907517226 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,180 +1,223 @@ +## The main Github Actions workflow name: CI -## Only run when: -## - manually triggered -## - PR's are (re)opened -## - push to master (i.e. merge develop -> master) - on: push: branches: - master - pull_request: + - develop + paths-ignore: + - "**.md" + - "**.yml" workflow_dispatch: inputs: tag: description: "The tag to create (optional)" required: false + pull_request_target: + types: + - opened + - reopened + - synchronize + paths-ignore: + - "**.md" + - "**.yml" + ## might be better to use inclusive v exclusive paths here, ex: + # paths: + # - "**.rs" + # - "**.clar" + pull_request_review: + types: + - submitted + +defaults: + run: + shell: bash concurrency: - group: ${{ github.head_ref || github.run_id }} + group: ci-${{ github.head_ref || github.ref }} + ## Always cancel duplicate jobs cancel-in-progress: true +run-name: ${{ inputs.tag }} + jobs: - ## rust format: Execute on every run + ## + ## Jobs to execute everytime workflow runs + ## do not run if the trigger is any of the following: + ## - PR review submitted (not approved) + ## and any of: + ## - PR review comment + ## - PR change is requested rustfmt: + if: | + !( + github.event_name == 'pull_request_review' && + github.event.action == 'submitted' && + ( + github.event.review.state == 'commented' || + github.event.review.state == 'changes_requested' + ) + ) name: Rust Format runs-on: ubuntu-latest steps: - name: Checkout the latest code id: git_checkout - uses: actions/checkout@v3 - - name: Define Rust Toolchain - id: define_rust_toolchain - run: echo "RUST_TOOLCHAIN=$(cat ./rust-toolchain)" >> $GITHUB_ENV + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Setup Rust Toolchain id: setup_rust_toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@f3c84ee10bf5a86e7a5d607d487bf17d57670965 # v1.5.0 with: - toolchain: ${{ env.RUST_TOOLCHAIN }} components: rustfmt - - name: Rustfmt + cache: false + + - name: Rustfmt id: rustfmt - uses: actions-rust-lang/rustfmt@v1 + uses: actions-rust-lang/rustfmt@2d1d4e9f72379428552fa1def0b898733fb8472d # v1.1.0 - ## Release tests: Execute on every run - release-tests: - name: Release Tests - uses: stacks-network/stacks-blockchain/.github/workflows/stacks-blockchain-tests.yml@master - - ## Checked for leaked credentials: Execute on every run - leaked-cred-test: - name: Leaked Credential Test - runs-on: ubuntu-latest - steps: - - name: Extract branch name - id: extract_branch - if: ${{ github.event_name != 'pull_request' }} - run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV - - name: Extract branch name - id: extract_branch_pr - if: ${{ github.event_name == 'pull_request' }} - run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF})" >> $GITHUB_ENV - - name: Branch name - run: echo running on branch ${{ env.BRANCH_NAME }} - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: TruffleHog Scan - id: trufflehog_check - uses: trufflesecurity/trufflehog@main - with: - path: ./ - base: ${{ env.BRANCH_NAME }} - head: HEAD - - ############################################### - ## Build Tagged Release - ############################################### - ## Build source binaries - ## Only run if: - ## - Tag is provided - ## - OR - ## - Not the default branch - ## - AND - ## - Not a PR - build-source: - if: ${{ inputs.tag != '' || (github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && !contains(github.ref, 'refs/pull')) }} - name: Build Binaries - uses: stacks-network/stacks-blockchain/.github/workflows/build-source-binary.yml@master + ###################################################################################### + ## Create a tagged github release + ## + ## Runs when the following is true: + ## - tag is provided + ## - workflow is building default branch (master) + create-release: + if: | + inputs.tag != '' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + name: Create Release needs: - rustfmt - - release-tests - - leaked-cred-test + uses: ./.github/workflows/github-release.yml with: tag: ${{ inputs.tag }} - parallel_jobs: 4 - arch: >- - ["linux-glibc-x64", "linux-musl-x64", "linux-glibc-arm64", "linux-musl-arm64", "macos-x64", "macos-arm64", "windows-x64"] + secrets: inherit - ## Create github release with binary archives - ## Only run if: - ## - Tag is provided - ## - OR - ## - Not the default branch - ## - AND - ## - Not a PR - github-release: - if: ${{ inputs.tag != '' || (github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && !contains(github.ref, 'refs/pull')) }} - name: Github Release - uses: stacks-network/stacks-blockchain/.github/workflows/github-release.yml@master - needs: build-source - with: - tag: ${{ inputs.tag }} - arch: >- - ["linux-glibc-x64", "linux-musl-x64", "linux-glibc-arm64", "linux-musl-arm64", "macos-x64", "macos-arm64", "windows-x64"] - secrets: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - - ## Create docker alpine images - ## Only run if: - ## - Tag is provided - ## - OR - ## - Not the default branch - ## - AND - ## - Not a PR - docker-alpine: - if: ${{ inputs.tag != '' || (github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && !contains(github.ref, 'refs/pull')) }} - name: Docker Alpine (Binary) - uses: stacks-network/stacks-blockchain/.github/workflows/image-build-alpine-binary.yml@master - needs: github-release - with: - tag: ${{ inputs.tag }} - docker_platforms: linux/arm64, linux/amd64, linux/amd64/v2, linux/amd64/v3 - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - - ## Create docker debian images - ## Only run if: - ## - Tag is provided - ## - OR - ## - Not the default branch - ## - AND - ## - Not a PR - docker-debian: - if: ${{ inputs.tag != '' || (github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && !contains(github.ref, 'refs/pull')) }} - name: Docker Debian (Binary) - uses: stacks-network/stacks-blockchain/.github/workflows/image-build-debian-binary.yml@master - needs: github-release - with: - tag: ${{ inputs.tag }} - docker_platforms: linux/amd64, linux/amd64/v2, linux/amd64/v3 - linux_version: debian - build_type: binary - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - - ############################################### - ## Build Branch/PR - ############################################### - ## Create docker debian images - ## Only run if: - ## - Tag is *not* provided - build-branch: - if: ${{ inputs.tag == '' }} - name: Docker Debian (Source) - uses: stacks-network/stacks-blockchain/.github/workflows/image-build-debian-source.yml@master + ## Build and push Debian image built from source + ## + ## Runs when: + ## - tag is not provided + ## and the following are not true: + ## - PR review submitted (not approved) + ## and any of: + ## - PR review comment + ## - PR change is requested + docker-image: + if: | + inputs.tag == '' && + !( + github.event_name == 'pull_request_review' && + github.event.action == 'submitted' && + ( + github.event.review.state == 'commented' || + github.event.review.state == 'changes_requested' + ) + ) + name: Docker Image (Source) + uses: ./.github/workflows/image-build-source.yml needs: - rustfmt - - leaked-cred-test - with: - docker_platforms: linux/amd64 - linux_version: debian - build_type: source - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + secrets: inherit + + ## Create a reusable cache for tests + ## + ## Runs when: + ## - tag is provided + ## or: + ## - no tag provided + ## and any of: + ## - PR is approved (any approval will trigger) + ## - this workflow is called manually + ## - PR is opened + ## - commit to either (development, master) branch + create-cache: + if: | + inputs.tag != '' || ( + inputs.tag == '' && ( + ( + github.event_name == 'pull_request_review' && + github.event.action == 'submitted' && + github.event.review.state == 'approved' + ) || + github.event_name == 'workflow_dispatch' || + github.event_name == 'pull_request_target' || + ( + contains(' + refs/heads/develoment + refs/heads/master + ', github.event.pull_request.head.ref) && + github.event_name == 'push' + ) + ) + ) + name: Create Test Cache + needs: + - rustfmt + uses: ./.github/workflows/create-cache.yml + + ## Tests to run regularly + ## + ## Runs when: + ## - tag is provided + ## either or of the following: + ## - tag is not provided + ## - PR is approved + stacks-blockchain-tests: + if: | + inputs.tag != '' || ( + inputs.tag == '' || ( + github.event_name == 'pull_request_review' && + github.event.action == 'submitted' && + github.event.review.state == 'approved' + ) + ) + name: Stacks Blockchain Tests + needs: + - rustfmt + - create-cache + uses: ./.github/workflows/stacks-blockchain-tests.yml + + bitcoin-tests: + if: | + inputs.tag != '' || ( + inputs.tag == '' || ( + github.event_name == 'pull_request_review' && + github.event.action == 'submitted' && + github.event.review.state == 'approved' + ) + ) + name: Bitcoin Tests + needs: + - rustfmt + - create-cache + uses: ./.github/workflows/bitcoin-tests.yml + + ## Test to run on a tagged release + ## + ## Runs when: + ## - tag is provided + atlas-tests: + if: inputs.tag != '' + name: Atlas Tests + needs: + - rustfmt + - create-cache + uses: ./.github/workflows/atlas-tests.yml + + epoch-tests: + if: inputs.tag != '' + name: Epoch Tests + needs: + - rustfmt + - create-cache + uses: ./.github/workflows/epoch-tests.yml + + slow-tests: + if: inputs.tag != '' + name: Slow Tests + needs: + - rustfmt + - create-cache + uses: ./.github/workflows/slow-tests.yml diff --git a/.github/workflows/clarity-js-sdk-pr.yml b/.github/workflows/clarity-js-sdk-pr.yml index 9ac0956a8..25e96f9f2 100644 --- a/.github/workflows/clarity-js-sdk-pr.yml +++ b/.github/workflows/clarity-js-sdk-pr.yml @@ -4,6 +4,10 @@ name: Open Clarity JS SDK PR +defaults: + run: + shell: bash + env: CLARITY_JS_SDK_REPOSITORY: stacks-network/clarity-js-sdk COMMIT_USER: Hiro DevOps @@ -24,22 +28,25 @@ jobs: steps: - name: Checkout latest clarity js sdk id: git_checkout - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: token: ${{ secrets.GH_TOKEN }} repository: ${{ env.CLARITY_JS_SDK_REPOSITORY }} ref: master + - name: Determine Release Version id: get_release_version run: | RELEASE_VERSION=$(echo ${GITHUB_REF#refs/*/} | tr / -) echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV + - name: Update SDK Tag id: update_sdk_tag run: sed -i "s@CORE_SDK_TAG = \".*\"@CORE_SDK_TAG = \"$RELEASE_VERSION\"@g" packages/clarity-native-bin/src/index.ts + - name: Create Pull Request id: create_pr - uses: peter-evans/create-pull-request@v4 + uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5.0.2 with: token: ${{ secrets.GH_TOKEN }} commit-message: "chore: update clarity-native-bin tag" diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml index b2a44f729..bb54c28cc 100644 --- a/.github/workflows/docs-pr.yml +++ b/.github/workflows/docs-pr.yml @@ -9,6 +9,10 @@ name: Open Docs PR +defaults: + run: + shell: bash + env: ROBOT_OWNER: kantai-robot ROBOT_REPO: docs.blockstack @@ -16,9 +20,8 @@ env: TARGET_REPO: docs TARGET_REPOSITORY: stacks-network/docs -# Only run when: -# - push to master - +## Only run when: +## - push to master on: push: branches: @@ -33,19 +36,22 @@ jobs: steps: - name: Checkout the latest code id: git_checkout - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Build docs id: build_docs env: DOCKER_BUILDKIT: 1 run: rm -rf docs-output && docker build -o docs-output -f ./.github/actions/docsgen/Dockerfile.docsgen . + - name: Checkout latest docs id: git_checkout_docs - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: token: ${{ secrets.DOCS_GITHUB_TOKEN }} repository: ${{ env.TARGET_REPOSITORY }} path: docs + - name: Branch and commit id: push run: | @@ -67,10 +73,11 @@ jobs: git push robot $ROBOT_BRANCH echo "::set-output name=open_pr::1" fi + - name: Open PR id: open_pr if: ${{ steps.push.outputs.open_pr == '1' }} - uses: actions/github-script@v6 + uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1 with: github-token: ${{ secrets.DOCS_GITHUB_TOKEN }} script: | diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml index c0683f51d..14e7117a9 100644 --- a/.github/workflows/github-release.yml +++ b/.github/workflows/github-release.yml @@ -1,50 +1,75 @@ -## -## Create the github release and store artifact files (with checksum) -## +## Github workflow to create a github release and upload binary artifacts name: Github Release -# Only run when: -# - manually triggered via the ci.yml workflow - on: workflow_call: inputs: tag: - required: true - type: string - arch: - description: "Stringified JSON object listing of platform matrix" + description: "Release Tag" required: true type: string secrets: GH_TOKEN: required: true +concurrency: + group: github-release-${{ github.head_ref || github.ref }} + ## Always cancel duplicate jobs + cancel-in-progress: true + +run-name: ${{ inputs.tag }} + jobs: + ## Build arch dependent binaries from source + ## + ## Runs when the following is true: + ## - tag is provided + ## - workflow is building default branch (master) + build-binaries: + if: | + inputs.tag != '' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + name: Build Binaries + uses: ./.github/workflows/create-source-binary.yml + with: + tag: ${{ inputs.tag }} + secrets: inherit + + ## Runs when the following is true: + ## - tag is provided + ## - workflow is building default branch (master) create-release: - if: ${{ inputs.tag != '' }} + if: | + inputs.tag != '' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) name: Create Release runs-on: ubuntu-latest + needs: + - build-binaries steps: + ## Downloads the artifacts built in `create-source-binary.yml` - name: Download Artifacts id: download_artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: artifact path: release - # Generate a checksums file to be added to the release page + + ## Generate a checksums file to be added to the release page - name: Generate Checksums id: generate_checksum - uses: jmgilman/actions-generate-checksum@v1 + uses: jmgilman/actions-generate-checksum@24a35957fba81c6cbaefeb1e3d59ee56e3db5077 # v1.0.0 with: + method: sha512 output: CHECKSUMS.txt patterns: | release/*.zip - # Upload the release archives with the checksums file + + ## Upload the release archives with the checksums file - name: Upload Release id: upload_release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 #v0.1.15 env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} with: @@ -56,3 +81,21 @@ jobs: files: | release/*.zip CHECKSUMS.txt + + ## Builds arch dependent Docker images from binaries + ## + ## Runs when the following is true: + ## - tag is provided + ## - workflow is building default branch (master) + docker-image: + if: | + inputs.tag != '' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + name: Docker Image (Binary) + uses: ./.github/workflows/image-build-binary.yml + needs: + - build-binaries + - create-release + with: + tag: ${{ inputs.tag }} + secrets: inherit diff --git a/.github/workflows/stacks-blockchain-tests.yml b/.github/workflows/stacks-blockchain-tests.yml index fb1dffc1a..fa415a9cd 100644 --- a/.github/workflows/stacks-blockchain-tests.yml +++ b/.github/workflows/stacks-blockchain-tests.yml @@ -1,95 +1,138 @@ -## -## Run tests for tagged releases -## +## Github workflow to run full genesis and unit tests -name: Tests - -# Only run when: -# - manually triggered via the ci.yml workflow +name: Tests::Stacks Blockchain on: workflow_call: +defaults: + run: + shell: bash + +## env vars are transferred to composite action steps +env: + BITCOIND_TEST: 1 + RUST_BACKTRACE: full + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + RETRIES: 3 + RETRY_DELAY: 10000 + TEST_TIMEOUT: 30 + TEST_RETRIES: 2 + +concurrency: + group: stacks-blockchain-tests-${{ github.head_ref || github.ref || github.run_id }} + ## Only cancel in progress if this is for a PR + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: - # Run full genesis test + # Full genesis test with code coverage full-genesis: name: Full Genesis Test runs-on: ubuntu-latest + strategy: + ## Continue with the test matrix even if we've had a failure + fail-fast: false + ## Run a maximum of 2 concurrent tests from the test matrix + max-parallel: 2 + matrix: + test-name: + - neon_integrations::bitcoind_integration_test steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Single full genesis integration test - id: full_genesis_test - env: - DOCKER_BUILDKIT: 1 - # Remove .dockerignore file so codecov has access to git info - run: | - rm .dockerignore - docker build -o coverage-output -f ./.github/actions/bitcoin-int-tests/Dockerfile.large-genesis . - - name: Large Genesis Codecov - id: full_genesis_codecov - uses: codecov/codecov-action@v3 + ## Setup test environment + - name: Setup Test Environment + id: setup_tests + uses: stacks-network/actions/stacks-core/testenv@main with: - files: ./coverage-output/lcov.info - name: large_genesis - fail_ci_if_error: false + genesis: true - # Run unit tests with code coverage + ## Run test matrix using restored cache of archive file + ## - Test will timeout after env.TEST_TIMEOUT minutes + - name: Run Tests + id: run_tests + timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT) }} + uses: stacks-network/actions/stacks-core/run-tests@main + with: + test-name: ${{ matrix.test-name }} + threads: 1 + archive-file: /tmp/genesis_archive.tar.zst + + ## Upload code coverage file + - name: Code Coverage + id: codecov + uses: stacks-network/actions/codecov@main + with: + test-name: large_genesis + filename: ./lcov.info + + - name: Status Output + run: | + echo "run_tests: ${{ steps.run_tests.outputs.status }}" + echo "codecov: ${{ steps.codecov.outputs.status }}" + + - name: Check Failures + if: steps.run_tests.outputs.status == 'failure' || steps.codecov.outputs.status == 'failure' + run: exit 1 + + # Unit tests with code coverage unit-tests: name: Unit Tests runs-on: ubuntu-latest + ## Continue the workflow in case a step fails (ex a single test fails) + continue-on-error: true + strategy: + ## Continue the workflow in case a step fails (ex a single test fails) + fail-fast: false + matrix: + ## Partition the tests into 8 jobs + ## - This is used in a later step when running `cargo nextest run ... --partition count:num/8` + partition: [1, 2, 3, 4, 5, 6, 7, 8] steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Run unit tests (with coverage) - id: unit_tests_codecov - env: - DOCKER_BUILDKIT: 1 - # Remove .dockerignore file so codecov has access to git info - run: | - rm .dockerignore - docker build -o coverage-output -f ./.github/actions/bitcoin-int-tests/Dockerfile.code-cov . - - name: Run unit tests - id: codedov - uses: codecov/codecov-action@v3 - with: - files: ./coverage-output/lcov.info - name: unit_tests - fail_ci_if_error: false + ## Setup test environment + - name: Setup Test Environment + id: setup_tests + uses: stacks-network/actions/stacks-core/testenv@main + ## Run test matrix using restored cache of archive file + ## - Test will timeout after env.TEST_TIMEOUT minutes + - name: Run Tests + id: run_tests + timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT) }} + uses: stacks-network/actions/stacks-core/run-tests/partition@main + with: + partition: ${{ matrix.partition }} + total-partitions: 8 + + ## Create and upload code coverage file + - name: Code Coverage + id: codecov + uses: stacks-network/actions/codecov@main + with: + test-name: ${{ matrix.test-name }} + + ## Generate and upload openapi html artifact open-api-validation: name: OpenAPI Validation runs-on: ubuntu-latest steps: - - name: Checkout the latest code - id: git_checkout - uses: actions/checkout@v3 - - name: Run units tests (with coverage) - id: api_codecov - env: - DOCKER_BUILDKIT: 1 - run: docker build -o dist/ -f .github/actions/open-api/Dockerfile.open-api-validate . - - name: Upload bundled html - id: upload_html_artifact - uses: actions/upload-artifact@v3 + - name: OpenAPI + id: openapi + uses: stacks-network/actions/openapi@main with: - name: open-api-bundle - path: | - dist + input: ./docs/rpc/openapi.yaml + output: ./open-api-docs.html - # Run net-tests + ## Disabled + ## - this test can take several hours to run nettest: - # disable this job/test for now, since we haven't seen this pass - # on github actions in a while, and the failures can take > 4 hours if: ${{ false }} name: Net-Test runs-on: ubuntu-latest steps: + ## checkout the code - name: Checkout the latest code id: git_checkout - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Run network relay tests id: nettest env: @@ -101,17 +144,23 @@ jobs: name: Core Contracts Test runs-on: ubuntu-latest steps: + ## Checkout the code - name: Checkout the latest code id: git_checkout - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + ## Use Clarinet to run contract unit-tests and create code coverage file - name: Execute core contract unit tests in Clarinet id: clarinet_unit_test - uses: docker://hirosystems/clarinet:1.1.0 + uses: docker://hirosystems/clarinet:1.8.0 with: args: test --coverage --manifest-path=./contrib/core-contract-tests/Clarinet.toml - - name: Export code coverage - id: clarinet_codecov - uses: codecov/codecov-action@v3 + + ## Upload code coverage file + - name: Code Coverage + id: codecov + uses: stacks-network/actions/codecov@main with: - files: ./coverage.lcov - verbose: true + test-name: ${{ matrix.test-name }} + upload-only: true + filename: ./coverage.lcov diff --git a/docs/ci-release.md b/docs/ci-release.md index 7025226d1..82fe164ec 100644 --- a/docs/ci-release.md +++ b/docs/ci-release.md @@ -3,148 +3,184 @@ All releases are built via a Github Actions workflow named `CI`, and is responsible for building binary archives, checksums, and resulting docker images. This workflow will also trigger any tests that need to be run, like integration tests. -1. Releases are only created if a tag is manually provided when the ci workflow is triggered. -2. Pushing a new feature branch: Nothing is triggered automatically. PR's are required, or the ci workflow can be triggered manually on a specific branch to build a docker image for the specified branch. - -The following workflow steps are currently disabled: - -- Clippy -- Net-test -- Crate audit +1. Releases are only created if a tag is **manually** provided when the ci workflow is triggered. +2. Pushing a new feature branch: Nothing is triggered automatically. A PR is required, or the ci workflow can be triggered manually on a specific branch to build a docker image (and run the standard PR tests) for the specified branch. +3. Caching is used to speed up testing - a cache is created based on the type of data (i.e. cargo) and the commit sha. tests can be retried quickly since the cache will persist until the cleanup job is run. +4. [nextest](https://nexte.st/) is used to run the tests from an archived file that is cached (using commit sha as a key)) + - Two [archives](https://nexte.st/book/reusing-builds.html) are created, one for genesis tests and one for generic tests (it is done this way to reduce the time spent building) + - Unit-tests are [partitioned](https://nexte.st/book/partitioning.html) and multi-threaded to speed up execution time ## TL;DR -1. A PR will produce a single image built from source on Debian with glibc with 2 tags: +1. An open/re-opened/synchronized PR will produce a single image built from source on Debian with glibc with 2 tags: - `stacks-blockchain:` - `stacks-blockchain:` 2. A merged PR from `develop` to the default branch will produce a single image built from source on Debian with glibc: - `stacks-blockchain:` 3. An untagged build of any branch will produce a single image built from source on Debian with glibc: - `stacks-blockchain:` -4. A tagged release on a non-default branch will produce 2 versions of the docker image (along with all binary archives): - - An Alpine image for several architectures tagged with: +4. A tagged release on a non-default branch will produces: + - Docker Alpine image for several architectures tagged with: - `stacks-blockchain:` - - An Debian image for several architectures tagged with: + - Docker Debian image for several architectures tagged with: - `stacks-blockchain:` -5. A tagged release on the default branch will produce 2 versions of the docker image (along with all binary archives): - - An Alpine image for several architectures tagged with: +5. A tagged release on the default branch will produce: + - Github Release of the specified tag with: + - Binary archives for several architectures + - Docker Alpine image for several architectures tagged with: - `stacks-blockchain:` - `stacks-blockchain:` - - An Debian image for several architectures tagged with: + - Docker Debian image for several architectures tagged with: - `stacks-blockchain:` - `stacks-blockchain:` -## Release workflow: +## Release workflow -1. Create a feature branch: `feat/112-fix-something` -2. PR `feat/112-fix-something` to the `develop` branch - 1. CI Workflow is automatically triggered, resulting in a pushed docker image tagged with the **branch name** and **PR number** -3. PR `develop` to the default branch - 1. CI Workflow is automatically triggered, resulting in a pushed docker image tagged with the **branch name** and **PR number** +1. Create a feature branch: `feat/fix-something` +2. PR `feat/fix-something` to the `develop` branch where the PR is numbered `112` + 1. Docker image tagged with the **branch name** and **PR number** + - ex: + - `stacks-blockchain:feat-fix-something` + - `stacks-blockchain:pr-112` + 2. CI tests are run +3. PR `develop` to the default branch where the PR is numbered `112` + 1. Docker image tagged with the **branch name** and **PR number** + - ex: + - `stacks-blockchain:feat-fix-something` + - `stacks-blockchain:pr-112` + 2. CI tests are run 4. Merge `develop` branch to the default branch - 1. CI Workflow is triggered, resulting in a pushed docker image tagged with the **default branch name** + 1. Docker image is tagged with the **default branch** `master` + - ex: + - `stacks-blockchain:master` + 2. CI tests are run 5. CI workflow is manually triggered on **non-default branch** with a version, i.e. `2.1.0.0.0-rc0` - 1. Github release for the manually input version is created with binaries - 2. Docker image pushed with tags of the **input version** and **branch** + 1. No Docker images/binaries are created + 2. All release tests are run 6. CI workflow is manually triggered on **default branch** with a version, i.e. `2.1.0.0.0` 1. Github release for the manually input version is created with binaries - 2. Docker image pushed with tags of the **input version** and **latest** + 2. All release tests are run + 3. Docker image pushed with tags of the **input version** and **latest** + - ex: + - `stacks-blockchain:2.1.0.0.0-debian` + - `stacks-blockchain:latest-debian` + - `stacks-blockchain:2.1.0.0.0` + - `stacks-blockchain:latest` -## PR a branch to develop: +## Tests -ex: Branch is named `feat/112-fix-something` and the PR is numbered `112` +Tests are separated into several different workflows, with the intention that they can be _conditionally_ run depending upon the triggering operation. For example, on a PR synchronize we don't want to run some identified "slow" tests, but we do want to run the [Stacks Blockchain Tests](../.github/workflows/stacks-blockchain-tests.yml) and [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml). -- Steps executed: - - Rust Format - - Integration Tests - - Leaked credential test - - Docker image is built from source on a debian distribution and pushed with the branch name and PR number as tags - - ex: - - `stacks-blockchain:feat-112-fix-something` - - `stacks-blockchain:pr-112` -- Steps _not_ executed: - - No binaries are built - - No github release - - No docker images built from binary artifacts +There are also 2 different methods in use with regard to running tests: -## Merging a branch to develop: +1. [Github Actions matrix](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs) +2. [nextest partitioning](https://nexte.st/book/partitioning.html) + +A matrix is used when there are several known tests that need to be run. Partitions (shards) are used when there is a large and unknown number of tests to run (ex: `cargo test` to run all tests). + +There is also a workflow designed to run tests that are manually triggered: [Standalone Tests](../.github/workflows/standalone-tests.yml). +This workflow requires you to select which test(s) you want to run, which then triggers a reusbale workflow via conditional. For example, selecting "Epoch Tests" will run the tests defined in [Epoch Tests](../.github/workflows/epoch-tests.yml). Likewise, selecting `Release Tests` will run the same tests as a release workflow. + +Files: + +- [Standalone Tests](../.github/workflows/standalone-tests.yml) +- [Stacks Blockchain Tests](../.github/workflows/stacks-blockchain-tests.yml) +- [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) +- [Atlas Tests](../.github/workflows/atlas-tests.yml) +- [Epoch Tests](../.github/workflows/epoch-tests.yml) +- [Slow Tests](../.github/workflows/slow-tests.yml) + +## Triggering a workflow + +### PR a branch to develop + +ex: Branch is named `feat/fix-something` and the PR is numbered `112` + +- [Rust format](../.github/workflows/ci.yml) +- [Create Test Cache](../.github/workflows/create-cache.yml) +- [Stacks Blockchain Tests](../.github/workflows/stacks-blockchain-tests.yml) +- [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) +- [Docker image](../.github/workflows/image-build-source.yml) is built from source on a debian distribution and pushed with the branch name and PR number as tags +- ex: + - `stacks-blockchain:feat-fix-something` + - `stacks-blockchain:pr-112` + +--- + +### Merging a branch to develop Nothing is triggered automatically -## PR develop to master branches: +--- + +### PR develop to master branches ex: Branch is named `develop` and the PR is numbered `113` -- Steps executed: - - Rust format - - Integration tests - - Leaked credential test - - Docker image is built from source on a debian distribution and pushed with the branch name and PR number as tags - - ex: - - `stacks-blockchain:develop` - - `stacks-blockchain:pr-113` -- Steps _not_ executed: - - No binaries are built - - No github release - - No docker images built from binary artifacts +- [Rust format](../.github/workflows/ci.yml) +- [Create Test Cache](../.github/workflows/create-cache.yml) +- [Stacks Blockchain Tests](../.github/workflows/stacks-blockchain-tests.yml) +- [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) +- [Docker image](../.github/workflows/image-build-source.yml) is built from source on a debian distribution and pushed with the branch name and PR number as tags +- ex: + - `stacks-blockchain:develop` + - `stacks-blockchain:pr-113` -## Merging a PR from develop to master: +--- -- Steps executed: - - Rust format - - Integration tests - - Leaked credential test - - Docker image is built from source on a debian distribution and pushed with the branch name as a tag - - ex: - - `stacks-blockchain:master` -- Steps _not_ executed: - - No binaries are built - - No github release - - No docker images built from binary artifacts +### Merging a PR from develop to master -## Manually triggering workflow without tag (any branch): +- [Rust format](../.github/workflows/ci.yml) +- [Create Test Cache](../.github/workflows/create-cache.yml) +- [Stacks Blockchain Tests](../.github/workflows/stacks-blockchain-tests.yml) +- [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) +- [Docker image](../.github/workflows/image-build-source.yml) is built from source on a debian distribution and pushed with the branch name as a tag +- ex: + - `stacks-blockchain:master` -- Steps executed: - - Rust format - - Integration tests - - Leaked credential test - - Docker image is built from source on a debian distribution and pushed with the branch name as a tag - - ex: - - `stacks-blockchain:` -- Steps _not_ executed: - - No binaries are built - - No github release - - No docker images built from binary artifacts +--- -## Manually triggering workflow with tag on a non-default branch (i.e. tag of `2.1.0.0.0-rc0`): +### Manually triggering workflow without tag (any branch) -- Steps executed: - - Rust format - - Integration tests - - Leaked credential test - - Binaries built for specified architectures - - Archive and checksum files added to github release - - Github release (with artifacts/checksum) is created using the manually input tag - - Docker image built from binaries on debian/alpine distributions and pushed with the provided input tag and `latest` - - ex: - - `stacks-blockchain:2.1.0.0.0-rc0` -- Steps _not_ executed: - - No docker images built from source +- [Rust format](../.github/workflows/ci.yml) +- [Create Test Cache](../.github/workflows/create-cache.yml) +- [Stacks Blockchain Tests](../.github/workflows/stacks-blockchain-tests.yml) +- [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) +- [Docker image](../.github/workflows/image-build-source.yml) is built from source on a debian distribution and pushed with the branch name as a tag +- ex: + - `stacks-blockchain:` -## Manually triggering workflow with tag on default branch (i.e. tag of `2.1.0.0.0`): +--- -- Steps executed: - - Rust format - - Integration tests - - Leaked credential test - - Binaries built for specified architectures - - Archive and checksum files added to github release - - Github release (with artifacts/checksum) is created using the manually input tag - - Docker image built from binaries on debian/alpine distributions and pushed with the provided input tag and `latest` - - ex: - - `stacks-blockchain:2.1.0.0.0-debian` - - `stacks-blockchain:latest-debian` - - `stacks-blockchain:2.1.0.0.0` - - `stacks-blockchain:latest` -- Steps _not_ executed: - - No docker images built from source +### Manually triggering workflow with tag on a non-default branch (i.e. tag of `2.1.0.0.0-rc0`) + +- [Rust format](../.github/workflows/ci.yml) +- [Create Test Cache](../.github/workflows/create-cache.yml) +- [Stacks Blockchain Tests](../.github/workflows/stacks-blockchain-tests.yml) +- [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) +- [Atlas Tests](../.github/workflows/atlas-tests.yml) +- [Epoch Tests](../.github/workflows/epoch-tests.yml) +- [Slow Tests](../.github/workflows/slow-tests.yml) + +--- + +### Manually triggering workflow with tag on default branch (i.e. tag of `2.1.0.0.0`) + +- [Rust format](../.github/workflows/ci.yml) +- [Create Test Cache](../.github/workflows/create-cache.yml) +- [Stacks Blockchain Tests](../.github/workflows/stacks-blockchain-tests.yml) +- [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) +- [Atlas Tests](../.github/workflows/atlas-tests.yml) +- [Epoch Tests](../.github/workflows/epoch-tests.yml) +- [Slow Tests](../.github/workflows/slow-tests.yml) +- [Binaries built for specified architectures](../.github/workflows/create-source-binary.yml) + - Archive and checksum files added to github release +- [Github release](../.github/workflows/github-release.yml) (with artifacts/checksum) is created using the manually input tag +- [Docker image](../.github/workflows/image-build-binary.yml) built from binaries on debian/alpine distributions and pushed with the provided input tag and `latest` +- ex: + - `stacks-blockchain:2.1.0.0.0-debian` + - `stacks-blockchain:latest-debian` + - `stacks-blockchain:2.1.0.0.0` + - `stacks-blockchain:latest` + +--- From 478b8dab8950896580cf6490f1f3340a74ef4f55 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Sun, 19 Nov 2023 22:37:55 -0800 Subject: [PATCH 15/24] (3847) - Adding new workflow files --- .github/workflows/atlas-tests.yml | 61 +++++++++++ .github/workflows/create-cache.yml | 63 +++++++++++ .github/workflows/create-source-binary.yml | 74 +++++++++++++ .github/workflows/epoch-tests.yml | 85 +++++++++++++++ .github/workflows/image-build-binary.yml | 92 +++++++++++++++++ .github/workflows/image-build-source.yml | 69 +++++++++++++ .github/workflows/slow-tests.yml | 63 +++++++++++ .github/workflows/standalone-tests.yml | 115 +++++++++++++++++++++ .github/workflows/workflow-cleanup.yml | 47 +++++++++ 9 files changed, 669 insertions(+) create mode 100644 .github/workflows/atlas-tests.yml create mode 100644 .github/workflows/create-cache.yml create mode 100644 .github/workflows/create-source-binary.yml create mode 100644 .github/workflows/epoch-tests.yml create mode 100644 .github/workflows/image-build-binary.yml create mode 100644 .github/workflows/image-build-source.yml create mode 100644 .github/workflows/slow-tests.yml create mode 100644 .github/workflows/standalone-tests.yml create mode 100644 .github/workflows/workflow-cleanup.yml diff --git a/.github/workflows/atlas-tests.yml b/.github/workflows/atlas-tests.yml new file mode 100644 index 000000000..21da3c830 --- /dev/null +++ b/.github/workflows/atlas-tests.yml @@ -0,0 +1,61 @@ +## Github workflow to run atlas tests + +name: Tests::Atlas + +on: + workflow_call: + +defaults: + run: + shell: bash + +## env vars are transferred to composite action steps +env: + BITCOIND_TEST: 1 + RUST_BACKTRACE: full + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 + RETRIES: 3 + RETRY_DELAY: 10000 + TEST_TIMEOUT: 30 + TEST_RETRIES: 2 + +concurrency: + group: atlas-tests-${{ github.head_ref || github.ref || github.run_id}} + ## Only cancel in progress if this is for a PR + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # Atlas integration tests with code coverage + atlas-tests: + name: Atlas Test + runs-on: ubuntu-latest + strategy: + ## Continue with the test matrix even if we've had a failure + fail-fast: false + ## Run a maximum of 2 concurrent tests from the test matrix + max-parallel: 2 + matrix: + test-name: + - tests::neon_integrations::atlas_integration_test + - tests::neon_integrations::atlas_stress_integration_test + steps: + ## Setup test environment + - name: Setup Test Environment + id: setup_tests + uses: stacks-network/actions/stacks-core/testenv@main + + ## Run test matrix using restored cache of archive file + ## - Test will timeout after env.TEST_TIMEOUT minutes + - name: Run Tests + id: run_tests + timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT) }} + uses: stacks-network/actions/stacks-core/run-tests@main + with: + test-name: ${{ matrix.test-name }} + + ## Create and upload code coverage file + - name: Code Coverage + id: codecov + uses: stacks-network/actions/codecov@main + with: + test-name: ${{ matrix.test-name }} diff --git a/.github/workflows/create-cache.yml b/.github/workflows/create-cache.yml new file mode 100644 index 000000000..8bb736dd0 --- /dev/null +++ b/.github/workflows/create-cache.yml @@ -0,0 +1,63 @@ +## Github workflow to create reusable caches + +name: Create Test Cache + +on: + workflow_dispatch: + workflow_call: + +defaults: + run: + shell: bash + +## env vars are transferred to composite action steps +env: + RUSTFLAGS: "-Cinstrument-coverage -Awarnings" + LLVM_PROFILE_FILE: "stacks-blockchain-%p-%m.profraw" + BTC_VERSION: "0.20.0" + +## +## Cache will exist longer than workflow execution so other runners have access +## ex: a failed job should have access to the cache for however long `cleanup.yml` is set to delete old caches +## however, this is only relevant if the commit sha does not change between runs +concurrency: + group: create-cache-${{ github.head_ref || github.ref || github.run_id}} + ## Only cancel in progress if this is for a PR + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + ## Cache cargo data + cargo: + name: Cargo + runs-on: ubuntu-latest + steps: + ## Perform a lookup to check if the cache already exists + - name: Cargo Cache + id: cargo-cache + uses: stacks-network/actions/stacks-core/cache/cargo@main + with: + action: save + + ## Cache the bitcoin binary + bitcoin-binary: + name: Bitcoin Binary + runs-on: ubuntu-latest + steps: + - name: Bitcoin Cache + id: bitcoin-cache + uses: stacks-network/actions/stacks-core/cache/bitcoin@main + with: + action: save + + ## Cache nextest archives for tests + nextest-archive: + name: Test Archive + runs-on: ubuntu-latest + needs: + - cargo + steps: + - name: Build Nexttest Cache + id: nextest-cache + uses: stacks-network/actions/stacks-core/cache/build-cache@main + with: + genesis: true diff --git a/.github/workflows/create-source-binary.yml b/.github/workflows/create-source-binary.yml new file mode 100644 index 000000000..a6e70756e --- /dev/null +++ b/.github/workflows/create-source-binary.yml @@ -0,0 +1,74 @@ +## Github workflow to create multiarch binaries from source + +name: Create Binaries + +on: + workflow_call: + inputs: + tag: + description: "Tag name of this release (x.y.z)" + required: true + type: string + arch: + description: "Stringified JSON object listing of platform matrix" + required: false + type: string + default: >- + ["linux-glibc-x64", "linux-musl-x64", "linux-glibc-arm64", "linux-glibc-armv7", "linux-musl-arm64", "linux-musl-armv7", "macos-x64", "macos-arm64", "windows-x64"] + +defaults: + run: + shell: bash + +## change the display name to the tag being built +run-name: ${{ inputs.tag }} + +concurrency: + group: create-binary-${{ github.head_ref || github.ref || github.run_id}} + ## Only cancel in progress if this is for a PR + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + ## Runs when the following is true: + ## - tag is provided + ## - workflow is building default branch (master) + artifact: + if: | + inputs.tag != '' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + name: Build Binaries + runs-on: ubuntu-latest + strategy: + ## Run a maximum of 10 builds concurrently, using the matrix defined in inputs.arch + max-parallel: 10 + matrix: + platform: ${{ fromJson(inputs.arch) }} + steps: + ## Setup Docker for the builds + - name: Docker setup + uses: stacks-network/actions/docker@main + + ## Build the binaries using defined dockerfiles + - name: Build Binary (${{ matrix.platform }}) + id: build_binaries + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # 5.0.0 + with: + file: build-scripts/Dockerfile.${{ matrix.platform }} + outputs: type=local,dest=./release/${{ matrix.platform }} + build-args: | + STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }} + OS_ARCH=${{ matrix.platform }} + GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} + GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} + + ## Compress the binary artifact + - name: Compress artifact + id: compress_artifact + run: zip --junk-paths ${{ matrix.platform }} ./release/${{ matrix.platform }}/* + + ## Upload the binary artifact to the github action (used in `github-release.yml` to create a release) + - name: Upload artifact + id: upload_artifact + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + path: ${{ matrix.platform }}.zip diff --git a/.github/workflows/epoch-tests.yml b/.github/workflows/epoch-tests.yml new file mode 100644 index 000000000..a46d1520c --- /dev/null +++ b/.github/workflows/epoch-tests.yml @@ -0,0 +1,85 @@ +## Github workflow to run epoch tests + +name: Tests::Epoch + +on: + workflow_call: + +defaults: + run: + shell: bash + +## env vars are transferred to composite action steps +env: + BITCOIND_TEST: 1 + RUST_BACKTRACE: full + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 + RETRIES: 3 + RETRY_DELAY: 10000 + TEST_TIMEOUT: 30 + TEST_RETRIES: 2 + +concurrency: + group: epoch-tests-${{ github.head_ref || github.ref || github.run_id }} + ## Only cancel in progress if this is for a PR + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # Epoch integration tests with code coverage + epoch-tests: + name: Epoch Tests + runs-on: ubuntu-latest + strategy: + ## Continue with the test matrix even if we've had a failure + fail-fast: false + ## Run a maximum of 32 concurrent tests from the test matrix + max-parallel: 32 + matrix: + test-name: + - tests::epoch_205::bigger_microblock_streams_in_2_05 + - tests::epoch_205::test_cost_limit_switch_version205 + - tests::epoch_205::test_dynamic_db_method_costs + - tests::epoch_205::test_exact_block_costs + - tests::epoch_205::transition_empty_blocks + - tests::epoch_21::test_pox_missing_five_anchor_blocks + - tests::epoch_21::test_pox_reorg_one_flap + - tests::epoch_21::test_pox_reorgs_three_flaps + - tests::epoch_21::test_sortition_divergence_pre_21 + - tests::epoch_21::test_v1_unlock_height_with_current_stackers + - tests::epoch_21::test_v1_unlock_height_with_delay_and_current_stackers + - tests::epoch_21::trait_invocation_cross_epoch + - tests::epoch_21::transition_adds_burn_block_height + - tests::epoch_21::transition_adds_get_pox_addr_recipients + - tests::epoch_21::transition_adds_mining_from_segwit + - tests::epoch_21::transition_adds_pay_to_contract + - tests::epoch_21::transition_empty_blocks + - tests::epoch_21::transition_fixes_bitcoin_rigidity + - tests::epoch_21::transition_removes_pox_sunset + - tests::epoch_22::disable_pox + - tests::epoch_22::pox_2_unlock_all + - tests::epoch_22::test_pox_reorg_one_flap + - tests::epoch_23::trait_invocation_behavior + - tests::epoch_24::fix_to_pox_contract + - tests::epoch_24::verify_auto_unlock_behavior + steps: + ## Setup test environment + - name: Setup Test Environment + id: setup_tests + uses: stacks-network/actions/stacks-core/testenv@main + + ## Run test matrix using restored cache of archive file + ## - Test will timeout after env.TEST_TIMEOUT minutes + - name: Run Tests + id: run_tests + timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT) }} + uses: stacks-network/actions/stacks-core/run-tests@main + with: + test-name: ${{ matrix.test-name }} + threads: 1 + + ## Create and upload code coverage file + - name: Code Coverage + id: codecov + uses: stacks-network/actions/codecov@main + with: + test-name: ${{ matrix.test-name }} diff --git a/.github/workflows/image-build-binary.yml b/.github/workflows/image-build-binary.yml new file mode 100644 index 000000000..291a60757 --- /dev/null +++ b/.github/workflows/image-build-binary.yml @@ -0,0 +1,92 @@ +## Github workflow to build a multiarch docker image from pre-built binaries + +name: Docker Image (Binary) + +on: + workflow_call: + inputs: + tag: + required: true + type: string + description: "Version tag for alpine images" + docker-org: + required: false + type: string + description: "Docker repo org for uploading images (defaults to github org)" + default: "${GITHUB_REPOSITORY_OWNER}" + +defaults: + run: + shell: bash + +## Define which docker arch to build for +env: + docker_platforms: "linux/arm64, linux/arm/v7, linux/amd64, linux/amd64/v2, linux/amd64/v3" + docker-org: ${{ github.repository_owner }} + +concurrency: + group: docker-image-binary-${{ github.head_ref || github.ref || github.run_id }} + ## Always cancel duplicate jobs + cancel-in-progress: true + +run-name: ${{ inputs.tag }} + +jobs: + ## Runs when the following is true: + ## - tag is provided + ## - workflow is building default branch (master) + image: + if: | + inputs.tag != '' && + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + name: Build Image + runs-on: ubuntu-latest + strategy: + fail-fast: false + ## Build a maximum of 2 images concurrently based on matrix.dist + max-parallel: 2 + matrix: + dist: + - alpine + - debian + steps: + ## Setup Docker for the builds + - name: Docker setup + uses: stacks-network/actions/docker@main + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + ## Set docker metatdata + ## - depending on the matrix.dist, different tags will be enabled + ## ex. alpine will have this tag: `type=ref,event=tag,enable=${{ matrix.dist == 'alpine' }}` + - name: Docker Metadata ( ${{matrix.dist}} ) + id: docker_metadata + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 #v5.0.0 + with: + images: | + ${{env.docker-org}}/${{ github.event.repository.name }} + ${{env.docker-org}}/stacks-core + tags: | + type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'alpine' }} + type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine'}} + type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine' }} + type=ref,event=tag,enable=${{ matrix.dist == 'alpine' }} + type=raw,value=latest-${{ matrix.dist }},enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'debian' }} + type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'debian' }} + + ## Build docker image for release + - name: Build and Push ( ${{matrix.dist}} ) + id: docker_build + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + file: ./.github/actions/dockerfiles/Dockerfile.${{ matrix.dist }}-binary + platforms: ${{ env.docker_platforms }} + tags: ${{ steps.docker_metadata.outputs.tags }} + labels: ${{ steps.docker_metadata.outputs.labels }} + build-args: | + REPO=${{ github.repository_owner }}/${{ github.event.repository.name }} + STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }} + GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} + GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} + push: ${{ env.DOCKER_PUSH }} diff --git a/.github/workflows/image-build-source.yml b/.github/workflows/image-build-source.yml new file mode 100644 index 000000000..d126336da --- /dev/null +++ b/.github/workflows/image-build-source.yml @@ -0,0 +1,69 @@ +## Github workflow to build a docker image from source + +name: Docker Image (Source) + +on: + workflow_dispatch: + workflow_call: + +defaults: + run: + shell: bash + +## Define which docker arch to build for +env: + docker_platforms: linux/amd64 + docker-org: ${{ github.repository_owner }} + +concurrency: + group: docker-image-source-${{ github.head_ref || github.ref || github.run_id }} + ## Always cancel duplicate jobs + cancel-in-progress: true + +jobs: + ## Runs anytime `ci.yml` runs or when manually called + image: + name: Build Image + runs-on: ubuntu-latest + strategy: + fail-fast: false + ## Build a maximum of 2 images concurrently based on matrix.dist + max-parallel: 2 + matrix: + dist: + - debian + steps: + ## Setup Docker for the builds + - name: Docker setup + uses: stacks-network/actions/docker@main + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + ## Set docker metatdata + - name: Docker Metadata ( ${{matrix.dist}} ) + id: docker_metadata + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 #v5.0.0 + with: + images: | + ${{env.docker-org}}/${{ github.event.repository.name }} + ${{env.docker-org}}/stacks-core + tags: | + type=raw,value=${{ env.BRANCH_NAME }} + type=ref,event=pr + + ## Build docker image + - name: Build and Push ( ${{matrix.dist}} ) + id: docker_build + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + file: ./.github/actions/dockerfiles/Dockerfile.${{matrix.dist}}-source + platforms: ${{ env.docker_platforms }} + tags: ${{ steps.docker_metadata.outputs.tags }} + labels: ${{ steps.docker_metadata.outputs.labels }} + build-args: | + REPO=${{ github.repository_owner }}/${{ github.event.repository.name }} + STACKS_NODE_VERSION=${{ env.GITHUB_SHA_SHORT }} + GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} + GIT_COMMIT=${{ env.GITHUB_SHA_SHORT }} + push: ${{ env.DOCKER_PUSH }} diff --git a/.github/workflows/slow-tests.yml b/.github/workflows/slow-tests.yml new file mode 100644 index 000000000..edc53713e --- /dev/null +++ b/.github/workflows/slow-tests.yml @@ -0,0 +1,63 @@ +## Github workflow to run slow tests + +name: Tests::Slow + +on: + workflow_call: + +defaults: + run: + shell: bash + +## env vars are transferred to composite action steps +env: + BITCOIND_TEST: 1 + RUST_BACKTRACE: full + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 + RETRIES: 3 + RETRY_DELAY: 10000 + TEST_TIMEOUT: 30 + TEST_RETRIES: 2 + +concurrency: + group: slow-tests-${{ github.head_ref || github.ref || github.run_id }} + ## Only cancel in progress if this is for a PR + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # Slow integration tests with code coverage + slow-tests: + name: Slow Tests + runs-on: ubuntu-latest + strategy: + ## Continue with the test matrix even if we've had a failure + fail-fast: false + ## Run a maximum of 2 concurrent tests from the test matrix + max-parallel: 2 + matrix: + ## Each of these tests should take ~20 minutes if they are successful + test-name: + - tests::epoch_21::test_pox_reorg_flap_duel + - tests::epoch_21::test_pox_reorg_flap_reward_cycles + steps: + ## Setup test environment + - name: Setup Test Environment + id: setup_tests + uses: stacks-network/actions/stacks-core/testenv@main + + ## Run test matrix using restored cache of archive file + ## - Test will timeout after env.TEST_TIMEOUT minutes + - name: Run Tests + id: run_tests + timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT) }} + uses: stacks-network/actions/stacks-core/run-tests@main + with: + test-name: ${{ matrix.test-name }} + threads: 1 + + ## Create and upload code coverage file + - name: Code Coverage + id: codecov + uses: stacks-network/actions/codecov@main + with: + test-name: ${{ matrix.test-name }} diff --git a/.github/workflows/standalone-tests.yml b/.github/workflows/standalone-tests.yml new file mode 100644 index 000000000..458ea4c5e --- /dev/null +++ b/.github/workflows/standalone-tests.yml @@ -0,0 +1,115 @@ +## Github workflow to run specified tests on demand + +name: Standalone Tests + +on: + workflow_call: + inputs: + workflow: + description: "Tests to run (required)" + required: true + type: string + workflow_dispatch: + inputs: + workflow: + description: "Tests to run (required)" + required: true + type: choice + options: + - Release Tests + - CI Tests + - Atlas Tests + - Bitcoin Tests + - Epoch Tests + - Slow Tests + - Stacks-Blockchain Tests + +## Set the display name to the test being run +run-name: ${{ inputs.workflow }} + +jobs: + ## Runs every time to either create the required caches or confirm they already exist + create-cache: + name: Create Test Cache + uses: ./.github/workflows/create-cache.yml + + ##################################################### + ## Runs when: + ## either or of the following: + ## - workflow is 'Release Tests' + ## - workflow is 'CI Tests' + ## - workflow is 'Stacks-Blockchain Tests' + stacks-blockchain-tests: + if: | + ( + inputs.workflow == 'Release Tests' || + inputs.workflow == 'CI Tests' || + inputs.workflow == 'Stacks-Blockchain Tests' + ) + name: Stacks Blockchain Tests + needs: + - create-cache + uses: ./.github/workflows/stacks-blockchain-tests.yml + + ## Runs when: + ## either or of the following: + ## - workflow is 'Release Tests' + ## - workflow is 'CI Tests' + ## - workflow is 'Bitcoin Tests' + bitcoin-tests: + if: | + ( + inputs.workflow == 'Release Tests' || + inputs.workflow == 'CI Tests' || + inputs.workflow == 'Bitcoin Tests' + ) + name: Bitcoin Tests + needs: + - create-cache + uses: ./.github/workflows/bitcoin-tests.yml + + ##################################################### + ## Runs when: + ## either or of the following: + ## - workflow is 'Release Tests' + ## - workflow is 'Atlas Tests' + atlas-tests: + if: | + ( + inputs.workflow == 'Release Tests' || + inputs.workflow == 'Atlas Tests' + ) + name: Atlas Tests + needs: + - create-cache + uses: ./.github/workflows/atlas-tests.yml + + ## Runs when: + ## either or of the following: + ## - workflow is 'Release Tests' + ## - workflow is 'Epoch Tests' + epoch-tests: + if: | + ( + inputs.workflow == 'Release Tests' || + inputs.workflow == 'Epoch Tests' + ) + name: Epoch Tests + needs: + - create-cache + uses: ./.github/workflows/epoch-tests.yml + + ## Runs when: + ## either or of the following: + ## - workflow is 'Release Tests' + ## - workflow is 'Slow Tests' + slow-tests: + if: | + ( + inputs.workflow == 'Release Tests' || + inputs.workflow == 'Slow Tests' + ) + name: Slow Tests + needs: + - create-cache + uses: ./.github/workflows/slow-tests.yml diff --git a/.github/workflows/workflow-cleanup.yml b/.github/workflows/workflow-cleanup.yml new file mode 100644 index 000000000..e9d79305d --- /dev/null +++ b/.github/workflows/workflow-cleanup.yml @@ -0,0 +1,47 @@ +## Github workflow to clean up old caches and workflow runs + +name: Workflow Cleanup + +on: + workflow_dispatch: + inputs: + cache-ttl: + description: "How many days to keep a cache (default: 7)" + required: false + default: "7" + workflow-ttl: + description: "How many days to keep a successful workflow (default: 30)" + required: false + default: "30" + failed-workflow-ttl: + description: "How many days to keep failed workflows (default: 15)" + required: false + default: "15" + schedule: + ## Run every day at 00:00:00 + - cron: "0 0 * * *" + +## env vars are transferred to composite action steps +env: + CACHE_TTL: 7 ## number of days to keep a cache + WORKFLOW_TTL: 30 ## number of days to keep a successful workflow + FAILED_WORKFLOW_TTL: 15 ## number of days to keep a failed workflow + +concurrency: + group: cleanup-${{ github.head_ref || github.ref }} + ## Always cancel duplicate jobs + cancel-in-progress: true + +jobs: + workflow-cleanup: + name: Workflow Cleanup + runs-on: ubuntu-latest + steps: + - name: Cleanup + id: cleanup + uses: stacks-network/actions/cleanup/workflows@main + with: + token: ${{ secrets.GH_TOKEN }} + cache-ttl: ${{ inputs.cache-ttl || env.CACHE_TTL}} + workflow-ttl: ${{ inputs.workflow-ttl || env.WORKFLOW_TTL}} + failed-workflow-ttl: ${{ inputs.failed-workflow-ttl || env.FAILED_WORKFLOW_TTL }} From 6f6d1ab5cc044caa880b32856e84602ebe63b8bb Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Mon, 20 Nov 2023 12:02:07 -0800 Subject: [PATCH 16/24] set docker repo name and add tag arg for build --- .github/workflows/image-build-binary.yml | 3 ++- .github/workflows/image-build-source.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/image-build-binary.yml b/.github/workflows/image-build-binary.yml index 291a60757..a821531dc 100644 --- a/.github/workflows/image-build-binary.yml +++ b/.github/workflows/image-build-binary.yml @@ -22,7 +22,7 @@ defaults: ## Define which docker arch to build for env: docker_platforms: "linux/arm64, linux/arm/v7, linux/amd64, linux/amd64/v2, linux/amd64/v3" - docker-org: ${{ github.repository_owner }} + docker-org: blockstack concurrency: group: docker-image-binary-${{ github.head_ref || github.ref || github.run_id }} @@ -85,6 +85,7 @@ jobs: tags: ${{ steps.docker_metadata.outputs.tags }} labels: ${{ steps.docker_metadata.outputs.labels }} build-args: | + TAG=${{ inputs.tag }} REPO=${{ github.repository_owner }}/${{ github.event.repository.name }} STACKS_NODE_VERSION=${{ inputs.tag || env.GITHUB_SHA_SHORT }} GIT_BRANCH=${{ env.GITHUB_REF_SHORT }} diff --git a/.github/workflows/image-build-source.yml b/.github/workflows/image-build-source.yml index d126336da..79c428073 100644 --- a/.github/workflows/image-build-source.yml +++ b/.github/workflows/image-build-source.yml @@ -13,7 +13,7 @@ defaults: ## Define which docker arch to build for env: docker_platforms: linux/amd64 - docker-org: ${{ github.repository_owner }} + docker-org: blockstack concurrency: group: docker-image-source-${{ github.head_ref || github.ref || github.run_id }} From 2af76d943741ed7f3b941d9578053ac6c6af9ba9 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Wed, 22 Nov 2023 11:35:48 -0800 Subject: [PATCH 17/24] Add checks/trigger for next/fix typo --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 907517226..60b5d967f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,7 @@ on: branches: - master - develop + - next paths-ignore: - "**.md" - "**.yml" @@ -145,8 +146,9 @@ jobs: github.event_name == 'pull_request_target' || ( contains(' - refs/heads/develoment refs/heads/master + refs/heads/develop + refs/heads/next ', github.event.pull_request.head.ref) && github.event_name == 'push' ) From 98f638616f451e5034b38d1a65b508f581177acc Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Wed, 22 Nov 2023 13:40:39 -0800 Subject: [PATCH 18/24] Update docker image name Co-authored-by: Charlie <2747302+CharlieC3@users.noreply.github.com> --- .github/workflows/image-build-binary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/image-build-binary.yml b/.github/workflows/image-build-binary.yml index a821531dc..35c130b55 100644 --- a/.github/workflows/image-build-binary.yml +++ b/.github/workflows/image-build-binary.yml @@ -66,7 +66,7 @@ jobs: with: images: | ${{env.docker-org}}/${{ github.event.repository.name }} - ${{env.docker-org}}/stacks-core + ${{env.docker-org}}/stacks-blockchain tags: | type=raw,value=latest,enable=${{ inputs.tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) ) && matrix.dist == 'alpine' }} type=raw,value=${{ inputs.tag }}-${{ matrix.dist }},enable=${{ inputs.tag != '' && matrix.dist == 'alpine'}} From ef07100409a6b3665c183de5b241c2a30b65ee8d Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Wed, 22 Nov 2023 13:48:16 -0800 Subject: [PATCH 19/24] spelling error Co-authored-by: Charlie <2747302+CharlieC3@users.noreply.github.com> --- docs/ci-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ci-release.md b/docs/ci-release.md index 82fe164ec..6a0bc7e1c 100644 --- a/docs/ci-release.md +++ b/docs/ci-release.md @@ -79,7 +79,7 @@ There are also 2 different methods in use with regard to running tests: A matrix is used when there are several known tests that need to be run. Partitions (shards) are used when there is a large and unknown number of tests to run (ex: `cargo test` to run all tests). There is also a workflow designed to run tests that are manually triggered: [Standalone Tests](../.github/workflows/standalone-tests.yml). -This workflow requires you to select which test(s) you want to run, which then triggers a reusbale workflow via conditional. For example, selecting "Epoch Tests" will run the tests defined in [Epoch Tests](../.github/workflows/epoch-tests.yml). Likewise, selecting `Release Tests` will run the same tests as a release workflow. +This workflow requires you to select which test(s) you want to run, which then triggers a reusable workflow via conditional. For example, selecting "Epoch Tests" will run the tests defined in [Epoch Tests](../.github/workflows/epoch-tests.yml). Likewise, selecting `Release Tests` will run the same tests as a release workflow. Files: From 7702eaa19210789d3dadedeabac233e59488627e Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Sun, 26 Nov 2023 18:14:10 -0800 Subject: [PATCH 20/24] address comments in 4079 --- .github/workflows/atlas-tests.yml | 7 ------- .github/workflows/bitcoin-tests.yml | 7 ------- .github/workflows/ci.yml | 2 +- .github/workflows/create-cache.yml | 4 ---- .github/workflows/create-source-binary.yml | 4 ---- .github/workflows/epoch-tests.yml | 7 ------- .github/workflows/image-build-binary.yml | 4 ---- .github/workflows/image-build-source.yml | 6 +----- .github/workflows/slow-tests.yml | 7 ------- .github/workflows/stacks-blockchain-tests.yml | 7 ------- 10 files changed, 2 insertions(+), 53 deletions(-) diff --git a/.github/workflows/atlas-tests.yml b/.github/workflows/atlas-tests.yml index 21da3c830..cb9f4a968 100644 --- a/.github/workflows/atlas-tests.yml +++ b/.github/workflows/atlas-tests.yml @@ -5,19 +5,12 @@ name: Tests::Atlas on: workflow_call: -defaults: - run: - shell: bash - ## env vars are transferred to composite action steps env: BITCOIND_TEST: 1 RUST_BACKTRACE: full SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 - RETRIES: 3 - RETRY_DELAY: 10000 TEST_TIMEOUT: 30 - TEST_RETRIES: 2 concurrency: group: atlas-tests-${{ github.head_ref || github.ref || github.run_id}} diff --git a/.github/workflows/bitcoin-tests.yml b/.github/workflows/bitcoin-tests.yml index 48e53915c..de1b16c26 100644 --- a/.github/workflows/bitcoin-tests.yml +++ b/.github/workflows/bitcoin-tests.yml @@ -5,19 +5,12 @@ name: Tests::Bitcoin on: workflow_call: -defaults: - run: - shell: bash - ## env vars are transferred to composite action steps env: BITCOIND_TEST: 1 RUST_BACKTRACE: full SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 - RETRIES: 3 - RETRY_DELAY: 10000 TEST_TIMEOUT: 30 - TEST_RETRIES: 2 concurrency: group: bitcoin-tests-${{ github.head_ref || github.ref || github.run_id}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60b5d967f..d4ae0ce80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ defaults: shell: bash concurrency: - group: ci-${{ github.head_ref || github.ref }} + group: ci-${{ github.head_ref || github.ref || github.run_id }} ## Always cancel duplicate jobs cancel-in-progress: true diff --git a/.github/workflows/create-cache.yml b/.github/workflows/create-cache.yml index 8bb736dd0..71e0c02a3 100644 --- a/.github/workflows/create-cache.yml +++ b/.github/workflows/create-cache.yml @@ -6,10 +6,6 @@ on: workflow_dispatch: workflow_call: -defaults: - run: - shell: bash - ## env vars are transferred to composite action steps env: RUSTFLAGS: "-Cinstrument-coverage -Awarnings" diff --git a/.github/workflows/create-source-binary.yml b/.github/workflows/create-source-binary.yml index a6e70756e..8e32893f7 100644 --- a/.github/workflows/create-source-binary.yml +++ b/.github/workflows/create-source-binary.yml @@ -16,10 +16,6 @@ on: default: >- ["linux-glibc-x64", "linux-musl-x64", "linux-glibc-arm64", "linux-glibc-armv7", "linux-musl-arm64", "linux-musl-armv7", "macos-x64", "macos-arm64", "windows-x64"] -defaults: - run: - shell: bash - ## change the display name to the tag being built run-name: ${{ inputs.tag }} diff --git a/.github/workflows/epoch-tests.yml b/.github/workflows/epoch-tests.yml index a46d1520c..b7d9bd6f0 100644 --- a/.github/workflows/epoch-tests.yml +++ b/.github/workflows/epoch-tests.yml @@ -5,19 +5,12 @@ name: Tests::Epoch on: workflow_call: -defaults: - run: - shell: bash - ## env vars are transferred to composite action steps env: BITCOIND_TEST: 1 RUST_BACKTRACE: full SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 - RETRIES: 3 - RETRY_DELAY: 10000 TEST_TIMEOUT: 30 - TEST_RETRIES: 2 concurrency: group: epoch-tests-${{ github.head_ref || github.ref || github.run_id }} diff --git a/.github/workflows/image-build-binary.yml b/.github/workflows/image-build-binary.yml index 35c130b55..cab5ff162 100644 --- a/.github/workflows/image-build-binary.yml +++ b/.github/workflows/image-build-binary.yml @@ -15,10 +15,6 @@ on: description: "Docker repo org for uploading images (defaults to github org)" default: "${GITHUB_REPOSITORY_OWNER}" -defaults: - run: - shell: bash - ## Define which docker arch to build for env: docker_platforms: "linux/arm64, linux/arm/v7, linux/amd64, linux/amd64/v2, linux/amd64/v3" diff --git a/.github/workflows/image-build-source.yml b/.github/workflows/image-build-source.yml index 79c428073..1936999b2 100644 --- a/.github/workflows/image-build-source.yml +++ b/.github/workflows/image-build-source.yml @@ -6,10 +6,6 @@ on: workflow_dispatch: workflow_call: -defaults: - run: - shell: bash - ## Define which docker arch to build for env: docker_platforms: linux/amd64 @@ -47,7 +43,7 @@ jobs: with: images: | ${{env.docker-org}}/${{ github.event.repository.name }} - ${{env.docker-org}}/stacks-core + ${{env.docker-org}}/stacks-blockchain tags: | type=raw,value=${{ env.BRANCH_NAME }} type=ref,event=pr diff --git a/.github/workflows/slow-tests.yml b/.github/workflows/slow-tests.yml index edc53713e..38fb20ac3 100644 --- a/.github/workflows/slow-tests.yml +++ b/.github/workflows/slow-tests.yml @@ -5,19 +5,12 @@ name: Tests::Slow on: workflow_call: -defaults: - run: - shell: bash - ## env vars are transferred to composite action steps env: BITCOIND_TEST: 1 RUST_BACKTRACE: full SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 - RETRIES: 3 - RETRY_DELAY: 10000 TEST_TIMEOUT: 30 - TEST_RETRIES: 2 concurrency: group: slow-tests-${{ github.head_ref || github.ref || github.run_id }} diff --git a/.github/workflows/stacks-blockchain-tests.yml b/.github/workflows/stacks-blockchain-tests.yml index fa415a9cd..de285f4b7 100644 --- a/.github/workflows/stacks-blockchain-tests.yml +++ b/.github/workflows/stacks-blockchain-tests.yml @@ -5,19 +5,12 @@ name: Tests::Stacks Blockchain on: workflow_call: -defaults: - run: - shell: bash - ## env vars are transferred to composite action steps env: BITCOIND_TEST: 1 RUST_BACKTRACE: full SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - RETRIES: 3 - RETRY_DELAY: 10000 TEST_TIMEOUT: 30 - TEST_RETRIES: 2 concurrency: group: stacks-blockchain-tests-${{ github.head_ref || github.ref || github.run_id }} From 6f29b3cbbb0107b8ce4eea60cbfd090f8fe78d70 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Sun, 26 Nov 2023 18:17:33 -0800 Subject: [PATCH 21/24] remove comment in dockerfile --- build-scripts/Dockerfile.linux-glibc-arm64 | 1 - build-scripts/Dockerfile.linux-glibc-x64 | 1 - 2 files changed, 2 deletions(-) diff --git a/build-scripts/Dockerfile.linux-glibc-arm64 b/build-scripts/Dockerfile.linux-glibc-arm64 index 61ff5f4a0..7ce50b6a6 100644 --- a/build-scripts/Dockerfile.linux-glibc-arm64 +++ b/build-scripts/Dockerfile.linux-glibc-arm64 @@ -24,4 +24,3 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \ FROM scratch AS export-stage COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node / -## comment diff --git a/build-scripts/Dockerfile.linux-glibc-x64 b/build-scripts/Dockerfile.linux-glibc-x64 index 9c68c58af..2db13cb51 100644 --- a/build-scripts/Dockerfile.linux-glibc-x64 +++ b/build-scripts/Dockerfile.linux-glibc-x64 @@ -21,4 +21,3 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \ FROM scratch AS export-stage COPY --from=build /out/stacks-inspect /out/blockstack-cli /out/clarity-cli /out/stacks-node / -## comment From dec30101dea72d69619a0cbfd17662efdc17690b Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Tue, 28 Nov 2023 20:10:46 -0800 Subject: [PATCH 22/24] renaming items to stacks-core --- .github/workflows/ci.yml | 6 +++--- .github/workflows/clarity-js-sdk-pr.yml | 4 ++-- .github/workflows/create-cache.yml | 2 +- .github/workflows/docs-pr.yml | 6 +++--- ...ks-blockchain-tests.yml => stacks-core-tests.yml} | 4 ++-- .github/workflows/standalone-tests.yml | 12 ++++++------ 6 files changed, 17 insertions(+), 17 deletions(-) rename .github/workflows/{stacks-blockchain-tests.yml => stacks-core-tests.yml} (97%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4ae0ce80..2c86c6dcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,7 +166,7 @@ jobs: ## either or of the following: ## - tag is not provided ## - PR is approved - stacks-blockchain-tests: + stacks-core-tests: if: | inputs.tag != '' || ( inputs.tag == '' || ( @@ -175,11 +175,11 @@ jobs: github.event.review.state == 'approved' ) ) - name: Stacks Blockchain Tests + name: Stacks Core Tests needs: - rustfmt - create-cache - uses: ./.github/workflows/stacks-blockchain-tests.yml + uses: ./.github/workflows/stacks-core-tests.yml bitcoin-tests: if: | diff --git a/.github/workflows/clarity-js-sdk-pr.yml b/.github/workflows/clarity-js-sdk-pr.yml index 25e96f9f2..452380841 100644 --- a/.github/workflows/clarity-js-sdk-pr.yml +++ b/.github/workflows/clarity-js-sdk-pr.yml @@ -22,7 +22,7 @@ jobs: run: name: Open Clarity JS SDK PR runs-on: ubuntu-latest - # This condition can be removed once the main `stacks-blockchain` workflow creates pre-releases + # This condition can be removed once the main `stacks-core` workflow creates pre-releases # when appropriate, instead of full releases for every tag passed in. if: "!contains(github.ref, '-rc')" steps: @@ -58,7 +58,7 @@ jobs: labels: | dependencies body: | - :robot: This is an automated pull request created from a new release in [stacks-blockchain](https://github.com/stacks-network/stacks-blockchain/releases). + :robot: This is an automated pull request created from a new release in [stacks-core](https://github.com/stacks-network/stacks-core/releases). Updates the clarity-native-bin tag. assignees: zone117x diff --git a/.github/workflows/create-cache.yml b/.github/workflows/create-cache.yml index 71e0c02a3..a67ecc8fa 100644 --- a/.github/workflows/create-cache.yml +++ b/.github/workflows/create-cache.yml @@ -9,7 +9,7 @@ on: ## env vars are transferred to composite action steps env: RUSTFLAGS: "-Cinstrument-coverage -Awarnings" - LLVM_PROFILE_FILE: "stacks-blockchain-%p-%m.profraw" + LLVM_PROFILE_FILE: "stacks-core-%p-%m.profraw" BTC_VERSION: "0.20.0" ## diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml index bb54c28cc..d3dbeaa45 100644 --- a/.github/workflows/docs-pr.yml +++ b/.github/workflows/docs-pr.yml @@ -69,7 +69,7 @@ jobs: echo "::set-output name=open_pr::0" else git remote add robot https://github.com/$ROBOT_OWNER/$ROBOT_REPO - git commit -m "auto: update Clarity references JSONs from stacks-blockchain@${GITHUB_SHA}" + git commit -m "auto: update Clarity references JSONs from stacks-core@${GITHUB_SHA}" git push robot $ROBOT_BRANCH echo "::set-output name=open_pr::1" fi @@ -109,6 +109,6 @@ jobs: let result = await github.pulls.create({ owner, repo, head, base: "master", - title: "Auto: Update API documentation from stacks-blockchain", - body: "Update API documentation from the latest in `stacks-blockchain`", + title: "Auto: Update API documentation from stacks-core", + body: "Update API documentation from the latest in `stacks-core`", }); diff --git a/.github/workflows/stacks-blockchain-tests.yml b/.github/workflows/stacks-core-tests.yml similarity index 97% rename from .github/workflows/stacks-blockchain-tests.yml rename to .github/workflows/stacks-core-tests.yml index de285f4b7..6d3eb8d39 100644 --- a/.github/workflows/stacks-blockchain-tests.yml +++ b/.github/workflows/stacks-core-tests.yml @@ -1,6 +1,6 @@ ## Github workflow to run full genesis and unit tests -name: Tests::Stacks Blockchain +name: Tests::Stacks Core on: workflow_call: @@ -13,7 +13,7 @@ env: TEST_TIMEOUT: 30 concurrency: - group: stacks-blockchain-tests-${{ github.head_ref || github.ref || github.run_id }} + group: stacks-core-tests-${{ github.head_ref || github.ref || github.run_id }} ## Only cancel in progress if this is for a PR cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/standalone-tests.yml b/.github/workflows/standalone-tests.yml index 458ea4c5e..5c2bbe86b 100644 --- a/.github/workflows/standalone-tests.yml +++ b/.github/workflows/standalone-tests.yml @@ -22,7 +22,7 @@ on: - Bitcoin Tests - Epoch Tests - Slow Tests - - Stacks-Blockchain Tests + - Stacks-Core Tests ## Set the display name to the test being run run-name: ${{ inputs.workflow }} @@ -38,18 +38,18 @@ jobs: ## either or of the following: ## - workflow is 'Release Tests' ## - workflow is 'CI Tests' - ## - workflow is 'Stacks-Blockchain Tests' - stacks-blockchain-tests: + ## - workflow is 'Stacks-Core Tests' + stacks-core-tests: if: | ( inputs.workflow == 'Release Tests' || inputs.workflow == 'CI Tests' || - inputs.workflow == 'Stacks-Blockchain Tests' + inputs.workflow == 'Stacks-Core Tests' ) - name: Stacks Blockchain Tests + name: Stacks Core Tests needs: - create-cache - uses: ./.github/workflows/stacks-blockchain-tests.yml + uses: ./.github/workflows/stacks-core-tests.yml ## Runs when: ## either or of the following: From bd35a46bbca86248a72bc64e386f4d2b9273c963 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Tue, 28 Nov 2023 20:11:31 -0800 Subject: [PATCH 23/24] update documentation --- .github/PULL_REQUEST_TEMPLATE.md | 4 +- docs/ci-release.md | 139 ++++++++++++++++++++----------- 2 files changed, 95 insertions(+), 48 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index abcdea725..69ef6240b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,17 +3,19 @@ Pull requests are ideal for making small changes to this project. However, they are NOT an appropriate venue to introducing non-trivial or breaking changes to the codebase. For introducing non-trivial or breaking changes to the codebase, please follow the SIP (Stacks Improvement Proposal) process documented here: - https://github.com/blockstack/stacks-blockchain/blob/master/sip/sip-000-stacks-improvement-proposal-process.md. + https://github.com/stacksgov/sips/blob/main/sips/sip-000/sip-000-stacks-improvement-proposal-process.md. --> ### Description ### Applicable issues + - fixes # ### Additional info (benefits, drawbacks, caveats) ### Checklist + - [ ] Test coverage for new or modified code paths - [ ] Changelog is updated - [ ] Required documentation changes (e.g., `docs/rpc/openapi.yaml` and `rpc-endpoints.md` for v2 endpoints, `event-dispatcher.md` for new events) diff --git a/docs/ci-release.md b/docs/ci-release.md index 6a0bc7e1c..f042b05ed 100644 --- a/docs/ci-release.md +++ b/docs/ci-release.md @@ -1,38 +1,42 @@ # Releases -All releases are built via a Github Actions workflow named `CI`, and is responsible for building binary archives, checksums, and resulting docker images. -This workflow will also trigger any tests that need to be run, like integration tests. +All releases are built via a Github Actions workflow named `CI` ([ci.yml](../.github/workflows/ci.yml)), and is responsible for: -1. Releases are only created if a tag is **manually** provided when the ci workflow is triggered. -2. Pushing a new feature branch: Nothing is triggered automatically. A PR is required, or the ci workflow can be triggered manually on a specific branch to build a docker image (and run the standard PR tests) for the specified branch. -3. Caching is used to speed up testing - a cache is created based on the type of data (i.e. cargo) and the commit sha. tests can be retried quickly since the cache will persist until the cleanup job is run. -4. [nextest](https://nexte.st/) is used to run the tests from an archived file that is cached (using commit sha as a key)) +- Verifying code is formatted correctly +- Building binary archives and checksums +- Docker images +- Triggering tests conditionally (different tests run for a release vs a PR) + +1. Releases are only created if a tag is **manually** provided when the [CI workflow](../.github/workflows/ci.yml) is triggered. +2. [Caching](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows) is used to speed up testing - a cache is created based on the type of data (i.e. cargo) and the commit sha. tests can be retried quickly since the cache will persist until the cleanup job is run. +3. [nextest](https://nexte.st/) is used to run the tests from an archived file that is cached (using commit sha as a key)) - Two [archives](https://nexte.st/book/reusing-builds.html) are created, one for genesis tests and one for generic tests (it is done this way to reduce the time spent building) - Unit-tests are [partitioned](https://nexte.st/book/partitioning.html) and multi-threaded to speed up execution time ## TL;DR -1. An open/re-opened/synchronized PR will produce a single image built from source on Debian with glibc with 2 tags: - - `stacks-blockchain:` - - `stacks-blockchain:` -2. A merged PR from `develop` to the default branch will produce a single image built from source on Debian with glibc: - - `stacks-blockchain:` -3. An untagged build of any branch will produce a single image built from source on Debian with glibc: - - `stacks-blockchain:` -4. A tagged release on a non-default branch will produces: - - Docker Alpine image for several architectures tagged with: - - `stacks-blockchain:` - - Docker Debian image for several architectures tagged with: - - `stacks-blockchain:` -5. A tagged release on the default branch will produce: - - Github Release of the specified tag with: - - Binary archives for several architectures - - Docker Alpine image for several architectures tagged with: - - `stacks-blockchain:` - - `stacks-blockchain:` - - Docker Debian image for several architectures tagged with: - - `stacks-blockchain:` - - `stacks-blockchain:` +- Pushing a feature branch will not trigger a workflow +- An open/re-opened/synchronized PR will produce a single image built from source on Debian with glibc with 2 tags: + - `stacks-core:` + - `stacks-core:` +- A merged PR into `default-branch` from `develop` will produce a single image built from source on Debian with glibc: + - `stacks-core:` +- An untagged build of any branch will produce a single image built from source on Debian with glibc: + - `stacks-core:` +- A tagged release on a non-default branch will produces: + - Docker Alpine image for several architectures tagged with: + - `stacks-core:` + - Docker Debian image for several architectures tagged with: + - `stacks-core:` +- A tagged release on the default branch will produce: + - Github Release of the specified tag with: + - Binary archives for several architectures + - Docker Alpine image for several architectures tagged with: + - `stacks-core:` + - `stacks-core:` + - Docker Debian image for several architectures tagged with: + - `stacks-core:` + - `stacks-core:` ## Release workflow @@ -40,19 +44,19 @@ This workflow will also trigger any tests that need to be run, like integration 2. PR `feat/fix-something` to the `develop` branch where the PR is numbered `112` 1. Docker image tagged with the **branch name** and **PR number** - ex: - - `stacks-blockchain:feat-fix-something` - - `stacks-blockchain:pr-112` + - `stacks-core:feat-fix-something` + - `stacks-core:pr-112` 2. CI tests are run 3. PR `develop` to the default branch where the PR is numbered `112` 1. Docker image tagged with the **branch name** and **PR number** - ex: - - `stacks-blockchain:feat-fix-something` - - `stacks-blockchain:pr-112` + - `stacks-core:feat-fix-something` + - `stacks-core:pr-112` 2. CI tests are run 4. Merge `develop` branch to the default branch 1. Docker image is tagged with the **default branch** `master` - ex: - - `stacks-blockchain:master` + - `stacks-core:master` 2. CI tests are run 5. CI workflow is manually triggered on **non-default branch** with a version, i.e. `2.1.0.0.0-rc0` 1. No Docker images/binaries are created @@ -62,10 +66,10 @@ This workflow will also trigger any tests that need to be run, like integration 2. All release tests are run 3. Docker image pushed with tags of the **input version** and **latest** - ex: - - `stacks-blockchain:2.1.0.0.0-debian` - - `stacks-blockchain:latest-debian` - - `stacks-blockchain:2.1.0.0.0` - - `stacks-blockchain:latest` + - `stacks-core:2.1.0.0.0-debian` + - `stacks-core:latest-debian` + - `stacks-core:2.1.0.0.0` + - `stacks-core:latest` ## Tests @@ -79,7 +83,7 @@ There are also 2 different methods in use with regard to running tests: A matrix is used when there are several known tests that need to be run. Partitions (shards) are used when there is a large and unknown number of tests to run (ex: `cargo test` to run all tests). There is also a workflow designed to run tests that are manually triggered: [Standalone Tests](../.github/workflows/standalone-tests.yml). -This workflow requires you to select which test(s) you want to run, which then triggers a reusable workflow via conditional. For example, selecting "Epoch Tests" will run the tests defined in [Epoch Tests](../.github/workflows/epoch-tests.yml). Likewise, selecting `Release Tests` will run the same tests as a release workflow. +This workflow requires you to select which test(s) you want to run, which then triggers a reusbale workflow via conditional. For example, selecting "Epoch Tests" will run the tests defined in [Epoch Tests](../.github/workflows/epoch-tests.yml). Likewise, selecting `Release Tests` will run the same tests as a release workflow. Files: @@ -90,6 +94,47 @@ Files: - [Epoch Tests](../.github/workflows/epoch-tests.yml) - [Slow Tests](../.github/workflows/slow-tests.yml) +### Adding/changing tests + +With the exception of `unit-tests` in [Stacks Blockchain Tests](../.github/workflows/stacks-blockchain-tests.yml), adding/removing a test requires a change to the workflow matrix. Example from [Atlas Tests](../.github/workflows/atlas-tests.yml): + +```yaml +atlas-tests: + name: Atlas Test + runs-on: ubuntu-latest + strategy: + ## Continue with the test matrix even if we've had a failure + fail-fast: false + ## Run a maximum of 2 concurrent tests from the test matrix + max-parallel: 2 + matrix: + test-name: + - tests::neon_integrations::atlas_integration_test + - tests::neon_integrations::atlas_stress_integration_test +``` + +Example of adding a new test `tests::neon_integrations::atlas_new_test`: + +```yaml + ... + matrix: + test-name: + - tests::neon_integrations::atlas_integration_test + - tests::neon_integrations::atlas_stress_integration_test + - tests::neon_integrations::atlas_new_test +``` + +The separation of tests (outside of [Slow Tests](../.github/workflows/slow-tests.yml)) is performed by creating a separate workflow for each _type_ of test that is being run. Using the example above, to add/remove any tests from being run - the `matrix` will need to be adjusted. + +ex: + +- `Atlas Tests`: Tests related to Atlas +- `Bitcoin Tests`: Tests relating to burnchain operations +- `Epoch Tests`: Tests related to epoch changes +- `Slow Tests`: These tests have been identified as taking longer than others. The threshold used is if a test takes longer than `10 minutes` to complete successfully (or times out semi-regularly), it should be added here. +- `Stacks Blockchain Tests`: + - `full-genesis`: Tests related to full genesis + ## Triggering a workflow ### PR a branch to develop @@ -102,8 +147,8 @@ ex: Branch is named `feat/fix-something` and the PR is numbered `112` - [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) - [Docker image](../.github/workflows/image-build-source.yml) is built from source on a debian distribution and pushed with the branch name and PR number as tags - ex: - - `stacks-blockchain:feat-fix-something` - - `stacks-blockchain:pr-112` + - `stacks-core:feat-fix-something` + - `stacks-core:pr-112` --- @@ -123,8 +168,8 @@ ex: Branch is named `develop` and the PR is numbered `113` - [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) - [Docker image](../.github/workflows/image-build-source.yml) is built from source on a debian distribution and pushed with the branch name and PR number as tags - ex: - - `stacks-blockchain:develop` - - `stacks-blockchain:pr-113` + - `stacks-core:develop` + - `stacks-core:pr-113` --- @@ -136,7 +181,7 @@ ex: Branch is named `develop` and the PR is numbered `113` - [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) - [Docker image](../.github/workflows/image-build-source.yml) is built from source on a debian distribution and pushed with the branch name as a tag - ex: - - `stacks-blockchain:master` + - `stacks-core:master` --- @@ -148,7 +193,7 @@ ex: Branch is named `develop` and the PR is numbered `113` - [Bitcoin Tests](../.github/workflows/bitcoin-tests.yml) - [Docker image](../.github/workflows/image-build-source.yml) is built from source on a debian distribution and pushed with the branch name as a tag - ex: - - `stacks-blockchain:` + - `stacks-core:` --- @@ -178,9 +223,9 @@ ex: Branch is named `develop` and the PR is numbered `113` - [Github release](../.github/workflows/github-release.yml) (with artifacts/checksum) is created using the manually input tag - [Docker image](../.github/workflows/image-build-binary.yml) built from binaries on debian/alpine distributions and pushed with the provided input tag and `latest` - ex: - - `stacks-blockchain:2.1.0.0.0-debian` - - `stacks-blockchain:latest-debian` - - `stacks-blockchain:2.1.0.0.0` - - `stacks-blockchain:latest` + - `stacks-core:2.1.0.0.0-debian` + - `stacks-core:latest-debian` + - `stacks-core:2.1.0.0.0` + - `stacks-core:latest` --- From 6bdc9d5f8f872afd91a56089d7cceb7dcb7ddf9b Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Wed, 29 Nov 2023 22:53:52 -0800 Subject: [PATCH 24/24] Update location of archive --- .github/workflows/stacks-core-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stacks-core-tests.yml b/.github/workflows/stacks-core-tests.yml index 6d3eb8d39..af0b1a934 100644 --- a/.github/workflows/stacks-core-tests.yml +++ b/.github/workflows/stacks-core-tests.yml @@ -47,7 +47,7 @@ jobs: with: test-name: ${{ matrix.test-name }} threads: 1 - archive-file: /tmp/genesis_archive.tar.zst + archive-file: ~/genesis_archive.tar.zst ## Upload code coverage file - name: Code Coverage