name: PR Differences Mutants on: pull_request: types: - opened - reopened - synchronize - ready_for_review paths: - "**.rs" workflow_dispatch: concurrency: group: pr-differences-${{ github.head_ref || github.ref || github.run_id }} # Always cancel duplicate jobs cancel-in-progress: true jobs: mutants: name: Mutation Testing runs-on: ubuntu-latest steps: # Cleanup Runner - name: Cleanup Runner id: runner_cleanup uses: stacks-network/actions/cleanup/disk@main - name: Checkout repo id: git_checkout uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 with: fetch-depth: 0 - name: Relative diff id: relative_diff run: | git diff $(git merge-base origin/${{ github.base_ref || 'main' }} HEAD)..HEAD > git.diff - name: Install cargo-mutants id: install_cargo_mutants run: | cargo install --version 25.0.0 cargo-mutants --locked - name: Install cargo-nextest id: install_cargo_nextest uses: taiki-e/install-action@2f990e9c484f0590cb76a07296e9677b417493e9 # v2.33.23 with: tool: nextest # Latest version - name: Update git diff id: update_git_diff run: | input_file="git.diff" temp_file="temp_diff_file.diff" # Check if the file exists and is not empty if [ ! -s "$input_file" ]; then echo "Diff file ($input_file) is missing or empty!" exit 1 fi # Remove all lines related to deleted files including the first 'diff --git' line awk ' /^diff --git/ { diff_line = $0 getline if ($0 ~ /^deleted file mode/) { in_deleted_file_block = 1 } else { if (diff_line != "") { print diff_line diff_line = "" } in_deleted_file_block = 0 } } !in_deleted_file_block ' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" # Remove 'diff --git' lines only when followed by 'similarity index', 'rename from', and 'rename to' awk ' /^diff --git/ { diff_line = $0 getline if ($0 ~ /^similarity index/) { getline if ($0 ~ /^rename from/) { getline if ($0 ~ /^rename to/) { next } } } print diff_line } { print } ' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" - name: Run docker-compose uses: hoverkraft-tech/compose-action@f1ca7fefe3627c2dab0ae1db43a106d82740245e # v2.0.2 with: compose-file: "./dockerfiles/docker-compose.dev.postgres.yml" - name: Run mutants id: run_mutants run: | # Disable immediate exit on error set +e cargo mutants --workspace --in-place --timeout-multiplier 1.5 --no-shuffle -vV --in-diff git.diff --output ./ --test-tool=nextest -- --all-targets --test-threads 1 exit_code=$? # Create the folder only containing the outcomes (.txt files) and make a file containing the exit code of the command mkdir mutants echo "$exit_code" > ./mutants/exit_code.txt mv ./mutants.out/*.txt mutants/ # Enable immediate exit on error again set -e - name: Print mutants id: print_tested_mutants shell: bash run: | # Info for creating the link that paths to the specific mutation tested server_url="${{ github.server_url }}" organisation="${{ github.repository_owner }}" repository="${{ github.event.repository.name }}" commit="${{ github.sha }}" # Function to write to github step summary with specific info depending on the mutation category write_section() { local section_title=$1 local file_name=$2 if [ -s "$file_name" ]; then if [[ "$section_title" != "" ]]; then echo "## $section_title" >> "$GITHUB_STEP_SUMMARY" fi if [[ "$section_title" == "Missed:" ]]; then echo "
" >> "$GITHUB_STEP_SUMMARY" echo "What are missed mutants?" >> "$GITHUB_STEP_SUMMARY" echo "
" >> "$GITHUB_STEP_SUMMARY" echo "No test failed with this mutation applied, which seems to indicate a gap in test coverage. Or, it may be that the mutant is undistinguishable from the correct code. You may wish to add a better test, or mark that the function should be skipped." >> "$GITHUB_STEP_SUMMARY" echo "
" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" elif [[ "$section_title" == "Timeout:" ]]; then echo "
" >> "$GITHUB_STEP_SUMMARY" echo "What are timeout mutants?" >> "$GITHUB_STEP_SUMMARY" echo "
" >> "$GITHUB_STEP_SUMMARY" echo "The mutation caused the test suite to run for a long time, until it was eventually killed. You might want to investigate the cause and potentially mark the function to be skipped." >> "$GITHUB_STEP_SUMMARY" echo "
" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" elif [[ "$section_title" == "Unviable:" ]]; then echo "
" >> "$GITHUB_STEP_SUMMARY" echo "What are unviable mutants?" >> "$GITHUB_STEP_SUMMARY" echo "
" >> "$GITHUB_STEP_SUMMARY" echo "The attempted mutation doesn't compile. This is inconclusive about test coverage and no action is needed, unless you wish to test the specific function, in which case you may wish to add a 'Default::default()' implementation for the specific return type." >> "$GITHUB_STEP_SUMMARY" echo "
" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" fi if [[ "$section_title" != "" ]]; then awk -F':' '{gsub("%", "%%"); printf "- [ ] [%s](%s/%s/%s/blob/%s/%s#L%d)\n\n", $0, "'"$server_url"'", "'"$organisation"'", "'"$repository"'", "'"$commit"'", $1, $2-1}' "$file_name" >> "$GITHUB_STEP_SUMMARY" else awk -F':' '{gsub("%", "%%"); printf "- [x] [%s](%s/%s/%s/blob/%s/%s#L%d)\n\n", $0, "'"$server_url"'", "'"$organisation"'", "'"$repository"'", "'"$commit"'", $1, $2-1}' "$file_name" >> "$GITHUB_STEP_SUMMARY" fi if [[ "$section_title" == "Missed:" ]]; then echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY" echo "- Modify or add tests including this function." >> "$GITHUB_STEP_SUMMARY" echo "- If you are absolutely certain that this function should not undergo mutation testing, add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY" elif [[ "$section_title" == "Timeout:" ]]; then echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY" echo "- Modify the tests that include this funcion." >> "$GITHUB_STEP_SUMMARY" echo "- Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY" elif [[ "$section_title" == "Unviable:" ]]; then echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY" echo "- Create 'Default::default()' implementation for the specific structure." >> "$GITHUB_STEP_SUMMARY" echo "- Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY" fi echo >> "$GITHUB_STEP_SUMMARY" fi } # Print uncaught (missed/timeout/unviable) mutants to summary if [ -s ./mutants/missed.txt -o -s ./mutants/timeout.txt -o -s ./mutants/unviable.txt ]; then echo "# Uncaught Mutants" >> "$GITHUB_STEP_SUMMARY" echo "[Documentation - How to treat Mutants Output](https://github.com/stacks-network/actions/tree/main/stacks-core/mutation-testing#how-mutants-output-should-be-treated)" >> "$GITHUB_STEP_SUMMARY" write_section "Missed:" "./mutants/missed.txt" write_section "Timeout:" "./mutants/timeout.txt" write_section "Unviable:" "./mutants/unviable.txt" fi # Print caught mutants to summary if [ -s ./mutants/caught.txt ]; then echo "# Caught Mutants" >> "$GITHUB_STEP_SUMMARY" write_section "" "./mutants/caught.txt" fi # Get exit code from the file and match it exit_code=$(<"mutants/exit_code.txt") yellow_bold="\033[1;33m" reset="\033[0m" summary_link_message="${yellow_bold}Click here for more information on how to fix:${reset} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#:~:text=Output%20Mutants%20summary" case $exit_code in 0) if [[ ! -f ./mutants/caught.txt && ! -f ./mutants/missed.txt && ! -f ./mutants/timeout.txt && ! -f ./mutants/unviable.txt ]]; then echo "No mutants found to test!" elif [[ -s ./mutants/unviable.txt ]]; then echo -e "$summary_link_message" echo "Found unviable mutants!" exit 5 else echo "All new and updated functions are caught!" fi ;; 1) echo -e "$summary_link_message" echo "Invalid command line arguments!" exit $exit_code ;; 2) echo -e "$summary_link_message" echo "Found missed mutants!" exit $exit_code ;; 3) echo -e "$summary_link_message" echo "Found timeout mutants!" exit $exit_code ;; 4) echo -e "$summary_link_message" echo "Building the packages failed without any mutations!" exit $exit_code ;; *) echo -e "$summary_link_message" echo "Unknown exit code: $exit_code" exit $exit_code ;; esac exit 0