Skip to content

Commit

Permalink
coverage: add initial implementation of coverage with llvm-cov
Browse files Browse the repository at this point in the history
Run coverage on tests without baseline
  • Loading branch information
danctorres committed Nov 29, 2024
1 parent 60ba2bb commit efc6835
Show file tree
Hide file tree
Showing 3 changed files with 162 additions and 0 deletions.
4 changes: 4 additions & 0 deletions tools/coverage/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
sh_binary(
name = "llvm_cov",
srcs = ["llvm_cov.sh"],
)
47 changes: 47 additions & 0 deletions tools/coverage/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Coverage tooling

This project uses `llvm-cov` to generate coverage reports.

## 1. Key Features

1. **Local or Docker Setup:**
You can generate coverage reports by either installing `llvm-cov` locally or using the project's Docker environment.

2. **Partial Coverage Reporting:**
Currently, the reports are generated **without a baseline**, meaning only files touched during test execution are included.

**TODO:** Implement baseline generation to include all project files in the coverage report, even those not covered by tests.

## 2. Setup

To set up the required tools, follow these steps:

### 2.1. Without Docker

1. Update packages:

```bash
sudo apt update
```

2. Install LLVM Install the default version of LLVM:

```bash
sudo apt install llvm
```

### 2.2. Within Docker

For Docker setup, refer to the [README.md](../../README.md)

## How to Run

To generate a coverage report, run the following command:

```bash
bazel run //tools:llvm_cov -- -t <test> -o <output_dir>
```

Replace <test> with the name of the test target and <output_dir> with the desired output directory for the report.

Open the generated `index.html` file in the output directory to review test coverage.
111 changes: 111 additions & 0 deletions tools/coverage/llvm_cov.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/bin/bash

set -euo pipefail
set -o noglob

usage() {
echo " -t BAZEL_TESTS The test target (e.g., //path/to/test-target)"
echo " -b BAZEL_OUTPUT The bazel output base (e.g., $user/cov_cache)"
echo " -s SKIP_HTML_REPORT Defines if html report generation should be skipped"
echo " -o OUTPUT_DIR Directory for output (default is temp directory)"
echo " -h Display this help message"
}

while getopts "t:c:o:sh" option; do
case "${option}" in
t) BAZEL_TESTS=${OPTARG} ;;
c) BAZEL_OUTPUT=${OPTARG} ;;
s) SKIP_HTML_REPORT=true ;;
o) OUTPUT_DIR=${OPTARG} ;;
h | *)
usage
exit 1
;;
esac
done

if [[ -z "${BAZEL_TESTS}" ]]; then
echo "ERROR: Missing required arguments."
usage
exit 1
fi

OUTPUT_DIR="${OUTPUT_DIR:-"$(mktemp -d)/cov_report"}"
BAZEL_OUTPUT=${BAZEL_OUTPUT:-"/home/${USER}/.cache/coverage"}
SKIP_HTML_REPORT=${SKIP_HTML_REPORT:-false}
echo "INFO: Bazel cache: ${BAZEL_OUTPUT}"
echo "INFO: Tests target: ${BAZEL_TESTS}"
echo "INFO: Output directory: ${OUTPUT_DIR}"

readonly WORKSPACE=$(cd "$(dirname "$(readlink -f "${0}")")" && bazel info workspace)

function run_tests() {
echo -e "\nInfo: run test targets
Bazel output: ${BAZEL_OUTPUT}
Test target: ${BAZEL_TESTS}\n"
bazel --output_base="${BAZEL_OUTPUT}" test \
-c dbg \
--cxxopt=-O0 \
--experimental_fetch_all_coverage_outputs \
--nocache_test_results \
--build_runfile_links \
--strategy=TestRunner=standalone \
--copt=-fcoverage-mapping \
--copt=-fprofile-instr-generate \
--linkopt=-fcoverage-mapping \
--linkopt=-fprofile-instr-generate \
--instrumentation_filter="[/:]" \
-- ${BAZEL_TESTS}
}

function create_profdata() {
PROFRAW_PATH=$(find "${BAZEL_OUTPUT}" -name "default.profraw")
echo -e "\nINFO: merge raw profile data
Bazel output: ${BAZEL_OUTPUT}
Profile file: $PROFRAW_PATH
Output file: ${BAZEL_OUTPUT}/coverage.profdata\n"
llvm-profdata merge \
-sparse "$PROFRAW_PATH" \
-o ${BAZEL_OUTPUT}/coverage.profdata
}

function create_html_report() {
echo -e "\nINFO: create html report
Bazel output: ${BAZEL_OUTPUT}
Profile files: ${BAZEL_OUTPUT}/coverage.profdata
Output report html: ${OUTPUT_DIR}/index.html\n"
SOURCE_FILES=$(find ${BAZEL_OUTPUT} \
-type f \
-name "*.o" |
grep -vE "external/|test/.*")
llvm-cov show "${SOURCE_FILES}" \
--instr-profile="${BAZEL_OUTPUT}/coverage.profdata" \
-format=html \
-output-dir="${OUTPUT_DIR}"
}

function show_summary() {
echo -e "\nINFO: show coverage summary
Bazel output: ${BAZEL_OUTPUT}
Profile files: ${BAZEL_OUTPUT}/coverage.profdata
Output report html: ${OUTPUT_DIR}/index.html\n"
SOURCE_FILES=$(find ${BAZEL_OUTPUT} \
-type f \
-name "*.o" |
grep -vE "external/|test/.*")
llvm-cov report "${SOURCE_FILES}" \
--instr-profile="${BAZEL_OUTPUT}/coverage.profdata"
}

function main() {
pushd "${WORKSPACE}" || return
run_tests
create_profdata
show_summary
if [ "${SKIP_HTML_REPORT}" = false ]; then
create_html_report
fi
popd || return
}

main

0 comments on commit efc6835

Please sign in to comment.