Improve coverage builds and fix coverage reporting

Author: anonrig

.bazelrc

@@ -68,9 +68,15 @@ build:v8-codegen-opt --per_file_copt=external/abseil-cpp@-O2
 # zlib and tcmalloc (for Linux) are also CPU-intensive, optimize them too.
 build:v8-codegen-opt --per_file_copt=external/tcmalloc@-O2
 build:v8-codegen-opt --per_file_copt=external/zlib@-O2
+# BoringSSL is CPU-intensive for crypto tests, optimize it too.
+build:v8-codegen-opt --per_file_copt=external/boringssl@-O2
+# simdutf is used for fast string encoding/decoding
+build:v8-codegen-opt --per_file_copt=external/+http+simdutf@-O2
 build:v8-codegen-opt-windows --per_file_copt=v8/src@/O2
 build:v8-codegen-opt-windows --per_file_copt=external/abseil-cpp@/O2
 build:v8-codegen-opt-windows --per_file_copt=external/zlib@/O2
+build:v8-codegen-opt-windows --per_file_copt=external/boringssl@/O2
+build:v8-codegen-opt-windows --per_file_copt=external/+http+simdutf@/O2
 
 # In Google projects, exceptions are not used as a rule. Disabling them is more consistent with the
 # canonical V8 build and improves code size. Paths are adjusted for bzlmod mangling – V8 and ICU use
@@ -415,29 +421,59 @@ build:coverage --test_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1
 # GCOV is used by rules_cc to merge raw profile data (.profraw) into indexed profile data (.profdata)
 build:coverage --action_env=GCOV=llvm-profdata
 build:coverage --test_env=GCOV=llvm-profdata
+# COVERAGE_GCOV_PATH is passed through from the environment. Users must set this to the absolute path
+# of llvm-profdata (used by collect_cc_coverage.sh for merging .profraw files).
+# Example: export COVERAGE_GCOV_PATH=$(which llvm-profdata)
+build:coverage --action_env=COVERAGE_GCOV_PATH
+build:coverage --test_env=COVERAGE_GCOV_PATH
 # BAZEL_LLVM_COV is used by rules_cc to generate coverage reports from profile data
 build:coverage --action_env=BAZEL_LLVM_COV=llvm-cov
 build:coverage --test_env=BAZEL_LLVM_COV=llvm-cov
-# Enable optimizations to reduce coverage overhead while maintaining good coverage quality
-build:coverage --copt=-O2
-build:coverage --copt=-DNDEBUG
-build:coverage --combined_report=lcov
+# LLVM_COV is used by collect_cc_coverage.sh for generating LCOV output
+build:coverage --test_env=LLVM_COV=llvm-cov
 build:coverage --experimental_use_llvm_covmap
 build:coverage --experimental_generate_llvm_lcov
+# Ensure that we fetch coverage data from remote cache
 build:coverage --experimental_fetch_all_coverage_outputs
+build:coverage --experimental_split_coverage_postprocessing
+# Allow coverage outputs to be writable for post-processing
+build:coverage --experimental_writable_outputs
 build:coverage --collect_code_coverage
-build:coverage --instrumentation_filter="^//,-//external"
+# Only instrument source code, not tests or tools - significantly speeds up coverage builds
+build:coverage --instrumentation_filter="^//src/workerd[/:],^//src/rust[/:]"
 build:coverage --instrument_test_targets
+# Disable coverage instrumentation for external dependencies to speed up compilation.
+# Coverage for V8/external code is not useful for our purposes.
+# These flags negate -fprofile-instr-generate and -fcoverage-mapping set by rules_cc.
+build:coverage --per_file_copt=external/.*@-fno-profile-instr-generate,-fno-coverage-mapping
 # KJ uses _exit() by default which bypasses atexit handlers and prevents LLVM profile runtime
 # from writing coverage data. KJ_CLEAN_SHUTDOWN forces use of normal exit() instead.
 build:coverage --test_env=KJ_CLEAN_SHUTDOWN=1
+# Use -O1 for faster compilation - coverage builds don't need heavy optimization
+build:coverage --copt=-O1
+# External dependencies can use -O2 since they're not instrumented anyway
+build:coverage --config=v8-codegen-opt
+# Run coverage-related actions locally with sandboxing - remote execution doesn't support LLVM coverage well
+build:coverage --strategy=TestRunner=worker,sandboxed,local
+build:coverage --strategy=CoverageReport=worker,sandboxed,local
+build:coverage --strategy=CoveragePostprocessing=worker,sandboxed,local
+# Reduce debug info for faster compilation and smaller binaries
+build:coverage --copt=-g1
+# Use limited coverage mode for smaller binaries and faster execution (used by Chromium)
+build:coverage --copt=-mllvm
+build:coverage --copt=-limited-coverage-experimental=true
+# Disable gc-sections for faster linking (coverage doesn't need dead code elimination)
+build:coverage --linkopt=-Wl,--no-gc-sections
+# Disable test result caching to ensure fresh coverage data is generated
+coverage --nocache_test_results
 coverage --test_tag_filters=-off-by-default,-requires-fuzzilli,-requires-container-engine,-lint,-benchmark,-workerd-benchmark,-no-coverage
 # Coverage instrumentation slows down test execution, so extend timeouts
 # We disable enormous tests due to the slowdown (CI jobs have a 6h max duration)
 coverage --test_size_filters=-enormous
-coverage --test_timeout=60,240,240,240
+coverage --test_timeout=240,240,240,240
 coverage --build_tests_only
 coverage --config=coverage
+coverage --combined_report=lcov
 
 # This config is defined internally and enabled on many machines.
 # Defining it as empty just so these machines can run build commands from the workerd repo

.github/actions/setup-runner/action.yml

@@ -23,6 +23,8 @@ runs:
           libclang-rt-19-dev \
           llvm-19
         sudo ln -s /usr/bin/llvm-symbolizer-19 /usr/bin/llvm-symbolizer
+        sudo ln -s /usr/bin/llvm-profdata-19 /usr/bin/llvm-profdata
+        sudo ln -s /usr/bin/llvm-cov-19 /usr/bin/llvm-cov
         echo "build:linux --action_env=CC=/usr/lib/llvm-19/bin/clang" >> .bazelrc
         echo "build:linux --host_action_env=CC=/usr/lib/llvm-19/bin/clang" >> .bazelrc
         echo "build:linux --linkopt=--ld-path=/usr/lib/llvm-19/bin/ld.lld" >> .bazelrc

.github/workflows/_bazel.yml

@@ -125,8 +125,6 @@ jobs:
         run: |
           bazel --nowindows_enable_symlinks build ${{ inputs.extra_bazel_args }} --config=ci --profile build-win-workaround.bazel-profile.gz --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev //src/wpt:wpt-all@tsproject //src/node:node@tsproject //src/pyodide:pyodide_static@tsproject
       - name: Bazel build
-        # timestamps are no longer being added here, the GitHub logs include timestamps (Use
-        # 'Show timestamps' on the web interface)
         run: |
           bazel build --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev  --config=ci ${{ inputs.extra_bazel_args }} //...
       - name: Build and load container images
@@ -142,23 +140,20 @@ jobs:
         if: inputs.run_tests
         run: |
           bazel test --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev --config=ci ${{ inputs.extra_bazel_args }} ${{ inputs.test_target }}
-      - name: Setup LLVM tools for coverage
-        if: inputs.run_coverage
-        run: |
-          sudo apt-get update && sudo apt-get install -y llvm
-          sudo ln -sf /usr/bin/llvm-profdata /usr/local/bin/llvm-profdata
-          sudo ln -sf /usr/bin/llvm-cov /usr/local/bin/llvm-cov
       - name: Bazel coverage
         if: inputs.run_coverage
         run: |
-          bazel coverage --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev --config=ci --config=coverage ${{ inputs.extra_bazel_args }} ${{ inputs.test_target }}
+          bazel coverage --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev --config=ci ${{ inputs.extra_bazel_args }} ${{ inputs.test_target }}
       - name: Upload coverage to Codecov
         if: inputs.run_coverage
         uses: codecov/codecov-action@v5
         with:
-          files: ./bazel-out/_coverage/_coverage_report.dat
+          # Use bazel info to get the actual output path, avoiding symlink issues
+          files: ${{ github.workspace }}/bazel-out/_coverage/_coverage_report.dat
           fail_ci_if_error: true
           verbose: true
+          disable_search: true
+          root_dir: ${{ github.workspace }}
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       - name: Parse headers

.github/workflows/test.yml

@@ -106,7 +106,7 @@ jobs:
       suffix: 'coverage'
       # Use ci-linux-common instead of ci-linux to avoid overriding coverage test_tag_filters
       # (ci-linux sets test_tag_filters that would include container tests which require docker)
-      extra_bazel_args: '--config=ci-test --config=ci-linux-common --config=ci-limit-storage'
+      extra_bazel_args: '--config=ci-test --config=ci-linux-common --config=ci-limit-storage --config=coverage'
       upload_test_logs: true
       upload_binary: false
       build_container_images: false

build/fixtures/BUILD.bazel

@@ -1 +1,6 @@
-exports_files(["kj_test.sh"])
+exports_files([
+    "kj_test.sh",
+    "wd_test.sh",
+    "wd_test_sidecar.sh",
+    "wd_test_snapshot.sh",
+])

build/fixtures/kj_test.sh

@@ -1,3 +1,8 @@
 #!/bin/bash
 set -euo pipefail
-"$@"
+
+if [ -n "${COVERAGE_DIR:-}" ]; then
+    export LLVM_PROFILE_FILE="$COVERAGE_DIR/%p-%m.profraw"
+fi
+
+exec "$@"

build/kj_test.bzl

@@ -50,18 +50,20 @@ def kj_test(
 
     sh_test(
         name = test_name + "@",
-        size = size,
         srcs = ["//build/fixtures:kj_test.sh"],
-        data = [cross_alias] + data,
-        args = ["$(location " + cross_alias + ")"],
+        args = ["$(location {})".format(cross_alias)],
+        data = data + [cross_alias],
         tags = tags,
+        size = size,
     )
+
     sh_test(
         name = test_name + "@all-autogates",
-        size = size,
-        env = {"WORKERD_ALL_AUTOGATES": "1"},
         srcs = ["//build/fixtures:kj_test.sh"],
-        data = [cross_alias] + data,
-        args = ["$(location " + cross_alias + ")"],
-        tags = tags,
+        args = ["$(location {})".format(cross_alias)],
+        data = data + [cross_alias],
+        env = {"WORKERD_ALL_AUTOGATES": "1"},
+        # Tag with no-coverage to reduce coverage CI time
+        tags = tags + ["no-coverage"],
+        size = size,
     )

build/wd_cc_binary.bzl

@@ -47,6 +47,5 @@ def wd_cc_binary(
             "@//build/config:prebuilt_binaries_arm64": "@//:bin.arm64/tmp/{}/{}.aarch64-linux-gnu".format(pkg, prebuilt_binary_name),
             "//conditions:default": name,
         }),
-        # Propagate InstrumentedFilesInfo through the alias for coverage support
         testonly = kwargs.get("testonly", False),
     )

build/wd_rust_binary.bzl

@@ -54,10 +54,6 @@ def wd_rust_binary(
         visibility = visibility,
         data = data,
         proc_macro_deps = proc_macro_deps,
-        experimental_use_cc_common_link = select({
-            "@platforms//os:windows": 0,
-            "//conditions:default": 1,
-        }),
         target_compatible_with = select({
             "@//build/config:no_build": ["@platforms//:incompatible"],
             "//conditions:default": [],
@@ -74,10 +70,6 @@ def wd_rust_binary(
             # our tests are usually very heavy and do not support concurrent invocation
             "RUST_TEST_THREADS": "1",
         },
-        experimental_use_cc_common_link = select({
-            "@platforms//os:windows": 0,
-            "//conditions:default": 1,
-        }),
         target_compatible_with = select({
             "@//build/config:no_build": ["@platforms//:incompatible"],
             "//conditions:default": [],

build/wd_rust_crate.bzl

@@ -148,10 +148,6 @@ def wd_rust_crate(
         crate_features = crate_features,
         deps = test_deps,
         proc_macro_deps = test_proc_macro_deps,
-        experimental_use_cc_common_link = select({
-            "@platforms//os:windows": 0,
-            "//conditions:default": 1,
-        }),
     )
 
     if len(proc_macro_deps) + len(cxx_bridge_srcs) > 0: