diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 312211a1..fa3e3ff0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -51,3 +51,13 @@ jobs: run: | export PATH=/tmp/bin:$PATH ./test/apply_fixes/execute_tests.py + + integration-tests-examples: + runs-on: ubuntu-22.04 + needs: [ pre-commit, fast-tests ] + steps: + - uses: actions/checkout@v4 + - name: Integration tests - Examples + run: | + cd examples + ./test.py diff --git a/examples/test.py b/examples/test.py new file mode 100755 index 00000000..d43941c9 --- /dev/null +++ b/examples/test.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import logging +import shlex +import subprocess +import sys +from dataclasses import dataclass + +logging.basicConfig(format="%(message)s", level=logging.INFO) + + +@dataclass +class Example: + cmd: str + expected_success: bool + + +EXAMPLES = [ + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu --output_groups=dwyu //basic_usage:correct_dependencies", + expected_success=True, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu --output_groups=dwyu //basic_usage:false_dependencies", + expected_success=False, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu --output_groups=dwyu //basic_usage:not_using_lib", + expected_success=False, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu_ignoring_includes --output_groups=dwyu //ignoring_includes:use_unavailable_headers", + expected_success=True, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu --output_groups=dwyu //recursion:use_lib", + expected_success=True, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu_recursive --output_groups=dwyu //recursion:use_lib", + expected_success=False, + ), + Example( + cmd="bazel build //rule_using_dwyu/...", + expected_success=False, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu --output_groups=dwyu //skipping_targets:bad_target", + expected_success=False, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu --output_groups=dwyu //skipping_targets:bad_target_skipped", + expected_success=True, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu_custom_skipping --output_groups=dwyu //skipping_targets:bad_target_custom_skip", + expected_success=True, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu_map_specific_deps --output_groups=dwyu //target_mapping:use_lib_b", + expected_success=True, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu_map_direct_deps --output_groups=dwyu //target_mapping:use_lib_b", + expected_success=True, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu_map_direct_deps --output_groups=dwyu //target_mapping:use_lib_c", + expected_success=False, + ), + Example( + cmd="bazel build --aspects=//:aspect.bzl%dwyu_map_transitive_deps --output_groups=dwyu //target_mapping:use_lib_c", + expected_success=True, + ), +] + + +@dataclass +class Result: + example: str + success: bool + + +def execute_example(example: Example) -> Result: + logging.info(f"\n##\n## Executing: {example.cmd}\n##\n") + + process = subprocess.run(shlex.split(example.cmd), check=False) + if (process.returncode == 0 and example.expected_success) or ( + process.returncode != 0 and not example.expected_success + ): + return Result(example=example.cmd, success=True) + return Result(example=example.cmd, success=False) + + +def main() -> int: + """ + Basic testing if the examples behave as desired. + + We ony look for the return code of a command. If a command fails as expected we do not analyze if it fails for the + correct reason. These kind of detailed testing is done in the integration tests. + """ + results = [execute_example(ex) for ex in EXAMPLES] + failed_examples = [res.example for res in results if not res.success] + + if failed_examples: + logging.info("\nFAILURE: The following examples did not behave as expected:") + logging.info("\n".join(f"- {failed}" for failed in failed_examples)) + return 1 + + logging.info("\nSUCCESS: All examples behaved as expected") + return 0 + + +if __name__ == "__main__": + sys.exit(main())