diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh index 2eb58e8dc75ec..e417dd5380fe1 100755 --- a/ci/scripts/integration_arrow.sh +++ b/ci/scripts/integration_arrow.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -set -ex +set -e arrow_dir=${1} build_dir=${2} @@ -30,8 +30,14 @@ gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration : ${ARROW_INTEGRATION_JAVA:=ON} : ${ARROW_INTEGRATION_JS:=ON} +echo "::group::Integration: Prepare: Archery" +set -x pip install -e $arrow_dir/dev/archery[integration] +set +x +echo "::endgroup::" +echo "::group::Integration: Prepare: Dependencies" +set -x # For C Data Interface testing if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then pip install pythonnet @@ -39,6 +45,8 @@ fi if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then pip install jpype1 fi +set +x +echo "::endgroup::" export ARROW_BUILD_ROOT=${build_dir} diff --git a/ci/scripts/integration_arrow_build.sh b/ci/scripts/integration_arrow_build.sh index 9b54049a2b803..f7c6749217365 100755 --- a/ci/scripts/integration_arrow_build.sh +++ b/ci/scripts/integration_arrow_build.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -set -ex +set -e arrow_dir=${1} build_dir=${2} @@ -28,22 +28,44 @@ build_dir=${2} : ${ARROW_INTEGRATION_JAVA:=ON} : ${ARROW_INTEGRATION_JS:=ON} +echo "::group::Integration: Build: Rust" +set -x ${arrow_dir}/ci/scripts/rust_build.sh ${arrow_dir} ${build_dir} +set +x +echo "::endgroup::" +echo "::group::Integration: Build: nanoarrow" +set -x ${arrow_dir}/ci/scripts/nanoarrow_build.sh ${arrow_dir} ${build_dir} +set +x +echo "::endgroup::" +echo "::group::Integration: Build: C++" +set -x if [ "${ARROW_INTEGRATION_CPP}" == "ON" ]; then ${arrow_dir}/ci/scripts/cpp_build.sh ${arrow_dir} ${build_dir} fi +set +x +echo "::endgroup::" +echo "::group::Integration: Build: C#" +set -x if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then ${arrow_dir}/ci/scripts/csharp_build.sh ${arrow_dir} ${build_dir} fi +set +x +echo "::endgroup::" +echo "::group::Integration: Build: Go" +set -x if [ "${ARROW_INTEGRATION_GO}" == "ON" ]; then ${arrow_dir}/ci/scripts/go_build.sh ${arrow_dir} ${build_dir} fi +set +x +echo "::endgroup::" +echo "::group::Integration: Build: Java" +set -x if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then export ARROW_JAVA_CDATA="ON" export JAVA_JNI_CMAKE_ARGS="-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DARROW_JAVA_JNI_ENABLE_C=ON" @@ -51,7 +73,13 @@ if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then ${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java ${arrow_dir}/ci/scripts/java_build.sh ${arrow_dir} ${build_dir} /tmp/dist/java fi +set +x +echo "::endgroup::" +echo "::group::Integration: Build: JavaScript" +set -x if [ "${ARROW_INTEGRATION_JS}" == "ON" ]; then ${arrow_dir}/ci/scripts/js_build.sh ${arrow_dir} ${build_dir} fi +set +x +echo "::endgroup::" diff --git a/dev/archery/archery/docker/cli.py b/dev/archery/archery/docker/cli.py index 23c565f7780ff..6a1303a8983d5 100644 --- a/dev/archery/archery/docker/cli.py +++ b/dev/archery/archery/docker/cli.py @@ -21,6 +21,7 @@ import click from ..utils.cli import validate_arrow_sources +from ..utils.logger import group from .core import DockerCompose, UndefinedImage @@ -82,11 +83,12 @@ def docker(ctx, src, dry_run, using_legacy_docker_compose, using_docker_cli, using_docker_cli |= using_docker_buildx compose_bin = ("docker-compose" if using_legacy_docker_compose else "docker compose") - compose = DockerCompose(config_path, params=os.environ, - using_docker=using_docker_cli, - using_buildx=using_docker_buildx, - debug=ctx.obj.get('debug', False), - compose_bin=compose_bin) + with group("Docker: Preppare"): + compose = DockerCompose(config_path, params=os.environ, + using_docker=using_docker_cli, + using_buildx=using_docker_buildx, + debug=ctx.obj.get('debug', False), + compose_bin=compose_bin) if dry_run: _mock_compose_calls(compose) ctx.obj['compose'] = compose @@ -229,10 +231,12 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build, env = dict(kv.split('=', 1) for kv in env) try: if force_pull: - compose.pull(image, pull_leaf=use_leaf_cache) + with group("Docker: Pull"): + compose.pull(image, pull_leaf=use_leaf_cache) if force_build: - compose.build(image, use_cache=use_cache, - use_leaf_cache=use_leaf_cache) + with group("Docker: Build"): + compose.build(image, use_cache=use_cache, + use_leaf_cache=use_leaf_cache) if build_only: return compose.run( diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py index 66c8721519ede..25623689e7378 100644 --- a/dev/archery/archery/integration/runner.py +++ b/dev/archery/archery/integration/runner.py @@ -39,6 +39,7 @@ from .tester_nanoarrow import NanoarrowTester from .util import guid, printer from .util import SKIP_C_ARRAY, SKIP_C_SCHEMA, SKIP_FLIGHT, SKIP_IPC +from ..utils.logger import group from ..utils.source import ARROW_ROOT_DEFAULT from . import datagen @@ -91,20 +92,22 @@ def run_ipc(self): self._compare_ipc_implementations( producer, consumer, self._produce_consume, self.json_files) + if self.gold_dirs: for gold_dir, consumer in itertools.product( self.gold_dirs, filter(lambda t: t.CONSUMER, self.testers)): - log('\n') - log('******************************************************') - log('Tests against golden files in {}'.format(gold_dir)) - log('******************************************************') - - def run_gold(_, consumer, test_case: datagen.File): - return self._run_gold(gold_dir, consumer, test_case) - self._compare_ipc_implementations( - consumer, consumer, run_gold, - self._gold_tests(gold_dir)) + with group(f"Integration: Test: IPC: Gold: {consumer.name}"): + log('\n') + log('******************************************************') + log('Tests against golden files in {}'.format(gold_dir)) + log('******************************************************') + + def run_gold(_, consumer, test_case: datagen.File): + return self._run_gold(gold_dir, consumer, test_case) + self._compare_ipc_implementations( + consumer, consumer, run_gold, + self._gold_tests(gold_dir)) log('\n') def run_flight(self): @@ -233,14 +236,15 @@ def _compare_ipc_implementations( """ Compare Arrow IPC for two implementations (one producer, one consumer). """ - log('##########################################################') - log('IPC: {0} producing, {1} consuming' - .format(producer.name, consumer.name)) - log('##########################################################') + with group(f"Integration: Test: IPC: {producer.name} -> {consumer.name}"): + log('##########################################################') + log('IPC: {0} producing, {1} consuming' + .format(producer.name, consumer.name)) + log('##########################################################') - case_runner = partial(self._run_ipc_test_case, - producer, consumer, run_binaries) - self._run_test_cases(case_runner, test_cases) + case_runner = partial(self._run_ipc_test_case, + producer, consumer, run_binaries) + self._run_test_cases(case_runner, test_cases) def _run_ipc_test_case( self, @@ -357,14 +361,15 @@ def _compare_flight_implementations( producer: Tester, consumer: Tester ): - log('##########################################################') - log('Flight: {0} serving, {1} requesting' - .format(producer.name, consumer.name)) - log('##########################################################') + with group(f"Integration: Test: Flight: {producer.name} -> {consumer.name}"): + log('##########################################################') + log('Flight: {0} serving, {1} requesting' + .format(producer.name, consumer.name)) + log('##########################################################') - case_runner = partial(self._run_flight_test_case, producer, consumer) - self._run_test_cases( - case_runner, self.json_files + self.flight_scenarios) + case_runner = partial(self._run_flight_test_case, producer, consumer) + self._run_test_cases( + case_runner, self.json_files + self.flight_scenarios) def _run_flight_test_case(self, producer: Tester, @@ -415,27 +420,32 @@ def _compare_c_data_implementations( producer: Tester, consumer: Tester ): - log('##########################################################') - log(f'C Data Interface: ' - f'{producer.name} exporting, {consumer.name} importing') - log('##########################################################') - - # Serial execution is required for proper memory accounting - serial = True - - with producer.make_c_data_exporter() as exporter: - with consumer.make_c_data_importer() as importer: - case_runner = partial(self._run_c_schema_test_case, - producer, consumer, - exporter, importer) - self._run_test_cases(case_runner, self.json_files, serial=serial) - - if producer.C_DATA_ARRAY_EXPORTER and consumer.C_DATA_ARRAY_IMPORTER: - case_runner = partial(self._run_c_array_test_cases, + with group("Integration: Test: C Data Interface: " + f"{producer.name} -> {consumer.name}"): + log('##########################################################') + log(f'C Data Interface: ' + f'{producer.name} exporting, {consumer.name} importing') + log('##########################################################') + + # Serial execution is required for proper memory accounting + serial = True + + with producer.make_c_data_exporter() as exporter: + with consumer.make_c_data_importer() as importer: + case_runner = partial(self._run_c_schema_test_case, producer, consumer, exporter, importer) self._run_test_cases(case_runner, self.json_files, serial=serial) + if producer.C_DATA_ARRAY_EXPORTER and \ + consumer.C_DATA_ARRAY_IMPORTER: + case_runner = partial(self._run_c_array_test_cases, + producer, consumer, + exporter, importer) + self._run_test_cases(case_runner, + self.json_files, + serial=serial) + def _run_c_schema_test_case(self, producer: Tester, consumer: Tester, exporter: CDataExporter, @@ -657,22 +667,23 @@ def run_all_tests(with_cpp=True, with_java=True, with_js=True, if run_c_data: runner.run_c_data() - fail_count = 0 - if runner.failures: - log("################# FAILURES #################") - for test_case, producer, consumer, exc_info in runner.failures: - fail_count += 1 - log("FAILED TEST:", end=" ") - log(test_case.name, producer.name, "producing, ", - consumer.name, "consuming") - if exc_info: - exc_type, exc_value, exc_tb = exc_info - log(f'{exc_type}: {exc_value}') - log() - - log(f"{fail_count} failures, {len(runner.skips)} skips") - if fail_count > 0: - sys.exit(1) + with group("Integration: Test: Result"): + fail_count = 0 + if runner.failures: + log("################# FAILURES #################") + for test_case, producer, consumer, exc_info in runner.failures: + fail_count += 1 + log("FAILED TEST:", end=" ") + log(test_case.name, producer.name, "producing, ", + consumer.name, "consuming") + if exc_info: + exc_type, exc_value, exc_tb = exc_info + log(f'{exc_type}: {exc_value}') + log() + + log(f"{fail_count} failures, {len(runner.skips)} skips") + if fail_count > 0: + sys.exit(1) def write_js_test_json(directory): diff --git a/dev/archery/archery/utils/logger.py b/dev/archery/archery/utils/logger.py index 9d0feda88e6ea..7b3e0a5612234 100644 --- a/dev/archery/archery/utils/logger.py +++ b/dev/archery/archery/utils/logger.py @@ -15,7 +15,9 @@ # specific language governing permissions and limitations # under the License. +from contextlib import contextmanager import logging +import os """ Global logger. """ logger = logging.getLogger("archery") @@ -27,3 +29,21 @@ def __init__(self, quiet=False): ctx = LoggingContext() + +in_github_actions = (os.environ.get("GITHUB_ACTIONS") == "true") + + +@contextmanager +def group(name): + """ + Group outputs in the given with block. + + This does nothing in non GitHub Actions environment for now. + """ + if in_github_actions: + print(f"::group::{name}") + try: + yield + finally: + if in_github_actions: + print("::endgroup::")