Skip to content

Commit

Permalink
apacheGH-44058: [CI][Integration] Group logs on GitHub Actions
Browse files Browse the repository at this point in the history
  • Loading branch information
@kou
kou committed Sep 11, 2024
1 parent 170ea04 commit 6e93a55
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 67 deletions.
10 changes: 9 additions & 1 deletion ci/scripts/integration_arrow.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# specific language governing permissions and limitations
# under the License.

set -ex
set -e

arrow_dir=${1}
build_dir=${2}
Expand All @@ -30,15 +30,23 @@ gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration
: ${ARROW_INTEGRATION_JAVA:=ON}
: ${ARROW_INTEGRATION_JS:=ON}

echo "::group::Integration: Prepare: Archery"
set -x
pip install -e $arrow_dir/dev/archery[integration]
set +x
echo "::endgroup::"

echo "::group::Integration: Prepare: Dependencies"
set -x
# For C Data Interface testing
if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then
pip install pythonnet
fi
if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then
pip install jpype1
fi
set +x
echo "::endgroup::"

export ARROW_BUILD_ROOT=${build_dir}

Expand Down
30 changes: 29 additions & 1 deletion ci/scripts/integration_arrow_build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# specific language governing permissions and limitations
# under the License.

set -ex
set -e

arrow_dir=${1}
build_dir=${2}
Expand All @@ -28,30 +28,58 @@ build_dir=${2}
: ${ARROW_INTEGRATION_JAVA:=ON}
: ${ARROW_INTEGRATION_JS:=ON}

echo "::group::Integration: Build: Rust"
set -x
${arrow_dir}/ci/scripts/rust_build.sh ${arrow_dir} ${build_dir}
set +x
echo "::endgroup::"

echo "::group::Integration: Build: nanoarrow"
set -x
${arrow_dir}/ci/scripts/nanoarrow_build.sh ${arrow_dir} ${build_dir}
set +x
echo "::endgroup::"

echo "::group::Integration: Build: C++"
set -x
if [ "${ARROW_INTEGRATION_CPP}" == "ON" ]; then
${arrow_dir}/ci/scripts/cpp_build.sh ${arrow_dir} ${build_dir}
fi
set +x
echo "::endgroup::"

echo "::group::Integration: Build: C#"
set -x
if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then
${arrow_dir}/ci/scripts/csharp_build.sh ${arrow_dir} ${build_dir}
fi
set +x
echo "::endgroup::"

echo "::group::Integration: Build: Go"
set -x
if [ "${ARROW_INTEGRATION_GO}" == "ON" ]; then
${arrow_dir}/ci/scripts/go_build.sh ${arrow_dir} ${build_dir}
fi
set +x
echo "::endgroup::"

echo "::group::Integration: Build: Java"
set -x
if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then
export ARROW_JAVA_CDATA="ON"
export JAVA_JNI_CMAKE_ARGS="-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DARROW_JAVA_JNI_ENABLE_C=ON"

${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java
${arrow_dir}/ci/scripts/java_build.sh ${arrow_dir} ${build_dir} /tmp/dist/java
fi
set +x
echo "::endgroup::"

echo "::group::Integration: Build: JavaScript"
set -x
if [ "${ARROW_INTEGRATION_JS}" == "ON" ]; then
${arrow_dir}/ci/scripts/js_build.sh ${arrow_dir} ${build_dir}
fi
set +x
echo "::endgroup::"
20 changes: 12 additions & 8 deletions dev/archery/archery/docker/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import click

from ..utils.cli import validate_arrow_sources
from ..utils.logger import group
from .core import DockerCompose, UndefinedImage


Expand Down Expand Up @@ -82,11 +83,12 @@ def docker(ctx, src, dry_run, using_legacy_docker_compose, using_docker_cli,
using_docker_cli |= using_docker_buildx
compose_bin = ("docker-compose" if using_legacy_docker_compose
else "docker compose")
compose = DockerCompose(config_path, params=os.environ,
using_docker=using_docker_cli,
using_buildx=using_docker_buildx,
debug=ctx.obj.get('debug', False),
compose_bin=compose_bin)
with group("Docker: Preppare"):
compose = DockerCompose(config_path, params=os.environ,
using_docker=using_docker_cli,
using_buildx=using_docker_buildx,
debug=ctx.obj.get('debug', False),
compose_bin=compose_bin)
if dry_run:
_mock_compose_calls(compose)
ctx.obj['compose'] = compose
Expand Down Expand Up @@ -229,10 +231,12 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
env = dict(kv.split('=', 1) for kv in env)
try:
if force_pull:
compose.pull(image, pull_leaf=use_leaf_cache)
with group("Docker: Pull"):
compose.pull(image, pull_leaf=use_leaf_cache)
if force_build:
compose.build(image, use_cache=use_cache,
use_leaf_cache=use_leaf_cache)
with group("Docker: Build"):
compose.build(image, use_cache=use_cache,
use_leaf_cache=use_leaf_cache)
if build_only:
return
compose.run(
Expand Down
125 changes: 68 additions & 57 deletions dev/archery/archery/integration/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from .tester_nanoarrow import NanoarrowTester
from .util import guid, printer
from .util import SKIP_C_ARRAY, SKIP_C_SCHEMA, SKIP_FLIGHT, SKIP_IPC
from ..utils.logger import group
from ..utils.source import ARROW_ROOT_DEFAULT
from . import datagen

Expand Down Expand Up @@ -91,20 +92,22 @@ def run_ipc(self):
self._compare_ipc_implementations(
producer, consumer, self._produce_consume,
self.json_files)

if self.gold_dirs:
for gold_dir, consumer in itertools.product(
self.gold_dirs,
filter(lambda t: t.CONSUMER, self.testers)):
log('\n')
log('******************************************************')
log('Tests against golden files in {}'.format(gold_dir))
log('******************************************************')

def run_gold(_, consumer, test_case: datagen.File):
return self._run_gold(gold_dir, consumer, test_case)
self._compare_ipc_implementations(
consumer, consumer, run_gold,
self._gold_tests(gold_dir))
with group(f"Integration: Test: IPC: Gold: {consumer.name}"):
log('\n')
log('******************************************************')
log('Tests against golden files in {}'.format(gold_dir))
log('******************************************************')

def run_gold(_, consumer, test_case: datagen.File):
return self._run_gold(gold_dir, consumer, test_case)
self._compare_ipc_implementations(
consumer, consumer, run_gold,
self._gold_tests(gold_dir))
log('\n')

def run_flight(self):
Expand Down Expand Up @@ -233,14 +236,15 @@ def _compare_ipc_implementations(
"""
Compare Arrow IPC for two implementations (one producer, one consumer).
"""
log('##########################################################')
log('IPC: {0} producing, {1} consuming'
.format(producer.name, consumer.name))
log('##########################################################')
with group(f"Integration: Test: IPC: {producer.name} -> {consumer.name}"):
log('##########################################################')
log('IPC: {0} producing, {1} consuming'
.format(producer.name, consumer.name))
log('##########################################################')

case_runner = partial(self._run_ipc_test_case,
producer, consumer, run_binaries)
self._run_test_cases(case_runner, test_cases)
case_runner = partial(self._run_ipc_test_case,
producer, consumer, run_binaries)
self._run_test_cases(case_runner, test_cases)

def _run_ipc_test_case(
self,
Expand Down Expand Up @@ -357,14 +361,15 @@ def _compare_flight_implementations(
producer: Tester,
consumer: Tester
):
log('##########################################################')
log('Flight: {0} serving, {1} requesting'
.format(producer.name, consumer.name))
log('##########################################################')
with group(f"Integration: Test: Flight: {producer.name} -> {consumer.name}"):
log('##########################################################')
log('Flight: {0} serving, {1} requesting'
.format(producer.name, consumer.name))
log('##########################################################')

case_runner = partial(self._run_flight_test_case, producer, consumer)
self._run_test_cases(
case_runner, self.json_files + self.flight_scenarios)
case_runner = partial(self._run_flight_test_case, producer, consumer)
self._run_test_cases(
case_runner, self.json_files + self.flight_scenarios)

def _run_flight_test_case(self,
producer: Tester,
Expand Down Expand Up @@ -415,27 +420,32 @@ def _compare_c_data_implementations(
producer: Tester,
consumer: Tester
):
log('##########################################################')
log(f'C Data Interface: '
f'{producer.name} exporting, {consumer.name} importing')
log('##########################################################')

# Serial execution is required for proper memory accounting
serial = True

with producer.make_c_data_exporter() as exporter:
with consumer.make_c_data_importer() as importer:
case_runner = partial(self._run_c_schema_test_case,
producer, consumer,
exporter, importer)
self._run_test_cases(case_runner, self.json_files, serial=serial)

if producer.C_DATA_ARRAY_EXPORTER and consumer.C_DATA_ARRAY_IMPORTER:
case_runner = partial(self._run_c_array_test_cases,
with group("Integration: Test: C Data Interface: "
f"{producer.name} -> {consumer.name}"):
log('##########################################################')
log(f'C Data Interface: '
f'{producer.name} exporting, {consumer.name} importing')
log('##########################################################')

# Serial execution is required for proper memory accounting
serial = True

with producer.make_c_data_exporter() as exporter:
with consumer.make_c_data_importer() as importer:
case_runner = partial(self._run_c_schema_test_case,
producer, consumer,
exporter, importer)
self._run_test_cases(case_runner, self.json_files, serial=serial)

if producer.C_DATA_ARRAY_EXPORTER and \
consumer.C_DATA_ARRAY_IMPORTER:
case_runner = partial(self._run_c_array_test_cases,
producer, consumer,
exporter, importer)
self._run_test_cases(case_runner,
self.json_files,
serial=serial)

def _run_c_schema_test_case(self,
producer: Tester, consumer: Tester,
exporter: CDataExporter,
Expand Down Expand Up @@ -657,22 +667,23 @@ def run_all_tests(with_cpp=True, with_java=True, with_js=True,
if run_c_data:
runner.run_c_data()

fail_count = 0
if runner.failures:
log("################# FAILURES #################")
for test_case, producer, consumer, exc_info in runner.failures:
fail_count += 1
log("FAILED TEST:", end=" ")
log(test_case.name, producer.name, "producing, ",
consumer.name, "consuming")
if exc_info:
exc_type, exc_value, exc_tb = exc_info
log(f'{exc_type}: {exc_value}')
log()

log(f"{fail_count} failures, {len(runner.skips)} skips")
if fail_count > 0:
sys.exit(1)
with group("Integration: Test: Result"):
fail_count = 0
if runner.failures:
log("################# FAILURES #################")
for test_case, producer, consumer, exc_info in runner.failures:
fail_count += 1
log("FAILED TEST:", end=" ")
log(test_case.name, producer.name, "producing, ",
consumer.name, "consuming")
if exc_info:
exc_type, exc_value, exc_tb = exc_info
log(f'{exc_type}: {exc_value}')
log()

log(f"{fail_count} failures, {len(runner.skips)} skips")
if fail_count > 0:
sys.exit(1)


def write_js_test_json(directory):
Expand Down
20 changes: 20 additions & 0 deletions dev/archery/archery/utils/logger.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
# specific language governing permissions and limitations
# under the License.

from contextlib import contextmanager
import logging
import os

""" Global logger. """
logger = logging.getLogger("archery")
Expand All @@ -27,3 +29,21 @@ def __init__(self, quiet=False):


ctx = LoggingContext()

in_github_actions = (os.environ.get("GITHUB_ACTIONS") == "true")


@contextmanager
def group(name):
"""
Group outputs in the given with block.
This does nothing in non GitHub Actions environment for now.
"""
if in_github_actions:
print(f"::group::{name}")
try:
yield
finally:
if in_github_actions:
print("::endgroup::")

0 comments on commit 6e93a55

Please sign in to comment.