Skip to content
linux-package-test

Remove ballast extension entirely from components. #3323

Sign in to view logs

Remove ballast extension entirely from components.

Remove ballast extension entirely from components. #3323

Workflow file for this run

.github/workflows/linux-package-test.yml at 4fa30d6
name: linux-package-test
on:
push:
branches:
- main
pull_request:
paths:
- '.github/workflows/linux-package-test.yml'
- 'cmd/otelcol/**'
- 'internal/**'
- 'Makefile'
- 'Makefile.Common'
- '!**.md'
concurrency:
group: linux-package-test-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
PYTHON_VERSION: '3.11'
PIP_VERSION: '22.0.4'
REQUIREMENTS_PATH: "internal/buildscripts/packaging/tests/requirements.txt"
GO_VERSION: 1.22.7
jobs:
setup-environment:
# Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
runs-on: ubuntu-20.04
steps:
- name: Check out the codebase.
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache-dependency-path: '**/go.sum'
- name: Installing dependency
run: |
make install-tools
cross-compile:
# Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
runs-on: ubuntu-20.04
needs: [setup-environment]
strategy:
matrix:
SYS_BINARIES: [ "binaries-linux_amd64", "binaries-linux_arm64", "binaries-linux_ppc64le" ]
steps:
- name: Check out the codebase.
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache-dependency-path: '**/go.sum'
- name: Build Collector
run: |
make ${{ matrix.SYS_BINARIES }}
- name: Uploading binaries
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.SYS_BINARIES }}
path: |
./bin/*
agent-bundle-linux:
runs-on: ${{ fromJSON('["ubuntu-20.04", "otel-arm64"]')[matrix.ARCH == 'arm64'] }}
strategy:
matrix:
ARCH: ["amd64", "arm64"]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
id: bundle-cache
with:
path: .cache/buildx/agent-bundle-${{ matrix.ARCH }}
key: agent-bundle-buildx-${{ matrix.ARCH }}-${{ hashFiles('internal/signalfx-agent/bundle/**') }}
restore-keys: |
agent-bundle-buildx-${{ matrix.ARCH }}-
- run: make -C internal/signalfx-agent/bundle agent-bundle-linux ARCH=${{ matrix.ARCH }}
env:
BUNDLE_CACHE_HIT: "${{ steps.bundle-cache.outputs.cache-hit }}"
- uses: actions/upload-artifact@v4
with:
name: agent-bundle-linux-${{ matrix.ARCH }}
path: ./dist/agent-bundle_linux_${{ matrix.ARCH }}.tar.gz
build-package:
# Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
runs-on: ubuntu-20.04
needs: [cross-compile, agent-bundle-linux]
strategy:
matrix:
SYS_PACKAGE: [ "deb", "rpm", "tar" ]
ARCH: [ "amd64", "arm64" ]
fail-fast: false
steps:
- name: Check out the codebase.
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache-dependency-path: '**/go.sum'
- name: Downloading binaries-linux_${{ matrix.ARCH }}
uses: actions/download-artifact@v4
with:
name: binaries-linux_${{ matrix.ARCH }}
path: ./bin
- uses: actions/download-artifact@v4
with:
name: agent-bundle-linux-${{ matrix.ARCH }}
path: ./dist
- name: Build ${{ matrix.SYS_PACKAGE }} ${{ matrix.ARCH }} package
run: make ${{ matrix.SYS_PACKAGE }}-package SKIP_COMPILE=true SKIP_BUNDLE=true VERSION="" ARCH="${{ matrix.ARCH }}"
- name: Uploading ${{ matrix.SYS_PACKAGE }} ${{ matrix.ARCH }} package artifacts
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.SYS_PACKAGE }}-${{ matrix.ARCH }}-package
path: ./dist/splunk-otel-collector*
test-package-matrix:
runs-on: ubuntu-20.04
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Get matrix
id: get-matrix
run: |
# create test matrix for distro and arch
dockerfiles=$(find internal/buildscripts/packaging/tests/images/ -name "Dockerfile.*" | cut -d '.' -f2- | sort -u)
if [ -z "$dockerfiles" ]; then
echo "Failed to get dockerfiles from internal/buildscripts/packaging/tests/images!" >&2
exit 1
fi
distro=$(for d in $dockerfiles; do echo -n "\"$d\","; done)
arch="\"amd64\", \"arm64\""
matrix="{\"DISTRO\": [${distro%,}], \"ARCH\": [${arch}]}"
echo "$matrix" | jq
echo "matrix=${matrix}" >> $GITHUB_OUTPUT
outputs:
matrix: ${{ steps.get-matrix.outputs.matrix }}
test-package:
# Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
runs-on: ${{ fromJSON('["ubuntu-20.04", "ubuntu-22.04"]')[matrix.DISTRO == 'amazonlinux-2023'] }}
timeout-minutes: 30
needs: [build-package, test-package-matrix]
strategy:
matrix: ${{ fromJSON(needs.test-package-matrix.outputs.matrix) }}
fail-fast: false
steps:
- name: Check out the codebase.
uses: actions/checkout@v4
- name: Get package type for ${{ matrix.DISTRO }}
run: |
for pkg in "deb" "rpm" "tar"; do
if [[ -f "internal/buildscripts/packaging/tests/images/${pkg}/Dockerfile.${{ matrix.DISTRO }}" ]]; then
echo "SYS_PACKAGE=${pkg}" >> $GITHUB_ENV
exit 0
fi
done
echo "Unknown distro '${{ matrix.DISTRO }}'!"
exit 1
- uses: actions/download-artifact@v4
with:
name: ${{ env.SYS_PACKAGE }}-${{ matrix.ARCH }}-package
path: ./dist
- uses: docker/setup-qemu-action@v3
if: ${{ matrix.ARCH != 'amd64' }}
with:
platforms: ${{ matrix.ARCH }}
image: tonistiigi/binfmt:qemu-v7.0.0
- name: Setup python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
cache-dependency-path: ${{ env.REQUIREMENTS_PATH }}
- name: Install pytest
run: |
if which pip; then
pip install --upgrade 'pip==${{ env.PIP_VERSION }}'
else
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python get-pip.py 'pip==${{ env.PIP_VERSION }}'
fi
pip install -r "${{ env.REQUIREMENTS_PATH }}"
- name: Test ${{ env.SYS_PACKAGE }} on ${{ matrix.DISTRO }} ${{ matrix.ARCH }}
id: pytest
continue-on-error: true
run: |
distro="${{ matrix.DISTRO }}"
if [[ "$distro" = "amazonlinux-2" ]]; then
# workaround for pytest substring matching
distro="amazonlinux-2 and not amazonlinux-2023"
fi
python3 -u -m pytest -s --verbose -m "${{ env.SYS_PACKAGE }}" \
-k "$distro and ${{ matrix.ARCH }}" \
internal/buildscripts/packaging/tests/package_test.py
# qemu, networking, running systemd in containers, etc., can be flaky
- name: Re-run failed tests
if: ${{ steps.pytest.outcome == 'failure' }}
run: |
distro="${{ matrix.DISTRO }}"
if [[ "$distro" = "amazonlinux-2" ]]; then
# workaround for pytest substring matching
distro="amazonlinux-2 and not amazonlinux-2023"
fi
python3 -u -m pytest -s --verbose -m "${{ env.SYS_PACKAGE }}" \
-k "$distro and ${{ matrix.ARCH }}" \
--last-failed \
internal/buildscripts/packaging/tests/package_test.py
docker-otelcol:
# Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
runs-on: ubuntu-20.04
needs: [cross-compile, agent-bundle-linux]
steps:
# Multiarch images require more disk space
- uses: jlumbroso/free-disk-space@v1.3.1
- name: Check out the codebase.
uses: actions/checkout@v4
# Required to export a multiarch manifest and images to the local image store
- name: Set up containerd image store
uses: crazy-max/ghaction-setup-docker@v3
with:
daemon-config: |
{
"features": {
"containerd-snapshotter": true
}
}
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache-dependency-path: '**/go.sum'
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: arm64,ppc64le
image: tonistiigi/binfmt:qemu-v7.0.0
- name: Downloading binaries-linux_amd64
uses: actions/download-artifact@v4
with:
name: binaries-linux_amd64
path: ./bin
- name: Downloading binaries-linux_arm64
uses: actions/download-artifact@v4
with:
name: binaries-linux_arm64
path: ./bin
- name: Downloading binaries-linux_ppc64le
uses: actions/download-artifact@v4
with:
name: binaries-linux_ppc64le
path: ./bin
- uses: actions/download-artifact@v4
with:
name: agent-bundle-linux-amd64
path: ./dist
- uses: actions/download-artifact@v4
with:
name: agent-bundle-linux-arm64
path: ./dist
- name: Build the multiarch docker image
run: make docker-otelcol SKIP_COMPILE=true SKIP_BUNDLE=true ARCH=amd64,arm64,ppc64le
- name: Save the multiarch image archive to be loaded by downstream jobs
run: mkdir -p docker-otelcol && docker save -o ./docker-otelcol/image.tar otelcol:latest
- uses: actions/upload-artifact@v4
with:
name: otelcol
path: ./docker-otelcol
docker-otelcol-verify:
# Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
runs-on: ubuntu-20.04
needs: [docker-otelcol]
strategy:
matrix:
ARCH: [ "amd64", "arm64", "ppc64le" ]
fail-fast: false
steps:
- name: Check out the codebase.
uses: actions/checkout@v4
# Required to load a multiarch archive to the local image store
- name: Set up containerd image store
uses: crazy-max/ghaction-setup-docker@v3
with:
daemon-config: |
{
"features": {
"containerd-snapshotter": true
}
}
- name: Set up QEMU
if: ${{ matrix.ARCH != 'amd64' }}
uses: docker/setup-qemu-action@v3
with:
platforms: ${{ matrix.ARCH }}
image: tonistiigi/binfmt:qemu-v7.0.0
- uses: actions/download-artifact@v4
with:
name: otelcol
path: ./docker-otelcol
- name: Load multiarch docker image
run: docker load -i ./docker-otelcol/image.tar
- name: Run docker image
run: |
# ensure the collector can start with the default config file
docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest
sleep 10
if [ -z "$( docker ps --filter=status=running --filter=name=otelcol -q )" ]; then
docker logs otelcol
echo "Failing job execution: fail to start otelcol docker container with the default config in 10 seconds."
exit 1
fi
# get all provided config files from the container
tmpdir=$(mktemp -d)
docker cp otelcol:/etc/otel/collector $tmpdir
docker rm -f otelcol
# ensure the collector can start with all provided config files
configs=$(ls ${tmpdir}/collector/ 2>/dev/null)
if [ -z "$configs" ]; then
echo "failed to get config files from otelcol:/etc/otel/collector"
exit 1
fi
for config in $configs; do
# TODO: set fake-token back to 12345 once https://github.com/open-telemetry/opentelemetry-collector/issues/10937 is resolved
docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_CONFIG=/etc/otel/collector/${config} -e SPLUNK_ACCESS_TOKEN=fake-token -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest
sleep 10
if [ -z "$( docker ps --filter=status=running --filter=name=otelcol -q )" ]; then
docker logs otelcol
echo "Failing job execution: fail to start otelcol docker container with ${config} in 10 seconds."
exit 1
fi
docker rm -f otelcol
done
- name: Check journalctl
run: |
# ensure journalctl can run with the collected libraries
docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest
docker exec otelcol /bin/journalctl
docker rm -f otelcol
- name: Check python and java
if: ${{ matrix.ARCH != 'ppc64le' }}
run: |
# ensure python and java can run with the collected libraries
docker run --platform linux/${{ matrix.ARCH }} -d -e SPLUNK_ACCESS_TOKEN=12345 -e SPLUNK_REALM=fake-realm --name otelcol otelcol:latest
docker exec otelcol /usr/lib/splunk-otel-collector/agent-bundle/bin/python --version
docker exec otelcol /usr/lib/splunk-otel-collector/agent-bundle/jre/bin/java -version
# ensure collectd-python plugins were installed
docker exec otelcol sh -c 'test -d /usr/lib/splunk-otel-collector/agent-bundle/collectd-python/'
if [[ "$(docker exec otelcol ls /usr/lib/splunk-otel-collector/agent-bundle/collectd-python/ | wc -l)" -eq 0 ]]; then
echo "/usr/lib/splunk-otel-collector/agent-bundle/collectd-python/ is empty!" >&2
exit 1
fi
docker rm -f otelcol