Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(ci): upload coredump #11401

Closed
wants to merge 1 commit into from
Closed

chore(ci): upload coredump #11401

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .ci/run_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
#!/usr/bin/env bash
set -e

ulimit -c unlimited
mkdir -p /tmp/cores
chmod 777 /tmp/cores
echo '/tmp/cores/core.%e.%p' | sudo tee /proc/sys/kernel/core_pattern
sudo sysctl -w fs.suid_dumpable=2
sudo sysctl -p

function cyan() {
echo -e "\033[1;36m$*\033[0m"
}
Expand Down
66 changes: 47 additions & 19 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ on:

env:
BUILD_ROOT: ${{ github.workspace }}/${{ inputs.relative-build-root }}
RELATIVE_BUILD_ROOT: ${{ inputs.relative-build-root }}

jobs:
build:
Expand All @@ -37,9 +38,18 @@ jobs:

- name: Lookup build cache
id: cache-deps
# Note we only use the artifact in build, which may actually contain
# lua source codes from a previous version. But busted
# is able to override those files with them under current directory
# so we don't need to rebuild it everytime. So that also means
# the "artifacts" here can't be used as a per-commit CD artifact; only
# the ones from release.yml could.
uses: actions/cache@v3
with:
path: ${{ env.BUILD_ROOT }}
path: |
artifacts
${{ env.RELATIVE_BUILD_ROOT }}
!${{ env.RELATIVE_BUILD_ROOT }}/kong-dev/**/*
key: ${{ steps.cache-key.outputs.cache-key }}

- name: Install packages
Expand All @@ -51,8 +61,38 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
run: |
make build-kong
chmod +rw -R "$BUILD_ROOT/kong-dev"
bazel build //build:kong --verbose_failures --action_env=INSTALL_DESTDIR=/usr/local
bazel build //build:venv --verbose_failures --action_env=INSTALL_DESTDIR=/usr/local
# debian package for artifacts to be used in reproducing in local environment
bazel build :kong_deb --verbose_failures --action_env=INSTALL_DESTDIR=/usr/local

- name: Bazel Outputs
uses: actions/upload-artifact@v3
if: failure()
with:
name: bazel-outputs
path: |
bazel-out/_tmp/actions
retention-days: 3

- name: Collect artifacts
if: steps.cache-deps.outputs.cache-hit != 'true'
run: |
mkdir artifacts
cp bazel-bin/pkg/kong.amd64.deb artifacts/

# bazel output directory without write access, we add it when creating archive
tar -zcvf artifacts/build.tar.gz -C bazel-bin/build/kong-dev --mode="a+rw" .

- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
name: build
path: |
artifacts
${{ env.RELATIVE_BUILD_ROOT }}
!${{ env.RELATIVE_BUILD_ROOT }}/kong-dev/**/*
retention-days: 3

- name: Update PATH
run: |
Expand All @@ -62,26 +102,14 @@ jobs:

- name: Debug (nginx)
run: |
find -name libluajit-5.1.so.2
find -name luajit
echo nginx: $(which nginx)
nginx -V 2>&1 | sed -re 's/ --/\n--/g'
ldd $(which nginx)

- name: Debug (luarocks)
run: |
echo luarocks: $(which luarocks)
luarocks --version
luarocks config

- name: Bazel Outputs
uses: actions/upload-artifact@v3
if: failure()
with:
name: bazel-outputs
path: |
bazel-out/_tmp/actions
retention-days: 3

- name: Build Dev Kong dependencies
if: steps.cache-deps.outputs.cache-hit != 'true'
run: |
make install-dev-rocks
luarocks --version || true
luarocks config || true
149 changes: 109 additions & 40 deletions .github/workflows/build_and_test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,35 +58,44 @@ jobs:
- name: Checkout Kong source code
uses: actions/checkout@v3

- name: Lookup build cache
id: cache-deps
uses: actions/cache@v3
- name: Download artifacts
uses: actions/download-artifact@v3
with:
path: ${{ env.BUILD_ROOT }}
key: ${{ needs.build.outputs.cache-key }}
name: build
path: .

- name: Extract the artifact contents
run: |
sudo tar -C /usr/local/ -xzvf artifacts/build.tar.gz
sudo mkdir -p ${{ env.BUILD_ROOT }}/kong-dev
sudo tar -C ${{ env.BUILD_ROOT }}/kong-dev -xzvf artifacts/build.tar.gz

- name: Install Kong dev
run: make dev LIBRARY_PREFIX=/usr/local/kong

- name: Check test-helpers doc generation
run: |
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
pushd ./spec && ldoc .
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
pushd ./spec && ldoc .

- name: Check autodoc generation
run: |
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
scripts/autodoc
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
scripts/autodoc

- name: Lint Lua code
run: |
make lint
sudo chmod +x bazel-bin/build/kong-dev-venv.sh
make lint

- name: Validate rockspec file
run: |
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
scripts/validate-rockspec
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
scripts/validate-rockspec

- name: Check spec file misspelling
run: |
scripts/check_spec_files_spelling.sh
scripts/check_spec_files_spelling.sh

- name: Check labeler configuration
run: scripts/check-labeler.pl .github/labeler.yml
Expand All @@ -96,12 +105,18 @@ jobs:
KONG_TEST_PG_DATABASE: kong
KONG_TEST_PG_USER: kong
run: |
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
TEST_CMD="bin/busted -v -o htest spec/01-unit"
if [[ $KONG_TEST_COVERAGE = true ]]; then
TEST_CMD="$TEST_CMD --coverage"
fi
$TEST_CMD
ulimit -c unlimited
mkdir -p /tmp/cores
chmod 777 /tmp/cores
echo '/tmp/cores/core.%e.%p' | sudo tee /proc/sys/kernel/core_pattern
sudo sysctl -w fs.suid_dumpable=2
sudo sysctl -p
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
TEST_CMD="bin/busted -v -o htest spec/01-unit"
if [[ $KONG_TEST_COVERAGE = true ]]; then
TEST_CMD="$TEST_CMD --coverage"
fi
$TEST_CMD

- name: Archive coverage stats file
uses: actions/upload-artifact@v3
Expand All @@ -116,6 +131,16 @@ jobs:
if: failure()
run: |
sudo dmesg -T
sudo chmod -R 777 /tmp/cores

- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v3
with:
name: core-dumps-${{ github.job }}-${{ github.run_id }}
retention-days: 3
path: |
/tmp/cores

integration-tests-postgres:
name: Postgres ${{ matrix.suite }} - ${{ matrix.split }} tests
Expand Down Expand Up @@ -162,12 +187,20 @@ jobs:
- name: Checkout Kong source code
uses: actions/checkout@v3

- name: Lookup build cache
id: cache-deps
uses: actions/cache@v3
- name: Download artifacts
uses: actions/download-artifact@v3
with:
path: ${{ env.BUILD_ROOT }}
key: ${{ needs.build.outputs.cache-key }}
name: build
path: .

- name: Extract the artifact contents
run: |
sudo tar -C /usr/local/ -xzvf artifacts/build.tar.gz
sudo mkdir -p ${{ env.BUILD_ROOT }}/kong-dev
sudo tar -C ${{ env.BUILD_ROOT }}/kong-dev -xzvf artifacts/build.tar.gz

- name: Install Kong dev
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are those Install Kong dev steps needed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@fffonion It will install some dev-rocks and some binary like grpcurl.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

iirc we already installed the dev rocks on build step for ce (and cached) 🤔

Copy link
Contributor Author

@ADD-SP ADD-SP Aug 17, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

- name: Build Dev Kong dependencies
   if: steps.cache-deps.outputs.cache-hit != 'true'
   run: |
      make install-dev-rocks

This line has been removed in this PR, to keep consistency with EE.

run: make dev LIBRARY_PREFIX=/usr/local/kong

- name: Build WASM Test Filters
uses: ./.github/actions/build-wasm-test-filters
Expand Down Expand Up @@ -236,9 +269,9 @@ jobs:
TEST_SUITE: ${{ matrix.suite }}
TEST_SPLIT: ${{ matrix.split }}
run: |
make dev # required to install other dependencies like bin/grpcurl
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
.ci/run_tests.sh
make dev # required to install other dependencies like bin/grpcurl
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
.ci/run_tests.sh

- name: Archive coverage stats file
uses: actions/upload-artifact@v3
Expand All @@ -253,6 +286,16 @@ jobs:
if: failure()
run: |
sudo dmesg -T
sudo chmod -R 777 /tmp/cores

- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v3
with:
name: core-dumps-${{ github.job }}-${{ matrix.suite }}-${{ matrix.split }}-${{ github.run_id }}
retention-days: 3
path: |
/tmp/cores

integration-tests-dbless:
name: DB-less integration tests
Expand All @@ -270,12 +313,20 @@ jobs:
- name: Checkout Kong source code
uses: actions/checkout@v3

- name: Lookup build cache
id: cache-deps
uses: actions/cache@v3
- name: Download artifacts
uses: actions/download-artifact@v3
with:
path: ${{ env.BUILD_ROOT }}
key: ${{ needs.build.outputs.cache-key }}
name: build
path: .

- name: Extract the artifact contents
run: |
sudo tar -C /usr/local/ -xzvf artifacts/build.tar.gz
sudo mkdir -p ${{ env.BUILD_ROOT }}/kong-dev
sudo tar -C ${{ env.BUILD_ROOT }}/kong-dev -xzvf artifacts/build.tar.gz

- name: Install Kong dev
run: make dev LIBRARY_PREFIX=/usr/local/kong

- name: Build WASM Test Filters
uses: ./.github/actions/build-wasm-test-filters
Expand Down Expand Up @@ -309,9 +360,9 @@ jobs:
KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json
TEST_SUITE: dbless
run: |
make dev # required to install other dependencies like bin/grpcurl
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
.ci/run_tests.sh
make dev # required to install other dependencies like bin/grpcurl
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
.ci/run_tests.sh

- name: Archive coverage stats file
uses: actions/upload-artifact@v3
Expand All @@ -326,6 +377,16 @@ jobs:
if: failure()
run: |
sudo dmesg -T
sudo chmod -R 777 /tmp/cores

- name: Upload core dumps
if: failure()
uses: actions/upload-artifact@v3
with:
name: core-dumps-${{ github.job }}-${{ github.run_id }}
retention-days: 3
path: |
/tmp/cores

pdk-tests:
name: PDK tests
Expand All @@ -336,12 +397,20 @@ jobs:
- name: Checkout Kong source code
uses: actions/checkout@v3

- name: Lookup build cache
id: cache-deps
uses: actions/cache@v3
- name: Download artifacts
uses: actions/download-artifact@v3
with:
path: ${{ env.BUILD_ROOT }}
key: ${{ needs.build.outputs.cache-key }}
name: build
path: .

- name: Extract the artifact contents
run: |
sudo tar -C /usr/local/ -xzvf artifacts/build.tar.gz
sudo mkdir -p ${{ env.BUILD_ROOT }}/kong-dev
sudo tar -C ${{ env.BUILD_ROOT }}/kong-dev -xzvf artifacts/build.tar.gz

- name: Install Kong dev
ADD-SP marked this conversation as resolved.
Show resolved Hide resolved
run: make dev LIBRARY_PREFIX=/usr/local/kong

- name: Install Test::Nginx
run: |
Expand Down