Skip to content

Commit

Permalink
Merge branch 'StackStorm:master' into stackstorm-guru-badge
Browse files Browse the repository at this point in the history
  • Loading branch information
@kursataktas
kursataktas authored Nov 28, 2024
2 parents c46e5ed + 53d641f commit 98cd1dc
Show file tree
Hide file tree
Showing 95 changed files with 3,768 additions and 1,566 deletions.
29 changes: 29 additions & 0 deletions .github/workflows/lint.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,32 @@ jobs:
name: pants-log-py${{ matrix.python-version }}
path: .pants.d/pants.log
if: always() # We want the log even on failures.

set_merge_ok:
name: Set Merge OK (Lint)
if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
needs:
- lint-checks
outputs:
merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }}
runs-on: ubuntu-latest
steps:
- id: set_merge_ok
run: echo 'merge_ok=true' >> ${GITHUB_OUTPUT}

merge_ok:
name: Merge OK (Lint)
if: always()
needs:
- set_merge_ok
runs-on: ubuntu-latest
steps:
- run: |
merge_ok="${{ needs.set_merge_ok.outputs.merge_ok }}"
if [[ "${merge_ok}" == "true" ]]; then
echo "Merge OK"
exit 0
else
echo "Merge NOT OK"
exit 1
fi
263 changes: 245 additions & 18 deletions .github/workflows/test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,17 @@ on:
# - cron: '0 0 * * *'

jobs:
test:
name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}'
pants-plugins-tests:
name: 'Pants Plugins Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}'
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
# NOTE: We need to use full Python version as part of Python deps cache key otherwise
# setup virtualenv step will fail.
include:
- name: 'Test (pants runs: pytest)'
python-version-short: '3.8'
python-version: '3.8.10'
- name: 'Test (pants runs: pytest)'
python-version-short: '3.9'
python-version: '3.9.14'
python:
# Pants itself uses only 3.9
- {version-short: '3.9', version: '3.9.14'}

services:
mongo:
Expand Down Expand Up @@ -87,11 +83,109 @@ jobs:
git submodule foreach 'git fetch --all --tags'
git submodule foreach 'git tag'
- name: 'Set up Python (${{ matrix.python-version }})'
- name: 'Set up Python (${{ matrix.python.version }})'
id: python
uses: actions/setup-python@v5
with:
python-version: '${{ matrix.python-version }}'
python-version: '${{ matrix.python.version }}'

- name: Cache and Install APT Dependencies
uses: ./.github/actions/apt-packages

- name: Initialize Pants and its GHA caches
uses: ./.github/actions/init-pants
with:
# To ignore a bad cache, bump the cache* integer.
gha-cache-key: cache0-py${{ matrix.python.version }}

- name: Test pants-plugins
env:
# Github Actions uses the 'runner' user, so use that instead of stanley.
ST2TESTS_SYSTEM_USER: 'runner'
ST2TESTS_REDIS_HOST: '127.0.0.1'
ST2TESTS_REDIS_PORT: '6379'
run: |
pants test pants-plugins/::
- name: Upload pants log
uses: actions/upload-artifact@v4
with:
name: pants-log-py${{ matrix.python.version }}-pants-plugins-tests
path: .pants.d/pants.log
if: always() # We want the log even on failures.

unit-tests:
name: 'Unit Tests Shard ${{ matrix.shard.k }}/${{ matrix.shard.n }} (pants runs: pytest) - Python ${{ matrix.python.version-short }}'
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
# NOTE: We need to use full Python version as part of Python deps cache key otherwise
# setup virtualenv step will fail.
python:
- {version-short: '3.8', version: '3.8.10'}
- {version-short: '3.9', version: '3.9.14'}
shard:
# Sharding of tests is handled by pants:
# https://www.pantsbuild.org/stable/docs/using-pants/advanced-target-selection#sharding-the-input-targets
- {k: '0', n: '4'}
- {k: '1', n: '4'}
- {k: '2', n: '4'}
- {k: '3', n: '4'}

services:
mongo:
image: mongo:7.0
ports:
- 27017:27017

rabbitmq:
image: rabbitmq:3.8-management
options: >-
--name rabbitmq
ports:
- 5671:5671/tcp # AMQP SSL port
- 5672:5672/tcp # AMQP standard port
- 15672:15672/tcp # Management: HTTP, CLI

redis:
# Docker Hub image
image: redis
# Set health checks to wait until redis has started
options: >-
--name "redis"
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 6379:6379/tcp

env:
COLUMNS: '120'

steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
# a test uses a submodule, and pants needs access to it to calculate deps.
submodules: 'recursive'
# sadly, the submodule will only have fetch-depth=1, which is what we want
# for st2.git, but not for the submodules. We still want actions/checkout
# to do the initial checkout, however, so that it adds auth for fetching
# in the submodule.

- name: Fetch repository submodules
run: |
git submodule status
git submodule foreach 'git fetch --all --tags'
git submodule foreach 'git tag'
- name: 'Set up Python (${{ matrix.python.version }})'
id: python
uses: actions/setup-python@v5
with:
python-version: '${{ matrix.python.version }}'

- name: Cache and Install APT Dependencies
uses: ./.github/actions/apt-packages
Expand All @@ -100,21 +194,154 @@ jobs:
uses: ./.github/actions/init-pants
with:
# To ignore a bad cache, bump the cache* integer.
gha-cache-key: cache0-py${{ matrix.python-version }}
gha-cache-key: cache0-py${{ matrix.python.version }}

- name: Test
- name: Unit Tests
env:
# Github Actions uses the 'runner' user, so use that instead of stanley.
ST2TESTS_SYSTEM_USER: 'runner'
# We do not support running pytest everywhere yet. When we do it will be simply:
# pants test ::
# Until then, we need to manually adjust this command line to test what we can.
ST2TESTS_REDIS_HOST: '127.0.0.1'
ST2TESTS_REDIS_PORT: '6379'
run: >
pants
--python-bootstrap-search-path=[]
--python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
--tag=unit
--test-shard=${{ matrix.shard.k }}/${{ matrix.shard.n }}
test '::'
- name: Upload pants log
uses: actions/upload-artifact@v4
with:
name: pants-log-py${{ matrix.python.version }}-unit-tests-shard-${{ matrix.shard.k }}_${{ matrix.shard.n }}
path: .pants.d/pants.log
if: always() # We want the log even on failures.

pack-tests:
name: 'Pack Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}'
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
# NOTE: We need to use full Python version as part of Python deps cache key otherwise
# setup virtualenv step will fail.
python:
- {version-short: '3.8', version: '3.8.10'}
- {version-short: '3.9', version: '3.9.14'}

services:
mongo:
image: mongo:7.0
ports:
- 27017:27017

rabbitmq:
image: rabbitmq:3.8-management
options: >-
--name rabbitmq
ports:
- 5671:5671/tcp # AMQP SSL port
- 5672:5672/tcp # AMQP standard port
- 15672:15672/tcp # Management: HTTP, CLI

redis:
# Docker Hub image
image: redis
# Set health checks to wait until redis has started
options: >-
--name "redis"
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 6379:6379/tcp

env:
COLUMNS: '120'

steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
# a test uses a submodule, and pants needs access to it to calculate deps.
submodules: 'recursive'
# sadly, the submodule will only have fetch-depth=1, which is what we want
# for st2.git, but not for the submodules. We still want actions/checkout
# to do the initial checkout, however, so that it adds auth for fetching
# in the submodule.

- name: Fetch repository submodules
run: |
pants test pylint_plugins/:: pants-plugins/::
git submodule status
git submodule foreach 'git fetch --all --tags'
git submodule foreach 'git tag'
- name: 'Set up Python (${{ matrix.python.version }})'
id: python
uses: actions/setup-python@v5
with:
python-version: '${{ matrix.python.version }}'

- name: Cache and Install APT Dependencies
uses: ./.github/actions/apt-packages

- name: Initialize Pants and its GHA caches
uses: ./.github/actions/init-pants
with:
# To ignore a bad cache, bump the cache* integer.
gha-cache-key: cache0-py${{ matrix.python.version }}

- name: Pack Tests
env:
# Github Actions uses the 'runner' user, so use that instead of stanley.
ST2TESTS_SYSTEM_USER: 'runner'
ST2TESTS_REDIS_HOST: '127.0.0.1'
ST2TESTS_REDIS_PORT: '6379'
run: >
pants
--python-bootstrap-search-path=[]
--python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
--tag=pack
test '::'
- name: Upload pants log
uses: actions/upload-artifact@v4
with:
name: pants-log-py${{ matrix.python-version }}
name: pants-log-py${{ matrix.python.version }}-pack-tests
path: .pants.d/pants.log
if: always() # We want the log even on failures.

#integration-tests: TODO: run integration tests

set_merge_ok:
name: Set Merge OK (Tests)
if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
needs:
- pants-plugins-tests
- unit-tests
- pack-tests
#- integration-tests
outputs:
merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }}
runs-on: ubuntu-latest
steps:
- id: set_merge_ok
run: echo 'merge_ok=true' >> ${GITHUB_OUTPUT}

merge_ok:
name: Merge OK (Tests)
if: always()
needs:
- set_merge_ok
runs-on: ubuntu-latest
steps:
- run: |
merge_ok="${{ needs.set_merge_ok.outputs.merge_ok }}"
if [[ "${merge_ok}" == "true" ]]; then
echo "Merge OK"
exit 0
else
echo "Merge NOT OK"
exit 1
fi
13 changes: 7 additions & 6 deletions BUILD.tools
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ python_requirement(
name="bandit-reqs",
resolve="bandit",
requirements=[
"bandit==1.7.0",
# https://github.com/pantsbuild/pants/blob/release_2.23.0rc0/src/python/pants/backend/python/lint/bandit/subsystem.py#L44-L52
"bandit>=1.7.0,<1.8",
"setuptools",
"GitPython==3.1.18",
"GitPython>=3.1.24",
],
)

Expand All @@ -23,9 +24,9 @@ python_requirement(
name="flake8-reqs",
resolve="flake8",
requirements=[
"flake8==4.0.1", # st2flake8 does not support flake8 v5
"flake8==7.0.0", # st2flake8 does not support flake8 v5
# license check plugin
"st2flake8==0.1.0", # TODO: remove in favor of regex-lint or preamble
"st2flake8>0.1.0", # TODO: remove in favor of regex-lint or preamble
],
)

Expand All @@ -34,13 +35,13 @@ python_requirement(

python_requirement(
name="pytest-reqs",
resolve="pytest",
resolve="st2",
requirements=[
"pytest==7.0.1", # copied from https://www.pantsbuild.org/v2.14/docs/reference-pytest#version
"pytest-benchmark[histogram]==3.4.1", # used for st2common/benchmarks
# "pytest-timer[colorama]", # report test timing (--with-timer ala nose-timer)
"pytest-icdiff", # make diff output easier to read
"pygments", # highlight code in tracebacks
# "pygments", # highlight code in tracebacks (already included in requirements-pants.txt)
#
# other possible plugins
# "pytest-timeout", # time limit on tests
Expand Down
Loading

0 comments on commit 98cd1dc

Please sign in to comment.