Skip to content

Commit

Permalink
automated testing optimization (#2753)
Browse files Browse the repository at this point in the history 
* initial commit for the rspec and cucumber GHA rewrite

* proper way to install legacy chrome + concurrency killer

* get location of custom chrome

* pin exact chrome version

* go to latest chrome

* adding rspec, increasing threads for cucumber

* push multitasking

* MORE

* MORE

* LAST PUSH

* MORE

* found the culprit of the slowness, components specs were loaded twice, changed to load via config instead of a spec/component file

* fixing error of loading everything on a specific run

* trying to split a big rspec file

* forcing easy test with big test

* try to suggest some easy files

* fixing the json

* just pin the slow one at the beginning

* trying to split the long file in 2

* split in 4 parts

* update all actions that were throwing warnings

* I wiped rubocop, so sorry, bringing it back

* fixing threaded name in both runs

* changing id for pr checks

* fixed branch for checks

* another wrong branch

* show rubocop on log

* split steps

* I need to see the error

* pleasing rubocop

* more rubocop

* removing extra end
  • Loading branch information
@polographer
polographer authored Sep 5, 2024
1 parent b08fc2e commit 63e0995
Show file tree
Hide file tree
Showing 15 changed files with 2,705 additions and 2,703 deletions.
28 changes: 28 additions & 0 deletions .github/workflows/pr-checks.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: pull request checks
on:
pull_request:

concurrency:
group: checks-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
rubocop:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: ruby/setup-ruby@v1
with:
bundler-cache: true
rubygems: 3.3.26
- name: Run rubocop
run: |
bundle exec rubocop-git origin/master | tee results.txt
# - name: Print results
# run: |
# cat results.txt
- name: Evaluate results
run: |
cat results.txt | grep "no offenses detected"
283 changes: 79 additions & 204 deletions .github/workflows/ruby.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,222 +4,97 @@ on:
schedule:
- cron: "0 6 * * *"

env: # https://stackoverflow.com/questions/59867124/how-can-i-access-github-action-environment-variables-within-a-bash-script-run-by
TEST_BOOSTERS_RSPEC_TEST_FILE_PATTERN: "{spec,components/benefit_markets/spec,components/benefit_sponsors,components/notifier,components/sponsored_benefits,components/transport_gateway,components/transport_profiles}/**/*_spec.rb"
TB_RSPEC_FORMATTER: "progress"
concurrency:
group: ruby-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
testing_matrix:
name: ${{matrix.command_configs.name}}
runs-on: ${{ matrix.os }}
rspec:
runs-on: ubuntu-latest
services:
mongo:
image: mongo:5.0
ports:
- 27017:27017
options: >-
--name "mongo"
--health-cmd mongo
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
node: [12.x]
mongodb-version: ["3.6"]
command_configs:
- name: rubocop
skip_js: "true"
test_command: |
git fetch --no-tags origin
bundle exec rubocop-git origin/master | grep "no offenses detected"
# - name: Brakeman In Main app
# skip_js: 'true'
# test_command: |
# brakeman -i config/brakeman.ignore
# - name: Brakeman In Components
# skip_js: 'true'
# test_command: |
# for test_dir in `ls -1 components`; do
# cd components/$test_dir
# brakeman -i config/brakeman.ignore
# cd ../..
# done
- name: rspec 1/12
test_command: rspec_booster --job 1/12
- name: rspec 2/12
test_command: rspec_booster --job 2/12
- name: rspec 3/12
test_command: rspec_booster --job 3/12
- name: rspec 4/12
test_command: rspec_booster --job 4/12
- name: rspec 5/12
test_command: rspec_booster --job 5/12
- name: rspec 6/12
test_command: rspec_booster --job 6/12
- name: rspec 7/12
test_command: rspec_booster --job 7/12
- name: rspec 8/12
test_command: rspec_booster --job 8/12
- name: rspec 9/12
test_command: rspec_booster --job 9/12
- name: rspec 10/12
test_command: rspec_booster --job 10/12
- name: rspec 11/12
test_command: rspec_booster --job 11/12
- name: rspec 12/12
test_command: rspec_booster --job 12/12
- name: cucumber 1/4
test_command: |
cp -f config/cucumber.yml config/cucumber.yml.bak
cp -f config/cucumber_split.yml config/cucumber.yml
cat config/cucumber.yml
export DISPLAY=:99
if cucumber_booster --job 1/4 features
then
echo "Cucumber passed the first time!"
exit 0
else
cat config/cucumber.yml
ls -ltr && ls -ltr tmp
echo "catting rerun.txt"
cat rerun.txt
echo "Give cucumber one more try"
if bundle exec cucumber @rerun.txt --profile first_rerun
then
echo "Cucumber worked on retry"
exit 0
else
echo "Give cucumber yet another try"
cat config/cucumber.yml
bundle exec cucumber @tmp/cucumber_failures_2.log --profile second_rerun
fi
fi
needs_chromedriver: true
- name: cucumber 2/4
test_command: |
cp -f config/cucumber.yml config/cucumber.yml.bak
cp -f config/cucumber_split.yml config/cucumber.yml
cat config/cucumber.yml
export DISPLAY=:99
if cucumber_booster --job 2/4 features
then
echo "Cucumber passed the first time!"
exit 0
else
cat config/cucumber.yml
ls -ltr && ls -ltr tmp
echo "catting rerun.txt"
cat rerun.txt
echo "Give cucumber one more try"
if bundle exec cucumber @rerun.txt --profile first_rerun
then
echo "Cucumber worked on retry"
exit 0
else
echo "Give cucumber yet another try"
cat config/cucumber.yml
bundle exec cucumber @tmp/cucumber_failures_2.log --profile second_rerun
fi
fi
needs_chromedriver: true
- name: cucumber 3/4
test_command: |
cp -f config/cucumber.yml config/cucumber.yml.bak
cp -f config/cucumber_split.yml config/cucumber.yml
cat config/cucumber.yml
export DISPLAY=:99
if cucumber_booster --job 3/4 features
then
echo "Cucumber passed the first time!"
exit 0
else
cat config/cucumber.yml
ls -ltr && ls -ltr tmp
echo "catting rerun.txt"
cat rerun.txt
echo "Give cucumber one more try"
if bundle exec cucumber @rerun.txt --profile first_rerun
then
echo "Cucumber worked on retry"
exit 0
else
echo "Give cucumber yet another try"
cat config/cucumber.yml
bundle exec cucumber @tmp/cucumber_failures_2.log --profile second_rerun
fi
fi
needs_chromedriver: true
- name: cucumber 4/4
test_command: |
cp -f config/cucumber.yml config/cucumber.yml.bak
cp -f config/cucumber_split.yml config/cucumber.yml
cat config/cucumber.yml
export DISPLAY=:99
if cucumber_booster --job 4/4 features
then
echo "Cucumber passed the first time!"
exit 0
else
cat config/cucumber.yml
cp -f config/cucumber.yml.bak config/cucumber.yml
cat config/cucumber.yml
ls -ltr && ls -ltr tmp
echo "catting rerun.txt"
cat rerun.txt
echo "Give cucumber one more try"
if bundle exec cucumber @rerun.txt --profile first_rerun
then
echo "Cucumber worked on retry"
exit 0
else
echo "Give cucumber yet another try"
bundle exec cucumber @tmp/cucumber_failures_2.log --profile second_rerun
fi
fi
needs_chromedriver: true
group:
[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 ]
steps:
- uses: actions/checkout@v3
- name: Set up Ruby
uses: ruby/setup-ruby@v1
- uses: actions/checkout@v4
- uses: ruby/setup-ruby@v1
with:
rubygems: 3.3.26
bundler-cache: true
- name: Setup Node.js for use with actions
uses: actions/setup-node@v1
if: matrix.command_configs.skip_js != 'true'
with:
# Version Spec of the version to use. Examples: 10.x, 10.15.1, >=10.15.0, lts
node-version: ${{ matrix.node }}
- name: Launch MongoDB
uses: wbari/start-mongoDB@v0.2
if: matrix.command_configs.skip_js != 'true'
rubygems: 3.3.26
- uses: actions/setup-node@v4
with:
mongoDBVersion: ${{ matrix.mongodb-version }}
- name: Set up Chromedriver
if: matrix.command_configs.needs_chromedriver
uses: nanasess/setup-chromedriver@master
- name: Cache Gems
uses: actions/cache@v1
node-version: '12'
- name: Restore Node Modules
id: npm-cache
uses: actions/cache@v4
with:
path: vendor/bundle
key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }}
path: node_modules
key: ${{ runner.os }}-node_modules-v12-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }}
- name: Cache Node Modules
${{ runner.os }}-node_modules-v12-${{ hashFiles('yarn.lock') }}
- name: Install node dependencies
if: steps.npm-cache.outputs.cache-hit != 'true'
run: yarn install
- name: Install Test boosters gem
run: gem install treye-semaphore_test_boosters --version '2.5.2'
- name: Run tests
run: rspec_booster --job ${{ matrix.group }}/${{ strategy.job-total }}
env:
RSPEC_SPLIT_CONFIGURATION_PATH: 'ci/rspec-split-config.json'
TEST_BOOSTERS_RSPEC_TEST_FILE_PATTERN: "{spec,components/benefit_markets/spec,components/benefit_sponsors/spec,components/notifier/spec,components/sponsored_benefits/spec,components/transport_gateway/spec,components/transport_profiles/spec}/**/*_spec.rb"
TB_RSPEC_FORMATTER: "progress"

cucumber:
runs-on: ubuntu-latest
services:
mongo:
image: mongo:5.0
ports:
- 27017:27017
options: >-
--name "mongo"
--health-cmd mongo
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
group:
[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 ]
steps:
- uses: actions/checkout@v4
- uses: ruby/setup-ruby@v1
with:
bundler-cache: true
rubygems: 3.3.26
- uses: actions/setup-node@v4
with:
node-version: '12'
- name: Restore Node Modules
id: npm-cache
if: matrix.command_configs.skip_js != 'true'
uses: actions/cache@v1
uses: actions/cache@v4
with:
path: node_modules
key: ${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }}
key: ${{ runner.os }}-node_modules-v12-${{ hashFiles('yarn.lock') }}
restore-keys: |
${{ runner.os }}-node_modules-${{ hashFiles('yarn.lock') }}
- name: bundle install
run: |
gem install bundler -v '1.17.3'
bundle config path vendor/bundle
bundle install --jobs 4 --retry 3
gem install treye-semaphore_test_boosters --version '2.5.1'
gem install brakeman -v 5.4.1
- name: yarn install
if: matrix.command_configs.skip_js != 'true' && steps.npm-cache.outputs.cache-hit != 'true'
run: |
yarn install
- name: run webpack
if: matrix.command_configs.skip_js != 'true'
run: |
NODE_ENV=test RAILS_ENV=test ./bin/webpack
- name: run tests
run: ${{matrix.command_configs.test_command}}
${{ runner.os }}-node_modules-v12-${{ hashFiles('yarn.lock') }}
- name: Install node dependencies
if: steps.npm-cache.outputs.cache-hit != 'true'
run: yarn install
- name: Install Test boosters gem
run: gem install treye-semaphore_test_boosters --version '2.5.2'
- name: Run tests
run: cucumber_booster --job ${{ matrix.group }}/${{ strategy.job-total }}
6 changes: 6 additions & 0 deletions ci/rspec-split-config.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[
{ "files": ["spec/models/census_employee_spec.rb"]},
{ "files": ["spec/models/census_employee_part2_spec.rb"]},
{ "files": ["spec/models/census_employee_part3_spec.rb"]},
{ "files": ["spec/models/census_employee_part4_spec.rb"]}
]
2 changes: 1 addition & 1 deletion config/cucumber_split.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ wip: --tags @wip:11 --wip features
rerun: -format rerun --out rerun.txt --strict --tags 'not @wip'
first_rerun: --format 'pretty' --format rerun --out tmp/cucumber_failures_2.log
second_rerun: --format 'pretty' --format rerun --out tmp/cucumber_failures_3.log
screenshots: -r features SCREENSHOTS=true
# screenshots: -r features SCREENSHOTS=true
semaphoreci: --format rerun --out rerun.txt --format json --out=cucumber_report.json
17 changes: 0 additions & 17 deletions spec/components/benefit_markets_spec.rb
View file

This file was deleted.

20 changes: 0 additions & 20 deletions spec/components/benefit_sponsors_spec.rb
View file

This file was deleted.

14 changes: 0 additions & 14 deletions spec/components/notifier_spec.rb
View file

This file was deleted.

Loading

0 comments on commit 63e0995

Please sign in to comment.