Skip to content

Commit

Permalink
Upgrade circleci config version and improve api/security tests (#11325)
Browse files Browse the repository at this point in the history 
* consolidate api tests and security tests
* update config version to 2.1
* use local image for security tests
* skip pulling image for docker scout
* remove dependency on frontend code
  • Loading branch information
@zainasir
zainasir authored Jan 7, 2025
1 parent 96829ad commit 600691b
Showing 1 changed file with 111 additions and 61 deletions.
172 changes: 111 additions & 61 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,41 @@ defaults: &defaults
docker:
- image: circleci/node:15.2.1-browsers

version: 2
version: 2.1
commands:
build_push_image:
parameters:
push:
description: Push image to DockerHub
type: string
default: "false"
steps:
- run:
name: Build cBioPortal docker image
environment:
DOCKER_REPO: cbioportal/cbioportal-dev
command: |
export DOCKER_TAG=$CIRCLE_SHA1
URL="https://hub.docker.com/v2/repositories/cbioportal/cbioportal-dev/tags/$DOCKER_TAG-web-shenandoah"
TAG_FOUND=$(curl -s $URL | jq -r .name)
if [ $TAG_FOUND = "$DOCKER_TAG-web-shenandoah" ]; then
echo "Image already exists. Skipping build step!"
exit 0
fi
cd cbioportal-test
./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=<<parameters.push>> --skip_web_and_data=true
if [ "<<parameters.push>>" = "false" ]; then
EXISTS=$(docker inspect --type=image $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?);
else
EXISTS=$(docker manifest inspect $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?);
fi
if [ $EXISTS -eq 0 ]; then
echo "Build succeeded!"
else
echo "Build failed!"
exit 1
fi
jobs:
build_backend:
docker:
Expand Down Expand Up @@ -258,53 +292,71 @@ jobs:
paths:
- cbioportal-frontend

build_push_image:
checkout_pr:
machine:
image: ubuntu-2204:2024.08.1
resource_class: medium
working_directory: /tmp/repos
steps:
- attach_workspace:
at: /tmp/repos
- checkout:
path: /tmp/repos/cbioportal
- persist_to_workspace:
root: /tmp/repos
paths:
- cbioportal

build_image:
machine:
image: ubuntu-2204:2024.08.1
resource_class: medium
working_directory: /tmp/repos
environment:
DOCKER_REPO: cbioportal/cbioportal-dev
steps:
- attach_workspace:
at: /tmp/repos
- build_push_image:
push: "false"
- run:
name: Build cBioPortal docker image
environment:
DOCKER_REPO: cbioportal/cbioportal-dev
name: Save cbioportal image as tar
command: |
export DOCKER_TAG=$CIRCLE_SHA1
URL="https://hub.docker.com/v2/repositories/cbioportal/cbioportal-dev/tags/$DOCKER_TAG-web-shenandoah"
TAG_FOUND=$(curl -s $URL | jq -r .name)
if [ $TAG_FOUND = "$DOCKER_TAG-web-shenandoah" ]; then
echo "Image already exists. Skipping build step!"
exit 0
fi
cd cbioportal-test
./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=true --skip_web_and_data=true
EXISTS=$(docker manifest inspect $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?)
if [ $EXISTS -eq 0 ]; then
echo "Build succeeded!"
else
echo "Build failed!"
exit 1
fi
docker save -o $DOCKER_TAG-web-shenandoah.tar $DOCKER_REPO:$DOCKER_TAG-web-shenandoah
- persist_to_workspace:
root: /tmp/repos
paths:
- "*.tar"

push_image:
machine:
image: ubuntu-2204:2024.08.1
resource_class: medium
working_directory: /tmp/repos
steps:
- attach_workspace:
at: /tmp/repos
- build_push_image:
push: "true"

run_api_tests:
machine:
image: ubuntu-2204:2024.08.1
docker_layer_caching: true
resource_class: large
working_directory: /tmp/repos
environment:
DOCKER_REPO: cbioportal/cbioportal-dev
steps:
- attach_workspace:
at: /tmp/repos
- checkout:
path: /tmp/repos/cbioportal
- run:
name: Load cbioportal image
command: |
export DOCKER_TAG=$CIRCLE_SHA1
docker load -i $DOCKER_TAG-web-shenandoah.tar
- run:
name: Instantiate a cbioportal instance
environment:
DOCKER_REPO: cbioportal/cbioportal-dev
APP_CLICKHOUSE_MODE: "true"
command: |
cd cbioportal-test
Expand Down Expand Up @@ -346,8 +398,6 @@ jobs:
yarn --ignore-engines
yarn run apitests
- store_artifacts:
path: /tmp/repos/cbioportal-test/web-metadata.json
- store_artifacts:
path: /tmp/repos/docker-compose-logs.txt

Expand All @@ -356,43 +406,43 @@ jobs:
image: ubuntu-2204:2024.08.1
docker_layer_caching: true
resource_class: medium
working_directory: /tmp/repos
environment:
BASE_REPO: cbioportal/cbioportal
DEV_REPO: cbioportal/cbioportal-dev
OUTPUT_FORMAT: '{severity: .cvss.severity, source_id: .source_id, vulnerable_range: .vulnerable_range, fixed_by: .fixed_by, url: .url, description: .description}'
SORT: 'sort_by(.severity | if . == "CRITICAL" then 0 elif . == "HIGH" then 1 elif . == "MEDIUM" then 2 elif . == "LOW" then 3 else 4 end)'
steps:
- attach_workspace:
at: /tmp/repos
- run:
name: Install Docker Scout
command: |
curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s -- -b /home/circleci/bin
- run:
name: Log in to Docker
command: |
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin;
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
- run:
name: Load cbioportal image
command: |
export DOCKER_TAG=$CIRCLE_SHA1
docker load -i $DOCKER_TAG-web-shenandoah.tar
- run:
name: Run Docker Scout on master
command: |
IMAGE=$BASE_REPO:master-web-shenandoah
docker pull $IMAGE
docker-scout cves $IMAGE --format sbom | jq -r "[.vulnerabilities[].vulnerabilities[] | $OUTPUT_FORMAT] | $SORT" > master_report.sbom
- run:
name: Wait for cbioportal docker images
name: Run Docker Scout on PR
command: |
URL="https://hub.docker.com/v2/repositories/$DEV_REPO/tags/$CIRCLE_SHA1-web-shenandoah"
while true; do
TAG_FOUND=$(curl -s $URL | jq -r .name)
if [ $TAG_FOUND = "$CIRCLE_SHA1-web-shenandoah" ]; then
echo "Image found!"
exit 0
fi
echo "Image not found yet. Waiting for API Tests to finish building. Retrying in 30 seconds..."
sleep 30
done
IMAGE=$DEV_REPO:$CIRCLE_SHA1-web-shenandoah
docker-scout cves $IMAGE --format sbom | jq -r "[.vulnerabilities[].vulnerabilities[] | $OUTPUT_FORMAT] | $SORT" > pr_report.sbom
- run:
name: Run Docker Scout vulnerability test
name: Analyze and report results
command: |
BASE_IMAGE=$BASE_REPO:master-web-shenandoah
PR_IMAGE=$DEV_REPO:$CIRCLE_SHA1-web-shenandoah
OUTPUT_FORMAT='{severity: .cvss.severity, source_id: .source_id, vulnerable_range: .vulnerable_range, fixed_by: .fixed_by, url: .url, description: .description}'
SORT='sort_by(.severity | if . == "CRITICAL" then 0 elif . == "HIGH" then 1 elif . == "MEDIUM" then 2 elif . == "LOW" then 3 else 4 end)'
docker pull $BASE_IMAGE
docker pull $PR_IMAGE
docker-scout cves $BASE_IMAGE --format sbom | jq -r "[.vulnerabilities[].vulnerabilities[] | $OUTPUT_FORMAT] | $SORT" > base_report.sbom
docker-scout cves $PR_IMAGE --format sbom | jq -r "[.vulnerabilities[].vulnerabilities[] | $OUTPUT_FORMAT] | $SORT" > pr_report.sbom
DIFF=$(jq -s 'map(map(.source_id)) | .[0] - .[1]' pr_report.sbom base_report.sbom)
DIFF=$(jq -s 'map(map(.source_id)) | .[0] - .[1]' pr_report.sbom master_report.sbom)
COUNT=$(echo $DIFF | jq 'length')
if [ "$COUNT" -gt 0 ]; then
printf "New vulnerabilities found: $COUNT\n"
Expand All @@ -404,7 +454,6 @@ jobs:
fi
workflows:
version: 2
end_to_end_tests:
jobs:
- build_backend
Expand All @@ -417,28 +466,29 @@ workflows:
- build_backend
- pull_frontend_codebase
- install_yarn
api_tests:
tests:
jobs:
- checkout_pr
- pull_cbioportal_test_codebase
- pull_cbioportal_frontend_codebase
- wait_for_approval:
type: approval
- build_image:
requires:
- checkout_pr
- pull_cbioportal_test_codebase
- pull_cbioportal_frontend_codebase
- build_push_image:
- push_image:
context:
- api-tests
requires:
- wait_for_approval
- checkout_pr
- pull_cbioportal_test_codebase
- run_api_tests:
context:
- api-tests
requires:
- build_push_image

security_tests:
jobs:
- build_image
- pull_cbioportal_frontend_codebase
- run_security_tests:
context:
- docker-scout
- docker-scout
requires:
- build_image

0 comments on commit 600691b

Please sign in to comment.