Skip to content

Commit bee2ffb

Browse files
maxcao13maxcao13
authored
chore(tests): use docker-compose with additional test scripts (#1480)
Signed-off-by: Max Cao <macao@redhat.com>
1 parent f6f79fa commit bee2ffb

12 files changed

+733
-0
lines changed

‎.env

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
COMPOSE_PROJECT_NAME=cryostat-test
2+
COMPOSE_FILE=compose/compose-cryostat.yaml

‎compose-test.bash

Lines changed: 306 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
#!/bin/bash
2+
# shellcheck disable=SC1091,SC2086
3+
4+
set -ex
5+
6+
source "$(dirname "$0")/.env"
7+
# podman-compose .env files are ignored https://github.com/containers/podman-compose/issues/475
8+
export COMPOSE_PROJECT_NAME
9+
export COMPOSE_FILE
10+
11+
# handle compose engine
12+
if [[ -z "${CONTAINER_ENGINE}" ]]; then
13+
CONTAINER_ENGINE="podman"
14+
fi
15+
16+
if [[ "${CONTAINER_ENGINE}" == "podman" ]]; then
17+
COMPOSE_ENGINE="podman-compose"
18+
elif [[ "${CONTAINER_ENGINE}" == "docker" ]]; then
19+
COMPOSE_ENGINE="docker compose"
20+
else
21+
echo "ERROR: Invalid container engine specified."
22+
exit 1
23+
fi
24+
25+
if [ -z "$COMPOSE_PROFILES" ]; then
26+
COMPOSE_PROFILES=""
27+
fi
28+
29+
compose_files=(
30+
"compose/compose-cryostat.yaml"
31+
"compose/compose-cryostat-reports.yaml"
32+
"compose/compose-cryostat-grafana.yaml"
33+
"compose/compose-jfr-datasource.yaml"
34+
"compose/compose-jmxquarkus.yaml"
35+
)
36+
37+
display_usage() {
38+
echo "Usage: $(basename "$0") [-a targets] [-j targets] [-d] [-i] [-p] [-r] [-s] [-A]"
39+
echo "Options:"
40+
echo " -a targets Sets # of agent targets"
41+
echo " -j targets Sets # of JMX targets"
42+
echo " -d Enables Cryostat duplicate"
43+
echo " -i Enables invalid targets"
44+
echo " -p Enables a postgres database"
45+
echo " -r Enables automated rule that matches all targets"
46+
echo " -s Enables periodic target scaling/restarting"
47+
echo " -A Enables all of the above; Sets targets to 10 each"
48+
}
49+
50+
51+
# Parse command-line options
52+
while getopts ":a:j:diprsA" opt; do
53+
case $opt in
54+
a)
55+
CT_AGENT_REPLICAS=$OPTARG
56+
;;
57+
j)
58+
CT_JMX_REPLICAS=$OPTARG
59+
;;
60+
d)
61+
CT_EN_DUPLICATE=true
62+
;;
63+
i)
64+
CT_EN_INVALID=true
65+
;;
66+
p)
67+
CT_EN_POSTGRES=true
68+
;;
69+
r)
70+
CT_EN_RULES=true
71+
;;
72+
s)
73+
CT_EN_SCALING=true
74+
;;
75+
A)
76+
# shellcheck disable=SC2034
77+
CT_AGENT_REPLICAS=10
78+
# shellcheck disable=SC2034
79+
CT_JMX_REPLICAS=10
80+
CT_EN_DUPLICATE=true
81+
CT_EN_INVALID=true
82+
CT_EN_POSTGRES=true
83+
CT_EN_RULES=true
84+
CT_EN_SCALING=true
85+
;;
86+
\?)
87+
echo "Invalid option: -$OPTARG"
88+
display_usage
89+
exit 1
90+
;;
91+
esac
92+
done
93+
94+
shift $((OPTIND - 1))
95+
96+
getPomProperty() {
97+
if command -v xpath > /dev/null 2>&1 ; then
98+
xpath -q -e "project/properties/$1/text()" pom.xml
99+
elif command -v mvnd > /dev/null 2>&1 ; then
100+
mvnd help:evaluate -o -B -q -DforceStdout -Dexpression="$1"
101+
else
102+
mvn help:evaluate -o -B -q -DforceStdout -Dexpression="$1"
103+
fi
104+
}
105+
106+
compose_down() {
107+
$COMPOSE_ENGINE down --remove-orphans
108+
if [ -n "$child_pid" ]; then
109+
kill "$child_pid"
110+
fi
111+
}
112+
113+
cleanup() {
114+
rm -f "$tmp_yaml"
115+
}
116+
117+
if [ -z "$CRYOSTAT_DISABLE_SSL" ]; then
118+
protocol="https"
119+
else
120+
protocol="http"
121+
fi
122+
123+
# grafana
124+
grafanaStream="$(getPomProperty cryostat.itest.grafana.imageStream)"
125+
grafanaTag="$(getPomProperty cryostat.itest.grafana.version)"
126+
GRAFANA_IMAGE="${grafanaStream}:${grafanaTag}"
127+
datasourceStream="$(getPomProperty cryostat.itest.jfr-datasource.imageStream)"
128+
datasourceTag="$(getPomProperty cryostat.itest.jfr-datasource.version)"
129+
DATASOURCE_IMAGE="${datasourceStream}:${datasourceTag}"
130+
export GRAFANA_IMAGE
131+
export DATASOURCE_IMAGE
132+
133+
# reports
134+
reportsStream="$(getPomProperty cryostat.itest.reports.imageStream)"
135+
reportsTag="$(getPomProperty cryostat.itest.reports.version)"
136+
REPORTS_RJMX_PORT=10000
137+
REPORTS_HTTP_PORT="$(getPomProperty cryostat.itest.reports.port)"
138+
REPORTS_IMAGE="${reportsStream}:${reportsTag}"
139+
export REPORTS_IMAGE
140+
export REPORTS_RJMX_PORT
141+
export REPORTS_HTTP_PORT
142+
143+
# agent app configuration
144+
CRYOSTAT_AGENT_AUTHORIZATION="Basic $(echo user:pass | base64)"
145+
CRYOSTAT_AGENT_BASEURI="${protocol}://cryostat:8181/"
146+
export CRYOSTAT_AGENT_AUTHORIZATION
147+
export CRYOSTAT_AGENT_BASEURI
148+
149+
# postgres
150+
if [ "$CT_EN_POSTGRES" = true ]; then
151+
if [ -z "${POSTGRES_IMAGE}" ]; then
152+
postgresStream="$(getPomProperty postgres.image)"
153+
postgresTag="$(getPomProperty postgres.version)"
154+
POSTGRES_IMAGE="${postgresStream}:${postgresTag}"
155+
export POSTGRES_IMAGE
156+
fi
157+
compose_files+=("compose/compose-postgres.yaml")
158+
fi
159+
160+
CRYOSTAT_LIVENESS_PATH="${protocol}://cryostat:8181/health/liveness"
161+
export CRYOSTAT_LIVENESS_PATH
162+
163+
if [ -z "$KEYSTORE_PATH" ] && [ -f "$(dirname "$0")/certs/cryostat-keystore.p12" ] ; then
164+
export KEYSTORE_PATH="/certs/cryostat-keystore.p12"
165+
KEYSTORE_PASS="$(cat "$(dirname "$0")"/certs/keystore.pass)"
166+
export KEYSTORE_PASS
167+
fi
168+
169+
child_process() {
170+
while true; do
171+
sleep 10
172+
echo "Scaling down scaled-app..."
173+
if [ "$CONTAINER_ENGINE" = "podman" ]; then
174+
podman-compose kill scaled-app
175+
else
176+
docker compose scale scaled-app=0
177+
fi
178+
179+
sleep 5
180+
181+
echo "Stopping stopped-app..."
182+
$COMPOSE_ENGINE stop stopped-app
183+
184+
sleep 10
185+
186+
echo "Scaling up scaled-app..."
187+
if [ "$CONTAINER_ENGINE" = "podman" ]; then
188+
podman-compose restart scaled-app
189+
else
190+
docker compose scale scaled-app=1
191+
fi
192+
193+
sleep 5
194+
195+
echo "Restarting stopped-app..."
196+
$COMPOSE_ENGINE restart stopped-app
197+
done
198+
}
199+
200+
# options post-handling
201+
202+
# testing automated rule
203+
if [ "$CT_EN_RULES" = true ]; then
204+
# generate automated-rule that matches vertx-fib-demos
205+
touch "$(dirname "$0")"/cryostat-compose-test.json
206+
echo "{\"name\":\"cryostat-compose-test\",\"description\":\"\",\"matchExpression\":\"target.alias == 'es.andrewazor.demo.Main'\",\"eventSpecifier\":\"template=Continuous,type=TARGET\",\"archivalPeriodSeconds\":30,\"initialDelaySeconds\":30,\"preservedArchives\":3,\"maxAgeSeconds\":35,\"maxSizeBytes\":0,\"enabled\":true}" >> "$(dirname "$0")"/cryostat-compose-test.json
207+
# move to conf.d
208+
mv "$(dirname "$0")"/cryostat-compose-test.json "$(dirname "$0")"/conf/rules/cryostat-compose-test.json
209+
else
210+
# delete that file if it exists
211+
rm -f "$(dirname "$0")"/conf/rules/cryostat-compose-test.json
212+
fi
213+
214+
# FIXME: podman-compose does not support COMPOSE_PROFILES yet https://github.com/containers/podman-compose/pull/592
215+
# testing invalid targets
216+
217+
PROFILE_ARGS=""
218+
if [ "$CT_EN_SCALING" = true ]; then
219+
PROFILE_ARGS+="--profile scaled "
220+
fi
221+
222+
if [ "$CT_EN_INVALID" = true ]; then
223+
# COMPOSE_PROFILES+=(",invalid")
224+
PROFILE_ARGS+="--profile invalid "
225+
fi
226+
227+
# testing duplicate Cryostat instance
228+
if [ "$CT_EN_DUPLICATE" = true ]; then
229+
# COMPOSE_PROFILES+=(",duplicate")
230+
PROFILE_ARGS+="--profile duplicate "
231+
fi
232+
233+
# trap on CTRL+C SIGINT (we don't want restart signals to tear down the test environment)
234+
trap compose_down INT TERM
235+
236+
merged_yaml="---
237+
services:"
238+
239+
sections=("vertx-jmx" "quarkus-agent")
240+
replicas_variables=("CT_JMX_REPLICAS" "CT_AGENT_REPLICAS")
241+
242+
non_zero_replicas=false
243+
244+
# Iterate over the sections
245+
for ((i=0; i<${#sections[@]}; i++))
246+
do
247+
section="${sections[i]}"
248+
replicas_var="${replicas_variables[i]}"
249+
replicas="${!replicas_var}"
250+
251+
if (( replicas > 0 )); then
252+
non_zero_replicas=true
253+
254+
for ((j=1; j<=replicas; j++))
255+
do
256+
current_yaml="$(sed "s/$section/$section-${j}/g" "compose/compose-$section.yaml")"
257+
current_yaml="${current_yaml//---/}"; current_yaml="${current_yaml//services:/}"
258+
merged_yaml+="$current_yaml"
259+
done
260+
fi
261+
done
262+
263+
if $non_zero_replicas; then
264+
trap cleanup EXIT
265+
tmp_yaml=$(mktemp)
266+
echo "$merged_yaml" >> "$tmp_yaml"
267+
compose_files+=("$tmp_yaml")
268+
fi
269+
270+
# handle compose_files arguments
271+
COMPOSE_ARGS=""
272+
for file in "${compose_files[@]}"; do
273+
COMPOSE_ARGS+=" -f $file"
274+
done
275+
276+
$COMPOSE_ENGINE $PROFILE_ARGS $COMPOSE_ARGS up -d --remove-orphans
277+
278+
# testing periodically scaling
279+
if [ "$CT_EN_SCALING" = true ]; then
280+
child_process &
281+
child_pid=$!
282+
fi
283+
284+
# handle cryostat container restart signals
285+
wait_on_cryostat() {
286+
timeout_duration=20
287+
polling_interval=1
288+
if timeout "$timeout_duration" bash -c "
289+
while [ \$($CONTAINER_ENGINE inspect cryostat --format='{{.State.ExitCode}}' 2>/dev/null || echo 1) -ne 0 ] ; do
290+
echo \"Waiting for cryostat to restart...\"
291+
sleep $polling_interval
292+
done
293+
"; then
294+
loop
295+
else
296+
echo "Cryostat timed out or failed..."
297+
exit 1
298+
fi
299+
}
300+
301+
loop() {
302+
$COMPOSE_ENGINE logs --tail 100 -f cryostat 2>&1 | tee cryostat-run.log
303+
wait_on_cryostat
304+
}
305+
306+
loop

‎compose/COMPOSE_TESTING_README.md

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Cryostat compose testing README (DEPRECATED/needs updates)
2+
3+
There are multiple yaml files in the base directory used for testing Cryostat that is compatible with the [Compose Specification](https://github.com/compose-spec/compose-spec/blob/master/spec.md). It is used by the `compose-test.bash` script to test various scenarios. The `test-restart.bash` script is used to test restarting the Cryostat container.
4+
5+
Read the Cryostat CONFIGURATION section of the main [README](./README.md) for more information on Cryostat-specific environment variables.
6+
7+
## Requirements
8+
* [podman-compose 1.0.7+](https://github.com/containers/podman-compose#installation) or [Docker Engine](https://docs.docker.com/engine/install/)
9+
* Bash
10+
11+
## Environment variables for testing
12+
13+
### General
14+
15+
- **COMPOSE_PROJECT_NAME**: The name of the project. Used to specify the containers within the project. Defaults to the `.env` file's `COMPOSE_PROJECT_NAME` variable. If not set, defaults to `cryostat`. More info [here](https://docs.docker.com/compose/environment-variables/envvars/#compose_project_name).
16+
17+
- **CONTAINER_ENGINE**: The container engine to use for the compose file. Defaults to `podman`. Set to `docker` to use `docker compose` instead.
18+
19+
### Testing scenarios
20+
The `CT` prefix stands for *Cryostat Test* or *Compose Test*.
21+
22+
- **CT_EN_RULES**: Use to test automated rules. Creates an enabled automated rule that will trigger on Cryostat startup that matches all targets.
23+
24+
- **CT_EN_SCALING**: Use to test targets that periodically scale, and restart.
25+
26+
- **CT_EN_INVALID**: Use to enable invalid targets. One target has an invalid Agent callback URI, and the other has an invalid `io.cryostat.connectUrl` Podman label.
27+
28+
- **CT_EN_DUPLICATE**: Use to test an environment with two instances of Cryostat. The second instance will be available at `localhost:8282`.
29+
30+
- **CT_EN_POSTGRES**: Use to test a Cryostat instance with a *Postgres* database. The database will be available at `localhost:5432`.
31+
32+
- **CT_JMX_REPLICAS**: Set the number of replicas for the JMX test app. Defaults to 1.
33+
34+
- **CT_AGENT_REPLICAS**: Set the number of replicas for the Agent test app. Defaults to 1.
35+
36+
## Scripts
37+
38+
- **compose-test.bash**: Runs the compose test with the default variables. The following options are available:
39+
* `-a targets`: Sets the number of agent targets.
40+
* `-j targets`: Sets the number of JMX targets.
41+
* `-d`: Enables Cryostat duplicate.
42+
* `-i`: Enables invalid targets.
43+
* `-p`: Enables *Postgres* database.
44+
* `-r`: Enables automated rule that matches all targets.
45+
* `-s`: Enables periodic target scaling/restarting.
46+
* `-A`: Enables all of the above; Sets number of external targets to 10 each.
47+
48+
- **test-restart.bash**: Restarts the Cryostat container, but keeps the other containers running. It assumes that the `compose-test.bash` script is currently being executed and that an instance of the Cryostat container is already running."
49+

‎compose/compose-cryostat-grafana.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
services:
3+
cryostat:
4+
environment:
5+
- GRAFANA_DASHBOARD_EXT_URL=http://localhost:3000
6+
- GRAFANA_DASHBOARD_URL=http://grafana:3000
7+
grafana:
8+
image: ${GRAFANA_IMAGE-quay.io/cryostat/cryostat-grafana-dashboard:latest}
9+
container_name: grafana
10+
restart: unless-stopped
11+
environment:
12+
- GF_INSTALL_PLUGINS=grafana-simple-json-datasource
13+
- GF_AUTH_ANONYMOUS_ENABLED=true
14+
- JFR_DATASOURCE_URL=http://jfr-datasource:8080
15+
ports:
16+
- "3000:3000"
17+
expose:
18+
- "3000"

0 commit comments

Comments
 (0)