-
Notifications
You must be signed in to change notification settings - Fork 18
/
Copy pathdocker-compose.yml
215 lines (204 loc) · 7.78 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2022-2023 Dell Inc, or its subsidiaries.
# Copyright (C) 2023 Intel Corporation
---
# version: "3.7" - version is obsolete and can be removed/commented out
services:
spdk:
image: docker.io/opiproject/spdk:v24.01
volumes:
- /dev/hugepages:/dev/hugepages
- /dev/shm:/dev/shm
- /proc:/proc
- /var/tmp:/var/tmp
ports:
- "9009:9009"
- "4444:4444"
- "5555:5555"
privileged: true
networks:
- opi
working_dir: /usr/libexec/spdk/scripts
command: |
sh -x -c 'sync; echo 1 > /proc/sys/vm/drop_caches && \
mkdir -p /mnt/huge && \
grep hugetlbfs /proc/mounts || mount -t hugetlbfs nodev /mnt/huge && \
echo 1024 > /proc/sys/vm/nr_hugepages && \
grep "" /sys/kernel/mm/hugepages/hugepages-*/nr_hugepages && \
dd if=/dev/zero of=/tmp/aio_bdev_file bs=512 count=64 && \
echo -n NVMeTLSkey-1:01:MDAxMTIyMzM0NDU1NjY3Nzg4OTlhYWJiY2NkZGVlZmZwJEiQ: > /tmp/opikey.txt && \
chmod 0600 /tmp/opikey.txt && \
/usr/local/bin/spdk_tgt -m 0x1 -s 512 --no-pci -S /var/tmp |& tee /tmp/spdk.log & \
for i in `seq 1 10`; do ./rpc.py spdk_get_version && break || sleep 1; done && \
./rpc.py bdev_malloc_create -b Malloc0 64 512 && \
./rpc.py bdev_malloc_create -b Malloc1 64 512 && \
./rpc.py nvmf_create_transport -t TCP -u 8192 -m 4 -c 0 && \
./rpc.py nvmf_create_transport -t VFIOUSER && \
./rpc.py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -d SPDK_Controller1 && \
./rpc.py nvmf_subsystem_allow_any_host nqn.2016-06.io.spdk:cnode1 --disable && \
./rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t tcp -a `hostname -i` -f ipv4 -s 4444 && \
./rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t tcp -a `hostname -i` -f ipv4 -s 5555 --secure-channel && \
./rpc.py nvmf_subsystem_add_host nqn.2016-06.io.spdk:cnode1 nqn.2014-08.org.nvmexpress:uuid:feb98abe-d51f-40c8-b348-2753f3571d3c --psk /tmp/opikey.txt && \
./rpc_http_proxy.py 0.0.0.0 9009 spdkuser spdkpass'
healthcheck:
test:
[
"CMD-SHELL",
"python3 /usr/libexec/spdk/scripts/rpc.py spdk_get_version || exit 1"
]
interval: 6s
retries: 5
start_period: 20s
timeout: 10s
jaeger:
image: jaegertracing/all-in-one:1.53.0
ports:
- "16686:16686"
- "4317:4317"
- "4318:4318"
environment:
- COLLECTOR_OTLP_ENABLED=true
networks:
- opi
healthcheck:
test:
[
"CMD-SHELL",
"nc -zv localhost 4317 && nc -zv localhost 4318 || exit 1"
]
interval: 6s
retries: 5
start_period: 20s
timeout: 10s
frr:
image: quay.io/frrouting/frr:9.1.0
cap_add:
- NET_ADMIN
- SYS_ADMIN
- SYS_MODULE
- CAP_NET_RAW
stdin_open: true
tty: true
volumes:
- ./network/conf/frr.conf:/etc/frr/frr.conf
networks:
- opi
command: |
sh -x -c 'touch /etc/frr/vtysh.conf && \
sed -i "s/bgpd=no/bgpd=yes/g" /etc/frr/daemons && \
sed -i "s/127.0.0.1/0.0.0.0/g" /etc/frr/daemons && \
ip link add name lo0 type dummy && \
ifconfig lo0 10.0.0.1 netmask 255.255.255.255 up && \
/etc/init.d/frr stop && \
/usr/lib/frr/watchfrr -d -F traditional zebra bgpd staticd && \
sleep infinity'
redis:
image: redis:7.2.7-alpine3.21
networks:
- opi
healthcheck:
test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
opi-spdk-server:
image: docker.io/opiproject/opi-spdk-bridge:main
environment:
- OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4317
volumes_from:
- spdk
networks:
- opi
depends_on:
redis:
condition: service_healthy
jaeger:
condition: service_healthy
spdk:
condition: service_healthy
command: /opi-spdk-bridge -grpc_port=50051 -http_port=8082 -spdk_addr=/var/tmp/spdk.sock -redis_addr="redis:6379"
healthcheck:
test: grpcurl -plaintext localhost:50051 list || exit 1
opi-smbios-server:
image: docker.io/opiproject/opi-smbios-bridge:main
environment:
- OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4317
networks:
- opi
depends_on:
jaeger:
condition: service_healthy
command: /opi-smbios-bridge -grpc_port=50051 -http_port=8082
healthcheck:
test: grpcurl -plaintext localhost:50051 list || exit 1
opi-evpn-server:
image: docker.io/opiproject/opi-evpn-bridge:main
environment:
- OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4317
cap_add:
- NET_ADMIN
networks:
- opi
command: /opi-evpn-bridge --grpcport=50151 --httpport=8082 --dbaddress="redis:6379" --database=redis
healthcheck:
test: grpcurl -plaintext localhost:50151 list || exit 1
depends_on:
redis:
condition: service_healthy
jaeger:
condition: service_healthy
frr:
condition: service_started
opi-evpn-test:
build:
context: .
networks:
- opi
depends_on:
opi-evpn-server:
condition: service_healthy
command: network evpn create-vrf --name blue --vni 100 --vtep 10.0.0.100/24 --loopback 10.100.0.1/24 --addr opi-evpn-server:50151
opi-smbios-test:
build:
context: .
networks:
- opi
depends_on:
opi-smbios-server:
condition: service_healthy
command: inventory get --addr opi-smbios-server:50051
opi-storage-test:
build:
context: .
networks:
- opi
depends_on:
opi-spdk-server:
condition: service_healthy
entrypoint: ["/bin/sh", "-c", "-x"]
command: |
'/dpu storage test --addr opi-spdk-server:50051 && \
nvmf0=$$(/dpu storage create backend nvme controller --addr=opi-spdk-server:50051 --id nvmf0 --multipath failover) && \
/dpu storage get backend nvme controller --addr=opi-spdk-server:50051 --name "$$nvmf0" && \
path0=$$(/dpu storage create backend nvme path tcp --addr=opi-spdk-server:50051 --controller "$$nvmf0" --id path0 --ip $$(getent hosts spdk | cut -d" " -f1) --port 4444 --nqn nqn.2016-06.io.spdk:cnode1 --hostnqn nqn.2014-08.org.nvmexpress:uuid:feb98abe-d51f-40c8-b348-2753f3571d3c) && \
/dpu storage get backend nvme path --addr=opi-spdk-server:50051 --name "$$path0" && \
ss0=$$(/dpu storage create frontend nvme subsystem --addr=opi-spdk-server:50051 --id subsys0 --nqn "nqn.2022-09.io.spdk:opitest1") && \
ctrl0=$$(/dpu storage create frontend nvme controller tcp --addr=opi-spdk-server:50051 --id ctrl0 --ip "127.0.0.1" --port 4420 --subsystem "$$ss0") && \
ns0=$$(/dpu storage create frontend nvme namespace --addr=opi-spdk-server:50051 --id namespace0 --volume "Malloc0" --subsystem "$$ss0") && \
/dpu storage delete frontend nvme namespace --addr=opi-spdk-server:50051 --name "$$ns0" && \
/dpu storage delete frontend nvme controller --addr=opi-spdk-server:50051 --name "$$ctrl0" && \
/dpu storage delete frontend nvme subsystem --addr=opi-spdk-server:50051 --name "$$ss0"' && \
/dpu storage delete backend nvme path --addr=opi-spdk-server:50051 --name $$path0 && \
/dpu storage delete backend nvme controller --addr=opi-spdk-server:50051 --name "$$nvmf0" && \
opi-test:
image: docker.io/library/alpine:3.21
networks:
- opi
depends_on:
opi-evpn-test:
condition: service_completed_successfully
opi-smbios-test:
condition: service_completed_successfully
opi-storage-test:
condition: service_completed_successfully
command: |
sh -c 'sleep 60 && exit 0'
networks:
opi: