Skip to content

Commit 19e012c

Browse files
jerabekjirijerabekjiri
committed
init 4.9 docker compose standalone profile with CI
No-Issue
1 parent 1322bf8 commit 19e012c

25 files changed

+1240
-1
lines changed

‎.github/workflows/ci-docker-compose-integration.yml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
---
2+
name: Docker Compose Integration
3+
on:
4+
pull_request:
5+
branches:
6+
- '**'
7+
push:
8+
branches:
9+
- '**'
10+
workflow_dispatch:
11+
12+
jobs:
13+
integration:
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
env:
18+
- { TEST_PROFILE: standalone, HUB_TEST_MARKS: "deployment_standalone or all" }
19+
20+
runs-on: ubuntu-latest
21+
steps:
22+
- uses: actions/checkout@v4
23+
24+
- uses: actions/setup-python@v4
25+
with:
26+
python-version: "3.10"
27+
28+
- name: (Linux) Install docker compose
29+
run: |
30+
curl -L -o /tmp/docker-compose https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64
31+
install /tmp/docker-compose /usr/local/bin/
32+
33+
- name: sudo apt update
34+
run: |
35+
sudo apt update
36+
37+
- name: Install python requirements and podman
38+
run: |
39+
sudo apt-get install -y libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev podman
40+
41+
- name: Spin up dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml
42+
run: |
43+
docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml up --detach
44+
45+
- name: Export environment variables to host
46+
if: ${{ matrix.env.TEST_PROFILE != 'certified-sync' }}
47+
run: |
48+
docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV
49+
50+
- name: Export environment variables to host (certified-sync)
51+
if: ${{ matrix.env.TEST_PROFILE == 'certified-sync' }}
52+
run: |
53+
docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec standalone-manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV
54+
55+
- run: cat $GITHUB_ENV
56+
57+
- name: Wait for API online status
58+
run: |
59+
max_runs=10
60+
for i in $(seq 1 $max_runs); do
61+
echo "$i: checking api status..."
62+
RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -u "${{ env.DJANGO_SUPERUSER_USERNAME }}:${{ env.DJANGO_SUPERUSER_PASSWORD }}" -LI http://localhost:5001/api/galaxy/pulp/api/v3/status/)
63+
if [ "$RESPONSE" -ne 200 ]; then
64+
echo "API is down. Retrying in 10 seconds..."
65+
sleep 10
66+
else
67+
echo "API online."
68+
exit 0
69+
fi
70+
done
71+
72+
- name: Install integration requirements
73+
run: |
74+
pip install -r integration_requirements.txt
75+
76+
- name: Run integration tests
77+
run: |
78+
pytest -v -r sx --color=yes -m '${{ matrix.env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration
79+
env:
80+
GALAXYKIT_SLEEP_SECONDS_POLLING: ".5"
81+
GALAXYKIT_SLEEP_SECONDS_ONETIME: ".5"
82+
GALAXYKIT_POLLING_MAX_ATTEMPTS: "50"
83+
GALAXY_SLEEP_SECONDS_POLLING: ".5"
84+
GALAXY_SLEEP_SECONDS_ONETIME: ".5"
85+
GALAXY_POLLING_MAX_ATTEMPTS: "50"

‎.github/workflows/ci_oci-env-integration.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
env:
1818
- TEST_PROFILE: ldap
1919
- TEST_PROFILE: keycloak
20-
- TEST_PROFILE: standalone
20+
# - TEST_PROFILE: standalone
2121
- TEST_PROFILE: rbac
2222
- TEST_PROFILE: certified-sync
2323
- TEST_PROFILE: insights

‎dev/compose/Dockerfile.dev

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# This Dockerfile is meant to extend the main Dockerfile.rhel8
2+
# Moving files to the required places
3+
# Installing extra development libraries and tools
4+
FROM localhost/galaxy_ng/galaxy_ng:base
5+
6+
ARG GNUPGHOME
7+
ENV GNUPGHOME $GNUPGHOME
8+
9+
USER root
10+
11+
RUN /app/dev/compose/signing/setup_gpg_workarounds.sh
12+
13+
RUN set -ex; \
14+
install -Dm 0775 /app/dev/compose/database/database_fields.symmetric.key /etc/pulp/certs/database_fields.symmetric.key && \
15+
install -Dm 0775 /app/dev/compose/signing/signing-secret.key /etc/pulp/certs/signing-secret.key && \
16+
install -Dm 0775 /app/dev/compose/signing/signing-secret.key.password.txt /etc/pulp/certs/signing-secret.key.password.txt && \
17+
install -Dm 0775 /app/dev/compose/signing/signing-secret.key /etc/pulp/certs/signing-public.key && \
18+
install -Dm 0775 /app/dev/compose/signing/collection_sign.sh /var/lib/pulp/scripts/collection_sign.sh && \
19+
install -Dm 0775 /app/dev/compose/signing/container_sign.sh /var/lib/pulp/scripts/container_sign.sh && \
20+
chmod +x /var/lib/pulp/scripts/*_sign.sh
21+
22+
RUN /app/dev/compose/signing/setup_gpg_keys.sh
23+
24+
RUN ${VIRTUAL_ENV}/bin/python3 -m pip install ipython ipdb django-extensions

‎dev/compose/README.md

Lines changed: 263 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,263 @@
1+
# Galaxy Simplified Compose Stack
2+
3+
Profiles:
4+
5+
- `standalone` - Run galaxy_ng for integration with Ansible Automation Platform
6+
7+
## Requirements
8+
9+
- `docker compose` version `>=2`
10+
11+
## Usage
12+
13+
Pick a profile as needed and on the root of `galaxy_ng` repository.
14+
15+
> Examples assumes `standalone` as the profile, change as needed.
16+
17+
Build images
18+
```bash
19+
docker compose -f dev/compose/standalone.yaml build
20+
```
21+
22+
Run the stack
23+
```bash
24+
docker compose -f dev/compose/standalone.yaml up
25+
# optionally pass `-d` to release the terminal
26+
```
27+
28+
Exec commands on the `manager` service
29+
30+
Bash
31+
```console
32+
$ docker compose -f dev/compose/standalone.yaml exec manager /bin/bash
33+
bash-4.4#
34+
```
35+
Django Admin
36+
```console
37+
$ docker compose -f dev/compose/standalone.yaml exec manager pulpcore-manager
38+
Type 'pulpcore-manager help <subcommand>' for help on a specific subcommand.
39+
40+
Available subcommands:
41+
42+
[app]
43+
add-signing-service
44+
analyze-publication
45+
...
46+
```
47+
48+
Settings
49+
```console
50+
$ docker compose -f dev/compose/standalone.yaml exec manager dynaconf get DATABASES | python -m json.tool
51+
{
52+
"default": {
53+
"ENGINE": "django.db.backends.postgresql",
54+
"HOST": "postgres",
55+
"NAME": "galaxy_ng",
56+
"PASSWORD": "galaxy_ng",
57+
"PORT": 5432,
58+
"USER": "galaxy_ng"
59+
}
60+
}
61+
```
62+
```console
63+
$ docker compose -f dev/compose/standalone.yaml exec manager dynaconf list
64+
CONTENT_ORIGIN<str> 'https://localhost'
65+
CACHE_ENABLED<bool> False
66+
CACHE_SETTINGS<dict> {'EXPIRES_TTL': 600}
67+
ALLOWED_CONTENT_CHECKSUMS<list> ['sha224', 'sha256', 'sha384', 'sha512']
68+
...
69+
```
70+
71+
Stopping
72+
```console
73+
$ docker compose -f dev/compose/standalone.yaml down
74+
# add -v to stop and remove volumes
75+
```
76+
77+
> [!TIP]
78+
> Stop with Ctrl + C if running without `-d` and then execute the `down` command.
79+
80+
## API Access
81+
82+
Galaxy API is available on:
83+
84+
[http://localhost:5001/api/galaxy/v3/swagger-ui/](http://localhost:5001/api/galaxy/v3/swagger-ui/)
85+
86+
AAP UI and API will be available only if started separately on:
87+
88+
[https://localhost](https://localhost)
89+
90+
91+
## Running UI for standalone development
92+
93+
Ansible Hub UI can be started separately as a standalone `npm` run.
94+
95+
```console
96+
# Assuming galaxy_ng is running on standalone compose.
97+
98+
$ git clone https://github.com/ansible/ansible-hub-ui ~/projects/ansible-hub-ui
99+
$ git clone https://github.com/ansible/galaxy_ng ~/projects/galaxy_ng
100+
```
101+
Open 2 terminals:
102+
103+
On the first terminal:
104+
105+
```console
106+
$ cd galaxy_ng
107+
$ docker compose -f dev/compose/standalone.yaml up
108+
```
109+
110+
On the second terminal:
111+
112+
```console
113+
cd ansible-hub-ui
114+
npm install
115+
npm run start-standalone
116+
```
117+
118+
UI will be available on http://localhost:8002 and API on http://localhost:5001
119+
120+
121+
## Auto Reload and Local Checkouts
122+
123+
To have the services `api`, `worker` and `content` to automatically reload when
124+
source code changes it is required to set which paths the `reloader` must watch for changes.
125+
126+
Set the variable `DEV_SOURCE_PATH` with the packages you want to add to the reloader list.
127+
128+
Those repositories must be local checkouts located on the same level of the `galaxy_ng` repository.
129+
130+
Example:
131+
132+
Get the repositories locally in the same root path.
133+
```console
134+
$ git clone https://github.com/dynaconf/dynaconf ~/projects/dynaconf
135+
$ git clone https://github.com/pulp/pulp_ansible ~/projects/pulp_ansible
136+
$ git clone https://github.com/ansible/galaxy_ng ~/projects/galaxy_ng
137+
```
138+
139+
> **IMPORTANT** Ensure all the repos are checked out to compatible branches.
140+
> for example. you may be on galaxy_ng:master and reading `setup.py` you
141+
> see that it requires `pulp_ansible>2.10,<3` then ensure you checkout `pulp_ansible`
142+
> to a compatible branch.
143+
144+
Start the compose setting the desired editable paths.
145+
146+
```console
147+
$ cd ~/projects/galaxy_ng
148+
$ export DEV_SOURCE_PATH="dynaconf:pulp_ansible:galaxy_ng"
149+
$ docker compose -f dev/compose/standalone.yaml up --build
150+
```
151+
152+
Optionally it can be informed in a single line:
153+
154+
```console
155+
$ DEV_SOURCE_PATH="dynaconf:pulp_ansible:galaxy_ng" docker compose -f dev/compose/app.yaml up --build
156+
```
157+
158+
> [!NOTE]
159+
> if passing on the call line, remember to repass the same variable every time you interact with
160+
>`docker compose` using the `run` command, usually `exec,logs,stats` doesn't require, but commands
161+
> that starts the service container from scratch needs the variables.
162+
163+
Now when changes are detected on `.py` and `.yaml` files on any of the `DEV_SOURCE_PATH`
164+
directories it will trigger reload of `api`, `worker`, and `content` services.
165+
166+
167+
## Troubleshooting
168+
169+
### VersionConflict error
170+
171+
Example:
172+
```bash
173+
api-1 | raise VersionConflict(dist, req).with_context(dependent_req)
174+
api-1 | pkg_resources.VersionConflict: (pkg_foo 3.2.6 (/venv/lib/python3.11/site-packages), Requirement.parse('pkg_foo<3.1.13,>=3.1.12'))
175+
```
176+
177+
Solution 1:
178+
Clean up local build files:
179+
180+
```bash
181+
cd ~/projects/galaxy_ng
182+
rm -rf .eggs
183+
rm -rf build
184+
rm -rf galaxy_ng.egg-info
185+
```
186+
187+
Solution 2:
188+
189+
- Ensure `LOCK_REQUIREMENTS` is set to `0`
190+
- Ensure all your local checkouts are checked out to compatible branches
191+
192+
### LLB definition error
193+
194+
```bash
195+
failed to solve: rpc error: code = Unknown desc = failed to solve with frontend dockerfile.v0: failed to create LLB definition: failed to do request: Head "http://localhost/v2/galaxy_ng/galaxy_ng/manifests/base": dial tcp [::1]:80: connect: connection refused
196+
```
197+
198+
Solution
199+
200+
```bash
201+
export DOCKER_BUILDKIT=0
202+
export COMPOSE_DOCKER_CLI_BUILD=0
203+
```
204+
205+
## Tests
206+
207+
### Unit tests
208+
Run unit tests with docker compose [Running unit tests](../../galaxy_ng//tests/unit/README.md)
209+
210+
### Integration tests
211+
Run integration tests with docker compose, check the [Running integration tests](../../galaxy_ng/tests/integration/README.md)
212+
213+
Run with legacy oci-env, check the [Running oci-env integration tests](../../docs/dev/integration_tests.md)
214+
215+
## Tips and Tricks.
216+
217+
---
218+
219+
### Debugging
220+
221+
#### Step 1 - Add the breakpoint
222+
223+
Edit the file you want to debug and add a breakpoint
224+
225+
```python
226+
__import__("ipdb").set_trace()
227+
```
228+
229+
> [!TIP]
230+
> Replace if you are using a different debugger, however the images has only **pdb** and **ipdb** installed.
231+
232+
If you discover other ways of debugging, like connecting dap protocol or vscode debugger, please update this file!
233+
234+
#### Step 2 - Now execute your stack or just the container you are trying to debug.
235+
236+
Example:
237+
238+
```bash
239+
$ export DEV_SOURCE_PATH=galaxy_ng
240+
$ docker compose -f dev/compose/standalone.yaml up migrations
241+
# The container will keep running stopped on the breakpoint.
242+
```
243+
244+
#### Step 3 - Attach
245+
246+
```bash
247+
$ docker compose -f dev/compose/standalone.yaml attach migrations
248+
ipdb>
249+
```
250+
251+
> [!IMPORTANT]
252+
> To detach from the container DO NOT use <kbd>Ctrl+c</kbd>,
253+
> instead, use <kbd>Ctrl-p Ctrl-q</kbd>
254+
255+
---
256+
257+
**TBD**
258+
259+
### Connecting to Database
260+
261+
### Dumping and restoring the database
262+
263+
### Populating testing data

0 commit comments

Comments
 (0)