From b32724e9a08d381105cb3685aebdf8c396e82a38 Mon Sep 17 00:00:00 2001 From: Yashvardhan Nanavati Date: Wed, 10 Dec 2025 11:03:49 -0800 Subject: [PATCH] Add dev-env for containerized workflow Signed-off-by: Yashvardhan Nanavati Assisted-by: Cursor Signed-off-by: Yashvardhan Nanavati --- .env.containerized.template | 94 +++++++ docker/Dockerfile-workers | 2 +- docker/containerized/README.md | 280 ++++++++++++++++++++ docker/containerized/konflux-ca.crt.example | 3 + docker/containerized/worker_config.py | 122 +++++++++ podman-compose-containerized.yml | 160 +++++++++++ 6 files changed, 660 insertions(+), 1 deletion(-) create mode 100644 .env.containerized.template create mode 100644 docker/containerized/README.md create mode 100644 docker/containerized/konflux-ca.crt.example create mode 100644 docker/containerized/worker_config.py create mode 100644 podman-compose-containerized.yml diff --git a/.env.containerized.template b/.env.containerized.template new file mode 100644 index 000000000..2eb34a442 --- /dev/null +++ b/.env.containerized.template @@ -0,0 +1,94 @@ +# IIB Containerized Workflow Environment Configuration +# Copy this file to .env.containerized and fill in your values +# DO NOT commit .env.containerized to git (it's already in .gitignore) + +# =================================================================== +# Konflux Cluster Configuration +# =================================================================== +# These settings are required for the worker to connect to your Konflux dev cluster + +# Konflux cluster API URL (e.g., https://api.konflux-dev.example.com:6443) +IIB_KONFLUX_CLUSTER_URL= + +# Konflux cluster service account token +# To create a token: +# 1. Create a service account: kubectl create serviceaccount iib-worker -n +# 2. Create a role with permissions to read/list pipelineruns +# 3. Create a rolebinding to bind the role to the service account +# 4. Get the token: kubectl create token iib-worker -n --duration=720h +IIB_KONFLUX_CLUSTER_TOKEN= + +# Konflux cluster CA certificate path (relative to this file) +# This should point to the file mounted at /etc/iib/konflux-ca.crt +# You can get the CA cert with: kubectl config view --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d > docker/containerized/konflux-ca.crt +IIB_KONFLUX_CLUSTER_CA_CERT=/etc/iib/konflux-ca.crt + +# Namespace where Konflux pipelines run +IIB_KONFLUX_NAMESPACE= + +# Pipeline timeout in seconds (default: 1800 = 30 minutes) +IIB_KONFLUX_PIPELINE_TIMEOUT=1800 + +# =================================================================== +# GitLab Configuration +# =================================================================== +# Required for pushing commits and creating merge requests + +# GitLab tokens for different repositories +# Format: {"repo_url": {"token_name": "ENV_VAR_NAME", "token": "actual_token"}} +# Example: +# IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP='{"https://gitlab.example.com/catalogs/v4.19": {"token_name": "GITLAB_TOKEN_V419", "token": "glpat-xxxxxxxxxxxxx"}}' +IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP= + +# =================================================================== +# Registry Configuration +# =================================================================== +# Configuration for the IIB output registry + +# Registry where built images will be pushed +IIB_REGISTRY=registry:8443 + +# Template for pushing built images +# Available placeholders: {registry}, {request_id} +IIB_IMAGE_PUSH_TEMPLATE={registry}/iib-build:{request_id} + +# =================================================================== +# Index DB Artifact Configuration +# =================================================================== +# Configuration for index.db artifact storage + +# Registry for index.db artifacts (usually Quay.io) +IIB_INDEX_DB_ARTIFACT_REGISTRY= + +# Registry for index.db ImageStream cache +IIB_INDEX_DB_IMAGESTREAM_REGISTRY= + +# Template for index.db artifact storage +IIB_INDEX_DB_ARTIFACT_TEMPLATE={registry}/index-db:{tag} + +# =================================================================== +# Optional Configuration +# =================================================================== + +# AWS S3 bucket for storing artifacts (optional) +# IIB_AWS_S3_BUCKET_NAME= + +# Greenwave URL for gating (optional) +# IIB_GREENWAVE_URL= + +# Log level (DEBUG, INFO, WARNING, ERROR) +IIB_LOG_LEVEL=DEBUG + +# Request logs directory (inside container) +IIB_REQUEST_LOGS_DIR=/var/log/iib/requests + +# Skopeo timeout +IIB_SKOPEO_TIMEOUT=300s + +# Total retry attempts for operations +IIB_TOTAL_ATTEMPTS=5 + +# Retry configuration +IIB_RETRY_DELAY=10 +IIB_RETRY_JITTER=10 +IIB_RETRY_MULTIPLIER=5 diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index fccec097f..92d339de2 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -46,7 +46,7 @@ RUN curl -L "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4. RUN curl -L "https://github.com/oras-project/oras/releases/download/v1.3.0/oras_1.3.0_linux_amd64.tar.gz" -o /tmp/oras.tar.gz && \ tar -xvzf /tmp/oras.tar.gz -C /usr/bin/ && \ - rm /tmp/oc_client.tar.gz /usr/bin/LICENSE + rm /tmp/oras.tar.gz /usr/bin/LICENSE RUN git config --global user.email "exd-guild-hello-operator+iib-dev-env@redhat.com" RUN git config --global user.name "IIB dev-env" diff --git a/docker/containerized/README.md b/docker/containerized/README.md new file mode 100644 index 000000000..375a1465e --- /dev/null +++ b/docker/containerized/README.md @@ -0,0 +1,280 @@ +# IIB Containerized Workflow Development Environment + +This directory contains configuration for running IIB in containerized mode, where build operations are executed in an external Konflux cluster instead of locally in the worker. + +## Architecture Overview + +In the containerized workflow: + +1. **IIB Worker** receives a request (e.g., remove operators) +2. Worker clones the Git repository containing the catalog +3. Worker makes changes to the catalog locally +4. Worker commits and pushes changes to GitLab (either to a branch or creates an MR) +5. GitLab push triggers a **Konflux PipelineRun** in the external cluster +6. Worker monitors the PipelineRun status via Kubernetes API +7. When the PipelineRun completes, worker copies the built image from Konflux to IIB registry +8. Worker updates the index.db artifact and completes the request + +## Prerequisites + +1. **Konflux Cluster Access** + - A Konflux dev cluster with pipelines configured + - Service account with permissions to read/list PipelineRuns + - Cluster CA certificate + - Cluster API URL + +2. **GitLab Access** + - GitLab repositories for catalog storage + - GitLab access tokens with write permissions + +3. **Container Runtime** + - Podman installed and configured + - podman-compose installed + +## Setup Instructions + +### 1. Get Konflux Cluster Credentials + +#### Get the Cluster API URL + +```bash +kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' +``` + +#### Create a Service Account + +```bash +# Set your namespace +NAMESPACE="your-namespace" + +# Create service account +kubectl create serviceaccount iib-worker -n $NAMESPACE + +# Create role with PipelineRun permissions +cat < konflux-ca.crt +``` + +This will save the CA certificate to `konflux-ca.crt` in the current directory. + +### 2. Configure Environment Variables + +1. Copy the template file: + ```bash + cp .env.containerized.template .env.containerized + ``` + +2. Edit `.env.containerized` and fill in the required values: + + ```bash + # Konflux Cluster Configuration + IIB_KONFLUX_CLUSTER_URL=https://api.konflux-dev.example.com:6443 + IIB_KONFLUX_CLUSTER_TOKEN=eyJhbGc... # Token from above + IIB_KONFLUX_CLUSTER_CA_CERT=/etc/iib/konflux-ca.crt + IIB_KONFLUX_NAMESPACE=your-namespace + + # GitLab Configuration + IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP='{"https://gitlab.example.com/catalogs/v4.19": GITLAB_TOKEN_V419:glpat-xxxxxxxxxxxxx"}' + + # Registry Configuration + IIB_REGISTRY=registry:8443 + IIB_IMAGE_PUSH_TEMPLATE={registry}/iib-build:{request_id} + + # Index DB Artifact Configuration + IIB_INDEX_DB_ARTIFACT_REGISTRY=quay.io/your-org + IIB_INDEX_DB_IMAGESTREAM_REGISTRY=image-registry.openshift-image-registry.svc:5000 + ``` + +### 3. Place the Konflux CA Certificate + +Copy the CA certificate you downloaded to the correct location: + +```bash +cp konflux-ca.crt docker/containerized/konflux-ca.crt +``` + +### 4. Start the Development Environment + +```bash +# Start all services +podman-compose -f podman-compose-containerized.yml up -d + +# View logs +podman-compose -f podman-compose-containerized.yml logs -f iib-worker-containerized + +# Stop all services +podman-compose -f podman-compose-containerized.yml down +``` + +## Testing the Setup + +### 1. Verify Services are Running + +```bash +podman-compose -f podman-compose-containerized.yml ps +``` + +You should see: +- `iib-api` (running) +- `iib-worker-containerized` (running) +- `db` (running) +- `rabbitmq` (running) +- `registry` (running) +- `memcached` (running) +- `message-broker` (running) +- `minica` (exited 0) + +### 2. Check Worker Logs + +```bash +podman-compose -f podman-compose-containerized.yml logs iib-worker-containerized +``` + +Look for: +- "Configuring Kubernetes client for cross-cluster access to https://..." +- No errors about missing Konflux configuration + +### 3. Submit a Test Request + +```bash +# Using the IIB API +curl -X POST http://localhost:8080/api/v1/builds/rm \ + -H "Content-Type: application/json" \ + -d '{ + "from_index": "registry.example.com/catalog:v4.19", + "operators": ["test-operator"], + "index_to_gitlab_push_map": { + "registry.example.com/catalog:v4.19": "https://gitlab.example.com/catalogs/v4.19" + }, + "overwrite_from_index": false + }' +``` + +### 4. Monitor the Request + +Watch the worker logs to see: +1. Cloning the Git repository +2. Removing operators from the catalog +3. Committing and pushing to GitLab +4. Waiting for Konflux pipeline +5. Pipeline completion +6. Copying built image to IIB registry +7. Request completion + +## Troubleshooting + +### Worker Can't Connect to Konflux Cluster + +**Symptoms:** Error messages about Kubernetes client initialization + +**Solution:** +1. Verify the cluster URL is correct and accessible +2. Check that the token is valid (not expired) +3. Ensure the CA certificate is correct +4. Test connection manually: + ```bash + kubectl --server= --token= \ + --certificate-authority=docker/containerized/konflux-ca.crt \ + get pipelineruns -n + ``` + +### Permission Denied Errors + +**Symptoms:** Kubernetes API errors about permissions + +**Solution:** +1. Verify the service account has the correct role binding +2. Check that the role includes `get`, `list`, and `watch` verbs for `pipelineruns` +3. Ensure you're using the correct namespace + +### GitLab Authentication Errors + +**Symptoms:** Errors cloning or pushing to GitLab + +**Solution:** +1. Verify the GitLab token has correct permissions (read_repository, write_repository) +2. Check the token hasn't expired +3. Ensure the repository URL in `index_to_gitlab_push_map` is correct +4. Test the token manually: + ```bash + git clone https://oauth2:@gitlab.example.com/catalogs/v4.19.git + ``` + +### Pipeline Timeout + +**Symptoms:** "Timeout waiting for pipelinerun to complete" + +**Solution:** +1. Increase `IIB_KONFLUX_PIPELINE_TIMEOUT` in `.env.containerized` +2. Check the Konflux pipeline logs to see why it's taking long +3. Verify the pipeline isn't stuck or failing silently + +## Configuration Reference + +### Environment Variables + +All environment variables are documented in `.env.containerized.template`. + +### Worker Configuration + +The worker configuration is in `docker/containerized/worker_config.py`. This file: +- Reads environment variables from `.env.containerized` +- Extends the base `DevelopmentConfig` +- Includes the containerized task modules +- Validates required configuration on startup + +## Differences from Traditional Workflow + +| Aspect | Traditional Workflow | Containerized Workflow | +|--------|---------------------|------------------------| +| Build Location | Local in worker container | External Konflux cluster | +| Worker Privileges | Privileged (for building) | Unprivileged | +| Container Storage | Requires large volumes | Minimal storage needed | +| Git Operations | Optional | Required | +| External Dependencies | Local tools (buildah, podman) | Konflux cluster, GitLab | +| Scalability | Limited by worker resources | Limited by Konflux capacity | + +## Additional Resources + +- [IIB Documentation](../../docs/) +- [Konflux Documentation](https://konflux-ci.dev/) +- [GitLab API Documentation](https://docs.gitlab.com/ee/api/) +- [Tekton PipelineRuns](https://tekton.dev/docs/pipelines/pipelineruns/) + +## Contributing + +If you encounter issues or have improvements: + +1. Check existing issues and documentation +2. Test your changes locally +3. Submit a pull request with clear description diff --git a/docker/containerized/konflux-ca.crt.example b/docker/containerized/konflux-ca.crt.example new file mode 100644 index 000000000..c475c24ae --- /dev/null +++ b/docker/containerized/konflux-ca.crt.example @@ -0,0 +1,3 @@ +-----BEGIN CERTIFICATE----- +.... +-----END CERTIFICATE----- diff --git a/docker/containerized/worker_config.py b/docker/containerized/worker_config.py new file mode 100644 index 000000000..68db1ea68 --- /dev/null +++ b/docker/containerized/worker_config.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +""" +IIB Worker Configuration for Containerized Workflow. + +This configuration is used when running IIB in containerized mode where builds +are executed in an external Konflux cluster instead of locally in the worker. +""" +import json +import os +from typing import Optional + +from iib.workers.config import DevelopmentConfig + + +class ContainerizedConfig(DevelopmentConfig): + """Configuration for IIB worker in containerized mode.""" + + # =================================================================== + # Konflux Cluster Configuration + # =================================================================== + # These are read from environment variables set in .env.containerized + iib_konflux_cluster_url: Optional[str] = os.getenv('IIB_KONFLUX_CLUSTER_URL') + iib_konflux_cluster_token: Optional[str] = os.getenv('IIB_KONFLUX_CLUSTER_TOKEN') + iib_konflux_cluster_ca_cert: Optional[str] = os.getenv( + 'IIB_KONFLUX_CLUSTER_CA_CERT', '/etc/iib/konflux-ca.crt' + ) + iib_konflux_namespace: Optional[str] = os.getenv('IIB_KONFLUX_NAMESPACE') + iib_konflux_pipeline_timeout: int = int(os.getenv('IIB_KONFLUX_PIPELINE_TIMEOUT', '1800')) + + # =================================================================== + # GitLab Configuration + # =================================================================== + # Parse GitLab tokens from environment variable + _gitlab_tokens_str = os.getenv('IIB_INDEX_CONFIGS_GITLAB_TOKENS_MAP') + iib_index_configs_gitlab_tokens_map = ( + json.loads(_gitlab_tokens_str) if _gitlab_tokens_str else None + ) + + # =================================================================== + # Registry Configuration + # =================================================================== + iib_registry: str = os.getenv('IIB_REGISTRY', 'registry:8443') + iib_image_push_template: str = os.getenv( + 'IIB_IMAGE_PUSH_TEMPLATE', '{registry}/iib-build:{request_id}' + ) + # Docker config template for reset_docker_config() + # Points to the mounted auth config so symlink creation works correctly + iib_docker_config_template: str = '/etc/containers/auth.json' + + # =================================================================== + # Index DB Artifact Configuration + # =================================================================== + iib_index_db_artifact_registry: Optional[str] = os.getenv('IIB_INDEX_DB_ARTIFACT_REGISTRY') + iib_index_db_imagestream_registry: Optional[str] = os.getenv( + 'IIB_INDEX_DB_IMAGESTREAM_REGISTRY' + ) + iib_index_db_artifact_template: str = os.getenv( + 'IIB_INDEX_DB_ARTIFACT_TEMPLATE', '{registry}/index-db:{tag}' + ) + + # =================================================================== + # Task Routing Configuration + # =================================================================== + # Include containerized task modules + include = DevelopmentConfig.include + [ + 'iib.workers.tasks.build_containerized_rm', + ] + + # =================================================================== + # Logging Configuration + # =================================================================== + iib_log_level: str = os.getenv('IIB_LOG_LEVEL', 'DEBUG') + iib_request_logs_dir: Optional[str] = os.getenv( + 'IIB_REQUEST_LOGS_DIR', '/var/log/iib/requests' + ) + + # =================================================================== + # Optional Configuration + # =================================================================== + iib_aws_s3_bucket_name: Optional[str] = os.getenv('IIB_AWS_S3_BUCKET_NAME') + iib_greenwave_url: Optional[str] = os.getenv('IIB_GREENWAVE_URL') + iib_skopeo_timeout: str = os.getenv('IIB_SKOPEO_TIMEOUT', '300s') + iib_total_attempts: int = int(os.getenv('IIB_TOTAL_ATTEMPTS', '5')) + iib_retry_delay: int = int(os.getenv('IIB_RETRY_DELAY', '10')) + iib_retry_jitter: int = int(os.getenv('IIB_RETRY_JITTER', '10')) + iib_retry_multiplier: int = int(os.getenv('IIB_RETRY_MULTIPLIER', '5')) + + # =================================================================== + # Validation + # =================================================================== + @classmethod + def validate(cls): + """ + Validate that required configuration is present. + + :raises ValueError: If required configuration is missing + """ + required_configs = { + 'iib_konflux_cluster_url': cls.iib_konflux_cluster_url, + 'iib_konflux_cluster_token': cls.iib_konflux_cluster_token, + 'iib_konflux_cluster_ca_cert': cls.iib_konflux_cluster_ca_cert, + 'iib_konflux_namespace': cls.iib_konflux_namespace, + } + + missing = [name for name, value in required_configs.items() if not value] + + if missing: + raise ValueError( + f"Missing required Konflux configuration: {', '.join(missing)}. " + "Please set these in your .env.containerized file." + ) + + +# Validate configuration on import +ContainerizedConfig.validate() + +# Export config as module-level variables for Celery to pick up +# This is required because Celery's exec() loading expects module-level vars, not a class +_config = ContainerizedConfig() +for _attr in dir(_config): + if not _attr.startswith('_') and _attr not in globals(): + globals()[_attr] = getattr(_config, _attr) diff --git a/podman-compose-containerized.yml b/podman-compose-containerized.yml new file mode 100644 index 000000000..6aa029ccc --- /dev/null +++ b/podman-compose-containerized.yml @@ -0,0 +1,160 @@ +--- +version: '3' +services: + # This "service" generates the certificate for the registry. Then, + # it exits with status code 0. + minica: + image: registry.access.redhat.com/ubi8/go-toolset:latest + command: + - /bin/sh + - -c + - >- + go install github.com/jsha/minica@latest && + cd /opt/app-root/certs && + namei -l /opt/app-root && + /opt/app-root/src/bin/minica --domains registry + environment: + GOPATH: /opt/app-root/src + volumes: + - registry-certs-volume:/opt/app-root/certs:z + + registry: + image: registry:2 + ports: + - 8443:8443 + environment: + REGISTRY_HTTP_ADDR: 0.0.0.0:8443 + REGISTRY_HTTP_TLS_CERTIFICATE: /certs/registry/cert.pem + REGISTRY_HTTP_TLS_KEY: /certs/registry/key.pem + REGISTRY_AUTH: htpasswd + REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd + REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm + volumes: + - ./iib_data/registry:/var/lib/registry + - registry-certs-volume:/certs:z + - ./docker/registry/auth:/auth + + db: + image: postgres:9.6 + environment: + POSTGRES_USER: iib + POSTGRES_PASSWORD: iib + POSTGRES_DB: iib + POSTGRES_INITDB_ARGS: "--auth='ident' --auth='trust'" + + memcached: + image: memcached + ports: + - 11211:11211 + + rabbitmq: + image: rabbitmq:3.7-management + environment: + RABBITMQ_DEFAULT_USER: iib + RABBITMQ_DEFAULT_PASS: iib + # Avoid port conflict with ActiveMQ broker when using podman-compose. + # Even though the port is not exposed, podman-compose's use of a pod + # requires the ports to be unique across all containers within the pod. + RABBITMQ_NODE_PORT: 5673 + ports: + # The RabbitMQ management console + - 8081:15672 + + iib-api: + build: + context: . + dockerfile: ./docker/Dockerfile-api + command: + - /bin/sh + - -c + - >- + mkdir -p /etc/iib && + pip3 uninstall -y iib && + python3 setup.py develop --no-deps && + iib wait-for-db && + iib db upgrade && + flask run --reload --host 0.0.0.0 --port 8080 + environment: + FLASK_ENV: development + FLASK_APP: iib/web/wsgi.py + REQUESTS_CA_BUNDLE: /etc/pki/tls/certs/ca-bundle.crt + IIB_DEV: 'true' + volumes: + - ./:/src + - ./docker/message_broker/certs:/broker-certs + - request-logs-volume:/var/log/iib/requests:z + - request-related-bundles-volume:/var/lib/requests/related_bundles:z + - request-recursive-related-bundles-volume:/var/lib/requests/recursive_related_bundles:z + ports: + - 8080:8080 + depends_on: + - db + - message-broker + + # IIB Worker for containerized workflow (connects to external Konflux cluster) + iib-worker-containerized: + build: + context: . + dockerfile: ./docker/Dockerfile-workers + command: > + bash -c " + mkdir -p /root/.docker && + ln -sf /etc/containers/auth.json /root/.docker/config.json && + echo 'Created symlink for docker config' && + exec celery -A iib.workers.tasks worker --loglevel=info + " + environment: + IIB_DEV: 'false' + IIB_CELERY_CONFIG: /etc/iib/settings.py + REQUESTS_CA_BUNDLE: /etc/pki/tls/certs/ca-chain.crt + GIT_SSL_CAINFO: /etc/pki/tls/certs/ca-chain.crt + env_file: + # This file contains Konflux cluster credentials and other sensitive configuration + - .env.containerized + # Enable privileged mode for podman-in-podman support + privileged: true + security_opt: + - seccomp=unconfined + - label=disable + cap_add: + - SYS_ADMIN + - MKNOD + volumes: + - ./:/src + - registry-certs-volume:/registry-certs + - request-logs-volume:/var/log/iib/requests:z + - request-related-bundles-volume:/var/lib/requests/related_bundles:z + - request-recursive-related-bundles-volume:/var/lib/requests/recursive_related_bundles:z + # Mount custom worker configuration + - ./docker/containerized/worker_config.py:/etc/iib/settings.py:z + # Mount Docker auth config for registry authentication (in a location IIB won't try to delete) + - ./docker/config.json:/etc/containers/auth.json:ro + # Mount local registry CA certificate for podman (not system-wide to avoid interfering with Git) + - registry-certs-volume:/tmp/registry-certs:ro + # Mount Konflux CA chain for GitLab SSL verification + - ./docker/containerized/konflux-ca.crt:/tmp/host-ca-chain.crt:ro + depends_on: + - rabbitmq + - registry + - minica + - memcached + + # This is an external message broker used to publish messages about state changes + message-broker: + build: + context: . + dockerfile: ./docker/message_broker/Dockerfile + volumes: + - message-broker-volume:/opt/activemq/data:z + - ./docker/message_broker/certs:/broker-certs + ports: + - 5671:5671 # amqp+ssl + - 5672:5672 # amqp + - 8161:8161 # web console + +volumes: + registry-certs-volume: + message-broker-volume: + request-logs-volume: + request-recursive-related-bundles-volume: + request-related-bundles-volume: