Arch: refactor and add unit tests for EventStreamRuntime docker image build (All-Hands-AI#2908)

* deprecating recall action

* fix integration tests

* fix integration tests

* refractor runtime to use async

* remove search memory

* rename .initialize to .ainit

* draft of runtime image building (separate from img agnostic)

* refractor runtime build into separate file and add unit tests for it

* fix image agnostic tests

* Update opendevin/runtime/utils/runtime_build.py

Co-authored-by: Mingzhang Zheng <649940882@qq.com>

---------

Co-authored-by: Mingzhang Zheng <649940882@qq.com>

Author: xingyaoww

opendevin/runtime/client/runtime.py

@@ -33,7 +33,7 @@
 )
 from opendevin.runtime.runtime import Runtime
 from opendevin.runtime.utils import find_available_tcp_port
-from opendevin.runtime.utils.image_agnostic import get_od_sandbox_image
+from opendevin.runtime.utils.runtime_build import build_runtime_image
 
 
 class EventStreamRuntime(Runtime):
@@ -72,8 +72,13 @@ def __init__(
         self.action_semaphore = asyncio.Semaphore(1)  # Ensure one action at a time
 
     async def ainit(self):
-        self.container_image = get_od_sandbox_image(
-            self.container_image, self.docker_client, is_eventstream_runtime=True
+        self.container_image = build_runtime_image(
+            self.container_image,
+            self.docker_client,
+            # NOTE: You can need set DEBUG=true to update the source code
+            # inside the container. This is useful when you want to test/debug the
+            # latest code in the runtime docker container.
+            update_source_code=config.debug,
         )
         self.container = await self._init_container(
             self.sandbox_workspace_dir,

opendevin/runtime/utils/image_agnostic.py

@@ -1,13 +1,15 @@
-import os
-import shutil
+"""
+This module contains functions for building and managing the agnostic sandbox image.
+
+This WILL BE DEPRECATED when EventStreamRuntime is fully implemented and adopted.
+"""
+
 import tempfile
 
 import docker
 
 from opendevin.core.logger import opendevin_logger as logger
 
-from .source import create_project_source_dist
-
 
 def generate_dockerfile(base_image: str) -> str:
     """
@@ -36,122 +38,29 @@ def generate_dockerfile(base_image: str) -> str:
     return dockerfile_content
 
 
-def generate_dockerfile_for_eventstream_runtime(
-    base_image: str, temp_dir: str, skip_init: bool = False
-) -> str:
-    """
-    Generate the Dockerfile content for the eventstream runtime image based on user-provided base image.
-
-    NOTE: This is only tested on debian yet.
-    """
-    if skip_init:
-        dockerfile_content = f'FROM {base_image}\n'
-    else:
-        dockerfile_content = (
-            f'FROM {base_image}\n'
-            # FIXME: make this more generic / cross-platform
-            'RUN apt update && apt install -y wget sudo\n'
-            'RUN apt-get update && apt-get install -y libgl1-mesa-glx\n'  # Extra dependency for OpenCV
-            'RUN mkdir -p /opendevin && mkdir -p /opendevin/logs && chmod 777 /opendevin/logs\n'
-            'RUN echo "" > /opendevin/bash.bashrc\n'
-            'RUN if [ ! -d /opendevin/miniforge3 ]; then \\\n'
-            '        wget --progress=bar:force -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \\\n'
-            '        bash Miniforge3.sh -b -p /opendevin/miniforge3 && \\\n'
-            '        rm Miniforge3.sh && \\\n'
-            '        chmod -R g+w /opendevin/miniforge3 && \\\n'
-            '        bash -c ". /opendevin/miniforge3/etc/profile.d/conda.sh && conda config --set changeps1 False && conda config --append channels conda-forge"; \\\n'
-            '    fi\n'
-            'RUN /opendevin/miniforge3/bin/mamba install python=3.11 -y\n'
-            'RUN /opendevin/miniforge3/bin/mamba install conda-forge::poetry -y\n'
-        )
-
-    tarball_path = create_project_source_dist()
-    filename = os.path.basename(tarball_path)
-    filename = filename.removesuffix('.tar.gz')
-
-    # move the tarball to temp_dir
-    _res = shutil.copy(tarball_path, os.path.join(temp_dir, 'project.tar.gz'))
-    if _res:
-        os.remove(tarball_path)
-    logger.info(
-        f'Source distribution moved to {os.path.join(temp_dir, "project.tar.gz")}'
-    )
-
-    # Copy the project directory to the container
-    dockerfile_content += 'COPY project.tar.gz /opendevin\n'
-    # remove /opendevin/code if it exists
-    dockerfile_content += (
-        'RUN if [ -d /opendevin/code ]; then rm -rf /opendevin/code; fi\n'
-    )
-    # unzip the tarball to /opendevin/code
-    dockerfile_content += (
-        'RUN cd /opendevin && tar -xzvf project.tar.gz && rm project.tar.gz\n'
-    )
-    dockerfile_content += f'RUN mv /opendevin/{filename} /opendevin/code\n'
-    # install (or update) the dependencies
-    dockerfile_content += (
-        'RUN cd /opendevin/code && '
-        '/opendevin/miniforge3/bin/mamba run -n base poetry env use python3.11 && '
-        '/opendevin/miniforge3/bin/mamba run -n base poetry install\n'
-        # for browser (update if needed)
-        'RUN apt-get update && cd /opendevin/code && /opendevin/miniforge3/bin/mamba run -n base poetry run playwright install --with-deps chromium\n'
-    )
-    return dockerfile_content
-
-
 def _build_sandbox_image(
-    base_image: str,
-    target_image_name: str,
-    docker_client: docker.DockerClient,
-    eventstream_runtime: bool = False,
-    skip_init: bool = False,
+    base_image: str, target_image_name: str, docker_client: docker.DockerClient
 ):
     try:
         with tempfile.TemporaryDirectory() as temp_dir:
-            if eventstream_runtime:
-                dockerfile_content = generate_dockerfile_for_eventstream_runtime(
-                    base_image, temp_dir, skip_init=skip_init
-                )
-            else:
-                dockerfile_content = generate_dockerfile(base_image)
+            dockerfile_content = generate_dockerfile(base_image)
 
-            if skip_init:
-                logger.info(
-                    f'Reusing existing od_sandbox image [{target_image_name}] but will update the source code in it.'
-                )
-                logger.info(
-                    (
-                        f'===== Dockerfile content =====\n'
-                        f'{dockerfile_content}\n'
-                        f'==============================='
-                    )
-                )
-            else:
-                logger.info(f'Building agnostic sandbox image: {target_image_name}')
-                logger.info(
-                    (
-                        f'===== Dockerfile content =====\n'
-                        f'{dockerfile_content}\n'
-                        f'==============================='
-                    )
+            logger.info(f'Building agnostic sandbox image: {target_image_name}')
+            logger.info(
+                (
+                    f'===== Dockerfile content =====\n'
+                    f'{dockerfile_content}\n'
+                    f'==============================='
                 )
+            )
             with open(f'{temp_dir}/Dockerfile', 'w') as file:
                 file.write(dockerfile_content)
 
             api_client = docker_client.api
             build_logs = api_client.build(
-                path=temp_dir,
-                tag=target_image_name,
-                rm=True,
-                decode=True,
-                # do not use cache when skip_init is True (i.e., when we want to update the source code in the existing image)
-                nocache=skip_init,
+                path=temp_dir, tag=target_image_name, rm=True, decode=True
             )
 
-            if skip_init:
-                logger.info(
-                    f'Rebuilding existing od_sandbox image [{target_image_name}] to update the source code.'
-                )
             for log in build_logs:
                 if 'stream' in log:
                     print(log['stream'].strip())
@@ -169,14 +78,8 @@ def _build_sandbox_image(
         raise e
 
 
-def _get_new_image_name(
-    base_image: str, is_eventstream_runtime: bool, dev_mode: bool = False
-) -> str:
+def _get_new_image_name(base_image: str) -> str:
     prefix = 'od_sandbox'
-    if is_eventstream_runtime:
-        prefix = 'od_eventstream_runtime'
-    if dev_mode:
-        prefix += '_dev'
     if ':' not in base_image:
         base_image = base_image + ':latest'
 
@@ -185,11 +88,7 @@ def _get_new_image_name(
     return f'{prefix}:{repo}__{tag}'
 
 
-def get_od_sandbox_image(
-    base_image: str,
-    docker_client: docker.DockerClient,
-    is_eventstream_runtime: bool = False,
-) -> str:
+def get_od_sandbox_image(base_image: str, docker_client: docker.DockerClient) -> str:
     """Return the sandbox image name based on user-provided base image.
 
     The returned sandbox image is assumed to contains all the required dependencies for OpenDevin.
@@ -199,52 +98,18 @@ def get_od_sandbox_image(
     if 'ghcr.io/opendevin/sandbox' in base_image:
         return base_image
 
-    new_image_name = _get_new_image_name(base_image, is_eventstream_runtime)
+    new_image_name = _get_new_image_name(base_image)
 
     # Detect if the sandbox image is built
-    image_exists = False
     images = docker_client.images.list()
     for image in images:
         if new_image_name in image.tags:
             logger.info('Found existing od_sandbox image, reuse:' + new_image_name)
-            image_exists = True
-            break
-
-    skip_init = False
-    if image_exists:
-        if is_eventstream_runtime:
-            # An eventstream runtime image is already built for the base image (with poetry and dev dependencies)
-            # but it might not contain the latest version of the source code and dependencies.
-            # So we need to build a new (dev) image with the latest source code and dependencies.
-            # FIXME: In production, we should just build once (since the source code will not change)
-            base_image = new_image_name
-            new_image_name = _get_new_image_name(
-                base_image, is_eventstream_runtime, dev_mode=True
-            )
-
-            # Delete the existing image named `new_image_name` if any
-            images = docker_client.images.list()
-            for image in images:
-                if new_image_name in image.tags:
-                    docker_client.images.remove(image.id, force=True)
-
-            # We will reuse the existing image but will update the source code in it.
-            skip_init = True
-            logger.info(
-                f'Reusing existing od_sandbox image [{base_image}] but will update the source code into [{new_image_name}]'
-            )
-        else:
             return new_image_name
-    else:
-        # If the sandbox image is not found, build it
-        logger.info(
-            f'od_sandbox image is not found for {base_image}, will build: {new_image_name}'
-        )
-    _build_sandbox_image(
-        base_image,
-        new_image_name,
-        docker_client,
-        is_eventstream_runtime,
-        skip_init=skip_init,
+
+    # If the sandbox image is not found, build it
+    logger.info(
+        f'od_sandbox image is not found for {base_image}, will build: {new_image_name}'
     )
+    _build_sandbox_image(base_image, new_image_name, docker_client)
     return new_image_name