[RAPTOR-11834] DR base python image is derived from an official Pytho…

…n image (#1229)

* [RAPTOR-11834] DR base python image is derived from an official Python image + harness cleanups

* Update docker/dropin_env_base/build_image.sh

Co-authored-by: Alex Shoop <alexander.shoop@datarobot.com>

* Small fix to README

---------

Co-authored-by: Alex Shoop <alexander.shoop@datarobot.com>

docker/dropin_env_base/Dockerfile

@@ -1,21 +1,10 @@
 # This is the default base image for use with user models and workflows.
-FROM openjdk:11.0.16-jre-slim-bullseye
+ARG BASE_ROOT_IMAGE=python:3.11.11-slim-bookworm
 
-ARG DATAROBOT_MLOPS_VERSION
+FROM ${BASE_ROOT_IMAGE}
 
-RUN apt-get update && apt-get upgrade -y && apt-get install -y python3 python3-pip python3-dev python3-distutils fontconfig && \
-    pip3 install --upgrade pip && \
-    pip3 install --no-cache-dir wheel setuptools && \
-    apt-get install -y python3-setuptools
-
-COPY requirements.txt requirements.txt
-COPY datarobot-mlops-*.jar /opt/jars/
-COPY mlops-agent-*.jar /opt/jars/
-
-ENV DRUM_JAVA_SHARED_JARS=/opt/jars/*
-ENV MLOPS_MONITORING_AGENT_JAR_PATH=/opt/jars/mlops-agent-${DATAROBOT_MLOPS_VERSION}.jar
+# Upgrade setuptools to a secure version (CVE-2024-6345)
+RUN pip install --no-cache-dir --upgrade pip setuptools>=70.0.0
 
 # This makes print statements show up in the logs API
 ENV PYTHONUNBUFFERED=1
-
-RUN pip3 install -U pip && pip3 install -r requirements.txt

docker/dropin_env_base/README.md

@@ -1,17 +1,15 @@
-# Python/Java(JRE) drop-in environments base image
+# Python drop-in environments base image
 Repository name: **datarobot/dropin-env-base**
-Latest date: 12-01-2023
+Latest date: January 22, 2025
 Dockerfile: https://github.com/datarobot/datarobot-user-models/blob/master/docker/dropin_env_base/Dockerfile
 
 ## Description
-This image is used as a base for Python and Java drop in environments.
-Based on openjdk:11.0.16-jre-slim-bullseye
+This image is used as a base for Python drop in environments.
+It is based on python:3.11-slim-bookworm, with some vulnerability fixes.
+
 Contains
-* Debian 11
-* JRE 11
-* Python 3.9
-* DRUM 1.10.14
-* datarobot-mlops 9.2.8
+* Debian 12.9
+* Python 3.11
 
 ## Guidelines
 DataRobot guidelines for publishing images to Docker Hub

docker/dropin_env_base/build_image.sh

@@ -6,8 +6,15 @@
 #
 # Released under the terms of DataRobot Tool and Utility Agreement.
 
+DATAROBOT_BASE_IMAGE_VERSION=1.0.0
+
 while [[ $# -gt 0 ]]; do
   case $1 in
+    --version)
+      shift
+      DATAROBOT_BASE_IMAGE_VERSION=$1
+      shift # past argument
+      ;;
     --push)
       PUSH=1
       shift # past argument
@@ -27,21 +34,19 @@ IMAGE_ORG_DATAROBOTDEV=datarobotdev
 
 IMAGE_REPO=dropin-env-base
 
-IMAGE_TAG=debian11-py3.9-jre11.0.16-drum1.13.0-mlops9.2.8
+BASE_ROOT_IMAGE_TAG=$(grep -E '^ARG[[:space:]]+BASE_ROOT_IMAGE=' Dockerfile | sed -E 's/^ARG[[:space:]]+BASE_ROOT_IMAGE=//')
+BASE_ROOT_IMAGE_TAG="${BASE_ROOT_IMAGE_TAG//:/-}"
+IMAGE_TAG=${DATAROBOT_BASE_IMAGE_VERSION}-${BASE_ROOT_IMAG_TAG}
 
 IMAGE_NAME_DATAROBOT=${IMAGE_ORG_DATAROBOT}/${IMAGE_REPO}:${IMAGE_TAG}
 IMAGE_NAME_DATAROBOTDEV=${IMAGE_ORG_DATAROBOTDEV}/${IMAGE_REPO}:${IMAGE_TAG}
 
 pwd
 
-echo "Building docker image: ${IMAGE_NAME_DATAROBOTDEV}:${IMAGE_TAG}"
-export DATAROBOT_MLOPS_VERSION=9.2.8
-${SCRIPT_DIR}/pull_artifacts.sh
-
-
+echo "Building docker image: ${IMAGE_NAME_DATAROBOTDEV}"
 
 # Build and save in the local registry. (In the harness pipeline we run trivy on it)
-docker build --no-cache --build-arg DATAROBOT_MLOPS_VERSION=${DATAROBOT_MLOPS_VERSION} -t ${IMAGE_NAME_DATAROBOTDEV} -t ${IMAGE_NAME_DATAROBOT} .
+docker build --no-cache -t ${IMAGE_NAME_DATAROBOTDEV} -t ${IMAGE_NAME_DATAROBOT} .
 
 # this is command to build images for specified platforms.
 # For more info: https://docs.docker.com/build/building/multi-platform/

harness_scripts/common/common.sh

@@ -0,0 +1,4 @@
+
+function title {
+    echo -e "\n== $1 =="
+}

harness_scripts/functional_by_framework/check_by_framework_run1_step_entrypoint.sh

@@ -1,40 +1,53 @@
 #!/bin/bash
 
+set -e
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. ${script_dir}/../common/common.sh
+
 FRAMEWORK=$1
 [ -z $FRAMEWORK ] && echo "Environment variable 'FRAMEWORK' is missing" && exit 1
 
-echo "== Assuming running integration tests in framework container (inside Docker), for env: '$FRAMEWORK' =="
+title "Assuming running integration tests in framework container (inside Docker), for env: '$FRAMEWORK'"
+
 ROOT_DIR="$(pwd)"
+PUBLIC_ENVS_DIR="${ROOT_DIR}/public_dropin_environments"
+REQ_FILE_PATH="${PUBLIC_ENVS_DIR}/${FRAMEWORK}/requirements.txt"
+[ ! -f $REQ_FILE_PATH ] && echo "Requirements file not found: $REQ_FILE_PATH" && exit 1
 
 script_dir=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
 . ${script_dir}/../../tools/update-python-to-meet-requirements.sh
 
-echo "== Installing pytest =="
+title "Installing pytest"
 pip install pytest pytest-xdist
-echo "== Uninstalling datarobot-drum =="
+
+title "Uninstalling datarobot-drum"
 pip uninstall datarobot-drum datarobot-mlops -y
-cd custom_model_runner
-echo "== Install datarobot-drum from source =="
+
+title "Installing datarobot-drum from source"
+
+pushd custom_model_runner
+
 if [ "$FRAMEWORK" = "java_codegen" ]; then
     make java_components
 fi
 
-PUBLIC_ENVS_DIR="${ROOT_DIR}/public_dropin_environments"
-REQ_FILE_PATH="${PUBLIC_ENVS_DIR}/${FRAMEWORK}/requirements.txt"
-
 # remove DRUM from requirements file to be able to install it from source
 sed -i "s/^datarobot-drum.*//" ${REQ_FILE_PATH}
 
 [ "$FRAMEWORK" = "r_lang" ] && EXTRA="[R]" || EXTRA=""
-pip install --upgrade --force-reinstall -r ${REQ_FILE_PATH} .$EXTRA
+pip install --force-reinstall -r ${REQ_FILE_PATH} .$EXTRA
+rm -rf build datarobot_drum.egg-info dist
+
+popd  # custom_model_runner
 
-cd -
 TESTS_TO_RUN="tests/functional/test_inference_per_framework.py \
               tests/functional/test_fit_per_framework.py \
               tests/functional/test_other_cases_per_framework.py \
               tests/functional/test_unstructured_mode_per_framework.py
              "
 
+title "Start testing"
 if [ "$FRAMEWORK" = "r_lang" ]; then
     Rscript -e "install.packages('pack', Ncpus=4)"
     TESTS_TO_RUN+="tests/integration/datarobot_drum/drum/language_predictors/test_language_predictors.py::TestRPredictor \
@@ -43,4 +56,4 @@ if [ "$FRAMEWORK" = "r_lang" ]; then
                   "
 fi
 
-pytest ${TESTS_TO_RUN} --framework-env $FRAMEWORK
+pytest ${TESTS_TO_RUN} --framework-env $FRAMEWORK

harness_scripts/functional_general/custom_java_predictor_entrypoint.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 
+set -e
+
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. ${script_dir}/../common/common.sh
 . ${script_dir}/../../tools/create-and-source-venv.sh
 
 pushd ./tests/functional/custom_java_predictor
@@ -16,7 +19,4 @@ popd
 echo "Installing requirements for all the tests: requirements_test.txt"
 pip install -r requirements_test.txt
 
-echo "Installing datarobot_drum from source code"
-pip3 install -e custom_model_runner/
-
 pytest tests/functional/test_inference_custom_java_predictor.py

harness_scripts/functional_general/general_tests_entrypoint.sh

@@ -1,13 +1,16 @@
 #!/usr/bin/env bash
 
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. ${script_dir}/../common/common.sh
+
 DOCKER_HUB_SECRET=$1
 if [ -n "$HARNESS_BUILD_ID" ]; then
-  echo "Running within a Harness pipeline."
+  title "Running within a Harness pipeline."
   [ -z $DOCKER_HUB_SECRET ] && echo "Docker HUB secret is expected as an input argument" && exit 1
   docker login -u datarobotread2 -p $DOCKER_HUB_SECRET || { echo "Docker login failed"; exit 1; }
 fi
 
-echo "== Build image for tests =="
+title "Build image for tests"
 tmp_py3_sklearn_env_dir=$(mktemp -d)
 cp -r public_dropin_environments/python3_sklearn/* $tmp_py3_sklearn_env_dir
 cp -r custom_model_runner/ $tmp_py3_sklearn_env_dir
@@ -25,17 +28,17 @@ docker build -t $image_name .
 popd
 
 docker images
-echo "== Image build succeeded: '$image_name' =="
+echo "Image build succeeded: '$image_name'"
 
-echo "== Preparing to test =="
+title "Preparing to test"
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 . ${script_dir}/../../tools/create-and-source-venv.sh
 
-echo "== Installing requirements for all the tests: requirements_test.txt =="
+title "Installing requirements for all the tests: requirements_test.txt"
 pip install -r requirements_test.txt
 
 pushd custom_model_runner
-echo "== Install drum from source =="
+title "Install drum from source"
 pip install .
 popd
 

harness_scripts/functional_general/mlops_reporting_entrypoint.sh

@@ -1,17 +1,18 @@
 #!/usr/bin/env bash
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. ${script_dir}/../common/common.sh
 . ${script_dir}/../../tools/create-and-source-venv.sh
 
-echo "== Preparing to test =="
+title "Preparing to test"
 apt-get update && apt-get install -y curl
 
-echo "Installing requirements for all the tests: requirements_test.txt"
+title "Installing requirements for all the tests: requirements_test.txt"
 pip install -r requirements_test.txt
 
 pushd custom_model_runner
-echo "== Install drum from source =="
+title "Install drum from source"
 pip install .
 popd
 
-pytest tests/functional/test_mlops_monitoring.py
+pytest tests/functional/test_mlops_monitoring.py

public_dropin_environments/python3_sklearn/Dockerfile

@@ -1,6 +1,7 @@
 # This is the default base image for use with user models and workflows.
 # It contains a variety of common useful data-science packages and tools.
 FROM datarobot/dropin-env-base:debian11-py3.9-jre11.0.16-drum1.13.0-mlops9.2.8
+
 # Install the list of core requirements, e.g. sklearn, numpy, pandas, flask.
 # **Don't modify this file!**
 COPY requirements.txt requirements.txt

tools/common.sh

@@ -0,0 +1,4 @@
+
+function title {
+    echo -e "\n\n== $1 =="
+}