feat: allow ilab model download to pull from OCI registries

Check if repository supplied during ilab model download meets the URL
structure of an OCI registry. If so, leverage skopeo to copy image layers
into cache, and apply mapping logic to move the model files into the
permanent models directory.

This PR also refactors existing ilab model download code to introduce base
and implementation classes to support multiple downloading backends
(HF vs OCI). Assumes skopeo is installed on the system and uses OCI v1.1

Signed-off-by: Jaideep Rao <jrao@redhat.com>

.github/workflows/test.yml

@@ -29,6 +29,7 @@ on:
 
 env:
   LC_ALL: en_US.UTF-8
+  REGISTRY_AUTH_FILE: /run/containers/1001/auth.json
 
 defaults:
   run:
@@ -67,15 +68,15 @@ jobs:
         if: matrix.platform != 'macos-latest'
         uses: ./.github/actions/free-disk-space
 
-      - name: Install the expect package
+      - name: Install the expect and skopeo package
         if: startsWith(matrix.platform, 'ubuntu')
         run: |
-          sudo apt-get install -y expect
+          sudo apt-get install -y expect skopeo
 
       - name: Install tools on MacOS
         if: startsWith(matrix.platform, 'macos')
         run: |
-          brew install expect coreutils bash
+          brew install expect coreutils bash skopeo
 
       - name: Setup Python ${{ matrix.python }}
         uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1

scripts/functional-tests.sh

@@ -161,6 +161,31 @@ fi
 # download the latest version of the ilab
 ilab model download
 
+test_oci_model_download_with_vllm_backend(){
+   # Enable globstar for recursive globbing
+    shopt -s globstar
+
+    # Run the ilab model download command with REGISTRY_AUTH_FILE
+    REGISTRY_AUTH_FILE=$HOME/auth.json ilab model download --repository docker://quay.io/ai-lab/models/granite-7b-lab --release latest --model-dir models/instructlab
+
+    patterns=(
+        "models/instructlab/config.json"
+        "models/instructlab/tokenizer.json"
+        "models/instructlab/tokenizer_config.json"
+        "models/instructlab/*.safetensors"
+    )
+
+    for pattern in "${patterns[@]}"
+    do
+        matching_files=("$pattern")
+        if [ ${#matching_files[@]} -eq 0 ]
+        then
+            echo "No files found matching pattern: $pattern"
+            exit 1
+        fi
+    done
+}
+
 # check that ilab model serve is working
 test_bind_port(){
     local formatted_script
@@ -546,6 +571,8 @@ test_server_chat_template() {
 # MAIN #
 ########
 # call cleanup in-between each test so they can run without conflicting with the server/chat process
+test_oci_model_download_with_vllm_backend
+cleanup
 test_bind_port
 cleanup
 test_ctx_size

src/instructlab/configuration.py

@@ -43,6 +43,7 @@ class STORAGE_DIR_NAMES:
     ILAB = "instructlab"
     DATASETS = "datasets"
     CHECKPOINTS = "checkpoints"
+    OCI = "oci"
     MODELS = "models"
     TAXONOMY = "taxonomy"
     INTERNAL = (
@@ -98,6 +99,10 @@ def _reset(self):
     def CHECKPOINTS_DIR(self) -> str:
         return path.join(self._data_dir, STORAGE_DIR_NAMES.CHECKPOINTS)
 
+    @property
+    def OCI_DIR(self) -> str:
+        return path.join(self._cache_home, STORAGE_DIR_NAMES.OCI)
+
     @property
     def DATASETS_DIR(self) -> str:
         return path.join(self._data_dir, STORAGE_DIR_NAMES.DATASETS)
@@ -545,6 +550,7 @@ def ensure_storage_directories_exist():
         DEFAULTS._data_dir,
         DEFAULTS.CHATLOGS_DIR,
         DEFAULTS.CHECKPOINTS_DIR,
+        DEFAULTS.OCI_DIR,
         DEFAULTS.DATASETS_DIR,
         DEFAULTS.EVAL_DATA_DIR,
         DEFAULTS.INTERNAL_DIR,