Merge pull request #378 from backend-developers-ltd/hf-transfer

Enable huggingface hf_transfer
backend-developers-ltd · Jan 24, 2025 · 1b9fc5f · 1b9fc5f
2 parents 1cb2235 + 1b440ee
commit 1b9fc5f
Show file tree

Hide file tree

Showing 14 changed files with 117 additions and 89 deletions.
diff --git a/.github/workflows/executor_ci.yml b/.github/workflows/executor_ci.yml
@@ -25,9 +25,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox
@@ -50,9 +50,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox
@@ -76,9 +76,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox

diff --git a/.github/workflows/integration_ci.yml b/.github/workflows/integration_ci.yml
@@ -23,9 +23,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox

diff --git a/.github/workflows/library_ci.yml b/.github/workflows/library_ci.yml
@@ -25,9 +25,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox
@@ -50,9 +50,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox
@@ -74,9 +74,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox

diff --git a/.github/workflows/miner_ci.yml b/.github/workflows/miner_ci.yml
@@ -25,9 +25,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox
@@ -50,9 +50,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox
@@ -76,9 +76,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox

diff --git a/.github/workflows/validator_ci.yml b/.github/workflows/validator_ci.yml
@@ -25,9 +25,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox
@@ -50,9 +50,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox
@@ -76,9 +76,9 @@ jobs:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
           cache: "pip"
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.5.3"
+          version: "latest"
           enable-cache: true
       - name: Install dependencies
         run: python -m pip install --upgrade nox

diff --git a/executor/app/envs/prod/Dockerfile b/executor/app/envs/prod/Dockerfile
@@ -41,6 +41,7 @@ RUN apt-get update \
 WORKDIR /root/src/
 ENV PYTHONUNBUFFERED=1
 ENV PATH="/root/src/.venv/bin:$PATH"
+ENV HF_HUB_ENABLE_HF_TRANSFER=1
 
 COPY --from=base-image /root/src/ /root/src/
 

diff --git a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py
@@ -3,6 +3,7 @@
 import csv
 import io
 import logging
+import os
 import pathlib
 import random
 import re
@@ -389,6 +390,9 @@ def download_from_huggingface(
         self, relative_path: pathlib.Path, repo_id: str, revision: str | None
     ):
         try:
+            # enable hf_transfer download acceleration package
+            # https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubenablehftransfer
+            os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
             snapshot_download(
                 repo_id=repo_id,
                 revision=revision,

diff --git a/executor/pyproject.toml b/executor/pyproject.toml
@@ -25,7 +25,7 @@ dependencies = [
     "prometheus-client~=0.17.0",
     "django-prometheus==2.3.1",
     "django-business-metrics @ git+https://github.com/reef-technologies/django-business-metrics.git@9d08ddb3a9d26e8a7e478110d7c8c34c3aa03a01",
-    "huggingface-hub>=0.26.1",
+    "huggingface-hub[hf_transfer]>=0.26.1",
 ]
 
 [tool.uv.sources]

diff --git a/executor/uv.lock b/executor/uv.lock
diff --git a/miner/pyproject.toml b/miner/pyproject.toml
@@ -24,7 +24,7 @@ dependencies = [
     "prometheus-client~=0.17.0",
     "django-prometheus==2.3.1",
     "django-business-metrics @ git+https://github.com/reef-technologies/django-business-metrics.git@9d08ddb3a9d26e8a7e478110d7c8c34c3aa03a01",
-    "django-constance[database]==3.1.0",
+    "django-constance==3.1.0",
 ]
 
 [tool.uv.sources]