Skip to content

Commit

Permalink
add huggingface mirrors
Browse files Browse the repository at this point in the history
  • Loading branch information
sidecus committed Jul 22, 2024
1 parent 215c0ac commit d8a5218
Show file tree
Hide file tree
Showing 5 changed files with 56 additions and 11 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ Automatically setup package manager mirrors, including ubuntu, pypi and apk etc.
"ubuntu_mirror": "<your-ubuntu-mirror-domain>", // only applicable to Ubuntu based images
"apk_mirror": "<your-alpine-apk-mirror-domain>", // only applicable to Alpine based images
"pypi_mirror": "https://<your-pypi-mirror-domain>/pypi/web/simple/",
"huggingface_mirror": "https://dummy-hf-mirror.com"
}
}
}
Expand Down
8 changes: 8 additions & 0 deletions src/magicmirror/devcontainer-feature.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,14 @@
],
"default": "",
"description": "Alpine apk mirror"
},
"huggingface_mirror": {
"type": "string",
"proposals": [
"https://hf-mirror.com"
],
"default": "",
"description": "Huggingface mirror"
}
},
"installsAfter": [
Expand Down
27 changes: 19 additions & 8 deletions src/magicmirror/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ echo "Activating feature 'magicmirror'"
# For more details, see https://containers.dev/implementors/features#user-env-var
echo "The effective dev container remoteUser is '$_REMOTE_USER'"
echo "The effective dev container remoteUser's home directory is '$_REMOTE_USER_HOME'"

echo "The effective dev container containerUser is '$_CONTAINER_USER'"
echo "The effective dev container containerUser's home directory is '$_CONTAINER_USER_HOME'"

Expand All @@ -23,18 +22,19 @@ UBUNTU_MIRROR=${UBUNTU_MIRROR:-""}
echo "UBUNTU_MIRROR: $UBUNTU_MIRROR"
if [ "${OS}" = '"Ubuntu"' ] && [ -n "${UBUNTU_MIRROR}" ]; then
echo "Creating /etc/apt/sources.list with mirror = ${UBUNTU_MIRROR}"
cp /etc/apt/sources.list /etc/apt/sources.list.bak
sed -i s/archive.ubuntu.com/$UBUNTU_MIRROR/g /etc/apt/sources.list
sed -i s/security.ubuntu.com/$UBUNTU_MIRROR/g /etc/apt/sources.list
sed -i s/ports.ubuntu.com/$UBUNTU_MIRROR/g /etc/apt/sources.list
cp /etc/apt/sources.list /etc/apt/sources.list.mm.bak
sed -i "s/archive.ubuntu.com/${UBUNTU_MIRROR}/g" /etc/apt/sources.list
sed -i "s/security.ubuntu.com/${UBUNTU_MIRROR}/g" /etc/apt/sources.list
sed -i "s/ports.ubuntu.com/${UBUNTU_MIRROR}/g" /etc/apt/sources.list
fi

# Pypi mirror. TODO[sidecus]: Check Python installation
PYPI_MIRROR=${PYPI_MIRROR:-""}
echo "PYPI_MIRROR: $PYPI_MIRROR"
if [ -n "${PYPI_MIRROR}" ]; then
echo "Creating /etc/pip.conf with index-url = ${PYPI_MIRROR}"
echo "[global]" > /etc/pip.conf
echo "Seting up pypi mirror via /etc/pip.conf: index-url = ${PYPI_MIRROR}"
touch /etc/pip.conf && cp /etc/pip.conf /etc/pip.conf.mm.bak
echo "[global]" >> /etc/pip.conf
echo "index-url = ${PYPI_MIRROR}" >> /etc/pip.conf
fi

Expand All @@ -43,5 +43,16 @@ APK_MIRROR=${APK_MIRROR:-""}
echo "APK_MIRROR: $APK_MIRROR"
if [ "${OS}" = '"Alpine Linux"' ] && [ -n "${APK_MIRROR}" ]; then
echo "Creating /etc/apk/repositories with mirror = ${APK_MIRROR}"
cp /etc/apk/repositories /etc/apk/repositories.mm.bak
sed -i "s/dl-cdn.alpinelinux.org/${APK_MIRROR}/g" /etc/apk/repositories
fi
fi

# Set Huggingface mirror if it's provided
HUGGINGFACE_MIRROR=${HUGGINGFACE_MIRROR:-""}
echo "HUGGINGFACE_MIRROR: $HUGGINGFACE_MIRROR"
if [ -n "${HUGGINGFACE_MIRROR}" ]; then
echo "Enabling Huggingface mirror: ${HUGGINGFACE_MIRROR}"
touch /etc/environment && cp /etc/environment /etc/environment.mm.bak
echo "HF_ENDPOINT=${HF_ENDPOINT}" >> /etc/environment
set -a; . /etc/environment; set +a;
fi
17 changes: 17 additions & 0 deletions test/magicmirror/hf.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash

# This test file will be executed against one of the scenarios devcontainer.json test that
# includes the 'magicmirror' feature with "huggingface_mirror" option.

set -e

# Optional: Import test library bundled with the devcontainer CLI
source dev-container-features-test-lib

# Feature-specific tests
# The 'check' command comes from the dev-container-features-test-lib.
check "test HF mirror" bash -c '[ "${HF_ENDPOINT}"="https://dummy-hf-mirror.com" ];'

# Report results
# If any of the checks above exited with a non-zero exit code, the test will fail.
reportResults
14 changes: 11 additions & 3 deletions test/magicmirror/scenarios.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,31 @@
"image": "mcr.microsoft.com/devcontainers/base:ubuntu",
"features": {
"magicmirror": {
"ubuntu_mirror": "dummy-ubuntu-mirror"
"ubuntu_mirror": "dummy-ubuntu-mirror.com"
}
}
},
"pypi": {
"image": "mcr.microsoft.com/devcontainers/base:ubuntu",
"features": {
"magicmirror": {
"pypi_mirror": "dummy-pypi-mirror"
"pypi_mirror": "https://dummy-pypi-mirror.com/pypi/web/simple/"
}
}
},
"apk": {
"image": "mcr.microsoft.com/devcontainers/base:alpine",
"features": {
"magicmirror": {
"apk_mirror": "dummy-apk-repo"
"apk_mirror": "dummy-apk-repo.com"
}
}
},
"hf": {
"image": "mcr.microsoft.com/devcontainers/base:ubuntu",
"features": {
"magicmirror": {
"huggingface_mirror": "https://dummy-hf-mirror.com"
}
}
}
Expand Down

0 comments on commit d8a5218

Please sign in to comment.