From 3d94e977d6a6355bba0be3e209b70e553caa4367 Mon Sep 17 00:00:00 2001 From: Micke Nordin Date: Tue, 31 Oct 2023 15:10:41 +0100 Subject: [PATCH] Start working on tabbyml --- facts.d/kernel_release.sh | 3 + manifests/packages/linux_headers.pp | 7 ++ .../packages/nvidia_container_toolkit.pp | 22 ++++++ manifests/packages/nvidia_cuda_drivers.pp | 26 +++++++ manifests/tabbyml.pp | 24 ++++++ templates/tabbyml/docker-compose.yml.erb | 76 +++++++++++++++++++ 6 files changed, 158 insertions(+) create mode 100755 facts.d/kernel_release.sh create mode 100644 manifests/packages/linux_headers.pp create mode 100644 manifests/packages/nvidia_container_toolkit.pp create mode 100644 manifests/packages/nvidia_cuda_drivers.pp create mode 100644 manifests/tabbyml.pp create mode 100644 templates/tabbyml/docker-compose.yml.erb diff --git a/facts.d/kernel_release.sh b/facts.d/kernel_release.sh new file mode 100755 index 000000000..8a676f3cd --- /dev/null +++ b/facts.d/kernel_release.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "kernel_release=$(uname -r)" diff --git a/manifests/packages/linux_headers.pp b/manifests/packages/linux_headers.pp new file mode 100644 index 000000000..d50b0771a --- /dev/null +++ b/manifests/packages/linux_headers.pp @@ -0,0 +1,7 @@ +# Nvida container toolkit +class sunet::packages::linux_headers { + package { "linux-headers-${facts['kernel_release']}": + ensure => installed, + provider => 'apt', + } +} diff --git a/manifests/packages/nvidia_container_toolkit.pp b/manifests/packages/nvidia_container_toolkit.pp new file mode 100644 index 000000000..446b268ea --- /dev/null +++ b/manifests/packages/nvidia_container_toolkit.pp @@ -0,0 +1,22 @@ +# Nvida container toolkit +class sunet::packages::nvidia_container_toolkit { + include sunet::packages::curl + include sunet::packages::gpg + exec { 'nvidia-container-toolkit-keyring': + cmd => 'curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg', + unless => 'test -f /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg' + } + exec { 'nvidia-container-toolkit-repo': + cmd => 'curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed "s_deb https://_deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://_g" > /etc/apt/sources.list.d/nvidia-container-toolkit.list', + unless => 'test -f /etc/apt/sources.list.d/nvidia-container-toolkit.list' + } + exec { 'nvidia-container-toolkit-update': + cmd => 'apt update', + require => Exec['nvidia-container-toolkit-keyring', 'nvidia-container-toolkit-repo'] + } + package { 'nvidia-container-toolkit': + ensure => installed, + provider => 'apt', + require => Exec['nvidia-container-toolkit-update'] + } +} diff --git a/manifests/packages/nvidia_cuda_drivers.pp b/manifests/packages/nvidia_cuda_drivers.pp new file mode 100644 index 000000000..c085b7295 --- /dev/null +++ b/manifests/packages/nvidia_cuda_drivers.pp @@ -0,0 +1,26 @@ +# Nvida container toolkit +class sunet::packages::nvidia_cuda_drivers { + include sunet::packages::curl + include sunet::packages::linux_headers + $distro = downcase($facts['os']['distro']['id']) + $major = $facts['os']['distro']['release']['major'] + $minor = $facts['os']['distro']['release']['minor'] + $cuda_keyring = 'cuda-keyring_1.0-1_all.deb' + exec { 'nvidia-cuda-drivers-keyring': + cmd => "curl https://developer.download.nvidia.com/compute/cuda/repos/${distro}${major}${minor}/x86_64/${cuda_keyring} -o /tmp/${cuda_keyring} && dpkg -i /tmp/${cuda_keyring}", + unless => "test -f /tmp/${cuda_keyring}" + } + exec { 'nvidia-cuda-drivers-update': + cmd => 'apt update', + require => Exec['nvidia-cuda-drivers-keyring'] + } + package { 'cuda-drivers': + ensure => installed, + provider => 'apt', + require => Exec['nvidia-cuda-drivers-update'] + } + file_line { 'cuda_env_path': + line => 'PATH=/opt/nvidia/nsight-compute/bin${PATH:+:${PATH}}', + match => '^PATH=', + } +} diff --git a/manifests/tabbyml.pp b/manifests/tabbyml.pp new file mode 100644 index 000000000..bccfe8a8d --- /dev/null +++ b/manifests/tabbyml.pp @@ -0,0 +1,24 @@ +# microk8s cluster node +class sunet::tabbyml( + String $interface = 'enp2s0', + String $tabby_model = 'CodeLlama-13B', + String $vhost = 'tabby-lab.sunet.se', +) { + include sunet::packages::nvidia_container_toolkit + include sunet::packages::nvidia_cuda_driver + sunet::docker_compose { 'tabbyml': + content => template('sunet/tabbyml/docker-compose.erb.yml'), + service_name => 'tabbyml', + compose_dir => '/opt', + compose_filename => 'docker-compose.yml', + description => 'tabbyml', + } + $ports = [80, 443] + $ports.each|$port| { + sunet::nftables::docker_expose { "port_${port}": + allow_clients => 'any', + port => $port, + iif => $interface, + } + } +} diff --git a/templates/tabbyml/docker-compose.yml.erb b/templates/tabbyml/docker-compose.yml.erb new file mode 100644 index 000000000..e2d116b31 --- /dev/null +++ b/templates/tabbyml/docker-compose.yml.erb @@ -0,0 +1,76 @@ + +version: "3.7" + +services: + nginx: + image: docker.io/nginxproxy/nginx-proxy:latest + container_name: nginx + networks: + - internal_network + - external_network + dns: + - 89.32.32.32 + ports: + - "80:80" + - "443:443" + labels: + com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy: "true" + volumes: + - /opt/mastodon_web/nginx/certs:/etc/nginx/certs:ro + - /opt/mastodon_web/nginx/conf:/etc/nginx/conf.d + - /opt/mastodon_web/nginx/dhparam:/etc/nginx/dhparam + - /opt/mastodon_web/nginx/html:/usr/share/nginx/html + - /opt/mastodon_web/nginx/vhost:/etc/nginx/vhost.d + - /var/run/docker.sock:/tmp/docker.sock:ro + environment: + - ENABLE_IPV6=true + restart: unless-stopped + + acme: + image: docker.io/nginxproxy/acme-companion:latest + container_name: acme + networks: + - external_network + dns: + - 89.32.32.32 + volumes: + - /opt/mastodon_web/nginx/acme:/etc/acme.sh + - /opt/mastodon_web/nginx/certs:/etc/nginx/certs:rw + - /opt/mastodon_web/nginx/conf:/etc/nginx/conf.d + - /opt/mastodon_web/nginx/dhparam:/etc/nginx/dhparam + - /opt/mastodon_web/nginx/html:/usr/share/nginx/html + - /opt/mastodon_web/nginx/vhost:/etc/nginx/vhost.d:rw + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + - NGINX_PROXY_CONTAINER=nginx + - DEFAULT_EMAIL=noc@sunet.se + depends_on: + - nginx + restart: unless-stopped + + tabby: + restart: always + image: tabbyml/tabby + command: serve --model TabbyML/<%= @tabby_model %> --device cuda: + networks: + - internal_network + volumes: + - /opt/tabbyml/data:/data + environment: + - VIRTUAL_HOST=<%= @vhost %> + - VIRTUAL_PATH=/ + - VIRTUAL_PORT=8080 + - LETSENCRYPT_HOST=<%= @vhost %> + - ES_ENABLED=false + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + +networks: + external_network: + internal_network: + internal: true