Start working on tabbyml

facts.d/kernel_release.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "kernel_release=$(uname -r)"

manifests/packages/linux_headers.pp

@@ -0,0 +1,7 @@
+# Nvida container toolkit
+class sunet::packages::linux_headers {
+    package { "linux-headers-${facts['kernel_release']}":
+      ensure   => installed,
+      provider => 'apt',
+    }
+}

manifests/packages/nvidia_container_toolkit.pp

@@ -0,0 +1,22 @@
+# Nvida container toolkit
+class sunet::packages::nvidia_container_toolkit {
+    include sunet::packages::curl
+    include sunet::packages::gpg
+    exec { 'nvidia-container-toolkit-keyring':
+      cmd    => 'curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg',
+      unless => 'test -f /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg'
+    }
+    exec { 'nvidia-container-toolkit-repo':
+      cmd    => 'curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed "s_deb https://_deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://_g" > /etc/apt/sources.list.d/nvidia-container-toolkit.list',
+      unless => 'test -f /etc/apt/sources.list.d/nvidia-container-toolkit.list'
+    }
+    exec { 'nvidia-container-toolkit-update':
+      cmd     => 'apt update',
+      require => Exec['nvidia-container-toolkit-keyring', 'nvidia-container-toolkit-repo']
+    }
+    package { 'nvidia-container-toolkit':
+      ensure   => installed,
+      provider => 'apt',
+      require  => Exec['nvidia-container-toolkit-update']
+    }
+}

manifests/packages/nvidia_cuda_drivers.pp

@@ -0,0 +1,26 @@
+# Nvida container toolkit
+class sunet::packages::nvidia_cuda_drivers {
+  include sunet::packages::curl
+  include sunet::packages::linux_headers
+  $distro = downcase($facts['os']['distro']['id'])
+  $major = $facts['os']['distro']['release']['major']
+  $minor = $facts['os']['distro']['release']['minor']
+  $cuda_keyring = 'cuda-keyring_1.0-1_all.deb'
+  exec { 'nvidia-cuda-drivers-keyring':
+    cmd    => "curl https://developer.download.nvidia.com/compute/cuda/repos/${distro}${major}${minor}/x86_64/${cuda_keyring} -o /tmp/${cuda_keyring} && dpkg -i /tmp/${cuda_keyring}",
+    unless => "test -f /tmp/${cuda_keyring}"
+  }
+  exec { 'nvidia-cuda-drivers-update':
+    cmd     => 'apt update',
+    require => Exec['nvidia-cuda-drivers-keyring']
+  }
+  package { 'cuda-drivers':
+    ensure   => installed,
+    provider => 'apt',
+    require  => Exec['nvidia-cuda-drivers-update']
+  }
+  file_line { 'cuda_env_path':
+    line  => 'PATH=/opt/nvidia/nsight-compute/bin${PATH:+:${PATH}}',
+    match => '^PATH=',
+  }
+}

manifests/tabbyml.pp

@@ -0,0 +1,24 @@
+# microk8s cluster node
+class sunet::tabbyml(
+  String $interface   = 'enp2s0',
+  String $tabby_model = 'CodeLlama-13B',
+  String $vhost       = 'tabby-lab.sunet.se',
+) {
+  include sunet::packages::nvidia_container_toolkit
+  include sunet::packages::nvidia_cuda_driver
+  sunet::docker_compose { 'tabbyml':
+    content          => template('sunet/tabbyml/docker-compose.erb.yml'),
+    service_name     => 'tabbyml',
+    compose_dir      => '/opt',
+    compose_filename => 'docker-compose.yml',
+    description      => 'tabbyml',
+  }
+  $ports = [80, 443]
+  $ports.each|$port| {
+    sunet::nftables::docker_expose { "port_${port}":
+      allow_clients => 'any',
+      port          => $port,
+      iif           => $interface,
+    }
+  }
+}

templates/tabbyml/docker-compose.yml.erb

@@ -0,0 +1,76 @@
+
+version: "3.7"
+
+services:
+  nginx:
+    image: docker.io/nginxproxy/nginx-proxy:latest
+    container_name: nginx
+    networks:
+      - internal_network
+      - external_network
+    dns:
+      - 89.32.32.32
+    ports:
+      - "80:80"
+      - "443:443"
+    labels:
+      com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy: "true"
+    volumes:
+      - /opt/mastodon_web/nginx/certs:/etc/nginx/certs:ro
+      - /opt/mastodon_web/nginx/conf:/etc/nginx/conf.d
+      - /opt/mastodon_web/nginx/dhparam:/etc/nginx/dhparam
+      - /opt/mastodon_web/nginx/html:/usr/share/nginx/html
+      - /opt/mastodon_web/nginx/vhost:/etc/nginx/vhost.d
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+    environment:
+      - ENABLE_IPV6=true
+    restart: unless-stopped
+
+  acme:
+    image: docker.io/nginxproxy/acme-companion:latest
+    container_name: acme
+    networks:
+      - external_network
+    dns:
+      - 89.32.32.32
+    volumes:
+      - /opt/mastodon_web/nginx/acme:/etc/acme.sh
+      - /opt/mastodon_web/nginx/certs:/etc/nginx/certs:rw
+      - /opt/mastodon_web/nginx/conf:/etc/nginx/conf.d
+      - /opt/mastodon_web/nginx/dhparam:/etc/nginx/dhparam
+      - /opt/mastodon_web/nginx/html:/usr/share/nginx/html
+      - /opt/mastodon_web/nginx/vhost:/etc/nginx/vhost.d:rw
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+      - NGINX_PROXY_CONTAINER=nginx
+      - DEFAULT_EMAIL=noc@sunet.se
+    depends_on:
+      - nginx
+    restart: unless-stopped
+
+  tabby:
+    restart: always
+    image: tabbyml/tabby
+    command: serve --model TabbyML/<%= @tabby_model %> --device cuda:
+    networks:
+      - internal_network
+    volumes:
+      - /opt/tabbyml/data:/data
+    environment:
+      - VIRTUAL_HOST=<%= @vhost %>
+      - VIRTUAL_PATH=/
+      - VIRTUAL_PORT=8080
+      - LETSENCRYPT_HOST=<%= @vhost %>
+      - ES_ENABLED=false
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+
+networks:
+  external_network:
+  internal_network:
+    internal: true