feat: now using traefik proxy for true zero downtime

.env.sample

@@ -1,8 +1,16 @@
 # uncomment in your prod env to make the repo update itself:
 # PYTHON_ENV=production
 
-# comment out in your prod env to get production certs:
-LE_STAGING=1
+# Trusted ips for proxy protocol, usually the ip of your router (sending to traefik)
+TRUSTED_IPS_CIDR=192.168.0.0/24
+
+# Traefik dashboard domain name
+TRAEFIK_DOMAIN=traefik.example.com
+# Traefic basic auth user+pass (htpasswd) WHICH NEEDS TO BE QUOTED!
+# TRAEFIK_ADMIN='admin:$xxx/yyy'
+
+# Uncomment next line for prod certs
+TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_CASERVER=https://acme-staging-v02.api.letsencrypt.org/directory
+TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_EMAIL=mail@example.com
 
-# LE_EMAIL=you@gmail.com
 # API_KEY=your_api_key

.gitignore

@@ -5,5 +5,6 @@ __pycache__
 .venv
 certs/*
 db.yml
-proxy/*.conf
+proxy/nginx/*.conf
+proxy/traefik/*.yml
 upstream

api/main.py

@@ -22,7 +22,7 @@
 )
 from lib.git import update_repo
 from lib.models import Env, PingPayload, Project, Service, WorkflowJobPayload
-from lib.proxy import reload_proxy, update_proxy, write_nginx
+from lib.proxy import reload_proxy, update_proxy, write_proxies
 from lib.upstream import check_upstream, update_upstream, write_upstreams
 
 dotenv.load_dotenv()
@@ -36,7 +36,7 @@ def _after_config_change(project: str, service: str = None) -> None:
     """Run after a project is updated"""
     info("Config change detected")
     get_certs(project)
-    write_nginx()
+    write_proxies()
     write_upstreams()
     update_upstream(project, service, rollout=True)
     update_proxy()

bin/apply.py

@@ -8,7 +8,7 @@
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
 from lib.certs import get_certs
-from lib.proxy import reload_proxy, write_nginx
+from lib.proxy import reload_proxy, write_proxies
 from lib.upstream import update_upstreams, write_upstreams
 
 load_dotenv()
@@ -17,7 +17,7 @@
     # if any argument is passed, we also do a rollout
     rollout = bool(sys.argv[1]) if len(sys.argv) > 1 else False
     get_certs()
-    write_nginx()
+    write_proxies()
     write_upstreams()
     update_upstreams(rollout)
     reload_proxy()

bin/start-api.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env sh
 . .venv/bin/activate
 
-kill $(fuser 8888/tcp 2>/dev/null | awk '{ print $1 }')
+kill $(fuser 8888/tcp 2>/dev/null | awk '{ print $1 }') 2>/dev/null
 
 PYTHONPATH=. python api/main.py main:app >logs/error.log 2>&1 &

bin/write-artifacts.py

@@ -8,12 +8,12 @@
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
 from lib.data import validate_db
-from lib.proxy import write_nginx
+from lib.proxy import write_proxies
 from lib.upstream import write_upstreams
 
 load_dotenv()
 
 if __name__ == "__main__":
     validate_db()
-    write_nginx()
+    write_proxies()
     write_upstreams()

lib/functions.sh

@@ -28,7 +28,7 @@ dcp() {
     docker compose --project-directory $dir -p $project -f $dir/docker-compose.yml pull
     cmd="$cmd --remove-orphans"
   fi
-  eval "docker compose --project-directory $dir -p $project -f $dir/docker-compose.yml $cmd $@"
+  eval "export HOST_GID=$(id -g daemon) && docker compose --project-directory $dir -p $project -f $dir/docker-compose.yml $cmd $@"
 }
 
 dcpx() {

lib/git.py

@@ -2,7 +2,7 @@
 
 from dotenv import load_dotenv
 
-from lib.proxy import reload_proxy, write_nginx
+from lib.proxy import reload_proxy, write_proxies
 from lib.upstream import update_upstreams, write_upstreams
 from lib.utils import run_command
 
@@ -15,7 +15,7 @@ def update_repo() -> None:
     if os.environ["PYTHON_ENV"] == "production":
         run_command("git fetch origin main".split(" "), cwd=".")
         run_command("git reset --hard origin/main".split(" "), cwd=".")
-    write_nginx()
+    write_proxies()
     write_upstreams()
     update_upstreams()
     reload_proxy()

lib/models.py

@@ -11,16 +11,27 @@ class Env(BaseModel):
 class Service(BaseModel):
     """Service model"""
 
-    name: str
-    port: int = 8080
-    """When set, the service will be exposed on this domain."""
-    image: str = None
     command: str = None
+    """The command to run in the service"""
+    env: Env = None
+    """A dictionary of environment variables to pass to the service"""
+    image: str = None
+    """The image name plus tag to use for the service"""
+    labels: List[str] = []
+    """Extra labels to add to the service. Should not interfere with generated traefik labels for ingress."""
+    name: str
     passthrough: bool = False
     """Wether or not traffic to this service is forwarded as-is (without terminating SSL)"""
+    path_prefix: str = None
+    """Should the service be exposed under a specific path?"""
+    path_remove: bool = False
+    """When set, the path prefix will be removed from the request before forwarding it to the service"""
+    port: int = 8080
+    """When set, the service will be exposed on this domain."""
+    proxyprotocol: bool = True
+    """When set, the service will be exposed using the PROXY protocol version 2"""
     volumes: List[str] = []
-    env: Env = None
-    """A dictionary of environment variables to pass to the service"""
+    """A list of volumes to mount in the service"""
 
 
 class Project(BaseModel):

lib/proxy.py

@@ -1,11 +1,15 @@
+import os
 from logging import info
 from typing import Dict, List
 
+from dotenv import load_dotenv
 from jinja2 import Template
 
 from lib.data import get_project, get_projects
 from lib.utils import run_command
 
+load_dotenv()
+
 
 def get_domains(project: str = None) -> List[str]:
     """Get all domains in use"""
@@ -44,11 +48,11 @@ def write_maps() -> None:
     internal = tpl.render(map=internal_map)
     passthrough = tpl.render(map=passthrough_map)
     terminate = tpl.render(map=terminate_map)
-    with open("proxy/map/internal.conf", "w", encoding="utf-8") as f:
+    with open("proxy/nginx/map/internal.conf", "w", encoding="utf-8") as f:
         f.write(internal)
-    with open("proxy/map/passthrough.conf", "w", encoding="utf-8") as f:
+    with open("proxy/nginx/map/passthrough.conf", "w", encoding="utf-8") as f:
         f.write(passthrough)
-    with open("proxy/map/terminate.conf", "w", encoding="utf-8") as f:
+    with open("proxy/nginx/map/terminate.conf", "w", encoding="utf-8") as f:
         f.write(terminate)
 
 
@@ -58,7 +62,7 @@ def write_proxy() -> None:
         t = f.read()
     tpl = Template(t)
     terminate = tpl.render(project=project)
-    with open("proxy/proxy.conf", "w", encoding="utf-8") as f:
+    with open("proxy/nginx/proxy.conf", "w", encoding="utf-8") as f:
         f.write(terminate)
 
 
@@ -68,14 +72,47 @@ def write_terminate() -> None:
         t = f.read()
     tpl = Template(t)
     terminate = tpl.render(domains=domains)
-    with open("proxy/terminate.conf", "w", encoding="utf-8") as f:
+    with open("proxy/nginx/terminate.conf", "w", encoding="utf-8") as f:
         f.write(terminate)
 
 
-def write_nginx() -> None:
+def write_routers() -> None:
+    projects = get_projects(filter=lambda p, s: not bool(p.entrypoint) or p.entrypoint == s.name)
+    with open("proxy/tpl/routers-web.yml.j2", encoding="utf-8") as f:
+        t = f.read()
+    tpl_routers_web = Template(t)
+    domain = os.environ.get("TRAEFIK_DOMAIN")
+    traefik_admin = os.environ.get("TRAEFIK_ADMIN")
+    routers_web = tpl_routers_web.render(
+        projects=projects, traefik_rule=f"Host(`{domain}`)", traefik_admin=traefik_admin
+    )
+    with open("proxy/traefik/routers-web.yml", "w", encoding="utf-8") as f:
+        f.write(routers_web)
+    with open("proxy/tpl/routers-tcp.yml.j2", encoding="utf-8") as f:
+        t = f.read()
+    tpl_routers_tcp = Template(t)
+    routers_tcp = tpl_routers_tcp.render(projects=projects, traefik_rule=f"HostSNI(`{domain}`)")
+    with open("proxy/traefik/routers-tcp.yml", "w", encoding="utf-8") as f:
+        f.write(routers_tcp)
+    with open("proxy/tpl/config-tcp.yml.j2", encoding="utf-8") as f:
+        t = f.read()
+    tpl_config_tcp = Template(t)
+    config_tcp = tpl_config_tcp.render(trusted_ips_cidr=os.environ.get("TRUSTED_IPS_CIDR"))
+    with open("proxy/traefik/config-tcp.yml", "w", encoding="utf-8") as f:
+        f.write(config_tcp)
+    with open("proxy/tpl/config-web.yml.j2", encoding="utf-8") as f:
+        t = f.read()
+    tpl_config_web = Template(t)
+    config_web = tpl_config_web.render(trusted_ips_cidr=os.environ.get("TRUSTED_IPS_CIDR"))
+    with open("proxy/traefik/config-web.yml", "w", encoding="utf-8") as f:
+        f.write(config_web)
+
+
+def write_proxies() -> None:
     write_maps()
     write_proxy()
     write_terminate()
+    write_routers()
 
 
 def update_proxy(

lib/proxy_test.py

@@ -14,7 +14,7 @@
     get_terminate_map,
     reload_proxy,
     write_maps,
-    write_nginx,
+    write_proxies,
     write_proxy,
     write_terminate,
 )
@@ -117,9 +117,9 @@ def test_write_maps(
             mock_open.call_args_list,
             [
                 mock.call("proxy/tpl/map.conf.j2", encoding="utf-8"),
-                mock.call("proxy/map/internal.conf", "w", encoding="utf-8"),
-                mock.call("proxy/map/passthrough.conf", "w", encoding="utf-8"),
-                mock.call("proxy/map/terminate.conf", "w", encoding="utf-8"),
+                mock.call("proxy/nginx/map/internal.conf", "w", encoding="utf-8"),
+                mock.call("proxy/nginx/map/passthrough.conf", "w", encoding="utf-8"),
+                mock.call("proxy/nginx/map/terminate.conf", "w", encoding="utf-8"),
             ],
         )
 
@@ -135,7 +135,7 @@ def test_write_proxy(self, _: Mock, mock_open: Mock) -> None:
             mock_open.call_args_list,
             [
                 call("proxy/tpl/proxy.conf.j2", encoding="utf-8"),
-                call("proxy/proxy.conf", "w", encoding="utf-8"),
+                call("proxy/nginx/proxy.conf", "w", encoding="utf-8"),
             ],
         )
 
@@ -153,16 +153,16 @@ def test_write_terminate(self, _: Mock, mock_get_domains: Mock, mock_open: Mock)
             mock_open.call_args_list,
             [
                 call("proxy/tpl/terminate.conf.j2", encoding="utf-8"),
-                call("proxy/terminate.conf", "w", encoding="utf-8"),
+                call("proxy/nginx/terminate.conf", "w", encoding="utf-8"),
             ],
         )
 
     @mock.patch("lib.proxy.write_maps")
     @mock.patch("lib.proxy.write_proxy")
     @mock.patch("lib.proxy.write_terminate")
-    def test_write_nginx(self, mock_write_terminate: Mock, mock_write_proxy: Mock, mock_write_maps: Mock) -> None:
+    def test_write_proxies(self, mock_write_terminate: Mock, mock_write_proxy: Mock, mock_write_maps: Mock) -> None:
         # Call the function under test
-        write_nginx()
+        write_proxies()
 
         # Assert that the write_maps, write_proxy, and write_terminate functions are called
         mock_write_maps.assert_called_once()

lib/upstream.py

@@ -1,34 +1,39 @@
 import os
 from logging import info
-from typing import List
 
+from dotenv import load_dotenv
 from jinja2 import Template
 
 from lib.data import get_project, get_projects, get_service
-from lib.models import Service
+from lib.models import Project
 from lib.utils import run_command
 
+load_dotenv()
 
-def write_upstream(project: str, services: List[Service]) -> None:
+
+def write_upstream(project: Project) -> None:
     with open("tpl/docker-compose.yml.j2", encoding="utf-8") as f:
         tpl = f.read()
-    content = Template(tpl).render(project=project, services=services)
-    with open(f"upstream/{project}/docker-compose.yml", "w", encoding="utf-8") as f:
+    if os.environ.get("PYTHON_ENV") != "production":
+        content = Template(tpl).render(project=project, domain=os.environ.get("TRAEFIK_DOMAIN"), env="development")
+    else:
+        content = Template(tpl).render(project=project, domain=project.domain)
+    with open(f"upstream/{project.name}/docker-compose.yml", "w", encoding="utf-8") as f:
         f.write(content)
 
 
-def write_upstream_volume_folders(project: str, services: List[Service]) -> None:
-    for svc in services:
-        for path in svc.volumes:
-            os.makedirs(f"upstream/{project}{path}", exist_ok=True)
+def write_upstream_volume_folders(project: Project) -> None:
+    for s in project.services:
+        for path in s.volumes:
+            os.makedirs(f"upstream/{project.name}{path}", exist_ok=True)
 
 
 def write_upstreams() -> None:
     # iterate over projects that have an entrypoint;
     for p in [project for project in get_projects() if project.entrypoint]:
         os.makedirs(f"upstream/{p.name}", exist_ok=True)
-        write_upstream(p.name, p.services)
-        write_upstream_volume_folders(p.name, p.services)
+        write_upstream(p)
+        write_upstream_volume_folders(p)
 
 
 def check_upstream(project: str, service: str = None) -> None:

lib/upstream_test.py

@@ -68,22 +68,22 @@ def test_write_upstream(
         mock_open: Mock,
     ) -> None:
 
-        services = [
-            Service(
-                image="morriz/hello-world:main",
-                name="master",
-                port=8080,
-                env=Env(**{"TARGET": "cost concerned people", "INFORMANT": "http://test-informant:8080"}),
-                volumes=["./data/bla:/data/bla", "./etc/dida:/etc/dida"],
-            ),
-            Service(image="morriz/hello-world:main", name="informant", port=8080, env=Env(**{"TARGET": "boss"})),
-        ]
+        project = Project(
+            name="test",
+            services=[
+                Service(
+                    image="morriz/hello-world:main",
+                    name="master",
+                    port=8080,
+                    env=Env(**{"TARGET": "cost concerned people", "INFORMANT": "http://test-informant:8080"}),
+                    volumes=["./data/bla:/data/bla", "./etc/dida:/etc/dida"],
+                ),
+                Service(image="morriz/hello-world:main", name="informant", port=8080, env=Env(**{"TARGET": "boss"})),
+            ],
+        )
 
         # Call the function under test
-        write_upstream(
-            "test",
-            services=services,
-        )
+        write_upstream(project)
 
         # Assert that the subprocess.Popen was called with the correct arguments
         mock_open.return_value.write.assert_called_once_with(_ret_tpl)

load-test.js

@@ -0,0 +1,17 @@
+import http from 'k6/http';
+import { check, sleep } from 'k6';
+
+export const options = {
+  insecureSkipTLSVerify: true,
+  stages: [
+    // { duration: '30s', target: 20 },
+    { duration: '0m20s', target: 1 },
+    // { duration: '20s', target: 20 },
+  ],
+};
+
+export default function () {
+  const res = http.get(`https://hello.srv.instrukt.ai`);
+  check(res, { 'status was 200': (r) => r.status == 200 });
+  sleep(1);
+}