feat: ingress list per service

.codiumai.toml

@@ -58,13 +58,13 @@ example_test = """
     @mock.patch("os.scandir")
     @mock.patch("lib.upstream.update_upstream")
     def test_update_upstreams(self, mock_update_upstream: Mock, mock_scandir: Mock):
-        mock_scandir.return_value = [DirEntry("upstream/my_project")]
+        mock_scandir.return_value = [DirEntry("upstream/my-project")]
 
         # Call the function under test
         update_upstreams()
 
         mock_update_upstream.assert_called_once_with(
-            "my_project",
+            "my-project",
             False,
         )
 

db.yml.sample

@@ -17,45 +17,66 @@ plugins:
 
 projects:
   - description: Home Assistant passthrough
-    domain: home.example.com
+    enabled: false
     name: home-assistant # keep this name as it also makes sure to forward port 80 for certbot
     services:
-      - name: 192.168.1.111
-        passthrough: true
-        port: 443
+      - ingress:
+        - domain: home.example.com
+          passthrough: true
+          port: 443
+        name: 192.168.1.111
   - description: itsUP API running on the host
-    domain: itsup.example.com
     name: itsUP
     services:
-      - name: host.docker.internal
-        port: 8888
+      - ingress:
+          - domain: itsup.example.com
+            port: 8888
+        name: host.docker.internal
+  - description: Minio service
+    enabled: false
+    name: minio
+    services:
+      - command: server --console-address ":9001" http://minio/data
+        env:
+          MINIO_ROOT_USER: root
+          MINIO_ROOT_PASSWORD: 83b01a6b8f210b5f5862943f3ebe257d
+          MINIO_DEFAULT_BUCKETS: ai-assistant
+        image: minio/minio:latest
+        ingress:
+          - domain: minio-api.example.com
+            port: 9000
+          - domain: minio-ui.example.com
+            port: 9001
+        name: app
+        volumes:
+          - /data
   - description: VPN server
-    domain: vpn.example.com
-    entrypoint: openvpn
+    enabled: false
     name: vpn
     services:
       - additional_properties:
           cap_add:
             - NET_ADMIN
         # change tag to x86_64 if not on ARM:
-        hostport: 1194
-        hostport: 1194
         image: nubacuk/docker-openvpn:aarch64
+        ingress:
+          - domain: vpn.example.com
+            hostport: 1194
+            port: 1194
+            protocol: udp
         name: openvpn
-        port: 1194
-        protocol: udp
         restart: always
         volumes:
           - /etc/openvpn
   - description: test project to demonstrate inter service connectivity
-    domain: hello.example.com
     name: test
-    entrypoint: master
     services:
       - env:
           TARGET: cost concerned people
           INFORMANT: http://test-informant:8080
         image: otomi/nodejs-helloworld:v1.2.13
+        ingress:
+          - domain: hello.example.com
         name: master
         volumes:
           - /data/bla
@@ -67,9 +88,9 @@ projects:
         image: otomi/nodejs-helloworld:v1.2.13
         name: informant
   - description: whoami service
-    domain: whoami.example.com
-    entrypoint: web
     name: whoami
     services:
       - image: traefik/whoami:latest
+        ingress:
+          - domain: whoami.example.com
         name: web

lib/data.py

@@ -2,11 +2,11 @@
 import inspect
 from logging import debug, info
 from modulefinder import Module
-from typing import Any, Callable, Dict, List, cast
+from typing import Any, Callable, Dict, List, Union, cast
 
 import yaml
 
-from lib.models import Env, Plugin, PluginRegistry, Project, Service
+from lib.models import Env, Ingress, Plugin, PluginRegistry, Project, Service
 
 
 def get_db() -> Dict[str, List[Dict[str, Any]] | Dict[str, Any]]:
@@ -69,7 +69,11 @@ def get_plugins(filter: Callable[[Plugin], bool] = None) -> List[Plugin]:
     return plugins
 
 
-def get_projects(filter: Callable[[Project, Service], bool] = None) -> List[Project]:
+def get_projects(
+    filter: Union[
+        Callable[[Project, Service, Ingress], bool], Callable[[Project, Service], bool], Callable[[Project], bool]
+    ] = None,
+) -> List[Project]:
     """Get all projects. Optionally filter the results."""
     debug("Getting projects" + (f" with filter {filter}" if filter else ""))
     db = get_db()
@@ -78,8 +82,23 @@ def get_projects(filter: Callable[[Project, Service], bool] = None) -> List[Proj
     for project in projects_raw:
         services = []
         p = Project(**project)
+        if not filter:
+            ret.append(p)
+            continue
         for s in p.services.copy():
-            if not filter or filter(p, s):
+            ingress = []
+            for i in s.ingress.copy():
+                if (
+                    (
+                        filter.__code__.co_argcount == 3
+                        and cast(Callable[[Project, Service, Ingress], bool], filter)(p, s, i)
+                    )
+                    or (filter.__code__.co_argcount == 2 and cast(Callable[[Project, Service], bool], filter)(p, s))
+                    or (filter.__code__.co_argcount == 1 and cast(Callable[[Project], bool], filter)(p))
+                ):
+                    ingress.append(i)
+            if len(ingress) > 0:
+                s.ingress = ingress
                 services.append(s)
         if len(services) > 0:
             p.services = services
@@ -125,7 +144,7 @@ def upsert_project(project: Project) -> None:
 def get_services(project: str = None) -> List[Service]:
     """Get all services or just for a particular project."""
     debug(f"Getting services for project {project}" if project else "Getting all services")
-    return [s for p in get_projects(lambda p, _: not bool(project) or p.name == project) for s in p.services]
+    return [s for p in get_projects(lambda p: not bool(project) or p.name == project) for s in p.services]
 
 
 def get_service(project: str | Project, service: str, throw: bool = True) -> Service:

lib/data_test.py

@@ -57,7 +57,7 @@ def test_write_projects(self, mock_write_db: Mock) -> None:
     def test_get_projects_with_filter(self, _: Mock) -> None:
 
         # Call the function under test
-        result = get_projects(lambda p, s: p.name == "test" and p.entrypoint == s.name)
+        result = get_projects(lambda p, s: p.name == "test" and s.ingress)
 
         # Assert the result
         expected_result = [
@@ -67,7 +67,7 @@ def test_get_projects_with_filter(self, _: Mock) -> None:
                 domain="hello.example.com",
                 entrypoint="master",
                 services=[
-                    test_projects[3].services[0],
+                    test_projects[4].services[0],
                 ],
             ),
         ]
@@ -79,6 +79,7 @@ def test_get_projects_with_filter(self, _: Mock) -> None:
         return_value=test_db.copy(),
     )
     def test_get_projects_no_filter(self, mock_get_db: Mock) -> None:
+        self.maxDiff = None
 
         # Call the function under test
         result = get_projects()
@@ -125,8 +126,8 @@ def test_upsert_nonexistent_project_fixed(self, mock_write_projects: Mock, mock_
         mock_write_projects.assert_called_once_with(test_projects + [new_project])
 
     # Upsert a project's service' env
-    @mock.patch("lib.data.get_project", return_value=test_projects[3].model_copy())
-    @mock.patch("lib.data.get_service", return_value=test_projects[3].services[1].model_copy())
+    @mock.patch("lib.data.get_project", return_value=test_projects[4].model_copy())
+    @mock.patch("lib.data.get_service", return_value=test_projects[4].services[1].model_copy())
     @mock.patch("lib.data.upsert_service")
     def test_upsert_env(self, mock_upsert_service: Mock, mock_get_service: Mock, mock_get_project: Mock) -> None:
 

lib/models.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from gc import enable
 from typing import Any, Dict, List
 
 from github_webhooks.schemas import WebhookCommonPayload
@@ -45,6 +46,34 @@ class Protocol(str, Enum):
     udp = "udp"
 
 
+class ProxyProtocol(str, Enum):
+    """ProxyProtocol enum"""
+
+    v1 = 1
+    v2 = 2
+
+
+class Ingress(BaseModel):
+    """Ingress model"""
+
+    domain: str
+    """The domain to use for the service"""
+    hostport: int = None
+    """The port to expose on the host"""
+    passthrough: bool = False
+    """Wether or not traffic to this service is forwarded as-is (without terminating SSL)"""
+    path_prefix: str = None
+    """Should the service be exposed under a specific path?"""
+    path_remove: bool = False
+    """When set, the path prefix will be removed from the request before forwarding it to the service"""
+    port: int = 8080
+    """The port to use for the service. If not defined will default to parent service port[idx]"""
+    protocol: Protocol = Protocol.tcp
+    """The protocol to use for the port"""
+    proxyprotocol: ProxyProtocol | None = ProxyProtocol.v2
+    """When set, the service is expected to accept the given PROXY protocol version. Explicitly set to null to disable."""
+
+
 class Service(BaseModel):
     """Service model"""
 
@@ -54,25 +83,14 @@ class Service(BaseModel):
     """The command to run in the service"""
     env: Env = None
     """A dictionary of environment variables to pass to the service"""
-    hostport: int = None
-    """The port to expose on the host"""
     image: str = None
-    """The image name plus tag to use for the service"""
+    """The full container image uri of the service"""
+    ingress: List[Ingress] = []
+    """Ingress configuration for the service. If a string is passed, it will be used as the domain."""
     labels: List[str] = []
     """Extra labels to add to the service. Should not interfere with generated traefik labels for ingress."""
     name: str
-    passthrough: bool = False
-    """Wether or not traffic to this service is forwarded as-is (without terminating SSL)"""
-    path_prefix: str = None
-    """Should the service be exposed under a specific path?"""
-    path_remove: bool = False
-    """When set, the path prefix will be removed from the request before forwarding it to the service"""
-    port: int = 8080
-    """When set, the service will be exposed on this domain."""
-    proxyprotocol: bool = True
-    """When set, the service will be exposed using the PROXY protocol version 2"""
-    protocol: Protocol = Protocol.tcp
-    """The protocol to use for the service"""
+    """The name of the service"""
     restart: str = "unless-stopped"
     """The restart policy to use for the service"""
     volumes: List[str] = []
@@ -82,11 +100,9 @@ class Service(BaseModel):
 class Project(BaseModel):
     """Project model"""
 
-    name: str
     description: str = None
-    domain: str = None
-    entrypoint: str = None
-    """When "entrypoint" is set it should point to a service in the "services" list that has an image"""
+    enabled: bool = True
+    name: str
     services: List[Service] = []
 
 

lib/proxy.py

@@ -6,16 +6,21 @@
 from jinja2 import Template
 
 from lib.data import get_plugin_registry, get_project, get_projects
-from lib.models import Protocol
+from lib.models import Protocol, ProxyProtocol
 from lib.utils import run_command
 
 load_dotenv()
 
 
 def get_domains(project: str = None) -> List[str]:
     """Get all domains in use"""
-    projects = get_projects(filter=lambda p, s: not s.passthrough and (not project or project == p.name))
-    return [p.domain for p in projects if p.domain]
+    projects = get_projects(filter=lambda p, _, i: not i.passthrough and (not project or project == p.name))
+    domains = []
+    for p in projects:
+        for s in p.services:
+            for i in s.ingress:
+                domains.append(i.domain)
+    return domains
 
 
 def get_internal_map() -> Dict[str, str]:
@@ -24,19 +29,24 @@ def get_internal_map() -> Dict[str, str]:
 
 
 def get_terminate_map() -> Dict[str, str]:
-    filtered = get_projects(
-        filter=lambda p, s: bool(p.domain) and not s.passthrough and (not bool(p.entrypoint) or p.entrypoint == s.name)
-    )
-    return {
-        p.domain: (f"{p.name}-" if p.entrypoint == s.name else "") + f"{s.name}:{s.port}"
-        for p in filtered
-        for s in p.services
-    }
+    projects = get_projects(filter=lambda _, _2, i: not i.passthrough)
+    map = {}
+    for p in projects:
+        for s in p.services:
+            prefix = f"{p.name}-" if s.image else ""
+            for i in s.ingress:
+                map[i.domain] = f"{prefix}{s.name}:{i.port}"
+    return map
 
 
 def get_passthrough_map() -> Dict[str, str]:
-    filtered = get_projects(filter=lambda _, s: s.passthrough is True)
-    return {p.domain: f"{s.name}:{s.port}" for p in filtered for s in p.services}
+    projects = get_projects(filter=lambda _, _2, i: i.passthrough)
+    map = {}
+    for p in projects:
+        for s in p.services:
+            for i in s.ingress:
+                map[i.domain] = f"{s.name}:{i.port}"
+    return map
 
 
 def write_maps() -> None:
@@ -78,9 +88,7 @@ def write_terminate() -> None:
 
 
 def write_routers() -> None:
-    projects_tcp = get_projects(
-        filter=lambda p, s: s.protocol == Protocol.tcp and (not bool(p.entrypoint) or p.entrypoint == s.name)
-    )
+    projects_tcp = get_projects(filter=lambda _, _2, i: i.protocol == Protocol.tcp)
     with open("proxy/tpl/routers-web.yml.j2", encoding="utf-8") as f:
         t = f.read()
     tpl_routers_web = Template(t)
@@ -97,10 +105,11 @@ def write_routers() -> None:
     with open("proxy/tpl/routers-tcp.yml.j2", encoding="utf-8") as f:
         t = f.read()
     tpl_routers_tcp = Template(t)
+    tpl_routers_tcp.globals["ProxyProtocol"] = ProxyProtocol
     routers_tcp = tpl_routers_tcp.render(projects=projects_tcp, traefik_rule=f"HostSNI(`{domain}`)")
     with open("proxy/traefik/routers-tcp.yml", "w", encoding="utf-8") as f:
         f.write(routers_tcp)
-    projects_udp = get_projects(filter=lambda _, s: s.protocol == Protocol.udp)
+    projects_udp = get_projects(filter=lambda _, _2, i: i.protocol == Protocol.udp)
     with open("proxy/tpl/routers-udp.yml.j2", encoding="utf-8") as f:
         t = f.read()
     tpl_routers_tcp = Template(t)
@@ -115,7 +124,7 @@ def write_config() -> None:
     tpl_config_tcp = Template(t)
     tpl_config_tcp.globals["Protocol"] = Protocol
     trusted_ips_cidrs = os.environ.get("TRUSTED_IPS_CIDRS").split(",")
-    projects_hostport = get_projects(filter=lambda _, s: bool(s.hostport))
+    projects_hostport = get_projects(filter=lambda _, _2, i: bool(i.hostport))
     config_tcp = tpl_config_tcp.render(projects=projects_hostport, trusted_ips_cidrs=trusted_ips_cidrs)
     with open("proxy/traefik/config-in.yml", "w", encoding="utf-8") as f:
         f.write(config_tcp)
@@ -141,7 +150,7 @@ def write_compose() -> None:
         t = f.read()
     tpl_compose = Template(t)
     tpl_compose.globals["Protocol"] = Protocol
-    projects_hostport = get_projects(filter=lambda _, s: bool(s.hostport))
+    projects_hostport = get_projects(filter=lambda _, _2, i: bool(i.hostport))
     compose = tpl_compose.render(projects=projects_hostport, plugin_registry=plugin_registry)
     with open("proxy/docker-compose.yml", "w", encoding="utf-8") as f:
         f.write(compose)