Replace telnet shell with http api

README.txt

@@ -74,13 +74,6 @@ configurable.
 Features
 ========
 
-Telnet shell
-------------
-
-Telnet into localhost port 6023 to get an interactive console. The console can
-currently be used to inspect the scheduler's live status.
-
-
 Self-check
 ----------
 

changelog.d/20230626_005413_jb_replace_telnet_with_http.rst

@@ -0,0 +1,5 @@
+- Replace prettytable with rich
+
+- Replace telnet shell with HTTP API
+
+- Migrate `backy check` to `backy client check` and use the new HTTP API

doc/backy.conf.example

@@ -2,6 +2,18 @@ global:
     base-dir: /my/backydir
     worker-limit: 3
     backup-completed-callback: /path/to/script.sh
+api:
+    addrs: "127.0.0.1, ::1"
+    port: 1234
+    tokens:
+        "test-token": "test-server"
+        "cli-token": "cli"
+    cli-default:
+        token: "cli-token"
+peers:
+    "test-server":
+        url: "https://example.com:1234"
+        token: "token2"
 schedules:
     default:
         daily:

doc/man-backy.rst

@@ -241,24 +241,6 @@ environment variables like **CEPH_CLUSTER** or **CEPH_ARGS**.
 **backy scheduler** processes exit cleanly on SIGTERM.
 
 
-Telnet shell
-------------
-
-The schedules opens a telnet server (default: localhost port 6023) for live
-inspection. The telnet interface accepts the following commands:
-
-jobs [REGEX]
-    Prints an overview of all configured jobs together with their last and
-    next backup run. An optional (extended) regular expression restricts output
-    to matching job names.
-
-status
-    Dumps internal server status details.
-
-quit
-    Exits the telnet shell.
-
-
 Files
 -----
 
@@ -271,7 +253,7 @@ structured key/value expression in YAML format.
 A description of top-level keys with their sub-keys follows. There is also a
 full example configuration in Section :ref:`example` below.
 
-config
+global
     Defines global scheduler options.
 
     base-dir
@@ -285,19 +267,23 @@ config
         Command/Script to invoke after the scheduler successfully completed a backup.
         The first argument is the job name. The output of `backy status --yaml` is available on stdin.
 
-    status-file
-        Path to a YAML status dump which is regularly updated by the scheduler
-        and evaluated by **backy check**. Defaults to `{base-dir}/status`.
+api
+    addrs
+        Comma-separated list of listen addresses for the api server
+        (default: 127.0.0.1, ::1).
+
+    port
+        Port number of the api server (default: 6023).
 
-    status-interval
-        Update status file every N seconds (default: 30).
+    tokens
+        A Token->Server-name mapping. Used for authenticating incoming api requests.
 
-    telnet-addrs
-        Comma-separated list of listen addresses for the telnet server
-        (default: 127.0.0.1, ::1).
+    cli-default
+        token
+            Default Token to use when issuing api requests via the `backy client` command.
 
-    telnet-port
-        Port number of the telnet server (default: 6023).
+peers
+    List of known backy servers with url and token. Currently used for synchronizing available revisions.
 
 .. _schedules:
 

lib.nix

@@ -18,9 +18,6 @@ let
       scriv = super.scriv.overrideAttrs (old: {
         buildInputs = (old.buildInputs or []) ++ [ super.setuptools ];
       });
-      telnetlib3 = super.telnetlib3.overrideAttrs (old: {
-        buildInputs = (old.buildInputs or []) ++ [ super.setuptools ];
-      });
       execnet = super.execnet.overrideAttrs (old: {
         buildInputs = (old.buildInputs or []) ++ [ super.hatchling super.hatch-vcs ];
       });
@@ -76,7 +73,7 @@ in
       src = ./.;
     } ''
       unpackPhase
-      cd *-source
+      cd $sourceRoot
       export BACKY_CMD=${poetryApplication}/bin/backy
       patchShebangs src
       pytest -vv -p no:cacheprovider --no-cov

pyproject.toml

@@ -46,18 +46,19 @@ consulate-fc-nix-test = "1.1.0a1"
 humanize = "^4.8.0"
 mmh3 = "^4.0"
 packaging = "^23.1"
-prettytable = "^3.6.0"
 python-lzo = "^1.15"
 requests = "^2.31.0"
 shortuuid = "^1.0.11"
 structlog = "^23.1.0"
-telnetlib3 = "^2.0.0"
 tzlocal = "^5.0"
 colorama = "^0.4.6"
+aiohttp = "^3.8.4"
+rich = "^13.3.2"
 
 [tool.poetry.dev-dependencies]
 pre-commit = "^3.3.3"
 pytest = "^7.4.0"
+pytest-aiohttp = "^1.0.4"
 pytest-asyncio = "^0.21.1"
 pytest-cache = "^1.0"
 pytest-cov = "^4.1.0"

setup.py

@@ -55,16 +55,16 @@ def version():
     install_requires=[
         "consulate",
         "packaging",
-        "prettytable",
         "tzlocal",
         "PyYaml",
         "setuptools",
         "shortuuid",
         "python-lzo",
-        "telnetlib3>=1.0",
         "humanize",
         "mmh3",
         "structlog",
+        "aiohttp",
+        "rich",
     ],
     extras_require={
         "test": [

src/backy/api.py

@@ -0,0 +1,146 @@
+import datetime
+import re
+from json import JSONEncoder
+from typing import Any, List, Tuple
+
+from aiohttp import hdrs, web
+from aiohttp.web_exceptions import HTTPAccepted, HTTPNotFound, HTTPUnauthorized
+from aiohttp.web_middlewares import middleware
+from aiohttp.web_runner import AppRunner, TCPSite
+from structlog.stdlib import BoundLogger
+
+import backy.daemon
+
+
+class BackyJSONEncoder(JSONEncoder):
+    def default(self, o: Any) -> Any:
+        if hasattr(o, "to_dict"):
+            return o.to_dict()
+        elif isinstance(o, datetime.datetime):
+            return o.isoformat()
+        else:
+            super().default(o)
+
+
+class BackyAPI:
+    daemon: "backy.daemon.BackyDaemon"
+    sites: dict[Tuple[str, int], TCPSite]
+    runner: AppRunner
+    tokens: dict
+    log: BoundLogger
+
+    def __init__(self, daemon, log):
+        self.log = log.bind(subsystem="api")
+        self.daemon = daemon
+        self.sites = {}
+        self.app = web.Application(
+            middlewares=[self.log_conn, self.require_auth, self.to_json]
+        )
+        self.app.add_routes(
+            [
+                web.get("/v1/status", self.get_status),
+                web.post("/v1/reload", self.reload_daemon),
+                web.get("/v1/jobs", self.get_jobs),
+                web.post("/v1/jobs/{job_name}/run", self.run_job),
+            ]
+        )
+
+    async def start(self):
+        self.runner = AppRunner(self.app)
+        await self.runner.setup()
+
+    async def stop(self):
+        await self.runner.cleanup()
+        self.sites = {}
+
+    async def reconfigure(
+        self, tokens: dict[str, str], addrs: List[str], port: int
+    ):
+        self.log.debug("reconfigure")
+        self.tokens = tokens
+        bind_addrs = [(addr, port) for addr in addrs if addr and port]
+        for bind_addr in bind_addrs:
+            if bind_addr in self.sites:
+                continue
+            self.sites[bind_addr] = site = TCPSite(
+                self.runner, bind_addr[0], bind_addr[1]
+            )
+            await site.start()
+            self.log.info("added-site", site=site.name)
+        for bind_addr, site in self.sites.items():
+            if bind_addr in bind_addrs:
+                continue
+            await site.stop()
+            del self.sites[bind_addr]
+            self.log.info("deleted-site", site=site.name)
+
+    @middleware
+    async def log_conn(self, request: web.Request, handler):
+        request["log"] = self.log.bind(
+            path=request.path, query=request.query_string
+        )
+        request["log"].debug("new-conn")
+        try:
+            resp = await handler(request)
+        except Exception as e:
+            if not isinstance(e, web.HTTPException):
+                request["log"].exception("error-handling-request")
+            else:
+                request["log"].debug(
+                    "request-result", status_code=e.status_code
+                )
+            raise
+        request["log"].debug(
+            "request-result", status_code=resp.status, response=resp.body
+        )
+        return resp
+
+    @middleware
+    async def require_auth(self, request: web.Request, handler):
+        token = request.headers.get(hdrs.AUTHORIZATION, "")
+        if not token.startswith("Bearer "):
+            request["log"].info("auth-invalid-token")
+            raise HTTPUnauthorized()
+        token = token.removeprefix("Bearer ")
+        client = self.tokens.get(token, None)
+        if not client:
+            request["log"].info("auth-token-unknown")
+            raise HTTPUnauthorized()
+        request["client"] = client
+        request["log"] = request["log"].bind(client=client)
+        request["log"].debug("auth-passed")
+        return await handler(request)
+
+    @middleware
+    async def to_json(self, request: web.Request, handler):
+        resp = await handler(request)
+        if isinstance(resp, web.Response):
+            return resp
+        elif resp is None:
+            raise web.HTTPNoContent()
+        else:
+            return web.json_response(resp, dumps=BackyJSONEncoder().encode)
+
+    async def get_status(self, request: web.Request):
+        filter = request.query.get("filter", "")
+        if filter:
+            filter = re.compile(filter)
+        return self.daemon.status(filter)
+
+    async def reload_daemon(self, request: web.Request):
+        self.daemon.reload()
+
+    async def get_jobs(self, request: web.Request):
+        return list(self.daemon.jobs.values())
+
+    async def get_job(self, request: web.Request):
+        try:
+            name = request.match_info.get("job_name", None)
+            return self.daemon.jobs[name]
+        except KeyError:
+            raise HTTPNotFound()
+
+    async def run_job(self, request: web.Request):
+        j = await self.get_job(request)
+        j.run_immediately.set()
+        raise HTTPAccepted()