chore: release

.github/workflows/docker-images.yaml

@@ -0,0 +1,59 @@
+name: docker-images
+
+on:
+  push:
+    paths:
+      - '.github/workflows/**'
+      - 'caddy/**'
+
+jobs:
+  main:
+    name: ${{ matrix.services }} service - ${{ matrix.environment }}
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        services: [caddy]
+        environment: [development, staging, production]
+        include:
+          - environment: development
+            redis_db: 0
+            api_endpoint: api.example.com
+          - environment: staging
+            redis_db: 1
+            api_endpoint: api.example.com
+          - environment: production
+            redis_db: 2
+            api_endpoint: api.example.com
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push cdn-${{ matrix.services }}
+        uses: docker/build-push-action@v3
+        with:
+          context: ./${{ matrix.services }}
+          file: ./${{ matrix.services }}/Dockerfile
+          platforms: linux/arm64
+          push: true
+          build-args: |
+            environment=${{ matrix.environment }}
+            redis_db=${{ matrix.redis_db }}
+            api_endpoint=${{ matrix.api_endpoint }}
+          tags: |
+            ghcr.io/storipress/cdn-${{ matrix.services }}:${{ matrix.environment }}

.gitignore

@@ -0,0 +1 @@
+/.idea

README.md

@@ -0,0 +1,13 @@
+# Storipress CDN Server
+
+![Docker](https://github.com/storipress/cdn-public/workflows/docker-images/badge.svg)
+
+## Server Setup
+
+1. clone this repo or copy `docker-compose.yml` and `server-setup.sh` files to target server.
+
+2. ensure `server-setup.sh` has executable permission.
+
+3. execute `server-setup.sh` script.
+
+4. done!

caddy/Dockerfile

@@ -0,0 +1,22 @@
+FROM ghcr.io/storipress/caddy:latest
+
+ARG redis_db
+ENV REDIS_DB=$redis_db
+ARG api_endpoint
+ENV API_ENDPOINT=$api_endpoint
+ARG environment
+ENV ENVIRONMENT=$environment
+
+WORKDIR /usr/local/caddy
+
+COPY src /usr/local/caddy
+COPY listener /usr/local/listener
+COPY supervisord.conf /etc/supervisord.conf
+
+RUN mkdir -p /usr/local/caddy/files && mkdir -p /usr/local/caddy/locks
+
+EXPOSE 80
+EXPOSE 443
+EXPOSE 2019
+
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]

caddy/listener/main.py

@@ -0,0 +1,32 @@
+import argparse
+import os
+import redis
+import sentry_sdk
+from utils import listen, download
+
+if __name__ == "__main__":
+    sentry_sdk.init(
+        dsn="SENTRY_DSN",
+        environment=os.getenv('ENVIRONMENT'),
+        traces_sample_rate=1.0
+    )
+
+    parser = argparse.ArgumentParser(description='redis handler.')
+    parser.add_argument('action', help='listen or download')
+
+    args = parser.parse_args()
+
+    # connect to redis
+    r = redis.StrictRedis(
+        host='redis.example.com',
+        db=int(os.getenv('REDIS_DB')),
+        socket_connect_timeout=5,
+        retry_on_timeout=True
+    )
+
+    if args.action == 'listen':
+        listen(r)
+    elif args.action == 'download':
+        download(r)
+    else:
+        print('[Debug] invalid argument', flush=True)

caddy/listener/utils.py

@@ -0,0 +1,194 @@
+import json
+import os
+from sentry_sdk import capture_message, push_scope
+
+caddyPath = '/usr/local/caddy'
+caddyFilesPath = '/usr/local/caddy/files'
+fileLockPath = '/usr/local/caddy/locks'
+
+caddyChannel = 'caddy_cdn'
+
+
+def caddy_reload():
+    # reload the caddy services
+    os.system('caddy reload --config ' + caddyPath + '/Caddyfile')
+
+
+def get_meta_key(tenant):
+    return 'cdn_meta_' + tenant
+
+
+def write_caddy_file(tenant, meta):
+    content = make_custom_content(meta['custom']['domain'], meta['reverse_path'], meta['custom']['redirect_domain'])
+
+    filename = caddyFilesPath + '/' + tenant
+
+    with open(filename, "w") as output:
+        output.write(content)
+
+
+def remove_caddy_file(tenant):
+    filename = caddyFilesPath + '/' + tenant
+    if os.path.exists(filename):
+        os.remove(filename)
+
+
+def make_custom_content(domain, reverse, redirect):
+    template_path = caddyPath + '/custom.caddy'
+    with open(template_path, 'r') as file:
+        content = file.read()
+
+    content = content.replace('REVERSE_PATH', reverse)
+    content = content.replace('DOMAIN', domain)
+
+    # contains redirect domain
+    if redirect == '':
+        return content
+
+    redirect_template_path = caddyPath + '/redirect.caddy'
+    with open(redirect_template_path, 'r') as file:
+        redirect_content = file.read()
+
+    redirect_content = redirect_content.replace('DOMAIN', redirect)
+    redirect_content = redirect_content.replace('REDIRECT', domain)
+    content = redirect_content + content
+
+    return content
+
+
+def get_meta_data(redis, tenant):
+    meta_key = get_meta_key(tenant)
+    message = redis.get(meta_key)
+
+    # publication meta not exists or domain not exists
+    if message is None:
+        sentry_capture('invalid meta data', {'content': 'none', 'key': meta_key})
+        return None
+
+    try:
+        meta = json.loads(message.decode())
+    except:
+        sentry_capture('invalid meta data', {'content': message, 'key': meta_key})
+        return None
+
+    return meta
+
+
+def sentry_capture(message, args):
+    with push_scope() as scope:
+        for key in args:
+            scope.set_extra(key, args[key])
+        capture_message(message)
+
+
+def terminate(tenant):
+    remove_caddy_file(tenant)
+    print("[Debug] terminate %s" % (tenant), flush=True)
+
+
+def sync(tenant, meta):
+    filename = fileLockPath + '/' + tenant
+    if os.path.exists(filename):
+        with open(filename, "r") as content:
+            timestamp = content.read()
+
+        if timestamp >= str(meta['timestamp']):
+            return
+
+    write_caddy_file(tenant, meta)
+
+    print("[Debug] sync %s %s" % (tenant, meta['timestamp']), flush=True)
+
+    with open(filename, "w") as output:
+        output.write(str(meta['timestamp']))
+
+
+def listen(redis):
+    # get redis pub/sub
+    sub = redis.pubsub()
+
+    # compatible for cdn_caddy
+    sub.subscribe(['cdn_caddy', 'cdn_caddy_' + os.getenv('ENVIRONMENT')])
+
+    try:
+        # listen redis caddy channel
+        for message in sub.listen():
+            # ignore the first message (subscribe message)
+            if message['type'] == 'subscribe':
+                continue
+
+            if not isinstance(message.get('data'), bytes):
+                continue
+
+            payload = json.loads(message['data'].decode())
+
+            if 'event' not in payload or 'tenant' not in payload:
+                sentry_capture('invalid payload', {'message': message})
+                continue
+
+            event = payload['event']
+            tenant = payload['tenant']
+
+            if event == 'terminate':
+                terminate(tenant)
+            # update caddyfile config
+            elif event == 'sync':
+                # get meta key
+                meta = get_meta_data(redis, tenant)
+
+                if meta is None:
+                    continue
+
+                if 'custom' not in meta:
+                    terminate(tenant)
+                    continue
+
+                sync(tenant, meta)
+            else:
+                sentry_capture('invalid event', {'event': event, 'payload': payload, 'tenant': tenant})
+                continue
+
+            # reload caddy
+            caddy_reload()
+    except KeyboardInterrupt:
+        redis.close()
+
+
+def download(redis):
+    print('[Debug] download start', flush=True)
+
+    keys = []
+    for key in redis.scan_iter(match='cdn_meta_*', count=100):
+        keys.append(key.decode())
+
+    for i in range(0, len(keys), 50):
+        group_keys = keys[i:i + 50]
+        contents = redis.mget(group_keys)
+
+        for idx, content in enumerate(contents):
+            key = group_keys[idx]
+            pattern = key.split('_')
+
+            if len(pattern) != 3:
+                sentry_capture('unknown key', {'key': key})
+                continue
+
+            try:
+                meta = json.loads(content.decode())
+            except:
+                sentry_capture('invalid meta data', {'content': content, 'key': key})
+                continue
+
+            tenant = pattern[2]
+
+            if 'custom' not in meta:
+                continue
+
+            sync(tenant, meta)
+
+    redis.close()
+
+    # reload caddy
+    caddy_reload()
+
+    print('[Debug] download end', flush=True)

caddy/src/Caddyfile

@@ -0,0 +1,24 @@
+{
+	admin 0.0.0.0:2019 {
+	}
+
+	servers {
+		metrics
+	}
+
+	storage redis {
+		host "redis.example.com"
+		db {$REDIS_DB}
+		key_prefix "caddytls"
+		value_prefix "caddy-storage-redis"
+		timeout 5
+		tls_enabled "false"
+		tls_insecure "true"
+	}
+
+	on_demand_tls {
+		ask "https://{$API_ENDPOINT}/caddy/on-demand-ask"
+	}
+}
+
+import ./files/*

caddy/src/custom.caddy

@@ -0,0 +1,35 @@
+DOMAIN {
+    tls {
+        on_demand
+
+        issuer acme {
+            disable_tlsalpn_challenge
+        }
+    }
+
+    header Server "storipress"
+
+    @v1 {
+        expression path_regexp('^/404$') != true
+        expression path_regexp('(?:\\.(?:html|css|js|webp|jpe?g|png|ico|svg|gif))$') != true
+        expression host('example.com') == true
+    }
+
+    @v2 `!host('example.com')`
+
+    uri @v2 strip_suffix /
+
+    uri @v1 path_regexp (\/[^\.\/]+?)$ $1/
+
+    reverse_proxy * REVERSE_PATH {
+        header_up Host {http.reverse_proxy.upstream.hostport}
+        header_down -Server
+
+        @error status 522
+        handle_response @error {
+            root    * /usr/local/caddy/pages
+            rewrite * /404.html
+            file_server
+        }
+    }
+}