feat: enable configurable namespaces, expose more options in helm chart, add stable tag, fix v2 API routing (#36)

* ci: tag stable release

* chore: refactor helm chart

* feat: add option for tolerations

* feat: make namespace configurable

* fix: correct syntax issues

* fix: address route registration issue

* feat: auto-detect pod namespace

* refactor: update var name

* fix: address if check

* tests: add root handler tests

* ci: reduce diff coverage target

Author: avtakkar

.github/.codecov.yml

@@ -11,7 +11,7 @@ coverage:
         target: 65%
     patch:
       default:
-        target: 65%
+        target: 55%
 
 ignore:
   - tests/**/*

.github/workflows/release.yml

@@ -107,3 +107,33 @@ jobs:
         run: |
           PEERD_IMAGE_TAG=${{ env.TAG }} make tests-deps-install ci-aks-streaming
 
+  tag:
+    name: Tag Release
+    runs-on: ubuntu-latest
+    needs: [publish, ciCtr, ciStreaming]
+    permissions:
+      contents: read
+      packages: write
+    env:
+      REGISTRY: ghcr.io
+      REPO_PREFIX: ${{ format('azure/acr/dev/') }}
+      SOURCE_TAG: ${{ needs.publish.outputs.git_tag }}
+      TARGET_TAG: stable
+    steps:
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull Tag ${{ env.SOURCE_TAG }}
+        if: ${{ success() }}
+        run: |
+          docker pull ${{ env.REGISTRY }}/${{ env.REPO_PREFIX }}peerd:${{ env.SOURCE_TAG }}
+
+      - name: Push Tag ${{ env.TARGET_TAG }}
+        if: ${{ success() }}
+        run: |
+          docker tag ${{ env.REGISTRY }}/${{ env.REPO_PREFIX }}peerd:${{ env.SOURCE_TAG }} ${{ env.REGISTRY }}/${{ env.REPO_PREFIX }}peerd:${{ env.TARGET_TAG }} && \
+            docker push ${{ env.REGISTRY }}/${{ env.REPO_PREFIX }}peerd:${{ env.TARGET_TAG }}

README.md

@@ -19,6 +19,33 @@ in a Kubernetes cluster. The source of the content could be another node in the
 This is **work in progress** and not yet production ready. We are actively working on this project and would love to
 hear your feedback. Please feel free to open an issue or a pull request.
 
+## Features
+
+* **Peer to Peer File Sharing**: Peerd allows a node to act as a mirror for files obtained from any HTTP upstream source
+  (such as an [Azure Blob] using a [SAS URL]), and can discover and serve a specified byte range of the file to/from
+  other nodes in the cluster. Peerd will first attempt to discover and serve this range from its peers. If not found, it
+  will  fallback to download the range from the upstream URL. Peerd caches downloaded ranges as well as optionally, can
+  prefetch the entire file.
+
+  With this facility, `peerd` can be used as the [p2p proxy] for [Overlaybd].
+
+  ```json
+  "p2pConfig": {
+    "enable": true,
+    "address": "localhost:30000/blobs"
+  }
+  ```
+
+* **Peer to Peer Container Image Sharing**: Pulling a container image to a node in Kubernetes is often a time consuming
+  process, especially in scenarios where the registry becomes a bottleneck, such as deploying a large cluster or scaling
+  out in response to bursty traffic. To increase throughput, nodes in the cluster which already have the image can be
+  used as an alternate image source. Peerd subscribes to events in the containerd content store, and advertises local
+  images to peers. When a node needs an image, it can query its peers for the image, and download it from them instead
+  of the registry. Containerd has a [mirror][containerd hosts] facility that can be used to configure Peerd as the 
+  mirror for container images.
+
+The APIs are described in the [swagger.yaml].
+
 ## Quickstart
 
 To see all available commands, run `make help`.
@@ -31,19 +58,17 @@ container images.
 
 ```bash
 CLUSTER_CONTEXT=<your-cluster-context> && \
-  TAG=<docker-image-tag> && \
   helm --kube-context=$CLUSTER_CONTEXT install --wait peerd ./build/package/peerd-helm \
-    --set peerd.image.ref=ghcr.io/azure/acr/dev/peerd:$TAG
+    --set peerd.image.ref=ghcr.io/azure/acr/dev/peerd:stable
 ```
 
-By default, `mcr.microsoft.com` and `ghcr.io` are mirrored, but this is configurable. For example, to mirror `docker.io`
-as well, run the following.
+By default, some well known registries are mirrored (see [values.yml]), but this is configurable. For example, to mirror
+`docker.io`, `mcr.microsoft.com` and `ghcr.io`, run the following.
 
 ```bash
 CLUSTER_CONTEXT=<your-cluster-context> && \
-  TAG=<docker-image-tag> && \
   helm --kube-context=$CLUSTER_CONTEXT install --wait peerd ./build/package/peerd-helm \
-    --set peerd.image.ref=ghcr.io/azure/acr/dev/peerd:$TAG
+    --set peerd.image.ref=ghcr.io/azure/acr/dev/peerd:stable
     --set peerd.hosts="mcr.microsoft.com ghcr.io docker.io"
 ```
 
@@ -63,39 +88,15 @@ kubectl --context=$CLUSTER_CONTEXT -n peerd-ns logs -l app=peerd -f
 
 ### Observe Metrics
 
-Peerd exposes metrics on the `/metrics/prometheus` endpoint. Mmetrics are prefixed with `peerd_`. `libp2p` metrics are
+Peerd exposes metrics on the `/metrics/prometheus` endpoint. Metrics are prefixed with `peerd_`. `libp2p` metrics are
 prefixed with `libp2p_`.
 
-#### Examples on a 5 node AKS cluster, node sizes: `Standard_D2s_v3` and `Standard_D8ds_v5`
+#### Example
 
-<img src="./assets/images/peer-metrics.png" alt="peer metrics" width="1000">
-
-## Features
-
-* **Peer to Peer File Sharing**: Peerd allows a node to act as a mirror for files obtained from any HTTP upstream source
-  (such as an [Azure Blob] using a [SAS URL]), and can discover and serve a specified byte range of the file to/from
-  other nodes in the cluster. Peerd will first attempt to discover and serve this range from its peers. If not found, it
-  will  fallback to download the range from the upstream URL. Peerd caches downloaded ranges as well as optionally, can
-  prefetch the entire file.
+On a 100 nodes AKS cluster of VM size `Standard_D2s_v3`, sample throughput observed by a single pod is shown below.
 
-  With this facility, `peerd` can be used as the [p2p proxy] for [Overlaybd].
 
-  ```json
-  "p2pConfig": {
-    "enable": true,
-    "address": "localhost:30000/blobs"
-  }
-  ```
-
-* **Peer to Peer Container Image Sharing**: Pulling a container image to a node in Kubernetes is often a time consuming
-  process, especially in scenarios where the registry becomes a bottleneck, such as deploying a large cluster or scaling
-  out in response to bursty traffic. To increase throughput, nodes in the cluster which already have the image can be
-  used as an alternate image source. Peerd subscribes to events in the containerd content store, and advertises local
-  images to peers. When a node needs an image, it can query its peers for the image, and download it from them instead
-  of the registry. Containerd has a [mirror][containerd hosts] facility that can be used to configure Peerd as the 
-  mirror for container images.
-
-The APIs are described in the [swagger.yaml].
+<img src="./assets/images/peer-metrics.png" alt="peer metrics" width="1000">
 
 ## Build
 
@@ -155,3 +156,4 @@ integration with [Overlaybd].
 [release-tag]: https://img.shields.io/github/v/tag/Azure/peerd?label=Docker%20Image%20Tag
 [peerd-pkgs]: https://github.com/Azure/peerd/pkgs/container/acr%2Fdev%2Fpeerd
 [build.md]: ./docs/build.md
+[values.yml]: ./build/package/peerd-helm/values.yaml

assets/images/peer-metrics.png

@@ -52,7 +52,7 @@ ifndef NODEPOOL
 	$(eval NODEPOOL := $(shell date +"prel%y%m%d"))
 endif
 ifndef PEERD_IMAGE_TAG
-	$(eval PEERD_IMAGE_TAG := "dev")
+	$(eval PEERD_IMAGE_TAG := "stable")
 endif
 	@echo "\033[92mRunning CI NODEPOOL: $(NODEPOOL)\033[0m"
 	@( PEERD_IMAGE_TAG=$(PEERD_IMAGE_TAG) PEERD_CONFIGURE_MIRRORS=true $(SCRIPTS_DIR)/azure.sh nodepool up -y $(NODEPOOL) )
@@ -66,7 +66,7 @@ ifndef NODEPOOL
 	$(eval NODEPOOL := $(shell date +"prels%y%m%d"))
 endif
 ifndef PEERD_IMAGE_TAG
-	$(eval PEERD_IMAGE_TAG := "dev")
+	$(eval PEERD_IMAGE_TAG := "stable")
 endif
 	@echo "\033[92mRunning CI NODEPOOL: $(NODEPOOL)\033[0m"
 	@( PEERD_IMAGE_TAG=$(PEERD_IMAGE_TAG) PEERD_CONFIGURE_MIRRORS=false $(SCRIPTS_DIR)/azure.sh nodepool up -y $(NODEPOOL) )

build/ci/Makefile

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.0.2-alpha
+version: 0.0.3-alpha
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.0.5-alpha"
+appVersion: "0.0.6-alpha"

build/package/peerd-helm/Chart.yaml

@@ -6,8 +6,12 @@ Expand the name of the chart.
 {{- end }}
 
 {{- define "peerd.namespace" -}}
+{{- if .Values.peerd.namespace.k8s }}
+{{- .Values.peerd.namespace.k8s }}
+{{- else }}
 {{ include "peerd.name" . }}-ns
 {{- end }}
+{{- end }}
 
 {{- define "peerd.serviceAccountName" -}}
 {{ include "peerd.name" . }}-sa

build/package/peerd-helm/templates/_helpers.tpl

@@ -0,0 +1,25 @@
+{{ if .Values.peerd.metrics.prometheus.aksAutoDiscovery }}
+kind: ConfigMap
+apiVersion: v1
+data:
+  prometheus-config: |-
+    global:
+      scrape_interval: 15s
+    scrape_configs:
+    - job_name: peerd
+      kubernetes_sd_configs:
+        - role: pod
+      relabel_configs:
+        - source_labels: [__meta_kubernetes_pod_label_app]
+          action: keep
+          regex: peerd
+        - source_labels: [__meta_kubernetes_pod_container_port_number]
+          action: keep
+          regex: "5004"
+      metrics_path: /metrics/prometheus
+metadata:
+  name: ama-metrics-prometheus-config
+  namespace: kube-system
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
+{{ end }}

build/package/peerd-helm/templates/ama-metrics-prometheus-config.yml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: node-reader-and-event-creator-clusterrole
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch", "update"]

build/package/peerd-helm/templates/events-clusterrole.yml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: node-reader-and-event-creator-clusterrole-binding
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "peerd.serviceAccountName" . }}
+  namespace: {{ include "peerd.namespace" . }}
+roleRef:
+  kind: ClusterRole
+  name: node-reader-and-event-creator-clusterrole
+  apiGroup: rbac.authorization.k8s.io

build/package/peerd-helm/templates/events-clusterrolebinding.yml

@@ -1,45 +1,16 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: {{ include "peerd.namespace" . }}
-  labels:
-    {{- include "peerd.labels" . | nindent 4 }}
----
-kind: ConfigMap
-apiVersion: v1
-data:
-  prometheus-config: |-
-    global:
-      scrape_interval: 15s
-    scrape_configs:
-    - job_name: peerd
-      kubernetes_sd_configs:
-        - role: pod
-      relabel_configs:
-        - source_labels: [__meta_kubernetes_pod_label_app]
-          action: keep
-          regex: peerd
-        - source_labels: [__meta_kubernetes_pod_container_port_number]
-          action: keep
-          regex: "5004"
-      metrics_path: /metrics/prometheus
-metadata:
-  name: ama-metrics-prometheus-config
-  namespace: kube-system
-  labels:
-    {{- include "peerd.labels" . | nindent 4 }}
----
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: &name {{ include "peerd.name" . }}
   namespace: {{ include "peerd.namespace" . }}
   labels:
     {{- include "peerd.labels" . | nindent 4 }}
+{{- if .Values.peerd.metrics.prometheus.aksAutoDiscovery }}
   annotations:
     prometheus.io/scrape: 'true'
     prometheus.io/path: '/metrics/prometheus'
     prometheus.io/port: '5004'
+{{- end }}
 spec:
   selector:
     matchLabels:
@@ -54,7 +25,7 @@ spec:
         - image: "{{ .Values.peerd.image.ref }}"
           imagePullPolicy: "{{ .Values.peerd.image.pullPolicy }}"
           args:
-            - "--log-level=debug"
+            - "--log-level={{ .Values.peerd.logLevel }}"
             - "run"
             - "--http-addr=0.0.0.0:5000"
             - "--add-mirror-configuration={{ .Values.peerd.configureMirrors }}"
@@ -95,35 +66,7 @@ spec:
           hostPath:
             path: /etc/containerd/certs.d
             type: DirectoryOrCreate
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: &name {{ include "peerd.name" . }}
-  namespace: {{ include "peerd.namespace" . }}
-  labels:
-    {{- include "peerd.labels" . | nindent 4 }}
-  annotations:
-    prometheus.io/scrape: 'true'
-    prometheus.io/path: '/metrics/prometheus'
-    prometheus.io/port: '30004'
-spec:
-  type: NodePort
-  selector:
-    app: *name
-  ports:
-    - name: http
-      protocol: TCP
-      port: 5000
-      nodePort: 30000
-      targetPort: http
-    - name: https
-      protocol: TCP
-      port: 5001
-      nodePort: 30001
-      targetPort: https
-    - name: metrics
-      protocol: TCP
-      port: 5004
-      nodePort: 30004
-      targetPort: metrics
+      {{- with .Values.peerd.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

build/package/peerd-helm/templates/app.yml renamed to build/package/peerd-helm/templates/peerd-ds.yml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "peerd.name" . }}-manager-role-binding
+  namespace: {{ include "peerd.namespace" . }}
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "peerd.name" . }}-manager-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "peerd.serviceAccountName" . }}
+    namespace: {{ include "peerd.namespace" . }}

build/package/peerd-helm/templates/peerd-manager-role-binding.yml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "peerd.name" . }}-manager-role
+  namespace: {{ include "peerd.namespace" . }}
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["configmaps"]
+    verbs: ["get", "watch", "list", "create", "update"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "watch", "list"]

build/package/peerd-helm/templates/peerd-manager-role.yml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ include "peerd.namespace" . }}
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}

build/package/peerd-helm/templates/peerd-ns.yml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "peerd.serviceAccountName" . }}-secret
+  namespace: {{ include "peerd.namespace" . }}
+  labels:
+    {{- include "peerd.labels" . | nindent 4 }}
+  annotations:
+    kubernetes.io/service-account.name: {{ include "peerd.serviceAccountName" . }}
+type: kubernetes.io/service-account-token