mvp for a production service

.github/workflows/ci.yaml

@@ -0,0 +1,141 @@
+---
+name: Build and Publish
+
+"on":
+  workflow_dispatch:
+  push:
+    branches:
+      - "**"
+    tags:
+      - "v*.*.*"
+  pull_request:
+    branches:
+      - "main"
+      - 'master'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  flake8:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v4
+      - name: Set up Python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install dependencies
+        run: pip install .
+      - name: flake8 Lint
+        uses: py-actions/flake8@v2
+
+  hadolint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hadolint/hadolint-action@v3.1.0
+        with:
+          dockerfile: Dockerfile
+
+  yamllint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run yamllint
+        uses: bewuethr/yamllint-action@v1
+
+  oci_image:
+    name: Build OCI Image
+    if: github.repository == 'lsst-it/gnocpush'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  helm-lint:
+    runs-on: ubuntu-latest
+    name: helm lint
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Run helm lint
+        run: cd charts/gnocpush && helm lint .
+
+
+  chart-release:
+    # only run when merged to main
+    if: github.ref == 'refs/heads/main'
+    name: Helm Chart Release
+    needs:
+      - oci_image
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.6.0
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.yamllint.yaml

@@ -0,0 +1,18 @@
+---
+extends: default
+
+ignore: |
+  charts/gnocpush/templates/*
+
+rules:
+  # 80 chars should be enough, but don't fail if a line is longer
+  line-length: false
+  # do not obsess over comment formatting
+  comments-indentation: false
+  comments:
+    require-starting-space: false
+  indentation:
+    spaces: consistent
+    indent-sequences: consistent
+  # ignore {{ .foo }} go templates
+  braces: false

Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.12
+
+WORKDIR /app
+COPY . .
+RUN pip install --no-cache-dir .
+ENTRYPOINT ["gnocpush"]

README.md

@@ -1,14 +1,181 @@
 # gnocpush
 
+A simple service to accept webhook payloads from [Prometheus Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) and to push those alerts on to [GlobalNOC's Alertmon](https://alertmon-stage.grnoc.iu.edu/alertmon2/).
+
+## Alert format
+
+`gnocpush` expects that alerts have labels taht match GlocalNOC's required parameter names.
+
+Note that group annotations/labels from the alert group are ignored.
+
+### Required annotations
+
+* `description` - A description of the alert.
+
+### Required labels
+
+* `node_name` - The name of the node that the alert is associated with.
+* `service_name` - The name of the service that the alert is associated with.
+* `severity` - The severity of the alert. One of: `Critical`, `Major`, `Minor`, `Unknown`, `OK`
+
+### Optional labels
+
+* `device` - The subcomponent of the node that is alarming.
+* `start_time` - The time that the alert started.
+
+### Example PrometheusRule
+
+```yaml
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    lsst.io/rule: "true"
+  name: net
+spec:
+  groups:
+    - name: net.rules
+      rules:
+        - alert: lhn_interface_up
+          annotations:
+            description: '{{ $labels.instance }} - {{ $labels.ifName }}|{{ $labels.ifAlias }} is down'
+          expr: ifOperStatus{ifAlias=~".*LHN.*"} != 1
+          for: 30s
+          labels:
+            severity: critical
+            node_name: '{{ $labels.instance }}'
+            device: '{{ $labels.ifName }}'
+            service_name: ifInErrors-{{ $labels.ifName}}
+            gnoc: "true"
+```
+
+
+### Example Payload
+
+```json
+{
+  "receiver": "gnocpush",
+  "status": "firing",
+  "alerts": [
+    {
+      "status": "firing",
+      "labels": {
+        "alertname": "ifInErrors",
+        "device": "Ethernet17/1",
+        "gnoc": "true",
+        "ifAlias": "rubinobs-br01 Et17/1 <--SCIENCE #1--> LS-DWDM Linecard001-Port2",
+        "ifDescr": "Ethernet17/1",
+        "ifIndex": "17001",
+        "ifName": "Ethernet17/1",
+        "instance": "new-rubinobs-br01",
+        "job": "snmp-network",
+        "node_name": "new-rubinobs-br01",
+        "prom": "dev/ruka",
+        "prometheus": "kube-prometheus-stack/kube-prometheus-stack-prometheus",
+        "service_name": "ifInErrors-Ethernet17/1",
+        "severity": "major",
+        "site": "dev"
+      },
+      "annotations": {
+        "description": "new-rubinobs-br01 - Ethernet17/1|rubinobs-br01 Et17/1 <--SCIENCE #1--> LS-DWDM Linecard001-Port2 has 12.2k input errors"
+      },
+      "startsAt": "2024-04-26T20:46:34.933Z",
+      "endsAt": "0001-01-01T00:00:00Z",
+      "generatorURL": "https://prometheus.example.org/graph?g0.expr=ifInErrors+%3E+1000&g0.tab=1",
+      "fingerprint": "46df8c14dbab758c"
+    }
+  ],
+  "groupLabels": {
+    "gnoc": "true"
+  },
+  "commonLabels": {
+    "gnoc": "true",
+    "job": "snmp-network",
+    "prom": "dev/ruka",
+    "prometheus": "kube-prometheus-stack/kube-prometheus-stack-prometheus",
+    "site": "dev"
+  },
+  "commonAnnotations": {},
+  "externalURL": "https://alertmanager.example.org",
+  "version": "4",
+  "groupKey": "{}/{gnoc=\"true\"}:{gnoc=\"true\"}",
+  "truncatedAlerts": 0
+}
+```
+
+## Alertmanager Configuration
+
+Note that `gnocpush` does not impose any alert grouping constraints.
+
+```yaml
+    config:
+      routes:
+        - receiver: gnocpush
+          continue: true
+          repeat_interval: 30s
+          group_interval: 30s
+          group_wait: 30s
+          group_by:
+            - gnoc
+          matchers:
+            - gnoc = "true"
+    receivers:
+      - name: gnocpush
+        webhook_configs:
+          - url: http://gnocpush.gnocpush:8080/alerts
+```
+
+## Deployment on Kubernetes
+
+```bash
+helm upgrade --install \
+  gnocpush ./charts/gnocpush \
+  --create-namespace --namespace gnocpush \
+  -f ./values.yaml
+```
+
+### Debugging a Kubernetes Deployment
+
+```bash
+k logs alertmanager-kube-prometheus-stack-alertmanager-0 --tail=100 -f
+
+k logs -l app.kubernetes.io/instance=gnocpush -f
+```
+
+```bash
+k -n gnocpush port-forward gnocpush-dc4d94d8-mqvqq 8080
+$ curl localhost:8080/metrics
+```
+
 ## Development
 
+### Local Development
+
 ```bash
 virtualenv venv
 . venv/bin/activate
 pip install --editable .
 ```
 
-## URLs
+### Testing with the OCI image
+
+```bash
+docker run \
+  -e GNOC_USERNAME=$GNOC_USERNAME \
+  -e GNOC_PASSWORD=$GNOC_PASSWORD \
+  -e GNOC_SERVER=$GNOC_SERVER \
+  -e GNOC_REALM=$GNOC_REALM \
+  --network=host ghcr.io/lsst-it/gnocpush
+```
+
+### Testing gnocpush with curl
+
+```bash
+curl http://localhost:8080/alerts -v --json @- < alerts.json
+```
+
+## Useful GlobalNOC URLs
 
 ### Stage
 

charts/gnocpush/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/gnocpush/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: gnocpush
+description: Push Prometheus Alertmanager webhook payloads to Global NOC
+type: application
+version: 0.1.0
+appVersion: 0.1.0