Lab 8, bonus task

app_go/Dockerfile

@@ -6,6 +6,7 @@ COPY go.mod *.go /usr/src/app/
 # No dependencies yet
 #RUN go mod download && go mod verify
 
+RUN ["env", "CGO_ENABLED=0", "go", "get"]
 RUN ["env", "CGO_ENABLED=0", "go", "build", "-o", "catfact_webapp", "."]
 
 

app_go/go.mod

@@ -1,3 +1,15 @@
 module catfact_webapp
 
 go 1.21.6
+
+require github.com/prometheus/client_golang v1.19.0
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
+)

app_go/go.sum

@@ -0,0 +1,16 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=

app_go/main.go

@@ -4,9 +4,14 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
-func handler(w http.ResponseWriter, r *http.Request) {
+func index(w http.ResponseWriter, r *http.Request) {
 	fact, err := catFact()
 	if err == nil {
         w.WriteHeader(http.StatusOK)
@@ -17,9 +22,55 @@ func handler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+
+var (
+	reqCnt = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "go_requests_count",
+		Help: "Number of HTTP requests",
+	})
+
+	reqHandleTime = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name: "go_request_handle_time",
+		Help: "Time to handle a request",
+	})
+)
+
+func noteTimeMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		defer func() {
+			var dtSec float64 = time.Since(start).Seconds()
+			reqCnt.Inc()
+			reqHandleTime.Observe(dtSec)
+		}()
+
+		next.ServeHTTP(w, r)
+	})
+}
+
+
 func main() {
-	http.HandleFunc("/", handler)
+	businessLogic := http.NewServeMux()
+	businessLogic.Handle("/", asHandler(index))
+	// Note: keeping /metrics under middleware too for consistency with app_py
+	businessLogic.Handle("/metrics", promhttp.Handler())
+
+	wrapped := noteTimeMiddleware(businessLogic)
+
 	hostPort := "0.0.0.0:5000"
 	_, _ = fmt.Println("Listening on http://" + hostPort)
-	log.Fatal(http.ListenAndServe(hostPort, nil))
+	log.Fatal(http.ListenAndServe(hostPort, wrapped))
+}
+
+
+type dummyHandler struct {
+	handlerFunc func (http.ResponseWriter, *http.Request)
+}
+
+func (h dummyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	h.handlerFunc(w, r)
+}
+
+func asHandler(handlerFunc func (http.ResponseWriter, *http.Request)) dummyHandler {
+	return dummyHandler{handlerFunc: handlerFunc}
 }

app_go/main_test.go

@@ -13,7 +13,7 @@ import (
 func TestFactLoads(t *testing.T) {
     w := httptest.NewRecorder()
 
-    handler(w, nil)
+    index(w, nil)
     resp := w.Result()
 
     if resp.StatusCode != http.StatusOK {

app_python/moscow_time/__init__.py

@@ -1,14 +1,31 @@
 import datetime
+from time import monotonic
 
-from flask import Flask
+from flask import Flask, request, Response
 import requests
+import prometheus_client
 
 from .cache import cache_for
 
 
 app = Flask(__name__)
 
 
+REQUEST_COUNT = prometheus_client.Counter('py_requests_count', 'Number of HTTP requests')
+REQUEST_HANDLE_TIME = prometheus_client.Histogram('py_request_handle_time', 'Time to handle a request')
+
+@app.before_request
+def note_request_start_time():
+    request.start_time = monotonic()
+
+@app.after_request
+def update_prometheus(response):
+    handle_time = monotonic() - request.start_time
+    REQUEST_COUNT.inc()
+    REQUEST_HANDLE_TIME.observe(handle_time)
+    return response
+
+
 # In case of high load, to avoid frequent requests, cache results for
 # one second
 @cache_for(1000)
@@ -30,3 +47,7 @@ def index():
     time = get_time()
     return f"In MSK it's {time.hour}:{time.minute}:{time.second}. " \
         "Have you brushed your teeth today yet?"
+
+@app.route('/metrics')
+def prometheus_metrics():
+    return Response(prometheus_client.generate_latest(), mimetype='text/plain')

app_python/requirements.txt

@@ -1,2 +1,3 @@
 Flask~=3.0.0
 requests~=2.31.0
+prometheus_client~=0.20.0

monitoring/METRICS.md

@@ -11,4 +11,16 @@
 ## Enhancements
 
 Grafana, Loki, and Promtail each have the RAM limit of 100MiB, Prometheus's memory is limited
-to 50MiB, limits for app_py and app_go are 30MiB and 6MiB respectively.
+to 50MiB, limits for app_py and app_go are 30MiB and 20MiB respectively.
+
+## Metrics from web apps
+
+The Python app exports metrics corresponding to the web app (requests count, request handle
+time), as well as metrics related to python runtime exposed by the `promteheus_client`, for instance:
+
+![Prometheus, python web_app](pics/prometheus_app_py_sample.png)
+
+The Go app exports metrics corresponding to the web app (requests count, request handle time),
+as well as metrics related to go runtime exposed by the prometheus client, for instance:
+
+![Prometheus, go web_app](pics/prometheus_app_go_sample.png)

monitoring/docker-compose.yml

@@ -7,24 +7,28 @@ volumes:
 
 services:
   app_py:
-    image: kolay0ne/app_py:lab6
+    image: kolay0ne/app_py:lab8
     ports:
       - "5000:5000"
     logging:
       options:
         max-size: 5m
     deploy:
       resources: {limits: {memory: 30M}}
+    networks:
+      - prometheus
 
   app_go:
-    image: kolay0ne/app_go:lab6
+    image: kolay0ne/app_go:lab8
     ports:
       - "5500:5000"
     logging:
       options:
         max-size: 5m
     deploy:
-      resources: {limits: {memory: 6M}}
+      resources: {limits: {memory: 20M}}
+    networks:
+      - prometheus
 
   loki:
     image: grafana/loki:2.9.2

monitoring/prometheus_config.yml

@@ -16,3 +16,11 @@ scrape_configs:
   - job_name: grafana
     static_configs:
       - targets: ['grafana:3000']
+
+  - job_name: app_py
+    static_configs:
+      - targets: ['app_py:5000']
+
+  - job_name: app_go
+    static_configs:
+      - targets: ['app_go:5000']