diff --git a/app_go/Dockerfile b/app_go/Dockerfile index 01c76e9961..b202a0b57f 100644 --- a/app_go/Dockerfile +++ b/app_go/Dockerfile @@ -6,6 +6,7 @@ COPY go.mod *.go /usr/src/app/ # No dependencies yet #RUN go mod download && go mod verify +RUN ["env", "CGO_ENABLED=0", "go", "get"] RUN ["env", "CGO_ENABLED=0", "go", "build", "-o", "catfact_webapp", "."] diff --git a/app_go/go.mod b/app_go/go.mod index b384e175b8..69568a6542 100644 --- a/app_go/go.mod +++ b/app_go/go.mod @@ -1,3 +1,15 @@ module catfact_webapp go 1.21.6 + +require github.com/prometheus/client_golang v1.19.0 + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + golang.org/x/sys v0.16.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect +) diff --git a/app_go/go.sum b/app_go/go.sum new file mode 100644 index 0000000000..cb062b6f83 --- /dev/null +++ b/app_go/go.sum @@ -0,0 +1,16 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/app_go/main.go b/app_go/main.go index 10fc818861..c775c49b5f 100644 --- a/app_go/main.go +++ b/app_go/main.go @@ -4,9 +4,14 @@ import ( "fmt" "log" "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" ) -func handler(w http.ResponseWriter, r *http.Request) { +func index(w http.ResponseWriter, r *http.Request) { fact, err := catFact() if err == nil { w.WriteHeader(http.StatusOK) @@ -17,9 +22,55 @@ func handler(w http.ResponseWriter, r *http.Request) { } } + +var ( + reqCnt = promauto.NewCounter(prometheus.CounterOpts{ + Name: "go_requests_count", + Help: "Number of HTTP requests", + }) + + reqHandleTime = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "go_request_handle_time", + Help: "Time to handle a request", + }) +) + +func noteTimeMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + defer func() { + var dtSec float64 = time.Since(start).Seconds() + reqCnt.Inc() + reqHandleTime.Observe(dtSec) + }() + + next.ServeHTTP(w, r) + }) +} + + func main() { - http.HandleFunc("/", handler) + businessLogic := http.NewServeMux() + businessLogic.Handle("/", asHandler(index)) + // Note: keeping /metrics under middleware too for consistency with app_py + businessLogic.Handle("/metrics", promhttp.Handler()) + + wrapped := noteTimeMiddleware(businessLogic) + hostPort := "0.0.0.0:5000" _, _ = fmt.Println("Listening on http://" + hostPort) - log.Fatal(http.ListenAndServe(hostPort, nil)) + log.Fatal(http.ListenAndServe(hostPort, wrapped)) +} + + +type dummyHandler struct { + handlerFunc func (http.ResponseWriter, *http.Request) +} + +func (h dummyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.handlerFunc(w, r) +} + +func asHandler(handlerFunc func (http.ResponseWriter, *http.Request)) dummyHandler { + return dummyHandler{handlerFunc: handlerFunc} } diff --git a/app_go/main_test.go b/app_go/main_test.go index d78a0dfee2..fef1008521 100644 --- a/app_go/main_test.go +++ b/app_go/main_test.go @@ -13,7 +13,7 @@ import ( func TestFactLoads(t *testing.T) { w := httptest.NewRecorder() - handler(w, nil) + index(w, nil) resp := w.Result() if resp.StatusCode != http.StatusOK { diff --git a/app_python/moscow_time/__init__.py b/app_python/moscow_time/__init__.py index a46d35dbb9..3055268770 100644 --- a/app_python/moscow_time/__init__.py +++ b/app_python/moscow_time/__init__.py @@ -1,7 +1,9 @@ import datetime +from time import monotonic -from flask import Flask +from flask import Flask, request, Response import requests +import prometheus_client from .cache import cache_for @@ -9,6 +11,21 @@ app = Flask(__name__) +REQUEST_COUNT = prometheus_client.Counter('py_requests_count', 'Number of HTTP requests') +REQUEST_HANDLE_TIME = prometheus_client.Histogram('py_request_handle_time', 'Time to handle a request') + +@app.before_request +def note_request_start_time(): + request.start_time = monotonic() + +@app.after_request +def update_prometheus(response): + handle_time = monotonic() - request.start_time + REQUEST_COUNT.inc() + REQUEST_HANDLE_TIME.observe(handle_time) + return response + + # In case of high load, to avoid frequent requests, cache results for # one second @cache_for(1000) @@ -30,3 +47,7 @@ def index(): time = get_time() return f"In MSK it's {time.hour}:{time.minute}:{time.second}. " \ "Have you brushed your teeth today yet?" + +@app.route('/metrics') +def prometheus_metrics(): + return Response(prometheus_client.generate_latest(), mimetype='text/plain') diff --git a/app_python/requirements.txt b/app_python/requirements.txt index b478fac35c..8eb3534ebe 100644 --- a/app_python/requirements.txt +++ b/app_python/requirements.txt @@ -1,2 +1,3 @@ Flask~=3.0.0 requests~=2.31.0 +prometheus_client~=0.20.0 diff --git a/monitoring/METRICS.md b/monitoring/METRICS.md index dbadaa4095..0111d36763 100644 --- a/monitoring/METRICS.md +++ b/monitoring/METRICS.md @@ -11,4 +11,16 @@ ## Enhancements Grafana, Loki, and Promtail each have the RAM limit of 100MiB, Prometheus's memory is limited -to 50MiB, limits for app_py and app_go are 30MiB and 6MiB respectively. +to 50MiB, limits for app_py and app_go are 30MiB and 20MiB respectively. + +## Metrics from web apps + +The Python app exports metrics corresponding to the web app (requests count, request handle +time), as well as metrics related to python runtime exposed by the `promteheus_client`, for instance: + +![Prometheus, python web_app](pics/prometheus_app_py_sample.png) + +The Go app exports metrics corresponding to the web app (requests count, request handle time), +as well as metrics related to go runtime exposed by the prometheus client, for instance: + +![Prometheus, go web_app](pics/prometheus_app_go_sample.png) diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index af9723eaae..30cb930e5c 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -7,7 +7,7 @@ volumes: services: app_py: - image: kolay0ne/app_py:lab6 + image: kolay0ne/app_py:lab8 ports: - "5000:5000" logging: @@ -15,16 +15,20 @@ services: max-size: 5m deploy: resources: {limits: {memory: 30M}} + networks: + - prometheus app_go: - image: kolay0ne/app_go:lab6 + image: kolay0ne/app_go:lab8 ports: - "5500:5000" logging: options: max-size: 5m deploy: - resources: {limits: {memory: 6M}} + resources: {limits: {memory: 20M}} + networks: + - prometheus loki: image: grafana/loki:2.9.2 diff --git a/monitoring/pics/prometheus_app_go_sample.png b/monitoring/pics/prometheus_app_go_sample.png new file mode 100644 index 0000000000..bb800214fb Binary files /dev/null and b/monitoring/pics/prometheus_app_go_sample.png differ diff --git a/monitoring/pics/prometheus_app_py_sample.png b/monitoring/pics/prometheus_app_py_sample.png new file mode 100644 index 0000000000..29d2e5a5d0 Binary files /dev/null and b/monitoring/pics/prometheus_app_py_sample.png differ diff --git a/monitoring/pics/prometheus_targets.png b/monitoring/pics/prometheus_targets.png index 94b99e28d5..463d06a9df 100644 Binary files a/monitoring/pics/prometheus_targets.png and b/monitoring/pics/prometheus_targets.png differ diff --git a/monitoring/prometheus_config.yml b/monitoring/prometheus_config.yml index 51e2bd9f3a..4512776995 100644 --- a/monitoring/prometheus_config.yml +++ b/monitoring/prometheus_config.yml @@ -16,3 +16,11 @@ scrape_configs: - job_name: grafana static_configs: - targets: ['grafana:3000'] + + - job_name: app_py + static_configs: + - targets: ['app_py:5000'] + + - job_name: app_go + static_configs: + - targets: ['app_go:5000']