From 1a7c4571889f38fd95f6c577d069a9fdf1038769 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Thu, 10 Aug 2023 15:31:41 +0530 Subject: [PATCH 01/29] Add Prometheus JMX Exporter jar --- enforcer-parent/enforcer/pom.xml | 9 +++++++-- enforcer-parent/enforcer/src/main/assembly/assembly.xml | 4 ++++ enforcer-parent/enforcer/src/main/resources/Dockerfile | 9 +++++++-- .../src/main/resources/prometheus_jmx_config.yml | 2 ++ resources/docker-compose/apim/docker-compose.yaml | 1 + resources/docker-compose/docker-compose.yaml | 1 + .../choreo-connect/choreo-connect-deployment.yaml | 1 + 7 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml diff --git a/enforcer-parent/enforcer/pom.xml b/enforcer-parent/enforcer/pom.xml index f9bf9a2890..fad2f0d286 100644 --- a/enforcer-parent/enforcer/pom.xml +++ b/enforcer-parent/enforcer/pom.xml @@ -189,10 +189,10 @@ ${project.build.directory}/ - org.apache.logging.log4j, org.json.wso2 + org.apache.logging.log4j, org.json.wso2, io.prometheus.jmx - log4j-api, log4j-core, log4j-jcl, json, log4j-slf4j2-impl + log4j-api, log4j-core, log4j-jcl, json, log4j-slf4j2-impl, jmx_prometheus_javaagent @@ -629,6 +629,11 @@ org.wso2.orbit.graphQL graphQL + + io.prometheus.jmx + jmx_prometheus_javaagent + 0.18.0 + diff --git a/enforcer-parent/enforcer/src/main/assembly/assembly.xml b/enforcer-parent/enforcer/src/main/assembly/assembly.xml index 44f55e57ec..48f585b815 100644 --- a/enforcer-parent/enforcer/src/main/assembly/assembly.xml +++ b/enforcer-parent/enforcer/src/main/assembly/assembly.xml @@ -67,5 +67,9 @@ LICENSE.txt + + src/main/resources/prometheus_jmx_config.yml + conf + diff --git a/enforcer-parent/enforcer/src/main/resources/Dockerfile b/enforcer-parent/enforcer/src/main/resources/Dockerfile index 893056fe60..726951bc56 100644 --- a/enforcer-parent/enforcer/src/main/resources/Dockerfile +++ b/enforcer-parent/enforcer/src/main/resources/Dockerfile @@ -65,6 +65,7 @@ ARG GRPC_HEALTH_PROBE_PATH=/bin/grpc_health_probe ENV VERSION=${MG_VERSION} ENV JAVA_OPTS="" +ENV JAVA_AGENT_OPTS="" ENV ENFORCER_HOME=${MG_USER_HOME} ENV ENFORCER_PRIVATE_KEY_PATH=/home/wso2/security/keystore/mg.key @@ -100,6 +101,7 @@ USER ${MG_USER} COPY maven/lib/ lib/ COPY maven/conf/log4j2.properties conf/log4j2.properties +COPY maven/conf/prometheus_jmx_config.yml conf/prometheus_jmx_config.yml COPY maven/security security COPY maven/check_health.sh . COPY maven/LICENSE.txt . @@ -108,6 +110,9 @@ COPY maven/LICENSE.txt . #Add the agent using JVM arg -javaagent:/home/wso2/conf/applicationinsights-agent-3.1.1.jar #Add the config file using System property -Dapplicationinsights.configuration.file=/home/wso2/conf/applicationinsights.json -EXPOSE 8081 9001 +# Add jmx_prometheus_javaagent +ENV JAVA_AGENT_OPTS="${JAVA_AGENT_OPTS} -javaagent:${ENFORCER_HOME}/lib/jmx_prometheus_javaagent-0.18.0.jar=19080:${ENFORCER_HOME}/conf/prometheus_jmx_config.yml" -CMD java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="${ENFORCER_HOME}/logs/heap-dump.hprof" $JAVA_OPTS -Dlog4j.configurationFile="${ENFORCER_HOME}/conf/log4j2.properties" -DtracingEnabled="true" -cp "lib/*:lib/dropins/*" org.wso2.choreo.connect.enforcer.server.AuthServer +EXPOSE 8081 9001 19080 + +CMD java $JAVA_AGENT_OPTS -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="${ENFORCER_HOME}/logs/heap-dump.hprof" $JAVA_OPTS -Dlog4j.configurationFile="${ENFORCER_HOME}/conf/log4j2.properties" -DtracingEnabled="true" -cp "lib/*:lib/dropins/*" org.wso2.choreo.connect.enforcer.server.AuthServer diff --git a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml new file mode 100644 index 0000000000..cb19fee9f8 --- /dev/null +++ b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml @@ -0,0 +1,2 @@ +rules: + - pattern: ".*" \ No newline at end of file diff --git a/resources/docker-compose/apim/docker-compose.yaml b/resources/docker-compose/apim/docker-compose.yaml index 56284a148f..bc28ad83cc 100644 --- a/resources/docker-compose/apim/docker-compose.yaml +++ b/resources/docker-compose/apim/docker-compose.yaml @@ -94,3 +94,4 @@ services: ports: - "8081:8081" - "9001:9001" + - "19080:19080" diff --git a/resources/docker-compose/docker-compose.yaml b/resources/docker-compose/docker-compose.yaml index c1a36b0179..b4eb9af679 100644 --- a/resources/docker-compose/docker-compose.yaml +++ b/resources/docker-compose/docker-compose.yaml @@ -80,3 +80,4 @@ services: ports: - "8081:8081" - "9001:9001" + - "19080:19080" diff --git a/resources/k8s-artifacts/choreo-connect/choreo-connect-deployment.yaml b/resources/k8s-artifacts/choreo-connect/choreo-connect-deployment.yaml index b97b052ad7..d2cebde822 100644 --- a/resources/k8s-artifacts/choreo-connect/choreo-connect-deployment.yaml +++ b/resources/k8s-artifacts/choreo-connect/choreo-connect-deployment.yaml @@ -83,6 +83,7 @@ spec: ports: - containerPort: 8081 - containerPort: 9001 + - containerPort: 19080 startupProbe: exec: command: [ "sh", "check_health.sh" ] From d4d218d73ba3ff8ede7346a027aaedab5cf27afc Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Fri, 11 Aug 2023 10:53:17 +0530 Subject: [PATCH 02/29] Add Prometheus Go client --- adapter/go.mod | 14 +++++++-- adapter/go.sum | 32 +++++++++++++++----- adapter/internal/adapter/adapter.go | 4 +++ adapter/pkg/metrics/metrics.go | 13 ++++++++ adapter/src/main/resources/Dockerfile | 2 +- resources/docker-compose/docker-compose.yaml | 1 + 6 files changed, 55 insertions(+), 11 deletions(-) create mode 100644 adapter/pkg/metrics/metrics.go diff --git a/adapter/go.mod b/adapter/go.mod index e4579b761c..b61117343a 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -17,13 +17,14 @@ require ( github.com/go-openapi/strfmt v0.21.1 github.com/go-openapi/swag v0.19.9 github.com/go-openapi/validate v0.19.11 - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/jessevdk/go-flags v1.5.0 github.com/lestrrat-go/jwx v1.1.3 github.com/mitchellh/mapstructure v1.3.3 github.com/pelletier/go-toml v1.8.1 + github.com/prometheus/client_golang v1.16.0 github.com/sirupsen/logrus v1.7.0 github.com/streadway/amqp v1.0.0 github.com/stretchr/testify v1.8.1 @@ -32,7 +33,7 @@ require ( golang.org/x/net v0.7.0 google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 google.golang.org/grpc v1.52.0 - google.golang.org/protobuf v1.28.1 + google.golang.org/protobuf v1.30.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -47,7 +48,9 @@ require ( github.com/acomagu/bufpipe v1.0.3 // indirect github.com/agnivade/levenshtein v1.0.1 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 // indirect @@ -68,14 +71,19 @@ require ( github.com/lestrrat-go/iter v1.0.0 // indirect github.com/lestrrat-go/option v1.0.0 // indirect github.com/mailru/easyjson v0.7.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect go.mongodb.org/mongo-driver v1.7.5 // indirect - golang.org/x/sys v0.5.0 // indirect + golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.7.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/adapter/go.sum b/adapter/go.sum index 34936ae045..ea54d19e30 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -35,9 +35,13 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:o github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc h1:PYXxkRUBGUMa5xgMVMDl62vEklZvKpVaxQeN9ie7Hfk= @@ -189,9 +193,10 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -227,8 +232,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -259,6 +264,8 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -281,10 +288,20 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -380,6 +397,7 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -407,8 +425,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= @@ -456,8 +474,8 @@ google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 0d104e6e88..16574c3c9a 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -43,6 +43,7 @@ import ( "github.com/wso2/product-microgateway/adapter/pkg/health" healthservice "github.com/wso2/product-microgateway/adapter/pkg/health/api/wso2/health/service" "github.com/wso2/product-microgateway/adapter/pkg/logging" + "github.com/wso2/product-microgateway/adapter/pkg/metrics" sync "github.com/wso2/product-microgateway/adapter/pkg/synchronizer" "github.com/wso2/product-microgateway/adapter/pkg/tlsutils" @@ -196,6 +197,9 @@ func Run(conf *config.Config) { }) } + // Start the metrics server + go metrics.StartMetricsServer("9999") + logger.LoggerMgw.Info("Starting adapter ....") cache := xds.GetXdsCache() enforcerCache := xds.GetEnforcerCache() diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go new file mode 100644 index 0000000000..deeae5b38c --- /dev/null +++ b/adapter/pkg/metrics/metrics.go @@ -0,0 +1,13 @@ +package metrics + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// StartMetricsServer initializes and starts the metrics server on the given port. +func StartMetricsServer(port string) { + http.Handle("/metrics", promhttp.Handler()) + http.ListenAndServe(":"+port, nil) +} diff --git a/adapter/src/main/resources/Dockerfile b/adapter/src/main/resources/Dockerfile index 8dec48bbec..04f97a2746 100644 --- a/adapter/src/main/resources/Dockerfile +++ b/adapter/src/main/resources/Dockerfile @@ -59,6 +59,6 @@ COPY maven/adapter-linux-amd64 adapter COPY maven/check_health.sh . COPY maven/LICENSE.txt . -EXPOSE 18000 9843 6060 +EXPOSE 18000 9843 6060 9999 CMD ./adapter diff --git a/resources/docker-compose/docker-compose.yaml b/resources/docker-compose/docker-compose.yaml index b4eb9af679..db7ed6ad97 100644 --- a/resources/docker-compose/docker-compose.yaml +++ b/resources/docker-compose/docker-compose.yaml @@ -49,6 +49,7 @@ services: ports: - "18000:18000" - "9843:9843" + - "9999:9999" enforcer: image: wso2/choreo-connect-enforcer:1.3.0-m1-SNAPSHOT logging: From 234576561df850d80294d5bd78ad9f0ba1936f1c Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Fri, 11 Aug 2023 17:57:33 +0530 Subject: [PATCH 03/29] Refactor Prometheus metrics-related changes --- adapter/internal/adapter/adapter.go | 2 +- enforcer-parent/enforcer/src/main/resources/Dockerfile | 7 ++----- resources/docker-compose/apim/docker-compose.yaml | 1 - resources/docker-compose/docker-compose.yaml | 2 -- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 16574c3c9a..00c20ad527 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -198,7 +198,7 @@ func Run(conf *config.Config) { } // Start the metrics server - go metrics.StartMetricsServer("9999") + go metrics.StartMetricsServer("19085") logger.LoggerMgw.Info("Starting adapter ....") cache := xds.GetXdsCache() diff --git a/enforcer-parent/enforcer/src/main/resources/Dockerfile b/enforcer-parent/enforcer/src/main/resources/Dockerfile index 726951bc56..60ce809f97 100644 --- a/enforcer-parent/enforcer/src/main/resources/Dockerfile +++ b/enforcer-parent/enforcer/src/main/resources/Dockerfile @@ -110,9 +110,6 @@ COPY maven/LICENSE.txt . #Add the agent using JVM arg -javaagent:/home/wso2/conf/applicationinsights-agent-3.1.1.jar #Add the config file using System property -Dapplicationinsights.configuration.file=/home/wso2/conf/applicationinsights.json -# Add jmx_prometheus_javaagent -ENV JAVA_AGENT_OPTS="${JAVA_AGENT_OPTS} -javaagent:${ENFORCER_HOME}/lib/jmx_prometheus_javaagent-0.18.0.jar=19080:${ENFORCER_HOME}/conf/prometheus_jmx_config.yml" +EXPOSE 8081 9001 -EXPOSE 8081 9001 19080 - -CMD java $JAVA_AGENT_OPTS -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="${ENFORCER_HOME}/logs/heap-dump.hprof" $JAVA_OPTS -Dlog4j.configurationFile="${ENFORCER_HOME}/conf/log4j2.properties" -DtracingEnabled="true" -cp "lib/*:lib/dropins/*" org.wso2.choreo.connect.enforcer.server.AuthServer +CMD java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="${ENFORCER_HOME}/logs/heap-dump.hprof" $JAVA_OPTS $JAVA_AGENT_OPTS -Dlog4j.configurationFile="${ENFORCER_HOME}/conf/log4j2.properties" -DtracingEnabled="true" -cp "lib/*:lib/dropins/*" org.wso2.choreo.connect.enforcer.server.AuthServer diff --git a/resources/docker-compose/apim/docker-compose.yaml b/resources/docker-compose/apim/docker-compose.yaml index bc28ad83cc..56284a148f 100644 --- a/resources/docker-compose/apim/docker-compose.yaml +++ b/resources/docker-compose/apim/docker-compose.yaml @@ -94,4 +94,3 @@ services: ports: - "8081:8081" - "9001:9001" - - "19080:19080" diff --git a/resources/docker-compose/docker-compose.yaml b/resources/docker-compose/docker-compose.yaml index db7ed6ad97..c1a36b0179 100644 --- a/resources/docker-compose/docker-compose.yaml +++ b/resources/docker-compose/docker-compose.yaml @@ -49,7 +49,6 @@ services: ports: - "18000:18000" - "9843:9843" - - "9999:9999" enforcer: image: wso2/choreo-connect-enforcer:1.3.0-m1-SNAPSHOT logging: @@ -81,4 +80,3 @@ services: ports: - "8081:8081" - "9001:9001" - - "19080:19080" From 919fa9b706ff5f0c80291c5d8495dfaaef1ff92e Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Fri, 11 Aug 2023 17:57:33 +0530 Subject: [PATCH 04/29] Refactor Prometheus metrics-related changes --- .../k8s-artifacts/choreo-connect/choreo-connect-deployment.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/resources/k8s-artifacts/choreo-connect/choreo-connect-deployment.yaml b/resources/k8s-artifacts/choreo-connect/choreo-connect-deployment.yaml index d2cebde822..b97b052ad7 100644 --- a/resources/k8s-artifacts/choreo-connect/choreo-connect-deployment.yaml +++ b/resources/k8s-artifacts/choreo-connect/choreo-connect-deployment.yaml @@ -83,7 +83,6 @@ spec: ports: - containerPort: 8081 - containerPort: 9001 - - containerPort: 19080 startupProbe: exec: command: [ "sh", "check_health.sh" ] From 694b30519f4e8931d92894903490f10e605e577a Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Mon, 14 Aug 2023 09:19:42 +0530 Subject: [PATCH 05/29] Add adapter config for metrics --- adapter/config/default_config.go | 5 +++++ adapter/config/types.go | 3 +++ adapter/internal/adapter/adapter.go | 7 ++++++- adapter/pkg/metrics/metrics.go | 10 +++++++--- adapter/src/main/resources/Dockerfile | 2 +- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/adapter/config/default_config.go b/adapter/config/default_config.go index 1264b48d04..a5a1466907 100644 --- a/adapter/config/default_config.go +++ b/adapter/config/default_config.go @@ -66,6 +66,11 @@ var defaultConfig = &Config{ MaxRetryCount: 20, ArtifactsDirectory: "/home/wso2/git-artifacts", }, + Metrics: metrics{ + Enabled: false, + Type: "prometheus", + Port: 19085, + }, }, Envoy: envoy{ ListenerHost: "0.0.0.0", diff --git a/adapter/config/types.go b/adapter/config/types.go index 56db0a5141..5a4ce40a91 100644 --- a/adapter/config/types.go +++ b/adapter/config/types.go @@ -99,6 +99,8 @@ type adapter struct { SoapErrorInXMLEnabled bool // SourceControl represents the configuration related to the repository where the api artifacts are stored SourceControl sourceControl + // Metric represents configurations to expose/export go metrics + Metrics metrics } // Envoy Listener Component related configurations. @@ -467,6 +469,7 @@ type tracing struct { type metrics struct { Enabled bool Type string + Port int32 } type analyticsAdapter struct { diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 00c20ad527..88795416be 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -198,7 +198,12 @@ func Run(conf *config.Config) { } // Start the metrics server - go metrics.StartMetricsServer("19085") + if conf.Adapter.Metrics.Enabled { + if conf.Adapter.Metrics.Type == metrics.PrometheusMetricType { + logger.LoggerMgw.Info("Starting Prometheus Metrics Server ....") + go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port) + } + } logger.LoggerMgw.Info("Starting adapter ....") cache := xds.GetXdsCache() diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index deeae5b38c..9d10fab6e9 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -2,12 +2,16 @@ package metrics import ( "net/http" + "strconv" "github.com/prometheus/client_golang/prometheus/promhttp" ) -// StartMetricsServer initializes and starts the metrics server on the given port. -func StartMetricsServer(port string) { +// PrometheusMetricType prometheus metric type +const PrometheusMetricType = "prometheus" + +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32) { http.Handle("/metrics", promhttp.Handler()) - http.ListenAndServe(":"+port, nil) + http.ListenAndServe(":"+strconv.FormatInt(int64(port), 10), nil) } diff --git a/adapter/src/main/resources/Dockerfile b/adapter/src/main/resources/Dockerfile index 04f97a2746..8dec48bbec 100644 --- a/adapter/src/main/resources/Dockerfile +++ b/adapter/src/main/resources/Dockerfile @@ -59,6 +59,6 @@ COPY maven/adapter-linux-amd64 adapter COPY maven/check_health.sh . COPY maven/LICENSE.txt . -EXPOSE 18000 9843 6060 9999 +EXPOSE 18000 9843 6060 CMD ./adapter From 5a73cf399ba19f63c5394cc48acef39b3b71485d Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Mon, 14 Aug 2023 11:58:47 +0530 Subject: [PATCH 06/29] Clean enforcer Dockerfile --- enforcer-parent/enforcer/src/main/resources/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/enforcer-parent/enforcer/src/main/resources/Dockerfile b/enforcer-parent/enforcer/src/main/resources/Dockerfile index 60ce809f97..f2c74dcfb8 100644 --- a/enforcer-parent/enforcer/src/main/resources/Dockerfile +++ b/enforcer-parent/enforcer/src/main/resources/Dockerfile @@ -65,7 +65,6 @@ ARG GRPC_HEALTH_PROBE_PATH=/bin/grpc_health_probe ENV VERSION=${MG_VERSION} ENV JAVA_OPTS="" -ENV JAVA_AGENT_OPTS="" ENV ENFORCER_HOME=${MG_USER_HOME} ENV ENFORCER_PRIVATE_KEY_PATH=/home/wso2/security/keystore/mg.key @@ -112,4 +111,4 @@ COPY maven/LICENSE.txt . EXPOSE 8081 9001 -CMD java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="${ENFORCER_HOME}/logs/heap-dump.hprof" $JAVA_OPTS $JAVA_AGENT_OPTS -Dlog4j.configurationFile="${ENFORCER_HOME}/conf/log4j2.properties" -DtracingEnabled="true" -cp "lib/*:lib/dropins/*" org.wso2.choreo.connect.enforcer.server.AuthServer +CMD java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="${ENFORCER_HOME}/logs/heap-dump.hprof" $JAVA_OPTS -Dlog4j.configurationFile="${ENFORCER_HOME}/conf/log4j2.properties" -DtracingEnabled="true" -cp "lib/*:lib/dropins/*" org.wso2.choreo.connect.enforcer.server.AuthServer From c9608b31a426c409d261d05bebddd8834b8a3e0d Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Tue, 15 Aug 2023 15:30:24 +0530 Subject: [PATCH 07/29] Change prometheus jmx config yaml --- .../main/resources/prometheus_jmx_config.yml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml index cb19fee9f8..ac3e05cd11 100644 --- a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml +++ b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml @@ -1,2 +1,20 @@ +# Config options: https://github.com/prometheus/jmx_exporter/blob/main/README.md +lowercaseOutputName: true +lowercaseOutputLabelNames: true rules: - - pattern: ".*" \ No newline at end of file + # WSO2 Choreo Connect related metrics + - pattern: 'org.wso2.choreo.connect.enforcer<>total_request_count: (.*)' + name: org_wso2_choreo_connect_enforcer_total_request_count + help: "WSO2 choreo connect enforcer total_request_count" + attrNameSnakeCase: true + type: COUNTER + - pattern: 'org.wso2.choreo.connect.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minutes): (.*)' + name: org_wso2_choreo_connect_enforcer_$1 + help: "WSO2 choreo connect enforcer $1" + attrNameSnakeCase: true + type: GAUGE + - pattern: 'org.wso2.choreo.connect.enforcer<>(\w+): (.*)' + name: org_wso2_choreo_connect_enforcer_thread_pool_$1 + help: "WSO2 choreo connect enforcer thread pool $1" + attrNameSnakeCase: true + type: GAUGE \ No newline at end of file From ee5744e223f4aac38749e12016779b5b7c4b9a14 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Wed, 16 Aug 2023 15:31:32 +0530 Subject: [PATCH 08/29] Add java lang OS metrics to prometheus jmx config yaml --- .../enforcer/src/main/resources/prometheus_jmx_config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml index ac3e05cd11..aad1733bc8 100644 --- a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml +++ b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml @@ -17,4 +17,10 @@ rules: name: org_wso2_choreo_connect_enforcer_thread_pool_$1 help: "WSO2 choreo connect enforcer thread pool $1" attrNameSnakeCase: true + type: GAUGE + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true type: GAUGE \ No newline at end of file From 03d0ff28b60e6eb0f359d1bf6a2ce22c2726db85 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Fri, 18 Aug 2023 15:20:00 +0530 Subject: [PATCH 09/29] Expose golang cpu,memory related metrics with gopsutil --- adapter/go.mod | 12 ++++- adapter/go.sum | 30 +++++++++++-- adapter/pkg/metrics/metrics.go | 82 ++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 6 deletions(-) diff --git a/adapter/go.mod b/adapter/go.mod index b61117343a..8a4159eb41 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -25,9 +25,10 @@ require ( github.com/mitchellh/mapstructure v1.3.3 github.com/pelletier/go-toml v1.8.1 github.com/prometheus/client_golang v1.16.0 + github.com/shirou/gopsutil/v3 v3.23.7 github.com/sirupsen/logrus v1.7.0 github.com/streadway/amqp v1.0.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.4 github.com/vektah/gqlparser/v2 v2.5.1 golang.org/x/crypto v0.1.0 // indirect golang.org/x/net v0.7.0 @@ -58,6 +59,7 @@ require ( github.com/emirpasic/gods v1.12.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/analysis v0.19.10 // indirect github.com/go-openapi/jsonpointer v0.19.3 // indirect github.com/go-openapi/jsonreference v0.19.3 // indirect @@ -70,20 +72,26 @@ require ( github.com/lestrrat-go/httpcc v1.0.0 // indirect github.com/lestrrat-go/iter v1.0.0 // indirect github.com/lestrrat-go/option v1.0.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect go.mongodb.org/mongo-driver v1.7.5 // indirect - golang.org/x/sys v0.8.0 // indirect + golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.7.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/adapter/go.sum b/adapter/go.sum index ea54d19e30..1a292eb26c 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -89,6 +89,8 @@ github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -202,7 +204,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -254,6 +258,8 @@ github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFe github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lestrrat-go/pdebug/v3 v3.0.1 h1:3G5sX/aw/TbMTtVc9U7IHBWRZtMvwvBziF1e4HoQtv8= github.com/lestrrat-go/pdebug/v3 v3.0.1/go.mod h1:za+m+Ve24yCxTEhR59N7UlnJomWwCiIqbJRmKeiADU4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -288,6 +294,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -305,6 +313,12 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -327,10 +341,14 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vektah/gqlparser/v2 v2.5.1 h1:ZGu+bquAY23jsxDRcYpWjttRZrUz07LbiY77gUOHcr4= github.com/vektah/gqlparser/v2 v2.5.1/go.mod h1:mPgqFBu/woKTVYWyNk8cO3kh4S/f4aRFZrvOnp3hmCs= @@ -344,6 +362,8 @@ github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHM github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= @@ -422,11 +442,13 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index 9d10fab6e9..9f5b5d7128 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -2,16 +2,98 @@ package metrics import ( "net/http" + "os" + "runtime" "strconv" + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v3/mem" + "github.com/shirou/gopsutil/v3/process" ) // PrometheusMetricType prometheus metric type const PrometheusMetricType = "prometheus" +var hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "host_info", + Help: "Host Info", +}, []string{"os"}) + +var availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_available_cpus", + Help: "Number of available CPUs", +}) + +var freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_free_physical_memory", + Help: "Amount of free physical memory in bytes", +}) + +var systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_system_cpu_load", + Help: "System-wide CPU usage as a percentage", +}) + +var processCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_process_cpu_load_percentage", + Help: "CPU usage of the current process as a percentage", +}) + +var loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "os_system_load_average", + Help: "Current load of CPU in the host system for the last {x} minutes", +}, []string{"duration"}) + +func init() { + prometheus.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, systemCPULoad, processCPULoad, loadAvg) +} + +// recordMetrics record custom golang metrics +func recordMetrics() { + for { + host, err := host.Info() + hostInfo.WithLabelValues(host.OS).Set(1) + + availableCPUs.Set(float64(runtime.NumCPU())) + + v, err := mem.VirtualMemory() + if err == nil { + freePhysicalMemory.Set(float64(v.Free)) + } + + percentages, err := cpu.Percent(0, false) + if err == nil && len(percentages) > 0 { + systemCPULoad.Set(percentages[0]) + } + + p, err := process.NewProcess(int32(os.Getpid())) + if err == nil { + percent, _ := p.CPUPercent() + processCPULoad.Set(percent) + } + + avg, err := load.Avg() + if err != nil { + return + } + loadAvg.WithLabelValues("1m").Set(avg.Load1) + loadAvg.WithLabelValues("5m").Set(avg.Load5) + loadAvg.WithLabelValues("15m").Set(avg.Load15) + + // Sleep 5s before the next measurement + time.Sleep(5 * time.Second) + } + +} + // StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. func StartPrometheusMetricsServer(port int32) { + go recordMetrics() http.Handle("/metrics", promhttp.Handler()) http.ListenAndServe(":"+strconv.FormatInt(int64(port), 10), nil) } From 9b8b51f4eac55fcce8a796ba0686cdcfb4807b82 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Thu, 24 Aug 2023 16:35:57 +0530 Subject: [PATCH 10/29] Fix jmx metric data types and avg resp time equation --- .../metrics/jmx/api/ExtAuthMetricsMXBean.java | 18 ++++++++-------- .../metrics/jmx/impl/ExtAuthMetrics.java | 21 ++++++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java index cfa5c15778..dada2e7f95 100644 --- a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java @@ -30,24 +30,24 @@ public interface ExtAuthMetricsMXBean { /** * Getter for average response time in milli seconds. - * - * @return long + * + * @return double */ - public long getAverageResponseTimeMillis(); + public double getAverageResponseTimeMillis(); /** * Getter for maximum response time in milliseconds. - * - * @return long + * + * @return double */ - public long getMaxResponseTimeMillis(); + public double getMaxResponseTimeMillis(); /** * Getter for mimnimum response time in milliseconds. - * - * @return long + * + * @return double */ - public long getMinResponseTimeMillis(); + public double getMinResponseTimeMillis(); /** * Resets all the metrics to thier initial values. diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java index 7e975e6d4b..7683815014 100644 --- a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java @@ -32,9 +32,9 @@ public class ExtAuthMetrics extends TimerTask implements ExtAuthMetricsMXBean { private long requestCountInLastFiveMinutes = 0; private long totalRequestCount = 0; - private long averageResponseTimeMillis = 0; - private long maxResponseTimeMillis = Long.MIN_VALUE; - private long minResponseTimeMillis = Long.MAX_VALUE; + private double averageResponseTimeMillis = 0; + private double maxResponseTimeMillis = Double.MIN_VALUE; + private double minResponseTimeMillis = Double.MAX_VALUE; private ExtAuthMetrics() { MBeanRegistrator.registerMBean(this); @@ -42,7 +42,7 @@ private ExtAuthMetrics() { /** * Getter for the Singleton ExtAuthMetrics instance. - * + * * @return ExtAuthMetrics */ public static ExtAuthMetrics getInstance() { @@ -64,24 +64,25 @@ public long getTotalRequestCount() { }; @Override - public long getAverageResponseTimeMillis() { + public double getAverageResponseTimeMillis() { return averageResponseTimeMillis; }; @Override - public long getMaxResponseTimeMillis() { + public double getMaxResponseTimeMillis() { return maxResponseTimeMillis; }; @Override - public long getMinResponseTimeMillis() { + public double getMinResponseTimeMillis() { return minResponseTimeMillis; }; public synchronized void recordMetric(long responseTimeMillis) { this.requestCountInLastFiveMinutes += 1; this.totalRequestCount += 1; - this.averageResponseTimeMillis = (this.averageResponseTimeMillis + responseTimeMillis) / totalRequestCount; + this.averageResponseTimeMillis = this.averageResponseTimeMillis + + (responseTimeMillis - this.averageResponseTimeMillis) / totalRequestCount; this.minResponseTimeMillis = Math.min(this.minResponseTimeMillis, responseTimeMillis); this.maxResponseTimeMillis = Math.max(this.maxResponseTimeMillis, responseTimeMillis); } @@ -90,8 +91,8 @@ public synchronized void recordMetric(long responseTimeMillis) { public synchronized void resetExtAuthMetrics() { this.totalRequestCount = 0; this.averageResponseTimeMillis = 0; - this.maxResponseTimeMillis = Long.MIN_VALUE; - this.minResponseTimeMillis = Long.MAX_VALUE; + this.maxResponseTimeMillis = Double.MIN_VALUE; + this.minResponseTimeMillis = Double.MAX_VALUE; } @Override From 77b5a98a85c54fcc66335b8db404a660e1771947 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Fri, 25 Aug 2023 11:44:29 +0530 Subject: [PATCH 11/29] Refactor metric names --- adapter/pkg/metrics/metrics.go | 14 +++++++------- .../src/main/resources/prometheus_jmx_config.yml | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index 9f5b5d7128..292dd84220 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -25,23 +25,23 @@ var hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ }, []string{"os"}) var availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_available_cpus", - Help: "Number of available CPUs", + Name: "os_available_cpu_total", + Help: "Number of available CPUs.", }) var freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_free_physical_memory", - Help: "Amount of free physical memory in bytes", + Name: "os_free_physical_memory_bytes", + Help: "Amount of free physical memory in bytes.", }) var systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_system_cpu_load", - Help: "System-wide CPU usage as a percentage", + Name: "os_system_cpu_load_percentage", + Help: "System-wide CPU usage as a percentage.", }) var processCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "os_process_cpu_load_percentage", - Help: "CPU usage of the current process as a percentage", + Help: "CPU usage of the current process as a percentage.", }) var loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ diff --git a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml index aad1733bc8..e9ba443e72 100644 --- a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml +++ b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml @@ -4,18 +4,18 @@ lowercaseOutputLabelNames: true rules: # WSO2 Choreo Connect related metrics - pattern: 'org.wso2.choreo.connect.enforcer<>total_request_count: (.*)' - name: org_wso2_choreo_connect_enforcer_total_request_count - help: "WSO2 choreo connect enforcer total_request_count" + name: org_wso2_choreo_connect_enforcer_request_count_total + help: "WSO2 choreo connect enforcer total request count." attrNameSnakeCase: true type: COUNTER - pattern: 'org.wso2.choreo.connect.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minutes): (.*)' name: org_wso2_choreo_connect_enforcer_$1 - help: "WSO2 choreo connect enforcer $1" + help: "WSO2 choreo connect enforcer $1." attrNameSnakeCase: true type: GAUGE - pattern: 'org.wso2.choreo.connect.enforcer<>(\w+): (.*)' name: org_wso2_choreo_connect_enforcer_thread_pool_$1 - help: "WSO2 choreo connect enforcer thread pool $1" + help: "WSO2 choreo connect enforcer thread pool $1." attrNameSnakeCase: true type: GAUGE # OS related metrics From b9383f95002763c319a93934b883784b594dbb3e Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Mon, 28 Aug 2023 15:36:17 +0530 Subject: [PATCH 12/29] Refactor adapter metrics --- adapter/pkg/metrics/metrics.go | 138 +++++++++++++++++++++++---------- 1 file changed, 96 insertions(+), 42 deletions(-) diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index 292dd84220..0ac6885d47 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -8,77 +8,123 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/procfs" "github.com/shirou/gopsutil/v3/cpu" "github.com/shirou/gopsutil/v3/host" "github.com/shirou/gopsutil/v3/load" "github.com/shirou/gopsutil/v3/mem" - "github.com/shirou/gopsutil/v3/process" + logger "github.com/wso2/product-microgateway/adapter/internal/loggers" ) // PrometheusMetricType prometheus metric type const PrometheusMetricType = "prometheus" -var hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "host_info", - Help: "Host Info", -}, []string{"os"}) - -var availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_available_cpu_total", - Help: "Number of available CPUs.", -}) - -var freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_free_physical_memory_bytes", - Help: "Amount of free physical memory in bytes.", -}) - -var systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_system_cpu_load_percentage", - Help: "System-wide CPU usage as a percentage.", -}) - -var processCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_process_cpu_load_percentage", - Help: "CPU usage of the current process as a percentage.", -}) - -var loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "os_system_load_average", - Help: "Current load of CPU in the host system for the last {x} minutes", -}, []string{"duration"}) +var ( + prometheusMetricRegistry = prometheus.NewRegistry() + + hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "host_info", + Help: "Host Info", + }, []string{"os"}) + + availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_available_cpu_total", + Help: "Number of available CPUs.", + }) + + freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_free_physical_memory_bytes", + Help: "Amount of free physical memory.", + }) + + totalVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_total_virtual_memory_bytes", + Help: "Amount of total virtual memory.", + }) + usedVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_used_virtual_memory_bytes", + Help: "Amount of used virtual memory.", + }) + + systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_system_cpu_load_percentage", + Help: "System-wide CPU usage as a percentage.", + }) + + loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "os_system_load_average", + Help: "Current load of CPU in the host system for the last {x} minutes", + }, []string{"duration"}) + + processStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_start_time_seconds", + Help: "Start time of the process since unix epoch in seconds.", + }) + + processOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_open_fds", + Help: "Number of open file descriptors.", + }) +) func init() { - prometheus.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, systemCPULoad, processCPULoad, loadAvg) + // Register the Go collector with the registry + goCollector := collectors.NewGoCollector() + prometheusMetricRegistry.MustRegister(goCollector) + + // Register other metrics + prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, + systemCPULoad, loadAvg, processStartTime, processOpenFDs) } // recordMetrics record custom golang metrics func recordMetrics() { for { host, err := host.Info() + if handleError(err, "Failed to get host info") { + return + } hostInfo.WithLabelValues(host.OS).Set(1) - availableCPUs.Set(float64(runtime.NumCPU())) v, err := mem.VirtualMemory() - if err == nil { - freePhysicalMemory.Set(float64(v.Free)) + if handleError(err, "Failed to read virtual memory metrics") { + return } + freePhysicalMemory.Set(float64(v.Free)) + usedVirtualMemory.Set(float64(v.Used)) + totalVirtualMemory.Set(float64(v.Total)) percentages, err := cpu.Percent(0, false) - if err == nil && len(percentages) > 0 { - systemCPULoad.Set(percentages[0]) + if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { + return } + systemCPULoad.Set(percentages[0]) - p, err := process.NewProcess(int32(os.Getpid())) - if err == nil { - percent, _ := p.CPUPercent() - processCPULoad.Set(percent) + pid := os.Getpid() + p, err := procfs.NewProc(pid) + if handleError(err, "Failed to get current process") { + return + } + stat, err := p.Stat() + if handleError(err, "Failed to get process stats") { + return + } + t, err := stat.StartTime() + if handleError(err, "Failed to read process start time") { + return } + processStartTime.Set(t) + fds, err := p.FileDescriptorsLen() + if handleError(err, "Failed to read file descriptor count") { + return + } + processOpenFDs.Set(float64(fds)) avg, err := load.Avg() - if err != nil { + if handleError(err, "Failed to read cpu load averages") { return } loadAvg.WithLabelValues("1m").Set(avg.Load1) @@ -91,9 +137,17 @@ func recordMetrics() { } +func handleError(err error, message string) bool { + if err != nil { + logger.LoggerMgw.Error(message) + return true + } + return false +} + // StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. func StartPrometheusMetricsServer(port int32) { go recordMetrics() - http.Handle("/metrics", promhttp.Handler()) + http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) http.ListenAndServe(":"+strconv.FormatInt(int64(port), 10), nil) } From 198f04edb33fb6f11f0aa9882609ce64d6747cd2 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Tue, 29 Aug 2023 10:24:36 +0530 Subject: [PATCH 13/29] Add enforcer request count window start time metric --- .../metrics/jmx/api/ExtAuthMetricsMXBean.java | 13 ++++++++++-- .../metrics/jmx/impl/ExtAuthMetrics.java | 20 +++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java index dada2e7f95..79eda95b85 100644 --- a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java @@ -55,8 +55,17 @@ public interface ExtAuthMetricsMXBean { public void resetExtAuthMetrics(); /** - * Resets all the metrics to thier initial values. + * Get request count in last five minutes window + * + * @return long + */ + public long getRequestCountInLastFiveMinuteWindow(); + + /** + * Get the start time of request count window + * + * @return */ - public long getRequestCountInLastFiveMinutes(); + public long getRequestCountWindowStartTimeMillis(); } diff --git a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java index 7683815014..77acffc721 100644 --- a/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java +++ b/enforcer-parent/enforcer/src/main/java/org/wso2/choreo/connect/enforcer/metrics/jmx/impl/ExtAuthMetrics.java @@ -30,7 +30,8 @@ public class ExtAuthMetrics extends TimerTask implements ExtAuthMetricsMXBean { private static final long REQUEST_COUNT_INTERVAL_MILLIS = 5 * 60 * 1000; private static ExtAuthMetrics extAuthMetricsMBean = null; - private long requestCountInLastFiveMinutes = 0; + private long requestCountInLastFiveMinuteWindow = 0; + private long requestCountWindowStartTimeMillis = System.currentTimeMillis(); private long totalRequestCount = 0; private double averageResponseTimeMillis = 0; private double maxResponseTimeMillis = Double.MIN_VALUE; @@ -51,7 +52,8 @@ public static ExtAuthMetrics getInstance() { if (extAuthMetricsMBean == null) { Timer timer = new Timer(); extAuthMetricsMBean = new ExtAuthMetrics(); - timer.schedule(extAuthMetricsMBean, 0, REQUEST_COUNT_INTERVAL_MILLIS); + extAuthMetricsMBean.requestCountWindowStartTimeMillis = System.currentTimeMillis(); + timer.schedule(extAuthMetricsMBean, REQUEST_COUNT_INTERVAL_MILLIS, REQUEST_COUNT_INTERVAL_MILLIS); } } } @@ -79,7 +81,7 @@ public double getMinResponseTimeMillis() { }; public synchronized void recordMetric(long responseTimeMillis) { - this.requestCountInLastFiveMinutes += 1; + this.requestCountInLastFiveMinuteWindow += 1; this.totalRequestCount += 1; this.averageResponseTimeMillis = this.averageResponseTimeMillis + (responseTimeMillis - this.averageResponseTimeMillis) / totalRequestCount; @@ -97,11 +99,17 @@ public synchronized void resetExtAuthMetrics() { @Override public synchronized void run() { - requestCountInLastFiveMinutes = 0; + requestCountWindowStartTimeMillis = System.currentTimeMillis(); + requestCountInLastFiveMinuteWindow = 0; } @Override - public long getRequestCountInLastFiveMinutes() { - return requestCountInLastFiveMinutes; + public long getRequestCountInLastFiveMinuteWindow() { + return requestCountInLastFiveMinuteWindow; + } + + @Override + public long getRequestCountWindowStartTimeMillis() { + return requestCountWindowStartTimeMillis; } } From 08ca01c914be7e436fe6a0d2c31095098e2e8476 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Tue, 29 Aug 2023 10:31:34 +0530 Subject: [PATCH 14/29] Add fix prometheus jmx config and refactor go modules --- adapter/go.mod | 4 ++-- .../enforcer/src/main/resources/prometheus_jmx_config.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/adapter/go.mod b/adapter/go.mod index 8a4159eb41..83291e02fe 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -25,12 +25,12 @@ require ( github.com/mitchellh/mapstructure v1.3.3 github.com/pelletier/go-toml v1.8.1 github.com/prometheus/client_golang v1.16.0 + github.com/prometheus/procfs v0.10.1 github.com/shirou/gopsutil/v3 v3.23.7 github.com/sirupsen/logrus v1.7.0 github.com/streadway/amqp v1.0.0 github.com/stretchr/testify v1.8.4 github.com/vektah/gqlparser/v2 v2.5.1 - golang.org/x/crypto v0.1.0 // indirect golang.org/x/net v0.7.0 google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 google.golang.org/grpc v1.52.0 @@ -82,7 +82,6 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect @@ -91,6 +90,7 @@ require ( github.com/xanzy/ssh-agent v0.3.0 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.mongodb.org/mongo-driver v1.7.5 // indirect + golang.org/x/crypto v0.1.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.7.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml index e9ba443e72..694ddd4393 100644 --- a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml +++ b/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml @@ -8,7 +8,7 @@ rules: help: "WSO2 choreo connect enforcer total request count." attrNameSnakeCase: true type: COUNTER - - pattern: 'org.wso2.choreo.connect.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minutes): (.*)' + - pattern: 'org.wso2.choreo.connect.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis): (.*)' name: org_wso2_choreo_connect_enforcer_$1 help: "WSO2 choreo connect enforcer $1." attrNameSnakeCase: true From e382ee83f3bc9eeea14acd050522ea08df6c2da0 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Thu, 31 Aug 2023 10:24:31 +0530 Subject: [PATCH 15/29] Remove jmx config from enforcer image and add license header --- adapter/pkg/metrics/metrics.go | 18 ++++++++++++++++++ .../enforcer/src/main/assembly/assembly.xml | 4 ---- .../enforcer/src/main/resources/Dockerfile | 1 - ...mx_config.yml => prometheus-jmx-config.yml} | 18 +++++++++++++++++- 4 files changed, 35 insertions(+), 6 deletions(-) rename enforcer-parent/enforcer/src/main/resources/{prometheus_jmx_config.yml => prometheus-jmx-config.yml} (61%) diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index 0ac6885d47..6fd5136b0f 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2023, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + // Package metrics holds the implementation for exposing adapter metrics to prometheus package metrics import ( diff --git a/enforcer-parent/enforcer/src/main/assembly/assembly.xml b/enforcer-parent/enforcer/src/main/assembly/assembly.xml index 48f585b815..44f55e57ec 100644 --- a/enforcer-parent/enforcer/src/main/assembly/assembly.xml +++ b/enforcer-parent/enforcer/src/main/assembly/assembly.xml @@ -67,9 +67,5 @@ LICENSE.txt - - src/main/resources/prometheus_jmx_config.yml - conf - diff --git a/enforcer-parent/enforcer/src/main/resources/Dockerfile b/enforcer-parent/enforcer/src/main/resources/Dockerfile index f2c74dcfb8..893056fe60 100644 --- a/enforcer-parent/enforcer/src/main/resources/Dockerfile +++ b/enforcer-parent/enforcer/src/main/resources/Dockerfile @@ -100,7 +100,6 @@ USER ${MG_USER} COPY maven/lib/ lib/ COPY maven/conf/log4j2.properties conf/log4j2.properties -COPY maven/conf/prometheus_jmx_config.yml conf/prometheus_jmx_config.yml COPY maven/security security COPY maven/check_health.sh . COPY maven/LICENSE.txt . diff --git a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml b/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml similarity index 61% rename from enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml rename to enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml index 694ddd4393..e303371eb5 100644 --- a/enforcer-parent/enforcer/src/main/resources/prometheus_jmx_config.yml +++ b/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml @@ -1,3 +1,19 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2023, WSO2 Inc. (http://wso2.com) All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ----------------------------------------------------------------------- + # Config options: https://github.com/prometheus/jmx_exporter/blob/main/README.md lowercaseOutputName: true lowercaseOutputLabelNames: true @@ -23,4 +39,4 @@ rules: name: os_$1 help: Operating System $1 attrNameSnakeCase: true - type: GAUGE \ No newline at end of file + type: GAUGE From 31d1e1e48b9293a8d56dc484202bd2198387d6b5 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Thu, 31 Aug 2023 15:21:04 +0530 Subject: [PATCH 16/29] Fix PR review changes --- adapter/internal/adapter/adapter.go | 9 ++++----- adapter/pkg/metrics/metrics.go | 10 ++++++++-- resources/conf/config.toml.template | 9 +++++++++ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 88795416be..fa372ebf75 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -198,11 +198,10 @@ func Run(conf *config.Config) { } // Start the metrics server - if conf.Adapter.Metrics.Enabled { - if conf.Adapter.Metrics.Type == metrics.PrometheusMetricType { - logger.LoggerMgw.Info("Starting Prometheus Metrics Server ....") - go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port) - } + if conf.Adapter.Metrics.Enabled && conf.Adapter.Metrics.Type == metrics.PrometheusMetricType { + logger.LoggerMgw.Info("Starting Prometheus Metrics Server ....") + go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port) + } logger.LoggerMgw.Info("Starting adapter ....") diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index 6fd5136b0f..d6eadfa442 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -15,10 +15,11 @@ * */ - // Package metrics holds the implementation for exposing adapter metrics to prometheus +// Package metrics holds the implementation for exposing adapter metrics to prometheus package metrics import ( + "fmt" "net/http" "os" "runtime" @@ -34,6 +35,7 @@ import ( "github.com/shirou/gopsutil/v3/load" "github.com/shirou/gopsutil/v3/mem" logger "github.com/wso2/product-microgateway/adapter/internal/loggers" + "github.com/wso2/product-microgateway/adapter/pkg/logging" ) // PrometheusMetricType prometheus metric type @@ -157,7 +159,11 @@ func recordMetrics() { func handleError(err error, message string) bool { if err != nil { - logger.LoggerMgw.Error(message) + logger.LoggerMgw.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintf(message, err.Error()), + Severity: logging.MINOR, + ErrorCode: 1109, + }) return true } return false diff --git a/resources/conf/config.toml.template b/resources/conf/config.toml.template index d589fcf7e8..c6375b68f4 100644 --- a/resources/conf/config.toml.template +++ b/resources/conf/config.toml.template @@ -91,6 +91,15 @@ soapErrorInXMLEnabled = false # Path to the private key used for authentication (Use "" in the case of a public repository (only for GitHub)) sshKeyFile = "/home/wso2/ssh-keys/id_ed25519" +# Configuration to expose adapter metrics +[adapter.metrics] + # Enable/Disable metrics + enabled = false + # The type of metric server, currently supports only prometheus + type = "prometheus" + # The port through which metrics are exposed + port = 19085 + # Configurations required for router to route the traffic from different clients to services [router] # -------------------------------------------------------- # Host for listener of Router From e92f7b3d7260ee1ac2069675c0ae31dd91edd1d2 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Thu, 31 Aug 2023 15:56:44 +0530 Subject: [PATCH 17/29] Move the prometheus jmx dependency version as a property to main pom.xml --- enforcer-parent/enforcer/pom.xml | 2 +- pom.xml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/enforcer-parent/enforcer/pom.xml b/enforcer-parent/enforcer/pom.xml index fad2f0d286..54cabc114f 100644 --- a/enforcer-parent/enforcer/pom.xml +++ b/enforcer-parent/enforcer/pom.xml @@ -632,7 +632,7 @@ io.prometheus.jmx jmx_prometheus_javaagent - 0.18.0 + ${io.prometheus.jmx.version} diff --git a/pom.xml b/pom.xml index 1aa89b1b36..c8574db44f 100644 --- a/pom.xml +++ b/pom.xml @@ -748,6 +748,7 @@ 2.6.3 1.0.0-beta.5 1.3.3 + 0.18.0 8.0.1 4 From c530fa7032f1ca72edc0e81a7ba15a2abccf42c9 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Fri, 1 Sep 2023 13:45:34 +0530 Subject: [PATCH 18/29] Update license header --- adapter/pkg/metrics/metrics.go | 2 +- .../enforcer/src/main/resources/prometheus-jmx-config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index d6eadfa442..79adfdeac0 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * Copyright (c) 2023, WSO2 Inc. (http://www.wso2.org). * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml b/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml index e303371eb5..eefbcad06f 100644 --- a/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml +++ b/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml @@ -1,5 +1,5 @@ # -------------------------------------------------------------------- -# Copyright (c) 2023, WSO2 Inc. (http://wso2.com) All Rights Reserved. +# Copyright (c) 2023, WSO2 Inc. (http://wso2.com). # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 05d1dcc6a2a0a183643eae9b7e48e1c315982600 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Mon, 11 Sep 2023 15:06:12 +0530 Subject: [PATCH 19/29] Add PrometheusMetricEndpointsTestCase --- integration/test-integration/pom.xml | 3 + .../CcWithPrometheusMetricsEnabled.java | 50 +++++++++++ .../PrometheusMetricEndpointsTestCase.java | 51 +++++++++++ .../dockerCompose/cc-prometheus-metrics.yaml | 85 +++++++++++++++++++ .../metricConfigs/prometheus-jmx-config.yml | 42 +++++++++ .../test/resources/testng-cc-standalone.xml | 6 ++ 6 files changed, 237 insertions(+) create mode 100644 integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java create mode 100644 integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java create mode 100644 integration/test-integration/src/test/resources/dockerCompose/cc-prometheus-metrics.yaml create mode 100644 integration/test-integration/src/test/resources/dockerVolumeMounts/metricConfigs/prometheus-jmx-config.yml diff --git a/integration/test-integration/pom.xml b/integration/test-integration/pom.xml index f369734ab3..e5c4da296a 100644 --- a/integration/test-integration/pom.xml +++ b/integration/test-integration/pom.xml @@ -80,6 +80,9 @@ + + + diff --git a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java new file mode 100644 index 0000000000..7d322bc229 --- /dev/null +++ b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023, WSO2 Inc. (http://www.wso2.org) + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.choreo.connect.tests.setup.standalone; + +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.wso2.choreo.connect.tests.context.CCTestException; +import org.wso2.choreo.connect.tests.context.CcInstance; +import org.wso2.choreo.connect.tests.util.ApictlUtils; + +import java.util.concurrent.TimeUnit; + +public class CcWithPrometheusMetricsEnabled { + CcInstance ccInstance; + + @BeforeTest(description = "initialise the setup") + void start() throws Exception { + ccInstance = new CcInstance.Builder().withNewDockerCompose("cc-prometheus-metrics.yaml") + .withNewConfig("prometheus-metrics-enabled.toml") + .withVolumeMountDir("metricConfigs") + .build(); + ccInstance.start(); + ApictlUtils.addEnv("test"); + ApictlUtils.login("test"); + TimeUnit.SECONDS.sleep(5); + + } + + @AfterTest(description = "stop the setup") + void stop() throws CCTestException { + ccInstance.stop(); + ApictlUtils.removeEnv("test"); + } +} diff --git a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java new file mode 100644 index 0000000000..f527a768ec --- /dev/null +++ b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023, WSO2 Inc. (http://www.wso2.org) + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.choreo.connect.tests.testcases.standalone.metrics; + +import org.testng.Assert; +import org.testng.annotations.Test; +import org.wso2.choreo.connect.tests.util.*; + +public class PrometheusMetricEndpointsTestCase { + @Test(description = "Test whether enforcer metric endpoint is up") + public void testEnforcerMetricEndpoint() throws Exception { + HttpResponse response = HttpClientRequest.doGet("http://localhost:19080/metrics"); + int responseCode = response.getResponseCode(); + Assert.assertTrue(response.getData().contains("jvm_"), "Metric information not found in expected format"); + Assert.assertEquals(200, responseCode, "Response code mismatched"); + } + + @Test(description = "Test whether adapter metric endpoint is up") + public void testAdapterMetricEndpoint() throws Exception { + HttpResponse response = HttpClientRequest.doGet("http://localhost:19085/metrics"); + int responseCode = response.getResponseCode(); + Assert.assertTrue(response.getData().contains("go_"), "Metric information not found in expected format"); + Assert.assertEquals(200, responseCode, "Response code mismatched"); + } + + + @Test(description = "Test whether router metric endpoint is up") + public void testRouterMetricEndpoint() throws Exception { + HttpResponse response = HttpClientRequest.doGet("http://localhost:9000/stats/prometheus"); + int responseCode = response.getResponseCode(); + Assert.assertTrue(response.getData().contains("envoy_"), "Metric information not found in expected format"); + Assert.assertEquals(200, responseCode, "Response code mismatched"); + } + +} diff --git a/integration/test-integration/src/test/resources/dockerCompose/cc-prometheus-metrics.yaml b/integration/test-integration/src/test/resources/dockerCompose/cc-prometheus-metrics.yaml new file mode 100644 index 0000000000..6fdd9d1f36 --- /dev/null +++ b/integration/test-integration/src/test/resources/dockerCompose/cc-prometheus-metrics.yaml @@ -0,0 +1,85 @@ +version: "3.7" +services: + router: + image: wso2/choreo-connect-router:1.2.0 + logging: + options: + max-size: "20m" + max-file: "5" + environment: + - ROUTER_ADMIN_HOST=0.0.0.0 + - ROUTER_ADMIN_PORT=9000 + - ROUTER_CLUSTER=default_cluster + - ROUTER_LABEL=Default + - ROUTER_PRIVATE_KEY_PATH=/home/wso2/security/keystore/mg.key + - ROUTER_PUBLIC_CERT_PATH=/home/wso2/security/keystore/mg.pem + - ADAPTER_HOST=adapter + - ADAPTER_PORT=18000 + - ADAPTER_CA_CERT_PATH=/home/wso2/security/truststore/mg.pem + - ENFORCER_HOST=enforcer + - ENFORCER_PORT=8081 + - ENFORCER_ANALYTICS_RECEIVER_PORT=18090 + - ENFORCER_CA_CERT_PATH=/home/wso2/security/truststore/mg.pem + - CONCURRENCY=2 + volumes: + - ../resources/router/security:/home/wso2/security + ports: + - "9095:9095" + - "9090:9090" + - "9000:9000" + links: + - adapter + - enforcer + adapter: + image: wso2/choreo-connect-adapter:1.2.0.4-SNAPSHOT + logging: + options: + max-size: "20m" + max-file: "5" + volumes: + - ../resources/adapter/security:/home/wso2/security + - ./conf/log_config.toml:/home/wso2/conf/log_config.toml + - ./conf/config.toml:/home/wso2/conf/config.toml + - ../resources/adapter/artifacts/apis:/home/wso2/artifacts/apis + environment: + - ADAPTER_PRIVATE_KEY_PATH=/home/wso2/security/keystore/mg.key + - ADAPTER_PUBLIC_CERT_PATH=/home/wso2/security/keystore/mg.pem + - cp_admin_pwd=admin + - adapter_admin_pwd=admin + ports: + - "18000:18000" + - "9843:9843" + - "19085:19085" + enforcer: + image: wso2/choreo-connect-enforcer:1.2.0.4-SNAPSHOT + logging: + options: + max-size: "20m" + max-file: "5" + volumes: + - ../resources/enforcer/security:/home/wso2/security + - ./conf/log4j2.properties:/home/wso2/conf/log4j2.properties + - ../resources/enforcer/dropins:/home/wso2/lib/dropins + - ./metricConfigs/:/home/wso2/conf/ + links: + - adapter + environment: + - ENFORCER_PRIVATE_KEY_PATH=/home/wso2/security/keystore/mg.key + - ENFORCER_PUBLIC_CERT_PATH=/home/wso2/security/keystore/mg.pem + - TRUSTED_CA_CERTS_PATH=/home/wso2/security/truststore + - ADAPTER_HOST_NAME=adapter + - ADAPTER_HOST=adapter + - ADAPTER_XDS_PORT=18000 + - ENFORCER_LABEL=Default + - ENFORCER_REGION=UNKNOWN + - XDS_MAX_MSG_SIZE=4194304 + - XDS_MAX_RETRIES=3 + - JAVA_OPTS=${JAVA_OPTS} -Dhttpclient.hostnameVerifier=AllowAll -Dchoreo.connect.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.18.0.jar=19080:/home/wso2/conf/prometheus-jmx-config.yml + - enforcer_admin_pwd=admin + - tm_admin_pwd=admin + - analytics_authURL=https://localhost:8080 + - analytics_authToken= + ports: + - "8081:8081" + - "9001:9001" + - "19080:19080" diff --git a/integration/test-integration/src/test/resources/dockerVolumeMounts/metricConfigs/prometheus-jmx-config.yml b/integration/test-integration/src/test/resources/dockerVolumeMounts/metricConfigs/prometheus-jmx-config.yml new file mode 100644 index 0000000000..b62d7a939f --- /dev/null +++ b/integration/test-integration/src/test/resources/dockerVolumeMounts/metricConfigs/prometheus-jmx-config.yml @@ -0,0 +1,42 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2023, WSO2 LLC. (https://wso2.com). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ----------------------------------------------------------------------- + +# Config options: https://github.com/prometheus/jmx_exporter/blob/main/README.md +lowercaseOutputName: true +lowercaseOutputLabelNames: true +rules: + # WSO2 Choreo Connect related metrics + - pattern: 'org.wso2.choreo.connect.enforcer<>total_request_count: (.*)' + name: org_wso2_choreo_connect_enforcer_request_count_total + help: "WSO2 choreo connect enforcer total request count." + attrNameSnakeCase: true + type: COUNTER + - pattern: 'org.wso2.choreo.connect.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis): (.*)' + name: org_wso2_choreo_connect_enforcer_$1 + help: "WSO2 choreo connect enforcer $1." + attrNameSnakeCase: true + type: GAUGE + - pattern: 'org.wso2.choreo.connect.enforcer<>(\w+): (.*)' + name: org_wso2_choreo_connect_enforcer_thread_pool_$1 + help: "WSO2 choreo connect enforcer thread pool $1." + attrNameSnakeCase: true + type: GAUGE + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE diff --git a/integration/test-integration/src/test/resources/testng-cc-standalone.xml b/integration/test-integration/src/test/resources/testng-cc-standalone.xml index d44d03456a..30f07c8c12 100644 --- a/integration/test-integration/src/test/resources/testng-cc-standalone.xml +++ b/integration/test-integration/src/test/resources/testng-cc-standalone.xml @@ -143,4 +143,10 @@ + + + + + + From 325f4fde8de45d9473e481041b47bc1b77e44cec Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Mon, 11 Sep 2023 15:11:02 +0530 Subject: [PATCH 20/29] Change PrometheusMetricEndpointsTestCase assertion order --- .../metrics/PrometheusMetricEndpointsTestCase.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java index f527a768ec..566f942c38 100644 --- a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java +++ b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java @@ -27,16 +27,16 @@ public class PrometheusMetricEndpointsTestCase { public void testEnforcerMetricEndpoint() throws Exception { HttpResponse response = HttpClientRequest.doGet("http://localhost:19080/metrics"); int responseCode = response.getResponseCode(); - Assert.assertTrue(response.getData().contains("jvm_"), "Metric information not found in expected format"); Assert.assertEquals(200, responseCode, "Response code mismatched"); + Assert.assertTrue(response.getData().contains("jvm_"), "Metric information not found in expected format"); } @Test(description = "Test whether adapter metric endpoint is up") public void testAdapterMetricEndpoint() throws Exception { HttpResponse response = HttpClientRequest.doGet("http://localhost:19085/metrics"); int responseCode = response.getResponseCode(); - Assert.assertTrue(response.getData().contains("go_"), "Metric information not found in expected format"); Assert.assertEquals(200, responseCode, "Response code mismatched"); + Assert.assertTrue(response.getData().contains("go_"), "Metric information not found in expected format"); } @@ -44,8 +44,8 @@ public void testAdapterMetricEndpoint() throws Exception { public void testRouterMetricEndpoint() throws Exception { HttpResponse response = HttpClientRequest.doGet("http://localhost:9000/stats/prometheus"); int responseCode = response.getResponseCode(); - Assert.assertTrue(response.getData().contains("envoy_"), "Metric information not found in expected format"); Assert.assertEquals(200, responseCode, "Response code mismatched"); + Assert.assertTrue(response.getData().contains("envoy_"), "Metric information not found in expected format"); } - } + From b5e1b9deef9e90592ba608d1741a490d6189379d Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Mon, 11 Sep 2023 15:17:17 +0530 Subject: [PATCH 21/29] Change image tag of prometheus metric test docker images --- .../test/resources/dockerCompose/cc-prometheus-metrics.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/test-integration/src/test/resources/dockerCompose/cc-prometheus-metrics.yaml b/integration/test-integration/src/test/resources/dockerCompose/cc-prometheus-metrics.yaml index 6fdd9d1f36..b57112399a 100644 --- a/integration/test-integration/src/test/resources/dockerCompose/cc-prometheus-metrics.yaml +++ b/integration/test-integration/src/test/resources/dockerCompose/cc-prometheus-metrics.yaml @@ -31,7 +31,7 @@ services: - adapter - enforcer adapter: - image: wso2/choreo-connect-adapter:1.2.0.4-SNAPSHOT + image: wso2/choreo-connect-adapter:1.2.0 logging: options: max-size: "20m" @@ -51,7 +51,7 @@ services: - "9843:9843" - "19085:19085" enforcer: - image: wso2/choreo-connect-enforcer:1.2.0.4-SNAPSHOT + image: wso2/choreo-connect-enforcer:1.2.0 logging: options: max-size: "20m" From 83e12c89f738e5b5d65398dd712439673a398778 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Mon, 11 Sep 2023 15:27:25 +0530 Subject: [PATCH 22/29] Add prometheus metrics test config toml --- .../configs/prometheus-metrics-enabled.toml | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 integration/test-integration/src/test/resources/configs/prometheus-metrics-enabled.toml diff --git a/integration/test-integration/src/test/resources/configs/prometheus-metrics-enabled.toml b/integration/test-integration/src/test/resources/configs/prometheus-metrics-enabled.toml new file mode 100644 index 0000000000..a03473487b --- /dev/null +++ b/integration/test-integration/src/test/resources/configs/prometheus-metrics-enabled.toml @@ -0,0 +1,81 @@ +[adapter] +[adapter.keystore] + certPath = "/home/wso2/security/keystore/mg.pem" + keyPath = "/home/wso2/security/keystore/mg.key" + +[adapter.truststore] + location = "/home/wso2/security/truststore" + +[router] + listenerPort = 9090 + securedListenerPort = 9095 + systemHost = "localhost" + useRemoteAddress = false + +[router.keystore] + certPath = "/home/wso2/security/keystore/mg.pem" + keyPath = "/home/wso2/security/keystore/mg.key" + +[router.upstream] +[router.upstream.tls] + trustedCertPath = "/etc/ssl/certs/ca-certificates.crt" + verifyHostName = true + disableSslVerification = false + +[enforcer] +[enforcer.jwtIssuer] + enabled = true + issuer = "https://localhost:9095/testkey" + +[enforcer.throttling] + enableGlobalEventPublishing = true + jmsConnectionProviderURL = "amqp://admin:$env{tm_admin_pwd}@carbon/carbon?brokerlist='tcp://apim:5673'" + [enforcer.throttling.publisher] + username = "admin" + password = "$env{tm_admin_pwd}" + [[enforcer.throttling.publisher.URLGroup]] + receiverURLs = ["tcp://apim:9612"] + authURLs = ["ssl://apim:9712"] + +[enforcer.security] +[[enforcer.security.tokenService]] + name = "Resident Key Manager" + issuer = "https://localhost:9444/oauth2/token" + certificateAlias = "wso2carbon" + jwksURL = "" + validateSubscription = false + consumerKeyClaim = "azp" + certificateFilePath = "/home/wso2/security/truststore/wso2carbon.pem" + +[[enforcer.security.tokenService]] + name = "MGW" + issuer = "https://localhost:9095/testkey" + certificateAlias = "mgw" + jwksURL = "" + validateSubscription = false + consumerKeyClaim = "" + certificateFilePath = "/home/wso2/security/truststore/mg.pem" + +[[enforcer.security.tokenService]] + name = "APIM Publisher" + issuer = "https://localhost:9444/publisher" + validateSubscription = true + certificateFilePath = "/home/wso2/security/truststore/wso2carbon.pem" + +[controlPlane] + enabled = true + serviceURL = "https://apim:9444/" + username = "admin" + password = "$env{cp_admin_pwd}" + environmentLabels = ["Default"] + skipSSLVerification = true + [controlPlane.brokerConnectionParameters] + eventListeningEndpoints = ["amqp://admin:$env{cp_admin_pwd}@apim:5673?retries='10'&connectdelay='30'"] + +[analytics] + enabled = false + +[adapter.metrics] + enabled = true + type = "prometheus" + port = 19085 From 06c7edab84ad9bc1ba79a128e139c97e5a8f3232 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Mon, 11 Sep 2023 15:52:44 +0530 Subject: [PATCH 23/29] Fix license header --- .../setup/standalone/CcWithPrometheusMetricsEnabled.java | 4 ++-- .../standalone/metrics/PrometheusMetricEndpointsTestCase.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java index 7d322bc229..e1ddbd8506 100644 --- a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java +++ b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java @@ -1,7 +1,7 @@ /* - * Copyright (c) 2023, WSO2 Inc. (http://www.wso2.org) + * Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) * - * WSO2 Inc. licenses this file to you under the Apache License, + * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except * in compliance with the License. * You may obtain a copy of the License at diff --git a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java index 566f942c38..87ae292194 100644 --- a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java +++ b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java @@ -1,7 +1,7 @@ /* - * Copyright (c) 2023, WSO2 Inc. (http://www.wso2.org) + * Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) * - * WSO2 Inc. licenses this file to you under the Apache License, + * WSO2 LLC. licenses this file to you under the Apache License, * Version 2.0 (the "License"); you may not use this file except * in compliance with the License. * You may obtain a copy of the License at From ef03a7a86ccd1b4f9f054b5e5970c0f2ca930c85 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Wed, 13 Sep 2023 15:50:24 +0530 Subject: [PATCH 24/29] Add response null assertion --- .../standalone/metrics/PrometheusMetricEndpointsTestCase.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java index 87ae292194..e1464d2722 100644 --- a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java +++ b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java @@ -26,6 +26,7 @@ public class PrometheusMetricEndpointsTestCase { @Test(description = "Test whether enforcer metric endpoint is up") public void testEnforcerMetricEndpoint() throws Exception { HttpResponse response = HttpClientRequest.doGet("http://localhost:19080/metrics"); + Assert.assertNotNull(response); int responseCode = response.getResponseCode(); Assert.assertEquals(200, responseCode, "Response code mismatched"); Assert.assertTrue(response.getData().contains("jvm_"), "Metric information not found in expected format"); @@ -34,6 +35,7 @@ public void testEnforcerMetricEndpoint() throws Exception { @Test(description = "Test whether adapter metric endpoint is up") public void testAdapterMetricEndpoint() throws Exception { HttpResponse response = HttpClientRequest.doGet("http://localhost:19085/metrics"); + Assert.assertNotNull(response); int responseCode = response.getResponseCode(); Assert.assertEquals(200, responseCode, "Response code mismatched"); Assert.assertTrue(response.getData().contains("go_"), "Metric information not found in expected format"); @@ -43,6 +45,7 @@ public void testAdapterMetricEndpoint() throws Exception { @Test(description = "Test whether router metric endpoint is up") public void testRouterMetricEndpoint() throws Exception { HttpResponse response = HttpClientRequest.doGet("http://localhost:9000/stats/prometheus"); + Assert.assertNotNull(response); int responseCode = response.getResponseCode(); Assert.assertEquals(200, responseCode, "Response code mismatched"); Assert.assertTrue(response.getData().contains("envoy_"), "Metric information not found in expected format"); From 3d0c00ae2550e2e70d96f1671a62f9103054caeb Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Thu, 14 Sep 2023 15:53:13 +0530 Subject: [PATCH 25/29] Add adapter metric collection in terval config and refactor code --- adapter/config/default_config.go | 7 +-- adapter/config/types.go | 7 +-- adapter/internal/adapter/adapter.go | 2 +- adapter/pkg/metrics/metrics.go | 49 ++++++++++++++++--- .../configs/prometheus-metrics-enabled.toml | 1 + resources/conf/config.toml.template | 2 + 6 files changed, 54 insertions(+), 14 deletions(-) diff --git a/adapter/config/default_config.go b/adapter/config/default_config.go index a5a1466907..a8e932e404 100644 --- a/adapter/config/default_config.go +++ b/adapter/config/default_config.go @@ -67,9 +67,10 @@ var defaultConfig = &Config{ ArtifactsDirectory: "/home/wso2/git-artifacts", }, Metrics: metrics{ - Enabled: false, - Type: "prometheus", - Port: 19085, + Enabled: false, + Type: "prometheus", + Port: 19085, + CollectionInterval: 5, }, }, Envoy: envoy{ diff --git a/adapter/config/types.go b/adapter/config/types.go index 5a4ce40a91..cd142c2585 100644 --- a/adapter/config/types.go +++ b/adapter/config/types.go @@ -467,9 +467,10 @@ type tracing struct { } type metrics struct { - Enabled bool - Type string - Port int32 + Enabled bool + Type string + Port int32 + CollectionInterval int32 } type analyticsAdapter struct { diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index fa372ebf75..fbf1400f43 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -200,7 +200,7 @@ func Run(conf *config.Config) { // Start the metrics server if conf.Adapter.Metrics.Enabled && conf.Adapter.Metrics.Type == metrics.PrometheusMetricType { logger.LoggerMgw.Info("Starting Prometheus Metrics Server ....") - go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port) + go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port, conf.Adapter.Metrics.CollectionInterval) } diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index 79adfdeac0..d46f2816e0 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -100,7 +100,7 @@ func init() { } // recordMetrics record custom golang metrics -func recordMetrics() { +var recordMetrics = func(collectionInterval int32) { for { host, err := host.Info() if handleError(err, "Failed to get host info") { @@ -151,8 +151,8 @@ func recordMetrics() { loadAvg.WithLabelValues("5m").Set(avg.Load5) loadAvg.WithLabelValues("15m").Set(avg.Load15) - // Sleep 5s before the next measurement - time.Sleep(5 * time.Second) + // Sleep before the next measurement + time.Sleep(time.Duration(collectionInterval) * time.Second) } } @@ -170,8 +170,43 @@ func handleError(err error, message string) bool { } // StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. -func StartPrometheusMetricsServer(port int32) { - go recordMetrics() - http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) - http.ListenAndServe(":"+strconv.FormatInt(int64(port), 10), nil) +func StartPrometheusMetricsServer(port int32, collectionInterval int32) { + done := make(chan struct{}) // Channel to indicate recordMetrics routine exit + + // Start the Prometheus metrics server + go func() { + http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) + err := http.ListenAndServe(":"+strconv.FormatInt(int64(port), 10), nil) + if err != nil { + logger.LoggerMgw.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, + }) + } + }() + + for { + // Start the recordMetrics goroutine + go func() { + defer func() { + if r := recover(); r != nil { + logger.LoggerMgw.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("recordMetrics goroutine exited with error:", r), + Severity: logging.MAJOR, + ErrorCode: 1111, + }) + } + }() + recordMetrics(collectionInterval) + done <- struct{}{} // Signal that the goroutine has completed + }() + + // Wait for the previous recordMetrics goroutine to complete + <-done + + // Log and restart the goroutine + logger.LoggerMgw.Info("Restarting recordMetrics goroutine...") + time.Sleep(3 * time.Second) + } } diff --git a/integration/test-integration/src/test/resources/configs/prometheus-metrics-enabled.toml b/integration/test-integration/src/test/resources/configs/prometheus-metrics-enabled.toml index a03473487b..0fab427827 100644 --- a/integration/test-integration/src/test/resources/configs/prometheus-metrics-enabled.toml +++ b/integration/test-integration/src/test/resources/configs/prometheus-metrics-enabled.toml @@ -79,3 +79,4 @@ enabled = true type = "prometheus" port = 19085 + collectionInterval = 10 diff --git a/resources/conf/config.toml.template b/resources/conf/config.toml.template index c6375b68f4..91441112c0 100644 --- a/resources/conf/config.toml.template +++ b/resources/conf/config.toml.template @@ -99,6 +99,8 @@ soapErrorInXMLEnabled = false type = "prometheus" # The port through which metrics are exposed port = 19085 + # The periodic metric collection interval in seconds + collectionInterval = 5 # Configurations required for router to route the traffic from different clients to services [router] # -------------------------------------------------------- From de379477a8a20379d29d4fa4a4e0bc48d3e0fb18 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Thu, 14 Sep 2023 17:12:47 +0530 Subject: [PATCH 26/29] Fix PR Review Changes --- adapter/internal/adapter/adapter.go | 2 +- adapter/pkg/metrics/metrics.go | 26 +++++++++---------- .../main/resources/prometheus-jmx-config.yml | 22 +++++++++------- .../CcWithPrometheusMetricsEnabled.java | 1 - .../PrometheusMetricEndpointsTestCase.java | 6 ++--- .../metricConfigs/prometheus-jmx-config.yml | 22 +++++++++------- resources/conf/config.toml.template | 4 +-- 7 files changed, 42 insertions(+), 41 deletions(-) diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index fbf1400f43..15d6dd48ba 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -198,7 +198,7 @@ func Run(conf *config.Config) { } // Start the metrics server - if conf.Adapter.Metrics.Enabled && conf.Adapter.Metrics.Type == metrics.PrometheusMetricType { + if conf.Adapter.Metrics.Enabled && strings.ToLower(conf.Adapter.Metrics.Type) == metrics.PrometheusMetricType { logger.LoggerMgw.Info("Starting Prometheus Metrics Server ....") go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port, conf.Adapter.Metrics.CollectionInterval) diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index d46f2816e0..f35b016a1a 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -1,18 +1,19 @@ /* - * Copyright (c) 2023, WSO2 Inc. (http://www.wso2.org). + * Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ // Package metrics holds the implementation for exposing adapter metrics to prometheus @@ -38,9 +39,6 @@ import ( "github.com/wso2/product-microgateway/adapter/pkg/logging" ) -// PrometheusMetricType prometheus metric type -const PrometheusMetricType = "prometheus" - var ( prometheusMetricRegistry = prometheus.NewRegistry() diff --git a/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml b/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml index eefbcad06f..7416aa7f3f 100644 --- a/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml +++ b/enforcer-parent/enforcer/src/main/resources/prometheus-jmx-config.yml @@ -1,17 +1,19 @@ # -------------------------------------------------------------------- -# Copyright (c) 2023, WSO2 Inc. (http://wso2.com). +# Copyright (c) 2023, WSO2 LLC. (http://www.wso2.com). # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # ----------------------------------------------------------------------- # Config options: https://github.com/prometheus/jmx_exporter/blob/main/README.md diff --git a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java index e1ddbd8506..14d0c78771 100644 --- a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java +++ b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/setup/standalone/CcWithPrometheusMetricsEnabled.java @@ -39,7 +39,6 @@ void start() throws Exception { ApictlUtils.addEnv("test"); ApictlUtils.login("test"); TimeUnit.SECONDS.sleep(5); - } @AfterTest(description = "stop the setup") diff --git a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java index e1464d2722..30d4586434 100644 --- a/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java +++ b/integration/test-integration/src/test/java/org/wso2/choreo/connect/tests/testcases/standalone/metrics/PrometheusMetricEndpointsTestCase.java @@ -23,7 +23,7 @@ import org.wso2.choreo.connect.tests.util.*; public class PrometheusMetricEndpointsTestCase { - @Test(description = "Test whether enforcer metric endpoint is up") + @Test(description = "Test whether Enforcer metric endpoint is up") public void testEnforcerMetricEndpoint() throws Exception { HttpResponse response = HttpClientRequest.doGet("http://localhost:19080/metrics"); Assert.assertNotNull(response); @@ -32,7 +32,7 @@ public void testEnforcerMetricEndpoint() throws Exception { Assert.assertTrue(response.getData().contains("jvm_"), "Metric information not found in expected format"); } - @Test(description = "Test whether adapter metric endpoint is up") + @Test(description = "Test whether Adapter metric endpoint is up") public void testAdapterMetricEndpoint() throws Exception { HttpResponse response = HttpClientRequest.doGet("http://localhost:19085/metrics"); Assert.assertNotNull(response); @@ -42,7 +42,7 @@ public void testAdapterMetricEndpoint() throws Exception { } - @Test(description = "Test whether router metric endpoint is up") + @Test(description = "Test whether Router metric endpoint is up") public void testRouterMetricEndpoint() throws Exception { HttpResponse response = HttpClientRequest.doGet("http://localhost:9000/stats/prometheus"); Assert.assertNotNull(response); diff --git a/integration/test-integration/src/test/resources/dockerVolumeMounts/metricConfigs/prometheus-jmx-config.yml b/integration/test-integration/src/test/resources/dockerVolumeMounts/metricConfigs/prometheus-jmx-config.yml index b62d7a939f..7416aa7f3f 100644 --- a/integration/test-integration/src/test/resources/dockerVolumeMounts/metricConfigs/prometheus-jmx-config.yml +++ b/integration/test-integration/src/test/resources/dockerVolumeMounts/metricConfigs/prometheus-jmx-config.yml @@ -1,17 +1,19 @@ # -------------------------------------------------------------------- -# Copyright (c) 2023, WSO2 LLC. (https://wso2.com). +# Copyright (c) 2023, WSO2 LLC. (http://www.wso2.com). # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # ----------------------------------------------------------------------- # Config options: https://github.com/prometheus/jmx_exporter/blob/main/README.md diff --git a/resources/conf/config.toml.template b/resources/conf/config.toml.template index 91441112c0..e3b6c2e227 100644 --- a/resources/conf/config.toml.template +++ b/resources/conf/config.toml.template @@ -99,8 +99,8 @@ soapErrorInXMLEnabled = false type = "prometheus" # The port through which metrics are exposed port = 19085 - # The periodic metric collection interval in seconds - collectionInterval = 5 + # The periodic metric collection interval in seconds + collectionInterval = 5 # Configurations required for router to route the traffic from different clients to services [router] # -------------------------------------------------------- From fbcc13eb57c69062a52a355b3f291170f2ca6007 Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Fri, 15 Sep 2023 08:57:11 +0530 Subject: [PATCH 27/29] Add adapter metrics types.go file --- adapter/pkg/metrics/types.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 adapter/pkg/metrics/types.go diff --git a/adapter/pkg/metrics/types.go b/adapter/pkg/metrics/types.go new file mode 100644 index 0000000000..43a534be9d --- /dev/null +++ b/adapter/pkg/metrics/types.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package metrics + +const ( + // PrometheusMetricType prometheus metric type + PrometheusMetricType = "prometheus" +) From aaef8bc230fc2a711059b5db668bf94e946eeb7e Mon Sep 17 00:00:00 2001 From: Avishka Shamendra <62060834+Avishka-Shamendra@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:19:39 +0530 Subject: [PATCH 28/29] Change log order Co-authored-by: SLahiruC <40932779+slahirucd7@users.noreply.github.com> --- adapter/internal/adapter/adapter.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 15d6dd48ba..ff8ecac542 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -197,6 +197,8 @@ func Run(conf *config.Config) { }) } +logger.LoggerMgw.Info("Starting adapter ....") + // Start the metrics server if conf.Adapter.Metrics.Enabled && strings.ToLower(conf.Adapter.Metrics.Type) == metrics.PrometheusMetricType { logger.LoggerMgw.Info("Starting Prometheus Metrics Server ....") @@ -204,7 +206,7 @@ func Run(conf *config.Config) { } - logger.LoggerMgw.Info("Starting adapter ....") + cache := xds.GetXdsCache() enforcerCache := xds.GetEnforcerCache() enforcerSubscriptionCache := xds.GetEnforcerSubscriptionCache() From 7b1668e3349545785f7702713c6e908a0e79b89f Mon Sep 17 00:00:00 2001 From: Avishka-Shamendra Date: Thu, 28 Sep 2023 11:44:58 +0530 Subject: [PATCH 29/29] Refactor code --- adapter/internal/adapter/adapter.go | 2 +- adapter/pkg/metrics/metrics.go | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 15d6dd48ba..50639d08d7 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -198,7 +198,7 @@ func Run(conf *config.Config) { } // Start the metrics server - if conf.Adapter.Metrics.Enabled && strings.ToLower(conf.Adapter.Metrics.Type) == metrics.PrometheusMetricType { + if conf.Adapter.Metrics.Enabled && strings.EqualFold(conf.Adapter.Metrics.Type, metrics.PrometheusMetricType) { logger.LoggerMgw.Info("Starting Prometheus Metrics Server ....") go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port, conf.Adapter.Metrics.CollectionInterval) diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index f35b016a1a..14326e0689 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -119,7 +119,12 @@ var recordMetrics = func(collectionInterval int32) { if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { return } - systemCPULoad.Set(percentages[0]) + totalPercentage := 0.0 + for _, p := range percentages { + totalPercentage += p + } + averagePercentage := totalPercentage / float64(len(percentages)) + systemCPULoad.Set(averagePercentage) pid := os.Getpid() p, err := procfs.NewProc(pid) @@ -174,7 +179,7 @@ func StartPrometheusMetricsServer(port int32, collectionInterval int32) { // Start the Prometheus metrics server go func() { http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) - err := http.ListenAndServe(":"+strconv.FormatInt(int64(port), 10), nil) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) if err != nil { logger.LoggerMgw.ErrorC(logging.ErrorDetails{ Message: fmt.Sprintln("Prometheus metrics server error:", err),