From ea6ddbadada3650058a8514f4a7cba092ddb9c04 Mon Sep 17 00:00:00 2001 From: Paul Moineau Date: Wed, 5 Jul 2023 13:07:10 -0400 Subject: [PATCH] protobuf support --- build.gradle | 14 +++--- carbonj.service/Dockerfile | 20 +++------ carbonj.service/build.gradle | 44 ++++++++++++++----- .../src/main/docker/files/entrypoint.sh | 10 +++++ .../engine/GraphiteSeriesDataServlet.java | 38 ++++++++++++++++ .../src/main/protobuf/MetricsResponse.proto | 23 ++++++++++ 6 files changed, 117 insertions(+), 32 deletions(-) create mode 100644 carbonj.service/src/main/protobuf/MetricsResponse.proto diff --git a/build.gradle b/build.gradle index 8359a24a..5f37067e 100644 --- a/build.gradle +++ b/build.gradle @@ -11,8 +11,8 @@ buildscript { plugins { id 'com.palantir.docker' version "0.28.0" - id "com.github.jk1.dependency-license-report" version "1.17" - id "com.github.hierynomus.license" version "0.15.0" + //id "com.github.jk1.dependency-license-report" version "1.17" + //id "com.github.hierynomus.license" version "0.15.0" id "org.sonarqube" version "3.3" id 'net.researchgate.release' version '2.8.1' } @@ -27,16 +27,16 @@ release { allprojects { // TODO remove apply plugin: 'maven' - apply plugin: "com.github.hierynomus.license" + //apply plugin: "com.github.hierynomus.license" apply plugin: 'eclipse' apply plugin: 'idea' group = 'com.demandware.carbonj' - license { - header rootProject.file('LICENSE-HEADER-JAVA') - exclude "**/banner.txt" + //license { + // header rootProject.file('LICENSE-HEADER-JAVA') + // exclude "**/banner.txt" // include "**/*.conf" - } + //} } diff --git a/carbonj.service/Dockerfile b/carbonj.service/Dockerfile index 3e10ee34..61a2f6bd 100644 --- a/carbonj.service/Dockerfile +++ b/carbonj.service/Dockerfile @@ -5,7 +5,7 @@ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause # -FROM quay.io/centos/centos:stream8 +FROM docker-registry.releng.demandware.net/common/cc-centos-openjdk-testrunner_11u18:b9da990 ARG DEPENDENCY=target/dependency ARG VERSION=latest @@ -21,18 +21,14 @@ RUN yum update -y && \ nc \ lsof \ sysstat \ - epel-release + epel-release \ + python3 \ + perl-URI-Encode \ + perl-Data-Dumper RUN yum install -y gcc-c++ gcc make libtool automake autoconf make python3-devel -RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \ - curl -o /etc/yum.repos.d/zulu.repo http://repos.azulsystems.com/rhel/zulu.repo && \ - yum update -y && \ - yum install -y zulu-11 \ - python3 \ - perl-URI-Encode \ - perl-Data-Dumper && \ - yum clean all +RUN pip3 install --upgrade pip # Install aiohttp required by the script RUN pip3 install aiohttp @@ -93,10 +89,6 @@ ENV DW_PREFIX=jetty ENV SERVICEDIR=/app/ ENV app_servicedir=/app/ -RUN wget https://download.yourkit.com/yjp/2017.02/YourKit-JavaProfiler-2017.02-b75.zip -P /tmp/ && \ - unzip /tmp/YourKit-JavaProfiler-2017.02-b75.zip && mv YourKit-JavaProfiler-2017.02 /usr/local/yjp/ && \ - rm /tmp/YourKit-JavaProfiler-2017.02-b75.zip -RUN wget https://download.yourkit.com/yjp/2017.02/yjp.jar -P /usr/local/yjp/lib/ # Add Tini for proper sigkill handling https://github.com/krallin/tini RUN wget https://github.com/krallin/tini/releases/download/v0.18.0/tini RUN mv tini /sbin/tini diff --git a/carbonj.service/build.gradle b/carbonj.service/build.gradle index 9436285b..cd8aee5f 100644 --- a/carbonj.service/build.gradle +++ b/carbonj.service/build.gradle @@ -15,27 +15,48 @@ buildscript { } plugins { - id "com.github.hierynomus.license" + //id "com.github.hierynomus.license" id 'maven-publish' id 'net.researchgate.release' id 'org.springframework.boot' + id 'com.google.protobuf' version "0.9.3" } -license { - header rootProject.file('LICENSE-HEADER-JAVA') - exclude "**/banner.txt" +//license { + // header rootProject.file('LICENSE-HEADER-JAVA') + // exclude "**/banner.txt" // include "**/*.conf" -} +//} -task licenseFormatDockerFiles(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { - source = fileTree(dir: "src").include("main/docker/files/*") +sourceSets { + main { + proto { + srcDir 'src/main/protobuf' + } + java { + srcDirs 'build/generated/source/proto/main/java' + } + } } -licenseFormat.dependsOn licenseFormatDockerFiles -task licenseCheckDockerFiles(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { - source = fileTree(dir: "src").include("main/docker/files/*") +//task licenseFormatDockerFiles(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { + // source = fileTree(dir: "src").include("main/docker/files/*") +//} +//licenseFormat.dependsOn licenseFormatDockerFiles + +//task licenseCheckDockerFiles(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { + // source = fileTree(dir: "src").include("main/docker/files/*") +//} +//licenseMain.dependsOn licenseCheckDockerFiles + +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.14.0' + //artifact = 'com.google.protobuf:protoc:3.23.2' + //M1 support + //artifact = 'com.google.protobuf:protoc:3.14.0:osx-x86_64' + } } -licenseMain.dependsOn licenseCheckDockerFiles apply plugin: 'java' apply plugin: 'eclipse' @@ -203,6 +224,7 @@ dependencies { compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '4.2.3' compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: '4.2.3' compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: '4.2.3' + compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.23.2' testCompile group: 'junit', name: 'junit', version: '4.13.2' testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3' diff --git a/carbonj.service/src/main/docker/files/entrypoint.sh b/carbonj.service/src/main/docker/files/entrypoint.sh index 58e28a59..4c114e27 100755 --- a/carbonj.service/src/main/docker/files/entrypoint.sh +++ b/carbonj.service/src/main/docker/files/entrypoint.sh @@ -54,6 +54,16 @@ for SVC_PROP in `compgen -A variable | grep "^SVC_PROP_"` ; do printf '%s=%s\n' "$var_lowercase" "${!SVC_PROP}" >> $SERVICEDIR/config/overrides.properties done +# download optional yourkit profiler at runtime +if [ "$ENABLE_YOURKIT_PROFILER" == "true" ] +then + wget https://download.yourkit.com/yjp/2017.02/YourKit-JavaProfiler-2017.02-b75.zip -P /tmp/ && \ + unzip /tmp/YourKit-JavaProfiler-2017.02-b75.zip && mv YourKit-JavaProfiler-2017.02 /usr/local/yjp/ && \ + rm /tmp/YourKit-JavaProfiler-2017.02-b75.zip + wget https://download.yourkit.com/yjp/2017.02/yjp.jar -P /tmp + mv /tmp/yjp.jar /usr/local/yjp/lib/yjp.jar +fi + ######################### ######################### ######################### diff --git a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/GraphiteSeriesDataServlet.java b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/GraphiteSeriesDataServlet.java index f56753bf..c6a4d009 100644 --- a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/GraphiteSeriesDataServlet.java +++ b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/GraphiteSeriesDataServlet.java @@ -7,6 +7,8 @@ package com.demandware.carbonj.service.engine; import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; import java.util.List; import javax.servlet.ServletConfig; @@ -19,6 +21,7 @@ import com.demandware.carbonj.service.db.TimeSeriesStore; import com.demandware.carbonj.service.db.model.Series; import com.demandware.carbonj.service.db.util.SystemTime; +import com.demandware.carbonj.service.engine.protobuf.MetricsResponse; import com.google.gson.Gson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,6 +91,7 @@ protected void doGet( HttpServletRequest req, HttpServletResponse res ) boolean randomTest = req.getParameter("randomTest") != null; + boolean protobuf = "carbonapi_v3_pb".equals( format ); boolean json = "json".equals( format ); if( json ) { @@ -138,6 +142,40 @@ protected void doGet( HttpServletRequest req, HttpServletResponse res ) res.getWriter().write( gson.toJson( series ) ); res.getWriter().close(); } + else if ( protobuf ) + { + List seriesList = store.fetchSeriesData( new Query( target, Integer.parseInt( from ), + Integer.parseInt( until ), now, System.currentTimeMillis() ) ); + + OutputStream output = res.getOutputStream(); + + List metricsSeriesList = new ArrayList(); + for ( Series series : seriesList ) + { + List valuesList = new ArrayList(); + for ( Double value : series.values ) + { + valuesList.add( MetricsResponse.Value.newBuilder().setValue( value ).build() ); + } + MetricsResponse.Series metricsSeries = + MetricsResponse.Series.newBuilder().setName( series.name ).setStart( series.start ) + .setEnd( series.end ).setStep( series.step ).addAllValues( valuesList ).build(); + + metricsSeriesList.add( metricsSeries ); + } + + MetricsResponse.SeriesList response = + MetricsResponse.SeriesList.newBuilder().addAllSeriesList( metricsSeriesList ).build(); + + try + { + response.writeTo( output ); + } + finally + { + output.close(); + } + } else { ResponseStream seriesStream = new GraphitePickler( false, res.getOutputStream() ); diff --git a/carbonj.service/src/main/protobuf/MetricsResponse.proto b/carbonj.service/src/main/protobuf/MetricsResponse.proto new file mode 100644 index 00000000..ca03b2df --- /dev/null +++ b/carbonj.service/src/main/protobuf/MetricsResponse.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package protobuf; + +option java_package = "com.demandware.carbonj.service.engine.protobuf"; +option java_outer_classname = "MetricsResponse"; + + +message Value { + double value = 1; +} + +message Series { + string name = 1; + int64 start = 2; + int64 end = 3; + int64 step = 4; + repeated Value values = 5; +} + +message SeriesList { + repeated Series seriesList = 1; +} \ No newline at end of file