From 709dcb3aa70e62e42cce2b24832e7ad4bfa584df Mon Sep 17 00:00:00 2001 From: Paul Moineau Date: Fri, 30 Jun 2023 14:35:48 -0400 Subject: [PATCH] W-13680300: add protobuf support --- build.gradle | 22 ++--- carbonj.service/build.gradle | 85 ++++++++++++------- .../engine/GraphiteSeriesDataServlet.java | 38 +++++++++ .../src/main/protobuf/MetricsResponse.proto | 23 +++++ 4 files changed, 128 insertions(+), 40 deletions(-) create mode 100644 carbonj.service/src/main/protobuf/MetricsResponse.proto diff --git a/build.gradle b/build.gradle index 8336af28..29bf3701 100644 --- a/build.gradle +++ b/build.gradle @@ -10,11 +10,11 @@ buildscript { } plugins { - id 'com.palantir.docker' version "0.26.0" - id "com.github.jk1.dependency-license-report" version "1.16" - id "com.github.hierynomus.license" version "0.15.0" - id "org.sonarqube" version "3.2.0" - id 'net.researchgate.release' version '2.8.1' + id 'com.palantir.docker' version "0.35.0" + //id "com.github.jk1.dependency-license-report" version "2.5" + //id "com.github.hierynomus.license" version "0.16.1" + id "org.sonarqube" version "4.2.1.3168" + id 'net.researchgate.release' version '3.0.2' } release { @@ -26,17 +26,17 @@ release { allprojects { // TODO remove - apply plugin: 'maven' - apply plugin: "com.github.hierynomus.license" + apply plugin: 'maven-publish' + //apply plugin: "com.github.hierynomus.license" apply plugin: 'eclipse' apply plugin: 'idea' group = 'com.demandware.carbonj' - license { - header rootProject.file('LICENSE-HEADER-JAVA') - exclude "**/banner.txt" + //license { + // header rootProject.file('LICENSE-HEADER-JAVA') + // exclude "**/banner.txt" // include "**/*.conf" - } + //} } diff --git a/carbonj.service/build.gradle b/carbonj.service/build.gradle index 72f59375..b75f03aa 100644 --- a/carbonj.service/build.gradle +++ b/carbonj.service/build.gradle @@ -15,27 +15,47 @@ buildscript { } plugins { - id "com.github.hierynomus.license" + //id "com.github.hierynomus.license" id 'maven-publish' id 'net.researchgate.release' id 'org.springframework.boot' + id 'com.google.protobuf' version "0.9.3" } -license { - header rootProject.file('LICENSE-HEADER-JAVA') - exclude "**/banner.txt" -// include "**/*.conf" -} - -task licenseFormatDockerFiles(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { - source = fileTree(dir: "src").include("main/docker/files/*") +sourceSets { + main { + proto { + srcDir 'src/main/protobuf' + } + java { + srcDirs 'build/generated/source/proto/main/java' + } + } } -licenseFormat.dependsOn licenseFormatDockerFiles -task licenseCheckDockerFiles(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { - source = fileTree(dir: "src").include("main/docker/files/*") +//license { +// header rootProject.file('LICENSE-HEADER-JAVA') +// exclude "**/banner.txt" +// include "**/*.conf" +//} + +//task licenseFormatDockerFiles(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { +// source = fileTree(dir: "src").include("main/docker/files/*") +//} +//licenseFormat.dependsOn licenseFormatDockerFiles + +//task licenseCheckDockerFiles(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { +// source = fileTree(dir: "src").include("main/docker/files/*") +//} +//licenseMain.dependsOn licenseCheckDockerFiles + +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.23.2' + //M1 support + //artifact = 'com.google.protobuf:protoc:3.14.0:osx-x86_64' + } } -licenseMain.dependsOn licenseCheckDockerFiles apply plugin: 'java' apply plugin: 'eclipse' @@ -187,22 +207,29 @@ description = 'CarbonJ Service Implementation' dependencies { compile project(':cc-metrics-reporter') // compile group: 'com.sfcc.um', name: 'metrics_reporter', version:'0.0.2' - compile group: 'org.apache.logging.log4j', name: 'log4j-1.2-api' - compile group: 'org.slf4j', name: 'slf4j-api' - compile group: 'org.rocksdb', name: 'rocksdbjni', version: '6.20.3' - compile group: 'com.google.code.gson', name: 'gson', version: '2.8.7' - compile group: 'com.google.code.externalsortinginjava', name: 'externalsortinginjava', version: '0.6.0' - compile group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' - compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' - compile group: 'commons-io', name: 'commons-io', version: '2.9.0' - compile group: 'joda-time', name: 'joda-time', version: '2.10.10' - compile group: 'io.netty', name: 'netty-all', version: '4.1.65.Final' - compile group: 'net.razorvine', name: 'pickle', version: '1.1' - compile group: 'org.python', name: 'jython-standalone', version: '2.7.2' - compile group: 'com.amazonaws', name: 'amazon-kinesis-client', version: '1.14.3' - compile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '4.2.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: '4.2.0' - compile group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: '4.2.0' + implementation group: 'org.apache.logging.log4j', name: 'log4j-1.2-api', version: '2.20.0' + implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0' + implementation group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: '2.20.0' + implementation group: 'org.slf4j', name: 'slf4j-api' + implementation group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' + implementation group: 'org.rocksdb', name: 'rocksdbjni', version: '8.3.2' + implementation group: 'com.google.code.gson', name: 'gson', version: '2.10.1' + implementation group: 'com.google.code.externalsortinginjava', name: 'externalsortinginjava', version: '0.6.2' + implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' + implementation group: 'commons-io', name: 'commons-io', version: '2.13.0' + implementation group: 'joda-time', name: 'joda-time', version: '2.12.5' + implementation group: 'io.netty', name: 'netty-all', version: '4.1.94.Final' + implementation group: 'net.razorvine', name: 'pickle', version: '1.4' + implementation group: 'org.python', name: 'jython-standalone', version: '2.7.3' + implementation group: 'com.amazonaws', name: 'amazon-kinesis-client', version: '1.15.0' + implementation group: 'io.dropwizard.metrics', name: 'metrics-core', version: '4.2.19' + implementation group: 'io.dropwizard.metrics', name: 'metrics-jvm', version: '4.2.19' + implementation group: 'io.dropwizard.metrics', name: 'metrics-graphite', version: '4.2.19' + implementation group: 'com.google.protobuf', name: 'protobuf-java', version: '3.23.2' + implementation 'jakarta.servlet:jakarta.servlet-api:6.0.0' + implementation 'org.eclipse.jetty:jetty-server' +// implementation 'org.mockito:mockito-core:4.11.0' testCompile group: 'junit', name: 'junit', version: '4.13.2' testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3' diff --git a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/GraphiteSeriesDataServlet.java b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/GraphiteSeriesDataServlet.java index f56753bf..c723f48e 100644 --- a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/GraphiteSeriesDataServlet.java +++ b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/GraphiteSeriesDataServlet.java @@ -7,6 +7,8 @@ package com.demandware.carbonj.service.engine; import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; import java.util.List; import javax.servlet.ServletConfig; @@ -19,6 +21,7 @@ import com.demandware.carbonj.service.db.TimeSeriesStore; import com.demandware.carbonj.service.db.model.Series; import com.demandware.carbonj.service.db.util.SystemTime; +import com.demandware.carbonj.service.engine.protobuf.MetricsResponse; import com.google.gson.Gson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,6 +91,7 @@ protected void doGet( HttpServletRequest req, HttpServletResponse res ) boolean randomTest = req.getParameter("randomTest") != null; + boolean protobuf = "carbonapi_v3_pb".equals( format ); boolean json = "json".equals( format ); if( json ) { @@ -138,6 +142,40 @@ protected void doGet( HttpServletRequest req, HttpServletResponse res ) res.getWriter().write( gson.toJson( series ) ); res.getWriter().close(); } + else if ( protobuf ) + { + List seriesList = store.fetchSeriesData( new Query( target, Integer.parseInt( from ), + Integer.parseInt( until ), now, System.currentTimeMillis() ) ); + + OutputStream output = res.getOutputStream(); + + List metricsSeriesList = new ArrayList(); + for ( Series series : seriesList ) + { + List valuesList = new ArrayList(); + for ( Double value : series.values ) + { + valuesList.add( MetricsResponse.Value.newBuilder().setValue( value ).build() ); + } + MetricsResponse.Series metricsSeries = + MetricsResponse.Series.newBuilder().setName( series.name ).setStart( series.start ) + .setEnd( series.end ).setStep( series.step ).addAllValues( valuesList ).build(); + + metricsSeriesList.add( metricsSeries ); + } + + MetricsResponse.SeriesList response = + MetricsResponse.SeriesList.newBuilder().addAllSeriesList( metricsSeriesList ).build(); + + try + { + response.writeTo( output ); + } + finally + { + output.close(); + } + } else { ResponseStream seriesStream = new GraphitePickler( false, res.getOutputStream() ); diff --git a/carbonj.service/src/main/protobuf/MetricsResponse.proto b/carbonj.service/src/main/protobuf/MetricsResponse.proto new file mode 100644 index 00000000..ca03b2df --- /dev/null +++ b/carbonj.service/src/main/protobuf/MetricsResponse.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package protobuf; + +option java_package = "com.demandware.carbonj.service.engine.protobuf"; +option java_outer_classname = "MetricsResponse"; + + +message Value { + double value = 1; +} + +message Series { + string name = 1; + int64 start = 2; + int64 end = 3; + int64 step = 4; + repeated Value values = 5; +} + +message SeriesList { + repeated Series seriesList = 1; +} \ No newline at end of file