Issue 17: Format commands as pretty JSON (#31)

Change the output format of commands that retrieve information from the system to pretty JSON.

Signed-off-by: Raúl Gracia <raul.gracia@emc.com>

pravega-cli/build.gradle

@@ -69,6 +69,8 @@ dependencies {
     compile group: 'io.pravega', name: 'pravega-shared-cluster', version: pravegaVersion
     compile group: 'io.pravega', name: 'pravega-test-integration', version: pravegaVersion
     compile group: 'javax.ws.rs', name: 'javax.ws.rs-api', version: javaxwsrsApiVersion
+    compile group: 'com.googlecode.json-simple', name: 'json-simple', version: jsonSimpleVersion
+    compile group: 'com.google.code.gson', name: 'gson', version: gsonVersion
 }
 
 tasks

pravega-cli/gradle.properties

@@ -11,14 +11,16 @@
 apacheCuratorVersion=4.0.1
 bookKeeperVersion=4.7.3
 checkstyleToolVersion=8.2
+gsonVersion=2.8.5
 hadoopVersion=2.7.3
 jacocoVersion=0.8.2
 javaxwsrsApiVersion=2.1
 junitVersion=4.12
+jsonSimpleVersion=1.1.1
 lombokVersion=1.18.4
 mockitoVersion=2.23.0
 nettyVersion=4.1.30.Final
-pravegaVersion=0.6.0-50.d6ec7cb-SNAPSHOT
+pravegaVersion=0.6.0-50.9f4fc18-SNAPSHOT
 slf4jApiVersion=1.7.25
 spotbugsPluginVersion=1.6.9
 spotbugsVersion=3.1.11

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/Command.java

@@ -9,9 +9,14 @@
  */
 package io.pravega.tools.pravegacli.commands;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParser;
 import io.pravega.common.Exceptions;
 import io.pravega.segmentstore.server.store.ServiceConfig;
 import io.pravega.tools.pravegacli.commands.bookkeeper.BookKeeperCleanupCommand;
@@ -127,6 +132,16 @@ protected void output(String messageTemplate, Object... args) {
         this.out.println(String.format(messageTemplate, args));
     }
 
+    protected void prettyJSONOutput(String jsonString) {
+        JsonElement je = new JsonParser().parse(jsonString);
+        output(new GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create().toJson(je));
+    }
+
+    protected void prettyJSONOutput(String key, Object value) {
+        JsonElement je = new JsonParser().parse(objectToJSON(new Tuple(key, value)));
+        output(new GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create().toJson(je));
+    }
+
     protected boolean confirmContinue() {
         output("Do you want to continue?[yes|no]");
         Scanner s = new Scanner(System.in);
@@ -305,5 +320,21 @@ private static class CommandInfo {
         private interface CommandCreator extends Function<CommandArgs, Command> {
         }
     }
+
+    @Data
+    private static class Tuple {
+        private final String key;
+        private final Object value;
+    }
+
+    private String objectToJSON(Object object) {
+        try {
+            return new ObjectMapper().writeValueAsString(object);
+        } catch (JsonProcessingException e) {
+            System.err.println("Exception parsing object: " + e.getMessage());
+        }
+        return "";
+    }
+
     //endregion
 }

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/bookkeeper/BookKeeperCommand.java

@@ -17,10 +17,7 @@
 import io.pravega.segmentstore.storage.impl.bookkeeper.ReadOnlyLogMetadata;
 import io.pravega.tools.pravegacli.commands.Command;
 import io.pravega.tools.pravegacli.commands.CommandArgs;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.val;
+import lombok.*;
 import org.apache.bookkeeper.client.BKException;
 import org.apache.bookkeeper.client.BookKeeperAdmin;
 import org.apache.curator.framework.CuratorFramework;
@@ -41,12 +38,12 @@ abstract class BookKeeperCommand extends Command {
      * @param logId The Log Id.
      * @param m     The Log Metadata for the given Log Id.
      */
-    protected void outputLogSummary(int logId, ReadOnlyLogMetadata m) {
+    void outputLogSummary(int logId, ReadOnlyLogMetadata m) {
         if (m == null) {
-            output("Log %d: No metadata.", logId);
+            prettyJSONOutput("log_no_metadata)", logId);
         } else {
-            output("Log %d: Epoch=%d, Version=%d, Enabled=%s, Ledgers=%d, Truncation={%s}", logId,
-                    m.getEpoch(), m.getUpdateVersion(), m.isEnabled(), m.getLedgers().size(), m.getTruncationAddress());
+            prettyJSONOutput("log_summary", new LogSummary(logId, m.getEpoch(), m.getUpdateVersion(), m.isEnabled(),
+                    m.getLedgers().size(), String.valueOf(m.getTruncationAddress())));
         }
     }
 
@@ -93,4 +90,15 @@ public void close() {
             Exceptions.handleInterrupted(this.bkAdmin::close);
         }
     }
+
+    @Data
+    @AllArgsConstructor
+    private static class LogSummary {
+        private int logId;
+        private long epoch;
+        private int version;
+        private boolean enabled;
+        private int ledgers;
+        private String truncation;
+    }
 }

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/bookkeeper/BookKeeperDetailsCommand.java

@@ -11,10 +11,11 @@
 
 import io.pravega.segmentstore.storage.impl.bookkeeper.LedgerMetadata;
 import io.pravega.tools.pravegacli.commands.CommandArgs;
-import java.util.stream.Collectors;
 import lombok.Cleanup;
+import lombok.Data;
 import lombok.val;
 import org.apache.bookkeeper.client.LedgerHandle;
+import java.util.stream.Collectors;
 
 /**
  * Fetches details about a BookKeeperLog.
@@ -56,11 +57,11 @@ public void execute() throws Exception {
             try {
                 lh = log.openLedgerNoFencing(lm);
                 val bkLm = context.bkAdmin.getLedgerMetadata(lh);
-                output("\tLedger %d: Seq=%d, Status=%s, LAC=%d, Length=%d, Bookies=%d, Frags=%d, E/W/A=%d/%d/%d, Ensembles=%s.",
-                        lm.getLedgerId(), lm.getSequence(), lm.getStatus(),
+                prettyJSONOutput("ledger_details", new LedgerDetails(lm.getLedgerId(), lm.getSequence(), String.valueOf(lm.getStatus()),
                         lh.getLastAddConfirmed(), lh.getLength(), lh.getNumBookies(), lh.getNumFragments(),
-                        bkLm.getEnsembleSize(), bkLm.getWriteQuorumSize(), bkLm.getAckQuorumSize(), getEnsembleDescription(bkLm));
+                        bkLm.getEnsembleSize(), bkLm.getWriteQuorumSize(), bkLm.getAckQuorumSize(), getEnsembleDescription(bkLm)));
             } catch (Exception ex) {
+                System.err.println("Exception executing BK details command: " + ex.getMessage());
                 output("\tLedger %d: Seq = %d, Status = %s. BK: %s",
                         lm.getLedgerId(), lm.getSequence(), lm.getStatus(), ex.getMessage());
             } finally {
@@ -73,7 +74,7 @@ public void execute() throws Exception {
 
     private String getEnsembleDescription(org.apache.bookkeeper.client.LedgerMetadata bkLm) {
         return bkLm.getEnsembles().entrySet().stream()
-                   .map(e -> String.format("%d:[%s]", e.getKey(), e.getValue().stream().map(Object::toString).collect(Collectors.joining(","))))
+                   .map(e -> String.format("%d: [%s]", e.getKey(), e.getValue().stream().map(Object::toString).collect(Collectors.joining(","))))
                    .collect(Collectors.joining(","));
     }
 
@@ -82,4 +83,19 @@ public static CommandDescriptor descriptor() {
                 "Lists metadata details about a BookKeeperLog, including BK Ledger information.",
                 new ArgDescriptor("log-id", "Id of the log to get details for."));
     }
+
+    @Data
+    private static class LedgerDetails {
+        private final long ledger;
+        private final int seq;
+        private final String status;
+        private final long lac;
+        private final long length;
+        private final long numBookies;
+        private final long numFragments;
+        private final int ensembleSize;
+        private final int writeQuorumSize;
+        private final int ackQuorumSize;
+        private final String ensembles;
+    }
 }

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/cluster/GetClusterNodesCommand.java

@@ -29,13 +29,10 @@ public void execute() {
         try {
             @Cleanup
             ZKHelper zkStoreHelper = ZKHelper.create(getServiceConfig().getZkURL(), getServiceConfig().getClusterName());
-            output("Cluster name: " + getServiceConfig().getClusterName());
-            output("Controller instances in the cluster:");
-            zkStoreHelper.getControllers().forEach(c -> output("> " + c));
-            output("Segment Store instances in the cluster:");
-            zkStoreHelper.getSegmentStores().forEach(ss -> output("> " + ss));
-            output("Bookies in the cluster:");
-            zkStoreHelper.getBookies().forEach(b -> output("> " + b));
+            prettyJSONOutput("cluster_name", getServiceConfig().getClusterName());
+            prettyJSONOutput("controllers", zkStoreHelper.getControllers());
+            prettyJSONOutput("segment_stores", zkStoreHelper.getSegmentStores());
+            prettyJSONOutput("bookies", zkStoreHelper.getBookies());
         } catch (Exception e) {
             System.err.println("Exception accessing to Zookeeper cluster metadata.");
         }

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/cluster/GetSegmentStoreByContainerCommand.java

@@ -32,7 +32,7 @@ public void execute() {
             @Cleanup
             ZKHelper zkStoreHelper = ZKHelper.create(getServiceConfig().getZkURL(), getServiceConfig().getClusterName());
             Optional<Host> host = zkStoreHelper.getHostForContainer(getIntArg(0));
-            output("Owner Segment Store: " + (host.isPresent() ? host.get() : "not found"));
+            prettyJSONOutput("owner_segment_store", host.get());
         } catch (Exception e) {
             System.err.println("Exception accessing to Zookeeper cluster metadata.");
         }

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/cluster/ListContainersCommand.java

@@ -28,9 +28,7 @@ public void execute() {
         try {
             @Cleanup
             ZKHelper zkStoreHelper = ZKHelper.create(getServiceConfig().getZkURL(), getServiceConfig().getClusterName());
-            output("Segment Store to Container map:");
-            zkStoreHelper.getCurrentHostMap().forEach((host, containers) ->
-                    output(">" + host + " -> " + containers + "\n"));
+            prettyJSONOutput("segment_store_container_map", zkStoreHelper.getCurrentHostMap());
         } catch (Exception e) {
             System.err.println("Exception accessing to Zookeeper cluster metadata: " + e.getMessage());
         }

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/controller/ControllerDescribeReaderGroupCommand.java

@@ -35,7 +35,7 @@ public void execute() {
         val context = createContext();
         String scope = getCommandArgs().getArgs().get(0);
         String readerGroup = getCommandArgs().getArgs().get(1);
-        output(executeRESTCall(context, "/v1/scopes/" + scope + "/readergroups/" + readerGroup));
+        prettyJSONOutput(executeRESTCall(context, "/v1/scopes/" + scope + "/readergroups/" + readerGroup));
     }
 
     public static CommandDescriptor descriptor() {

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/controller/ControllerDescribeScopeCommand.java

@@ -34,7 +34,7 @@ public void execute() {
         @Cleanup
         val context = createContext();
         // Print the response sent by the Controller.
-        output(executeRESTCall(context, "/v1/scopes/" + getCommandArgs().getArgs().get(0)));
+        prettyJSONOutput(executeRESTCall(context, "/v1/scopes/" + getCommandArgs().getArgs().get(0)));
     }
 
     public static CommandDescriptor descriptor() {

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/controller/ControllerDescribeStreamCommand.java

@@ -21,24 +21,19 @@
 import io.pravega.controller.store.host.HostMonitorConfig;
 import io.pravega.controller.store.host.HostStoreFactory;
 import io.pravega.controller.store.host.impl.HostMonitorConfigImpl;
-import io.pravega.controller.store.stream.ScaleMetadata;
 import io.pravega.controller.store.stream.StreamMetadataStore;
 import io.pravega.controller.store.stream.StreamStoreFactory;
-import io.pravega.controller.store.stream.VersionedMetadata;
 import io.pravega.controller.store.stream.records.ActiveTxnRecord;
-import io.pravega.controller.store.stream.records.EpochRecord;
-import io.pravega.controller.store.stream.records.StreamTruncationRecord;
 import io.pravega.controller.util.Config;
 import io.pravega.tools.pravegacli.commands.CommandArgs;
 import io.pravega.tools.pravegacli.commands.utils.CLIControllerConfig;
 import java.net.URI;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ScheduledExecutorService;
-import java.util.stream.Collectors;
+
 import lombok.Cleanup;
 import org.apache.curator.framework.CuratorFramework;
 
@@ -61,7 +56,6 @@ public void execute() {
         ensureArgCount(2);
         final String scope = getCommandArgs().getArgs().get(0);
         final String stream = getCommandArgs().getArgs().get(1);
-        StringBuilder responseBuilder = new StringBuilder();
 
         try {
             @Cleanup
@@ -82,53 +76,33 @@ public void execute() {
 
             // Output the configuration of this Stream.
             CompletableFuture<StreamConfiguration> streamConfig = store.getConfiguration(scope, stream, null, executor);
-            responseBuilder.append("Stream configuration: ").append(streamConfig.join().toString()).append("\n");
+            prettyJSONOutput("stream_config", streamConfig.join());
 
             // Output the state for this Stream.
-            responseBuilder.append("Stream state: ").append(store.getState(scope, stream, true, null,
-                    executor).join().toString()).append("\n");
+            prettyJSONOutput("stream_state", store.getState(scope, stream, true, null, executor).join());
 
             // Output the total number of segments for this Stream.
             Set<Long> segments = store.getAllSegmentIds(scope, stream, null, executor).join();
-            responseBuilder.append("Total number of Stream segments: ").append(segments.size()).append("\n");
+            prettyJSONOutput("segment_count", segments.size());
 
             // Check if the Stream is sealed.
-            responseBuilder.append("Is Stream sealed? ").append(store.isSealed(scope, stream, null, executor).join()).append("\n");
-
-            // Output the active epoch for this Stream.
-            EpochRecord epochRecord = store.getActiveEpoch(scope, stream, null, true, executor).join();
-            responseBuilder.append("Current Stream epoch: ").append(epochRecord.getEpoch()).append(", creation time: ")
-                           .append(epochRecord.getCreationTime()).append("\n");
+            prettyJSONOutput("is_sealed", store.isSealed(scope, stream, null, executor).join());
 
             // Output the active epoch for this Stream.
-            responseBuilder.append("Segments in active epoch: ").append("\n");
-            epochRecord.getSegments().forEach(s -> responseBuilder.append("> ").append(s.toString()).append("\n"));
+            prettyJSONOutput("active_epoch", store.getActiveEpoch(scope, stream, null, true, executor).join());
 
             // Output the number of active Transactions for ths Stream.
-            responseBuilder.append("Active Transactions in Stream: ").append("\n");
-            Map<UUID, ActiveTxnRecord> activeTxn = store.getActiveTxns(scope, stream, null,
-                    getCommandArgs().getState().getExecutor()).join();
-            activeTxn.forEach((txnId, txnRecord) -> responseBuilder.append("> TxnId: ").append(txnId).append(", TxnRecord: ")
-                                                                   .append(txnRecord.toString()).append("\n"));
+            Map<UUID, ActiveTxnRecord> activeTxn = store.getActiveTxns(scope, stream, null, getCommandArgs().getState().getExecutor()).join();
+            if (!activeTxn.isEmpty()) {
+                prettyJSONOutput("active_transactions", activeTxn);
+            }
 
             // Output Truncation point.
-            VersionedMetadata<StreamTruncationRecord> truncationRecord = store.getTruncationRecord(scope, stream,
-                    null, executor).join();
-            responseBuilder.append("Stream truncation record: lower epoch: ").append(truncationRecord.getObject().getSpanEpochLow())
-                           .append(", high epoch: ").append(truncationRecord.getObject().getSpanEpochHigh()).append(", deleted segments: ")
-                           .append(truncationRecord.getObject().getDeletedSegments().size()).append(", StreamCut: ")
-                           .append(truncationRecord.getObject().getStreamCut().toString()).append("\n");
+            prettyJSONOutput("truncation_record", store.getTruncationRecord(scope, stream, null, executor).join().getObject());
 
             // Output the metadata that describes all the scaling information for this Stream.
-            List<ScaleMetadata> scaleMetadata = store.getScaleMetadata(scope, stream, segments.stream().min(Long::compareTo).get(),
-                    segments.stream().max(Long::compareTo).get(), null, executor).join();
-            scaleMetadata.forEach(s -> responseBuilder.append("> Scale time: ").append(s.getTimestamp()).append(", splits: ")
-                                                      .append(s.getSplits()).append(", merges: ").append(s.getMerges()).append(", segments: ")
-                                                      .append(s.getSegments().stream()
-                                                               .map(segment -> String.valueOf(segment.getNumber()))
-                                                               .collect(Collectors.joining("-", "{", "}")))
-                                                      .append("\n"));
-            output(responseBuilder.toString());
+            prettyJSONOutput("scaling_info", store.getScaleMetadata(scope, stream, segments.stream().min(Long::compareTo).get(),
+                    segments.stream().max(Long::compareTo).get(), null, executor).join());
 
             // Cleanup resources.
             if (segmentHelper != null) {

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/controller/ControllerListReaderGroupsInScopeCommand.java

@@ -33,7 +33,7 @@ public void execute() {
         @Cleanup
         val context = createContext();
         String scope = getCommandArgs().getArgs().get(0);
-        output(executeRESTCall(context, "/v1/scopes/" + scope + "/readergroups"));
+        prettyJSONOutput(executeRESTCall(context, "/v1/scopes/" + scope + "/readergroups"));
     }
 
     public static CommandDescriptor descriptor() {

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/controller/ControllerListScopesCommand.java

@@ -32,7 +32,7 @@ public void execute() {
         ensureArgCount(0);
         @Cleanup
         val context = createContext();
-        output(executeRESTCall(context, "/v1/scopes/"));
+        prettyJSONOutput(executeRESTCall(context, "/v1/scopes/"));
     }
 
     public static CommandDescriptor descriptor() {

pravega-cli/src/main/java/io/pravega/tools/pravegacli/commands/controller/ControllerListStreamsInScopeCommand.java

@@ -33,7 +33,7 @@ public void execute() {
         @Cleanup
         val context = createContext();
         String scope = getCommandArgs().getArgs().get(0);
-        output(executeRESTCall(context, "/v1/scopes/" + scope + "/streams"));
+        prettyJSONOutput(executeRESTCall(context, "/v1/scopes/" + scope + "/streams"));
     }
 
     public static CommandDescriptor descriptor() {

pravega-cli/src/test/java/io/pravega/tools/pravegacli/BookkeeperCommandsTest.java

@@ -42,16 +42,17 @@ public void setUp() throws Exception {
         STATE.get().getConfigBuilder().include(bkProperties);
     }
 
-
     @Test
     public void testBookKeeperListCommand() throws Exception {
         String commandResult = TestUtils.executeCommand("bk list", STATE.get());
-        Assert.assertTrue(commandResult.contains("Log 0"));
+        System.err.println(commandResult);
+        Assert.assertTrue(commandResult.contains("log_no_metadata"));
     }
 
     @Test
     public void testBookKeeperDetailsCommand() throws Exception {
         String commandResult = TestUtils.executeCommand("bk details 0", STATE.get());
-        Assert.assertTrue(commandResult.contains("Log 0: No metadata"));
+        System.err.println(commandResult);
+        Assert.assertTrue(commandResult.contains("log_no_metadata"));
     }
 }