Skip to content

Commit 59a0f5d

Browse files
Pranshu-SPranshu-S
committed
Optimize NodeIndicesStats output behind flag
1 parent 93d507a commit 59a0f5d

File tree

4 files changed

+199
-17
lines changed

4 files changed

+199
-17
lines changed

‎server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010

1111
import org.opensearch.ExceptionsHelper;
1212
import org.opensearch.action.DocWriteResponse;
13+
import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
14+
import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
1315
import org.opensearch.action.bulk.BulkItemResponse;
1416
import org.opensearch.action.bulk.BulkRequest;
1517
import org.opensearch.action.bulk.BulkResponse;
@@ -19,21 +21,26 @@
1921
import org.opensearch.action.index.IndexResponse;
2022
import org.opensearch.action.update.UpdateRequest;
2123
import org.opensearch.action.update.UpdateResponse;
24+
import org.opensearch.cluster.ClusterState;
25+
import org.opensearch.common.settings.Settings;
2226
import org.opensearch.index.IndexNotFoundException;
2327
import org.opensearch.index.engine.DocumentMissingException;
2428
import org.opensearch.index.engine.VersionConflictEngineException;
2529
import org.opensearch.index.shard.IndexingStats.Stats.DocStatusStats;
30+
import org.opensearch.indices.IndicesService;
2631
import org.opensearch.test.OpenSearchIntegTestCase;
2732
import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
2833
import org.opensearch.test.OpenSearchIntegTestCase.Scope;
2934
import org.hamcrest.MatcherAssert;
3035

36+
import java.util.ArrayList;
3137
import java.util.Arrays;
3238
import java.util.Comparator;
3339
import java.util.Map;
3440
import java.util.concurrent.atomic.AtomicLong;
3541

3642
import static java.util.Collections.singletonMap;
43+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
3744
import static org.hamcrest.Matchers.equalTo;
3845
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
3946

@@ -243,6 +250,53 @@ public void testNodeIndicesStatsDocStatusStatsCreateDeleteUpdate() {
243250
}
244251
}
245252

253+
public void testNodeIndicesStatsResponseOptimised() {
254+
internalCluster().startNode();
255+
ensureGreen();
256+
String indexName = "test1";
257+
index(indexName, "type", "1", "f", "f");
258+
refresh();
259+
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
260+
261+
NodesStatsResponse response = client().admin().cluster().prepareNodesStats().get();
262+
response.getNodes().forEach(nodeStats -> {
263+
assertNotNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex()));
264+
assertNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex()));
265+
});
266+
267+
assertAcked(
268+
client().admin()
269+
.cluster()
270+
.prepareUpdateSettings()
271+
.setTransientSettings(Settings.builder().put(IndicesService.OPTIMIZED_NODES_STATS, true))
272+
.get()
273+
);
274+
275+
response = client().admin().cluster().prepareNodesStats().get();
276+
response.getNodes().forEach(nodeStats -> {
277+
assertNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex()));
278+
assertNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex()));
279+
280+
});
281+
ArrayList<String> level_indices = new ArrayList<>();
282+
level_indices.add("indices");
283+
CommonStatsFlags commonStatsFlags = new CommonStatsFlags().setLevels(level_indices.toArray(new String[0]));
284+
response = client().admin().cluster().prepareNodesStats().setIndices(commonStatsFlags).get();
285+
response.getNodes().forEach(nodeStats -> {
286+
assertNotNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex()));
287+
assertNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex()));
288+
});
289+
290+
ArrayList<String> level_shards = new ArrayList<>();
291+
level_shards.add("shards");
292+
commonStatsFlags = new CommonStatsFlags().setLevels(level_shards.toArray(new String[0]));
293+
response = client().admin().cluster().prepareNodesStats().setIndices(commonStatsFlags).get();
294+
response.getNodes().forEach(nodeStats -> {
295+
assertNotNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex()));
296+
assertNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex()));
297+
});
298+
}
299+
246300
private void assertDocStatusStats() {
247301
DocStatusStats docStatusStats = client().admin()
248302
.cluster()

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -749,7 +749,9 @@ public void apply(Settings value, Settings current, Settings previous) {
749749
RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING,
750750
RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS,
751751
RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA,
752-
SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING
752+
SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING,
753+
754+
IndicesService.OPTIMIZED_NODES_STATS_SETTING
753755
)
754756
)
755757
);

‎server/src/main/java/org/opensearch/indices/IndicesService.java

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,17 @@ public class IndicesService extends AbstractLifecycleComponent
237237
Setting.Property.NodeScope
238238
);
239239

240+
public static final String OPTIMIZED_NODES_STATS = "opensearch.experimental.optimization.nodes_stats.enabled";
241+
242+
public static final Setting<Boolean> OPTIMIZED_NODES_STATS_SETTING = Setting.boolSetting(
243+
OPTIMIZED_NODES_STATS,
244+
false,
245+
Setting.Property.NodeScope,
246+
Setting.Property.Dynamic
247+
);
248+
249+
public volatile boolean optimizedNodesStatsEnabled;
250+
240251
/**
241252
* Used to specify SEGMENT replication type as the default replication strategy for all indices in a cluster. By default, this is false.
242253
*/
@@ -433,6 +444,8 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon
433444
circuitBreakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(-sizeInBytes);
434445
}
435446
});
447+
this.optimizedNodesStatsEnabled = OPTIMIZED_NODES_STATS_SETTING.get(settings);
448+
clusterService.getClusterSettings().addSettingsUpdateConsumer(OPTIMIZED_NODES_STATS_SETTING, this::setOptimizedNodesStats);
436449
this.cleanInterval = INDICES_CACHE_CLEAN_INTERVAL_SETTING.get(settings);
437450
this.cacheCleaner = new CacheCleaner(indicesFieldDataCache, logger, threadPool, this.cleanInterval);
438451
this.metaStateService = metaStateService;
@@ -622,7 +635,9 @@ public NodeIndicesStats stats(CommonStatsFlags flags) {
622635
break;
623636
}
624637
}
625-
638+
if (optimizedNodesStatsEnabled) {
639+
return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats, flags.getLevels());
640+
}
626641
return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats);
627642
}
628643

@@ -1896,6 +1911,10 @@ private void setIdFieldDataEnabled(boolean value) {
18961911
this.idFieldDataEnabled = value;
18971912
}
18981913

1914+
private void setOptimizedNodesStats(boolean optimizedNodesStatsEnabled) {
1915+
this.optimizedNodesStatsEnabled = optimizedNodesStatsEnabled;
1916+
}
1917+
18991918
private void updateDanglingIndicesInfo(Index index) {
19001919
assert DiscoveryNode.isDataNode(settings) : "dangling indices information should only be persisted on data nodes";
19011920
assert nodeWriteDanglingIndicesInfo : "writing dangling indices info is not enabled";

‎server/src/main/java/org/opensearch/indices/NodeIndicesStats.java

Lines changed: 122 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
package org.opensearch.indices;
3434

35+
import org.opensearch.Version;
3536
import org.opensearch.action.admin.indices.stats.CommonStats;
3637
import org.opensearch.action.admin.indices.stats.IndexShardStats;
3738
import org.opensearch.action.admin.indices.stats.ShardStats;
@@ -63,6 +64,7 @@
6364

6465
import java.io.IOException;
6566
import java.util.ArrayList;
67+
import java.util.Arrays;
6668
import java.util.HashMap;
6769
import java.util.List;
6870
import java.util.Map;
@@ -75,22 +77,21 @@
7577
@PublicApi(since = "1.0.0")
7678
public class NodeIndicesStats implements Writeable, ToXContentFragment {
7779
private CommonStats stats;
80+
private Map<Index, CommonStats> statsByIndex;
7881
private Map<Index, List<IndexShardStats>> statsByShard;
7982

8083
public NodeIndicesStats(StreamInput in) throws IOException {
8184
stats = new CommonStats(in);
85+
if (in.getVersion().onOrAfter(Version.V_2_13_0)) {
86+
// contains statsByIndex
87+
if (in.readBoolean()) {
88+
statsByIndex = new HashMap<>();
89+
readStatsByIndex(in);
90+
}
91+
}
8292
if (in.readBoolean()) {
83-
int entries = in.readVInt();
8493
statsByShard = new HashMap<>();
85-
for (int i = 0; i < entries; i++) {
86-
Index index = new Index(in);
87-
int indexShardListSize = in.readVInt();
88-
List<IndexShardStats> indexShardStats = new ArrayList<>(indexShardListSize);
89-
for (int j = 0; j < indexShardListSize; j++) {
90-
indexShardStats.add(new IndexShardStats(in));
91-
}
92-
statsByShard.put(index, indexShardStats);
93-
}
94+
readStatsByShards(in);
9495
}
9596
}
9697

@@ -112,6 +113,57 @@ public NodeIndicesStats(CommonStats oldStats, Map<Index, List<IndexShardStats>>
112113
}
113114
}
114115

116+
public NodeIndicesStats(
117+
CommonStats oldStats,
118+
Map<Index, List<IndexShardStats>> statsByShard,
119+
SearchRequestStats searchRequestStats,
120+
String[] levels
121+
) {
122+
// make a total common stats from old ones and current ones
123+
this.stats = oldStats;
124+
for (List<IndexShardStats> shardStatsList : statsByShard.values()) {
125+
for (IndexShardStats indexShardStats : shardStatsList) {
126+
for (ShardStats shardStats : indexShardStats.getShards()) {
127+
stats.add(shardStats.getStats());
128+
}
129+
}
130+
}
131+
132+
if (this.stats.search != null) {
133+
this.stats.search.setSearchRequestStats(searchRequestStats);
134+
}
135+
136+
if (levels != null) {
137+
if (Arrays.stream(levels).anyMatch(NodeIndicesStats.levels.indices::equals)) {
138+
this.statsByIndex = createStatsByIndex(statsByShard);
139+
} else if (Arrays.stream(levels).anyMatch(NodeIndicesStats.levels.shards::equals)) {
140+
this.statsByShard = statsByShard;
141+
}
142+
}
143+
}
144+
145+
private void readStatsByIndex(StreamInput in) throws IOException {
146+
int indexEntries = in.readVInt();
147+
for (int i = 0; i < indexEntries; i++) {
148+
Index index = new Index(in);
149+
CommonStats commonStats = new CommonStats(in);
150+
statsByIndex.put(index, commonStats);
151+
}
152+
}
153+
154+
private void readStatsByShards(StreamInput in) throws IOException {
155+
int entries = in.readVInt();
156+
for (int i = 0; i < entries; i++) {
157+
Index index = new Index(in);
158+
int indexShardListSize = in.readVInt();
159+
List<IndexShardStats> indexShardStats = new ArrayList<>(indexShardListSize);
160+
for (int j = 0; j < indexShardListSize; j++) {
161+
indexShardStats.add(new IndexShardStats(in));
162+
}
163+
statsByShard.put(index, indexShardStats);
164+
}
165+
}
166+
115167
@Nullable
116168
public StoreStats getStore() {
117169
return stats.getStore();
@@ -195,7 +247,31 @@ public RecoveryStats getRecoveryStats() {
195247
@Override
196248
public void writeTo(StreamOutput out) throws IOException {
197249
stats.writeTo(out);
250+
251+
if (out.getVersion().onOrAfter(Version.V_2_13_0)) {
252+
out.writeBoolean(statsByIndex != null);
253+
if (statsByIndex != null) {
254+
writeStatsByIndex(out);
255+
}
256+
}
257+
198258
out.writeBoolean(statsByShard != null);
259+
if (statsByShard != null) {
260+
writeStatsByShards(out);
261+
}
262+
}
263+
264+
private void writeStatsByIndex(StreamOutput out) throws IOException {
265+
if (statsByIndex != null) {
266+
out.writeVInt(statsByIndex.size());
267+
for (Map.Entry<Index, CommonStats> entry : statsByIndex.entrySet()) {
268+
entry.getKey().writeTo(out);
269+
entry.getValue().writeTo(out);
270+
}
271+
}
272+
}
273+
274+
private void writeStatsByShards(StreamOutput out) throws IOException {
199275
if (statsByShard != null) {
200276
out.writeVInt(statsByShard.size());
201277
for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
@@ -222,16 +298,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
222298
builder.startObject(Fields.INDICES);
223299
stats.toXContent(builder, params);
224300

225-
if ("indices".equals(level)) {
226-
Map<Index, CommonStats> indexStats = createStatsByIndex();
301+
if (levels.indices.equals(level)) {
227302
builder.startObject(Fields.INDICES);
228-
for (Map.Entry<Index, CommonStats> entry : indexStats.entrySet()) {
303+
for (Map.Entry<Index, CommonStats> entry : statsByIndex.entrySet()) {
229304
builder.startObject(entry.getKey().getName());
230305
entry.getValue().toXContent(builder, params);
231306
builder.endObject();
232307
}
233308
builder.endObject();
234-
} else if ("shards".equals(level)) {
309+
} else if (levels.shards.equals(level)) {
235310
builder.startObject("shards");
236311
for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
237312
builder.startArray(entry.getKey().getName());
@@ -251,7 +326,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
251326
return builder;
252327
}
253328

254-
private Map<Index, CommonStats> createStatsByIndex() {
329+
private Map<Index, CommonStats> createStatsByIndex(Map<Index, List<IndexShardStats>> statsByShard) {
255330
Map<Index, CommonStats> statsMap = new HashMap<>();
256331
for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
257332
if (!statsMap.containsKey(entry.getKey())) {
@@ -276,6 +351,14 @@ public List<IndexShardStats> getShardStats(Index index) {
276351
}
277352
}
278353

354+
public CommonStats getIndexStats(Index index) {
355+
if (statsByIndex == null) {
356+
return null;
357+
} else {
358+
return statsByIndex.get(index);
359+
}
360+
}
361+
279362
/**
280363
* Fields used for parsing and toXContent
281364
*
@@ -284,4 +367,28 @@ public List<IndexShardStats> getShardStats(Index index) {
284367
static final class Fields {
285368
static final String INDICES = "indices";
286369
}
370+
371+
/**
372+
* Levels for the NodeIndicesStats
373+
*/
374+
public enum levels {
375+
nodes("nodes"),
376+
indices("indices"),
377+
shards("shards");
378+
379+
private final String name;
380+
381+
levels(String name) {
382+
this.name = name;
383+
}
384+
385+
@Override
386+
public String toString() {
387+
return name;
388+
}
389+
390+
public boolean equals(String value) {
391+
return this.name.equals(value);
392+
}
393+
}
287394
}

0 commit comments

Comments
 (0)