Skip to content

Commit 887e58d

Browse files
author
Peter Alfonsi
committed
add item_count to fielddata api
Signed-off-by: Peter Alfonsi <petealft@amazon.com>
1 parent c9eb7fb commit 887e58d

File tree

10 files changed

+156
-42
lines changed

10 files changed

+156
-42
lines changed

‎server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -842,7 +842,7 @@ public void testClusterStatsWithSelectiveMetricsFilterAndNoIndex() {
842842
assertEquals(new ByteSizeValue(0), statsResponseWithAllIndicesMetrics.getIndicesStats().getStore().getReservedSize());
843843
assertEquals(new ByteSizeValue(0), statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getMemorySize());
844844
assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getEvictions());
845-
assertNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getFields());
845+
assertNull(statsResponseWithAllIndicesMetrics.getIndicesStats().getFieldData().getFieldMemorySizes());
846846
assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments().getCount());
847847
assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments().getIndexWriterMemoryInBytes());
848848
assertEquals(0, statsResponseWithAllIndicesMetrics.getIndicesStats().getSegments().getVersionMapMemoryInBytes());

‎server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -234,20 +234,20 @@ public void testFieldDataStats() throws InterruptedException {
234234
greaterThan(0L)
235235
);
236236
assertThat(
237-
nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes()
237+
nodesStats.getNodes().get(0).getIndices().getFieldData().getFieldMemorySizes().get("field") + nodesStats.getNodes()
238238
.get(1)
239239
.getIndices()
240240
.getFieldData()
241-
.getFields()
241+
.getFieldMemorySizes()
242242
.get("field"),
243243
greaterThan(0L)
244244
);
245245
assertThat(
246-
nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes()
246+
nodesStats.getNodes().get(0).getIndices().getFieldData().getFieldMemorySizes().get("field") + nodesStats.getNodes()
247247
.get(1)
248248
.getIndices()
249249
.getFieldData()
250-
.getFields()
250+
.getFieldMemorySizes()
251251
.get("field"),
252252
lessThan(
253253
nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes()
@@ -267,9 +267,9 @@ public void testFieldDataStats() throws InterruptedException {
267267
.execute()
268268
.actionGet();
269269
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
270-
assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), greaterThan(0L));
270+
assertThat(indicesStats.getTotal().getFieldData().getFieldMemorySizes().get("field"), greaterThan(0L));
271271
assertThat(
272-
indicesStats.getTotal().getFieldData().getFields().get("field"),
272+
indicesStats.getTotal().getFieldData().getFieldMemorySizes().get("field"),
273273
lessThan(indicesStats.getTotal().getFieldData().getMemorySizeInBytes())
274274
);
275275

‎server/src/main/java/org/opensearch/common/FieldMemoryStats.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@
5252
*/
5353
@PublicApi(since = "1.0.0")
5454
public final class FieldMemoryStats implements Writeable, Iterable<Map.Entry<String, Long>> {
55-
5655
private final Map<String, Long> stats;
5756

5857
/**
@@ -101,6 +100,10 @@ public void toXContent(XContentBuilder builder, String key, String rawKey, Strin
101100
builder.endObject();
102101
}
103102

103+
public Map<String, Long> getStats() {
104+
return stats;
105+
}
106+
104107
/**
105108
* Creates a deep copy of this stats instance
106109
*/

‎server/src/main/java/org/opensearch/index/fielddata/FieldDataStats.java

Lines changed: 90 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
package org.opensearch.index.fielddata;
3434

35+
import org.opensearch.Version;
3536
import org.opensearch.common.FieldMemoryStats;
3637
import org.opensearch.common.Nullable;
3738
import org.opensearch.common.annotation.PublicApi;
@@ -44,6 +45,7 @@
4445

4546
import java.io.IOException;
4647
import java.util.Objects;
48+
import java.util.Set;
4749

4850
/**
4951
* Encapsulates heap usage for field data
@@ -58,10 +60,14 @@ public class FieldDataStats implements Writeable, ToXContentFragment {
5860
private static final String MEMORY_SIZE_IN_BYTES = "memory_size_in_bytes";
5961
private static final String EVICTIONS = "evictions";
6062
private static final String FIELDS = "fields";
63+
private static final String ITEM_COUNT = "item_count";
6164
private long memorySize;
6265
private long evictions;
66+
private long itemCount;
6367
@Nullable
64-
private FieldMemoryStats fields;
68+
private FieldMemoryStats fieldMemorySizes;
69+
@Nullable
70+
private FieldMemoryStats fieldItemCounts;
6571

6672
public FieldDataStats() {
6773

@@ -70,13 +76,40 @@ public FieldDataStats() {
7076
public FieldDataStats(StreamInput in) throws IOException {
7177
memorySize = in.readVLong();
7278
evictions = in.readVLong();
73-
fields = in.readOptionalWriteable(FieldMemoryStats::new);
79+
fieldMemorySizes = in.readOptionalWriteable(FieldMemoryStats::new);
80+
if (in.getVersion().onOrAfter(Version.V_3_3_0)) {
81+
itemCount = in.readVLong();
82+
fieldItemCounts = in.readOptionalWriteable(FieldMemoryStats::new);
83+
}
7484
}
7585

86+
/**
87+
* @deprecated Use the constructor that includes item counts instead.
88+
* @param memorySize the total memory size
89+
* @param evictions the number of evictions
90+
* @param fields the memory size for each field
91+
*/
92+
@Deprecated
7693
public FieldDataStats(long memorySize, long evictions, @Nullable FieldMemoryStats fields) {
94+
// TODO: Remove usage of this ctor in tests?
95+
this(memorySize, evictions, fields, 0, null);
96+
}
97+
98+
public FieldDataStats(
99+
long memorySize,
100+
long evictions,
101+
@Nullable FieldMemoryStats fieldMemorySizes,
102+
long itemCount,
103+
@Nullable FieldMemoryStats fieldItemCounts
104+
) {
77105
this.memorySize = memorySize;
78106
this.evictions = evictions;
79-
this.fields = fields;
107+
this.fieldMemorySizes = fieldMemorySizes;
108+
this.itemCount = itemCount;
109+
this.fieldItemCounts = fieldItemCounts;
110+
if (fieldMemorySizes != null && fieldItemCounts != null) {
111+
assert fieldMemorySizes.getStats().keySet().equals(fieldItemCounts.getStats().keySet());
112+
}
80113
}
81114

82115
public void add(FieldDataStats stats) {
@@ -85,11 +118,19 @@ public void add(FieldDataStats stats) {
85118
}
86119
this.memorySize += stats.memorySize;
87120
this.evictions += stats.evictions;
88-
if (stats.fields != null) {
89-
if (fields == null) {
90-
fields = stats.fields.copy();
121+
this.itemCount += stats.itemCount;
122+
if (stats.fieldMemorySizes != null) {
123+
if (fieldMemorySizes == null) {
124+
fieldMemorySizes = stats.fieldMemorySizes.copy();
125+
} else {
126+
fieldMemorySizes.add(stats.fieldMemorySizes);
127+
}
128+
}
129+
if (stats.fieldItemCounts != null) {
130+
if (fieldItemCounts == null) {
131+
fieldItemCounts = stats.fieldItemCounts.copy();
91132
} else {
92-
fields.add(stats.fields);
133+
fieldItemCounts.add(stats.fieldItemCounts);
93134
}
94135
}
95136
}
@@ -106,25 +147,57 @@ public long getEvictions() {
106147
return this.evictions;
107148
}
108149

150+
public long getItemCount() {
151+
return this.itemCount;
152+
}
153+
154+
@Nullable
155+
public FieldMemoryStats getFieldMemorySizes() {
156+
return fieldMemorySizes;
157+
}
158+
109159
@Nullable
110-
public FieldMemoryStats getFields() {
111-
return fields;
160+
public FieldMemoryStats getFieldItemCounts() {
161+
return fieldItemCounts;
112162
}
113163

114164
@Override
115165
public void writeTo(StreamOutput out) throws IOException {
116166
out.writeVLong(memorySize);
117167
out.writeVLong(evictions);
118-
out.writeOptionalWriteable(fields);
168+
out.writeOptionalWriteable(fieldMemorySizes);
169+
if (out.getVersion().onOrAfter(Version.V_3_3_0)) {
170+
out.writeVLong(itemCount);
171+
out.writeOptionalWriteable(fieldItemCounts);
172+
}
119173
}
120174

121175
@Override
122176
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
123177
builder.startObject(FIELDDATA);
124178
builder.humanReadableField(MEMORY_SIZE_IN_BYTES, MEMORY_SIZE, getMemorySize());
125179
builder.field(EVICTIONS, getEvictions());
180+
builder.field(ITEM_COUNT, getItemCount());
181+
Set<String> fields = null;
182+
// We've already asserted the two key sets are the same if both are not null
183+
if (fieldMemorySizes != null) {
184+
fields = fieldMemorySizes.getStats().keySet();
185+
} else if (fieldItemCounts != null) {
186+
fields = fieldItemCounts.getStats().keySet();
187+
}
126188
if (fields != null) {
127-
fields.toXContent(builder, FIELDS, MEMORY_SIZE_IN_BYTES, MEMORY_SIZE);
189+
builder.startObject(FIELDS);
190+
for (String field : fields) {
191+
builder.startObject(field);
192+
if (fieldMemorySizes != null) {
193+
builder.humanReadableField(MEMORY_SIZE_IN_BYTES, MEMORY_SIZE, new ByteSizeValue(fieldMemorySizes.get(field)));
194+
}
195+
if (fieldItemCounts != null) {
196+
builder.field(ITEM_COUNT, fieldItemCounts.get(field));
197+
}
198+
builder.endObject();
199+
}
200+
builder.endObject();
128201
}
129202
builder.endObject();
130203
return builder;
@@ -135,11 +208,15 @@ public boolean equals(Object o) {
135208
if (this == o) return true;
136209
if (o == null || getClass() != o.getClass()) return false;
137210
FieldDataStats that = (FieldDataStats) o;
138-
return memorySize == that.memorySize && evictions == that.evictions && Objects.equals(fields, that.fields);
211+
return memorySize == that.memorySize
212+
&& evictions == that.evictions
213+
&& Objects.equals(fieldMemorySizes, that.fieldMemorySizes)
214+
&& itemCount == that.itemCount
215+
&& Objects.equals(fieldItemCounts, that.fieldItemCounts);
139216
}
140217

141218
@Override
142219
public int hashCode() {
143-
return Objects.hash(memorySize, evictions, fields);
220+
return Objects.hash(memorySize, evictions, fieldMemorySizes, itemCount, fieldItemCounts);
144221
}
145222
}

‎server/src/main/java/org/opensearch/index/fielddata/ShardFieldData.java

Lines changed: 30 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.lucene.util.Accountable;
3636
import org.opensearch.common.FieldMemoryStats;
3737
import org.opensearch.common.annotation.PublicApi;
38+
import org.opensearch.common.collect.Tuple;
3839
import org.opensearch.common.metrics.CounterMetric;
3940
import org.opensearch.common.regex.Regex;
4041
import org.opensearch.common.util.concurrent.ConcurrentCollections;
@@ -55,37 +56,48 @@ public class ShardFieldData implements IndexFieldDataCache.Listener {
5556

5657
private final CounterMetric evictionsMetric = new CounterMetric();
5758
private final CounterMetric totalMetric = new CounterMetric();
58-
private final ConcurrentMap<String, CounterMetric> perFieldTotals = ConcurrentCollections.newConcurrentMap();
59+
// Tuple contains field memory size and field count, in that order
60+
private final ConcurrentMap<String, Tuple<CounterMetric, CounterMetric>> perFieldInfo = ConcurrentCollections.newConcurrentMap();
61+
private final CounterMetric countMetric = new CounterMetric();
5962

6063
public FieldDataStats stats(String... fields) {
6164
Map<String, Long> fieldTotals = null;
65+
Map<String, Long> fieldCounts = null;
6266
if (CollectionUtils.isEmpty(fields) == false) {
6367
fieldTotals = new HashMap<>();
64-
for (Map.Entry<String, CounterMetric> entry : perFieldTotals.entrySet()) {
68+
fieldCounts = new HashMap<>();
69+
for (Map.Entry<String, Tuple<CounterMetric, CounterMetric>> entry : perFieldInfo.entrySet()) {
6570
if (Regex.simpleMatch(fields, entry.getKey())) {
66-
fieldTotals.put(entry.getKey(), entry.getValue().count());
71+
fieldTotals.put(entry.getKey(), entry.getValue().v1().count());
72+
fieldCounts.put(entry.getKey(), entry.getValue().v2().count());
6773
}
6874
}
6975
}
7076
return new FieldDataStats(
7177
totalMetric.count(),
7278
evictionsMetric.count(),
73-
fieldTotals == null ? null : new FieldMemoryStats(fieldTotals)
79+
fieldTotals == null ? null : new FieldMemoryStats(fieldTotals),
80+
countMetric.count(),
81+
fieldCounts == null ? null : new FieldMemoryStats(fieldCounts)
7482
);
7583
}
7684

7785
@Override
7886
public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
7987
totalMetric.inc(ramUsage.ramBytesUsed());
80-
CounterMetric total = perFieldTotals.get(fieldName);
81-
if (total != null) {
82-
total.inc(ramUsage.ramBytesUsed());
88+
countMetric.inc();
89+
Tuple<CounterMetric, CounterMetric> fieldInfo = perFieldInfo.get(fieldName);
90+
if (fieldInfo != null) {
91+
fieldInfo.v1().inc(ramUsage.ramBytesUsed());
92+
fieldInfo.v2().inc();
8393
} else {
84-
total = new CounterMetric();
85-
total.inc(ramUsage.ramBytesUsed());
86-
CounterMetric prev = perFieldTotals.putIfAbsent(fieldName, total);
94+
fieldInfo = new Tuple<>(new CounterMetric(), new CounterMetric());
95+
fieldInfo.v1().inc(ramUsage.ramBytesUsed());
96+
fieldInfo.v2().inc();
97+
Tuple<CounterMetric, CounterMetric> prev = perFieldInfo.putIfAbsent(fieldName, fieldInfo);
8798
if (prev != null) {
88-
prev.inc(ramUsage.ramBytesUsed());
99+
prev.v1().inc(ramUsage.ramBytesUsed());
100+
prev.v2().inc();
89101
}
90102
}
91103
}
@@ -95,13 +107,16 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon
95107
if (wasEvicted) {
96108
evictionsMetric.inc();
97109
}
110+
Tuple<CounterMetric, CounterMetric> fieldInfo = perFieldInfo.get(fieldName);
98111
if (sizeInBytes != -1) {
99112
totalMetric.dec(sizeInBytes);
100-
101-
CounterMetric total = perFieldTotals.get(fieldName);
102-
if (total != null) {
103-
total.dec(sizeInBytes);
113+
if (fieldInfo != null) {
114+
fieldInfo.v1().dec(sizeInBytes);
104115
}
105116
}
117+
countMetric.dec();
118+
if (fieldInfo != null) {
119+
fieldInfo.v2().dec();
120+
}
106121
}
107122
}

‎server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,8 @@ private Table buildTable(final RestRequest request, final NodesStatsResponse nod
105105
Table table = getTableWithHeader(request);
106106

107107
for (NodeStats nodeStats : nodeStatses.getNodes()) {
108-
if (nodeStats.getIndices().getFieldData().getFields() != null) {
109-
for (var cursor : nodeStats.getIndices().getFieldData().getFields()) {
108+
if (nodeStats.getIndices().getFieldData().getFieldMemorySizes() != null) {
109+
for (var cursor : nodeStats.getIndices().getFieldData().getFieldMemorySizes()) {
110110
table.startRow();
111111
table.addCell(nodeStats.getNode().getId());
112112
table.addCell(nodeStats.getNode().getHostName());

‎server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1372,7 +1372,13 @@ private CommonStats createRandomCommonStats() {
13721372
commonStats.indexing = new IndexingStats();
13731373
commonStats.completion = new CompletionStats();
13741374
commonStats.flush = new FlushStats(randomLongBetween(0, 100), randomLongBetween(0, 100), randomLongBetween(0, 100));
1375-
commonStats.fieldData = new FieldDataStats(randomLongBetween(0, 100), randomLongBetween(0, 100), null);
1375+
commonStats.fieldData = new FieldDataStats(
1376+
randomLongBetween(0, 100),
1377+
randomLongBetween(0, 100),
1378+
null,
1379+
randomLongBetween(0, 100),
1380+
null
1381+
);
13761382
commonStats.queryCache = new QueryCacheStats(
13771383
randomLongBetween(0, 100),
13781384
randomLongBetween(0, 100),

‎server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodesTests.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,13 @@ private CommonStats createRandomCommonStats() {
367367
commonStats.indexing = new IndexingStats();
368368
commonStats.completion = new CompletionStats();
369369
commonStats.flush = new FlushStats(randomLongBetween(0, 100), randomLongBetween(0, 100), randomLongBetween(0, 100));
370-
commonStats.fieldData = new FieldDataStats(randomLongBetween(0, 100), randomLongBetween(0, 100), null);
370+
commonStats.fieldData = new FieldDataStats(
371+
randomLongBetween(0, 100),
372+
randomLongBetween(0, 100),
373+
null,
374+
randomLongBetween(0, 100),
375+
null
376+
);
371377
commonStats.queryCache = new QueryCacheStats(
372378
randomLongBetween(0, 100),
373379
randomLongBetween(0, 100),

‎server/src/test/java/org/opensearch/action/admin/cluster/stats/ClusterStatsResponseTests.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,13 @@ private CommonStats createRandomCommonStats() {
235235
commonStats.indexing = new IndexingStats();
236236
commonStats.completion = new CompletionStats();
237237
commonStats.flush = new FlushStats(randomLongBetween(0, 100), randomLongBetween(0, 100), randomLongBetween(0, 100));
238-
commonStats.fieldData = new FieldDataStats(randomLongBetween(0, 100), randomLongBetween(0, 100), null);
238+
commonStats.fieldData = new FieldDataStats(
239+
randomLongBetween(0, 100),
240+
randomLongBetween(0, 100),
241+
null,
242+
randomLongBetween(0, 100),
243+
null
244+
);
239245
commonStats.queryCache = new QueryCacheStats(
240246
randomLongBetween(0, 100),
241247
randomLongBetween(0, 100),

0 commit comments

Comments
 (0)