Skip to content

Commit 491b7c8

Browse files
kkewweikkewwei
committed
Add a dynamic setting to change skip_cache_factor and min_frequency for querycache
Signed-off-by: kkewwei <kewei.11@bytedance.com> Signed-off-by: kkewwei <kkewwei@163.com>
1 parent e634f71 commit 491b7c8

File tree

7 files changed

+135
-80
lines changed

7 files changed

+135
-80
lines changed

‎server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,10 @@ private void applyChanges(UpdateTask task, ClusterState previousClusterState, Cl
597597
logger.debug("completed calling listeners of cluster state for version {}", newClusterState.version());
598598
}
599599

600+
public ClusterSettings clusterSettings() {
601+
return clusterSettings;
602+
}
603+
600604
protected void connectToNodesAndWait(ClusterState newClusterState) {
601605
// can't wait for an ActionFuture on the cluster applier thread, but we do want to block the thread here, so use a CountDownLatch.
602606
final CountDownLatch countDownLatch = new CountDownLatch(1);

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,8 @@ public void apply(Settings value, Settings current, Settings previous) {
298298
IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING,
299299
IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING,
300300
IndicesQueryCache.INDICES_QUERIES_CACHE_SKIP_CACHE_FACTOR,
301+
IndicesQueryCache.INDICES_QUERY_CACHE_MIN_FREQUENCY,
302+
IndicesQueryCache.INDICES_QUERY_CACHE_COSTLY_MIN_FREQUENCY,
301303
IndicesService.CLUSTER_DEFAULT_INDEX_MAX_MERGE_AT_ONCE_SETTING,
302304
IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING,
303305
IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING,

‎server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
185185
IndexSettings.INDEX_SEARCH_IDLE_AFTER,
186186
IndexSettings.INDEX_SEARCH_THROTTLED,
187187
IndexSettings.INDEX_UNREFERENCED_FILE_CLEANUP,
188-
IndexSettings.INDEX_QUERY_CACHE_MIN_FREQUENCY,
189188
IndexFieldDataService.INDEX_FIELDDATA_CACHE_KEY,
190189
FieldMapper.IGNORE_MALFORMED_SETTING,
191190
FieldMapper.COERCE_SETTING,

‎server/src/main/java/org/opensearch/index/IndexSettings.java‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -783,14 +783,6 @@ public static IndexMergePolicy fromString(String text) {
783783
Property.IndexScope
784784
);
785785

786-
public static final Setting<Integer> INDEX_QUERY_CACHE_MIN_FREQUENCY = Setting.intSetting(
787-
"index.query_cache.min_frequency",
788-
5,
789-
1,
790-
Property.Dynamic,
791-
Property.IndexScope
792-
);
793-
794786
private final Index index;
795787
private final Version version;
796788
private final Logger logger;
@@ -840,7 +832,6 @@ public static IndexMergePolicy fromString(String text) {
840832
private final RemoteStorePathStrategy remoteStorePathStrategy;
841833
private final boolean isTranslogMetadataEnabled;
842834
private volatile boolean allowDerivedField;
843-
private volatile int queryCacheMinFrequency;
844835

845836
/**
846837
* The maximum age of a retention lease before it is considered expired.
@@ -1088,7 +1079,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
10881079
setEnableFuzzySetForDocId(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_ENABLED_SETTING));
10891080
setDocIdFuzzySetFalsePositiveProbability(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING));
10901081
isCompositeIndex = scopedSettings.get(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING);
1091-
queryCacheMinFrequency = scopedSettings.get(INDEX_QUERY_CACHE_MIN_FREQUENCY);
10921082
scopedSettings.addSettingsUpdateConsumer(
10931083
TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING,
10941084
tieredMergePolicyProvider::setNoCFSRatio
@@ -1211,15 +1201,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
12111201
IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING,
12121202
this::setRemoteStoreTranslogRepository
12131203
);
1214-
scopedSettings.addSettingsUpdateConsumer(INDEX_QUERY_CACHE_MIN_FREQUENCY, this::setQueryCacheMinFrequency);
1215-
}
1216-
1217-
private void setQueryCacheMinFrequency(int queryCacheMinFrequency) {
1218-
this.queryCacheMinFrequency = queryCacheMinFrequency;
1219-
}
1220-
1221-
public int getQueryCacheMinFrequency() {
1222-
return queryCacheMinFrequency;
12231204
}
12241205

12251206
private void setSearchIdleAfter(TimeValue searchIdleAfter) {

‎server/src/main/java/org/opensearch/index/shard/IndexShard.java‎

Lines changed: 2 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,10 @@
4343
import org.apache.lucene.index.LeafReader;
4444
import org.apache.lucene.index.SegmentInfos;
4545
import org.apache.lucene.index.Term;
46-
import org.apache.lucene.search.BooleanQuery;
47-
import org.apache.lucene.search.DisjunctionMaxQuery;
48-
import org.apache.lucene.search.MultiTermQuery;
4946
import org.apache.lucene.search.Query;
5047
import org.apache.lucene.search.QueryCachingPolicy;
5148
import org.apache.lucene.search.ReferenceManager;
5249
import org.apache.lucene.search.Sort;
53-
import org.apache.lucene.search.TermInSetQuery;
54-
import org.apache.lucene.search.UsageTrackingQueryCachingPolicy;
5550
import org.apache.lucene.store.AlreadyClosedException;
5651
import org.apache.lucene.store.BufferedChecksumIndexInput;
5752
import org.apache.lucene.store.ChecksumIndexInput;
@@ -192,6 +187,7 @@
192187
import org.opensearch.index.warmer.ShardIndexWarmerService;
193188
import org.opensearch.index.warmer.WarmerStats;
194189
import org.opensearch.indices.IndexingMemoryController;
190+
import org.opensearch.indices.IndicesQueryCache;
195191
import org.opensearch.indices.IndicesService;
196192
import org.opensearch.indices.RemoteStoreSettings;
197193
import org.opensearch.indices.cluster.IndicesClusterStateService;
@@ -499,7 +495,7 @@ public boolean shouldCache(Query query) {
499495
}
500496
};
501497
} else {
502-
cachingPolicy = new OpenseachUsageTrackingQueryCachingPolicy(() -> indexSettings.getQueryCacheMinFrequency());
498+
cachingPolicy = new IndicesQueryCache.OpenseachUsageTrackingQueryCachingPolicy(clusterApplierService.clusterSettings());
503499
}
504500
indexShardOperationPermits = new IndexShardOperationPermits(shardId, threadPool);
505501
readerWrapper = indexReaderWrapper;
@@ -5583,51 +5579,4 @@ public AsyncShardRefreshTask getRefreshTask() {
55835579
return refreshTask;
55845580
}
55855581

5586-
/**
5587-
* Custom caching policy for Opensearch.
5588-
*/
5589-
public static class OpenseachUsageTrackingQueryCachingPolicy extends UsageTrackingQueryCachingPolicy {
5590-
private Supplier<Integer> queryCacheMinFrequency;
5591-
5592-
public OpenseachUsageTrackingQueryCachingPolicy(Supplier<Integer> queryCacheMinFrequency) {
5593-
this.queryCacheMinFrequency = queryCacheMinFrequency;
5594-
}
5595-
5596-
@Override
5597-
protected int minFrequencyToCache(Query query) {
5598-
if (isCostly(query)) {
5599-
return 2;
5600-
}
5601-
int minFrequency = queryCacheMinFrequency.get();
5602-
if (query instanceof BooleanQuery || query instanceof DisjunctionMaxQuery) {
5603-
--minFrequency;
5604-
}
5605-
5606-
return Math.max(1, minFrequency);
5607-
}
5608-
5609-
/**
5610-
* Same to Lucene's UsageTrackingQueryCachingPolicy.isCostly, it's not public in Lucene.
5611-
* Given that lucene doesn't give the desired extensibility at this point.
5612-
* Also, we can extension point if needed.
5613-
*/
5614-
private boolean isCostly(Query query) {
5615-
return query instanceof MultiTermQuery
5616-
|| query.getClass().getSimpleName().equals("MultiTermQueryConstantScoreBlendedWrapper")
5617-
|| query.getClass().getSimpleName().equals("MultiTermQueryConstantScoreWrapper")
5618-
|| query instanceof TermInSetQuery
5619-
|| isPointQuery(query);
5620-
}
5621-
5622-
// Same to Lucene's UsageTrackingQueryCachingPolicy.isPointQuery
5623-
private boolean isPointQuery(Query query) {
5624-
for (Class<?> clazz = query.getClass(); clazz != Query.class; clazz = clazz.getSuperclass()) {
5625-
final String simpleName = clazz.getSimpleName();
5626-
if (simpleName.startsWith("Point") && simpleName.endsWith("Query")) {
5627-
return true;
5628-
}
5629-
}
5630-
return false;
5631-
}
5632-
}
56335582
}

‎server/src/main/java/org/opensearch/indices/IndicesQueryCache.java‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,16 @@
3535
import org.apache.logging.log4j.LogManager;
3636
import org.apache.logging.log4j.Logger;
3737
import org.apache.lucene.index.LeafReaderContext;
38+
import org.apache.lucene.search.BooleanQuery;
39+
import org.apache.lucene.search.DisjunctionMaxQuery;
3840
import org.apache.lucene.search.Explanation;
3941
import org.apache.lucene.search.LRUQueryCache;
42+
import org.apache.lucene.search.MultiTermQuery;
4043
import org.apache.lucene.search.Query;
4144
import org.apache.lucene.search.QueryCache;
4245
import org.apache.lucene.search.QueryCachingPolicy;
4346
import org.apache.lucene.search.ScorerSupplier;
47+
import org.apache.lucene.search.UsageTrackingQueryCachingPolicy;
4448
import org.apache.lucene.search.Weight;
4549
import org.opensearch.common.annotation.PublicApi;
4650
import org.opensearch.common.lucene.ShardCoreKeyMap;
@@ -101,6 +105,24 @@ public class IndicesQueryCache implements QueryCache, Closeable {
101105
Property.Dynamic
102106
);
103107

108+
// dynamic change the min frequency cache threshold for query
109+
public static final Setting<Integer> INDICES_QUERY_CACHE_MIN_FREQUENCY = Setting.intSetting(
110+
"indices.query_cache.min_frequency",
111+
5,
112+
1,
113+
Property.NodeScope,
114+
Property.Dynamic
115+
);
116+
117+
// dynamic change the min frequency cache threshold for costly query
118+
public static final Setting<Integer> INDICES_QUERY_CACHE_COSTLY_MIN_FREQUENCY = Setting.intSetting(
119+
"indices.query_cache.costly_min_frequency",
120+
2,
121+
1,
122+
Property.NodeScope,
123+
Property.Dynamic
124+
);
125+
104126
private final LRUQueryCache cache;
105127
private final ShardCoreKeyMap shardKeyMap = new ShardCoreKeyMap();
106128
private final Map<ShardId, Stats> shardStats = new ConcurrentHashMap<>();
@@ -428,4 +450,63 @@ protected void onMiss(Object readerCoreKey, Query filter) {
428450
shardStats.missCount += 1;
429451
}
430452
}
453+
454+
/**
455+
* Custom caching policy for Opensearch.
456+
*/
457+
public static class OpenseachUsageTrackingQueryCachingPolicy extends UsageTrackingQueryCachingPolicy {
458+
private int minFrequency;
459+
private int minFrequencyForCostly;
460+
461+
public OpenseachUsageTrackingQueryCachingPolicy(ClusterSettings clusterSettings) {
462+
minFrequency = clusterSettings.get(INDICES_QUERY_CACHE_MIN_FREQUENCY);
463+
minFrequencyForCostly = clusterSettings.get(INDICES_QUERY_CACHE_COSTLY_MIN_FREQUENCY);
464+
clusterSettings.addSettingsUpdateConsumer(INDICES_QUERY_CACHE_MIN_FREQUENCY, this::setMinFrequency);
465+
clusterSettings.addSettingsUpdateConsumer(INDICES_QUERY_CACHE_COSTLY_MIN_FREQUENCY, this::setMinFrequencyForCostly);
466+
}
467+
468+
@Override
469+
protected int minFrequencyToCache(Query query) {
470+
if (isCostly(query)) {
471+
return minFrequencyForCostly;
472+
}
473+
int minFrequency = this.minFrequency;
474+
if (query instanceof BooleanQuery || query instanceof DisjunctionMaxQuery) {
475+
--minFrequency;
476+
}
477+
478+
return Math.max(1, minFrequency);
479+
}
480+
481+
/**
482+
* Same to Lucene's UsageTrackingQueryCachingPolicy.isCostly, it's not public in Lucene.
483+
* Given that lucene doesn't give the desired extensibility at this point.
484+
* Also, we can extend it if needed.
485+
*/
486+
private boolean isCostly(Query query) {
487+
return query instanceof MultiTermQuery
488+
|| query.getClass().getSimpleName().equals("MultiTermQueryConstantScoreBlendedWrapper")
489+
|| query.getClass().getSimpleName().equals("MultiTermQueryConstantScoreWrapper")
490+
|| isPointQuery(query);
491+
}
492+
493+
// Same to Lucene's UsageTrackingQueryCachingPolicy.isPointQuery
494+
private boolean isPointQuery(Query query) {
495+
for (Class<?> clazz = query.getClass(); clazz != Query.class; clazz = clazz.getSuperclass()) {
496+
final String simpleName = clazz.getSimpleName();
497+
if (simpleName.startsWith("Point") && simpleName.endsWith("Query")) {
498+
return true;
499+
}
500+
}
501+
return false;
502+
}
503+
504+
public void setMinFrequency(int minFrequency) {
505+
this.minFrequency = minFrequency;
506+
}
507+
508+
public void setMinFrequencyForCostly(int minFrequencyForCostly) {
509+
this.minFrequencyForCostly = minFrequencyForCostly;
510+
}
511+
}
431512
}

‎server/src/test/java/org/opensearch/indices/IndicesQueryCacheTests.java‎

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,14 @@
4747
import org.apache.lucene.search.Explanation;
4848
import org.apache.lucene.search.IndexSearcher;
4949
import org.apache.lucene.search.MatchAllDocsQuery;
50+
import org.apache.lucene.search.MultiTermQuery;
5051
import org.apache.lucene.search.Query;
5152
import org.apache.lucene.search.QueryCachingPolicy;
5253
import org.apache.lucene.search.QueryVisitor;
5354
import org.apache.lucene.search.ScoreMode;
5455
import org.apache.lucene.search.Scorer;
5556
import org.apache.lucene.search.ScorerSupplier;
57+
import org.apache.lucene.search.TermInSetQuery;
5658
import org.apache.lucene.search.TermQuery;
5759
import org.apache.lucene.search.TotalHitCountCollector;
5860
import org.apache.lucene.search.Weight;
@@ -65,11 +67,15 @@
6567
import org.opensearch.core.index.shard.ShardId;
6668
import org.opensearch.index.cache.query.QueryCacheStats;
6769
import org.opensearch.index.mapper.KeywordFieldMapper;
68-
import org.opensearch.index.shard.IndexShard;
6970
import org.opensearch.test.OpenSearchTestCase;
7071

7172
import java.io.IOException;
72-
import java.util.concurrent.atomic.AtomicInteger;
73+
import java.util.ArrayList;
74+
import java.util.List;
75+
76+
import static org.opensearch.indices.IndicesQueryCache.OpenseachUsageTrackingQueryCachingPolicy;
77+
import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE;
78+
import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE;
7379

7480
public class IndicesQueryCacheTests extends OpenSearchTestCase {
7581

@@ -523,10 +529,13 @@ public void testDynamicChangeSettings() throws IOException {
523529
ShardId shard = new ShardId("index", "_na_", 0);
524530
r = OpenSearchDirectoryReader.wrap(r, shard);
525531

526-
AtomicInteger queryCacheMinFrequency = new AtomicInteger(5);
527532
IndexSearcher searcher = new IndexSearcher(r);
528-
searcher.setQueryCachingPolicy(new IndexShard.OpenseachUsageTrackingQueryCachingPolicy(() -> queryCacheMinFrequency.get()));
529533
Settings settings = Settings.builder().build();
534+
OpenseachUsageTrackingQueryCachingPolicy queryCachingPolicy = new OpenseachUsageTrackingQueryCachingPolicy(
535+
new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
536+
);
537+
538+
searcher.setQueryCachingPolicy(queryCachingPolicy);
530539
ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
531540
IndicesQueryCache cache = new IndicesQueryCache(settings, clusterSettings);
532541
searcher.setQueryCache(cache);
@@ -544,7 +553,7 @@ public void testDynamicChangeSettings() throws IOException {
544553
assertEquals(0L, stats.getHitCount());
545554
assertEquals(6L, stats.getMissCount());
546555

547-
queryCacheMinFrequency.set(2);
556+
queryCachingPolicy.setMinFrequency(2);
548557
searcher.search(booleanQuery.build(), collector);
549558
stats = cache.getStats(shard);
550559
// ensure result is cached
@@ -554,8 +563,8 @@ public void testDynamicChangeSettings() throws IOException {
554563

555564
// test changing skipCacheFactor
556565
{
557-
// make sure the TermInSetQuery can be cached, because queryCacheMinFrequency is 2
558-
queryCacheMinFrequency.set(4);
566+
// make sure the range query can be cached, because queryCacheMinFrequency is 2
567+
queryCachingPolicy.setMinFrequency(4);
559568
BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
560569
booleanQuery.add(IntPoint.newRangeQuery("age", 2, 9999), BooleanClause.Occur.FILTER);
561570
booleanQuery.add(new TermQuery(new Term("name", "3")), BooleanClause.Occur.MUST);
@@ -572,4 +581,34 @@ public void testDynamicChangeSettings() throws IOException {
572581
cache.onClose(shard);
573582
cache.close(); // this triggers some assertions
574583
}
584+
585+
public void testCostlyMinFrequencyToCache() throws IOException {
586+
Settings settings = Settings.builder().build();
587+
OpenseachUsageTrackingQueryCachingPolicy queryCachingPolicy = new OpenseachUsageTrackingQueryCachingPolicy(
588+
new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
589+
);
590+
int minFrequencyForCostly = 3;
591+
queryCachingPolicy.setMinFrequencyForCostly(minFrequencyForCostly);
592+
593+
// test MultiTermQuery
594+
List<BytesRef> terms = new ArrayList<>();
595+
terms.add(new BytesRef("foo"));
596+
TermInSetQuery termInSetQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, "field", terms);
597+
assertEquals(minFrequencyForCostly, queryCachingPolicy.minFrequencyToCache(termInSetQuery));
598+
599+
// test MultiTermQueryConstantScoreBlendedWrapper
600+
Query query = CONSTANT_SCORE_BLENDED_REWRITE.rewrite(null, termInSetQuery);
601+
assertEquals(minFrequencyForCostly, queryCachingPolicy.minFrequencyToCache(query));
602+
603+
// test MultiTermQueryConstantScoreWrapper
604+
query = CONSTANT_SCORE_REWRITE.rewrite(null, termInSetQuery);
605+
assertEquals(minFrequencyForCostly, queryCachingPolicy.minFrequencyToCache(query));
606+
607+
// test TermInSetQuery
608+
query = CONSTANT_SCORE_REWRITE.rewrite(null, termInSetQuery);
609+
assertEquals(minFrequencyForCostly, queryCachingPolicy.minFrequencyToCache(query));
610+
611+
query = IntPoint.newRangeQuery("age", 2, 9999);
612+
assertEquals(minFrequencyForCostly, queryCachingPolicy.minFrequencyToCache(query));
613+
}
575614
}

0 commit comments

Comments
 (0)