Open
Description
We see cluster state application take ≥10s in IndicesClusterStateService sometimes, and have recently started capturing hot threads when this happens. Looking at recent hot thread dumps captured in production I see that if it's still busy after 10s then the applier thread is often 100% busy within either MappingParser.parse or AnalysisRegistry.build:
Parsing mappings
100.0% [cpu=23.4%, other=76.6%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[es-es-index-69484dc5bc-h4xrn][clusterApplierService#updateTask][T#1]'
2/10 snapshots sharing following 45 elements
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.FieldMapper$Builder$$Lambda/0x000000001ea1b078.accept(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.FieldMapper$Parameter.validate(FieldMapper.java:946)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.FieldMapper$Builder.validate(FieldMapper.java:1432)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.FieldMapper$Builder.parse(FieldMapper.java:1592)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.FieldMapper$TypeParser.parse(FieldMapper.java:1707)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.FieldMapper$TypeParser.parse(FieldMapper.java:1657)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseProperties(ObjectMapper.java:419)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseObjectOrDocumentTypeProperties(ObjectMapper.java:332)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseObjectFields(ObjectMapper.java:293)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parse(ObjectMapper.java:283)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseProperties(ObjectMapper.java:419)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseObjectOrDocumentTypeProperties(ObjectMapper.java:332)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseObjectFields(ObjectMapper.java:293)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parse(ObjectMapper.java:283)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseProperties(ObjectMapper.java:419)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseObjectOrDocumentTypeProperties(ObjectMapper.java:332)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseObjectFields(ObjectMapper.java:293)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parse(ObjectMapper.java:283)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseProperties(ObjectMapper.java:419)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseObjectOrDocumentTypeProperties(ObjectMapper.java:332)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseObjectFields(ObjectMapper.java:293)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parse(ObjectMapper.java:283)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseProperties(ObjectMapper.java:419)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.ObjectMapper$TypeParser.parseObjectOrDocumentTypeProperties(ObjectMapper.java:332)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.RootObjectMapper.parse(RootObjectMapper.java:459)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.MappingParser.parse(MappingParser.java:125)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.MappingParser.parse(MappingParser.java:91)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.MapperService.parseMapping(MapperService.java:615)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.MapperService.updateMapping(MapperService.java:354)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexService.updateMapping(IndexService.java:866)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:607)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
6/10 snapshots sharing following 19 elements
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.MappingParser.parse(MappingParser.java:91)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.MapperService.parseMapping(MapperService.java:615)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.MapperService.updateMapping(MapperService.java:354)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexService.updateMapping(IndexService.java:866)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:607)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
2/10 snapshots sharing following 17 elements
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.mapper.MapperService.updateMapping(MapperService.java:359)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexService.updateMapping(IndexService.java:866)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:607)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
Constructing a Japanese tokenizer
100.0% [cpu=22.6%, other=77.4%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[es-es-search-6c44448496-fwrbw][clusterApplierService#updateTask][T#1]'
6/10 snapshots sharing following 34 elements
app/org.apache.lucene.core@10.2.1/org.apache.lucene.store.DataInput.readZInt(DataInput.java:137)
app/org.apache.lucene.analysis.common@10.2.1/org.apache.lucene.analysis.morph.ConnectionCosts.<init>(ConnectionCosts.java:53)
org.apache.lucene.analysis.ja.dict.ConnectionCosts.<init>(ConnectionCosts.java:56)
org.apache.lucene.analysis.ja.dict.ConnectionCosts.<init>(ConnectionCosts.java:52)
org.apache.lucene.analysis.ja.dict.ConnectionCosts$SingletonHolder.<clinit>(ConnectionCosts.java:75)
org.apache.lucene.analysis.ja.dict.ConnectionCosts.getInstance(ConnectionCosts.java:67)
org.apache.lucene.analysis.ja.JapaneseTokenizer.<init>(JapaneseTokenizer.java:181)
org.apache.lucene.analysis.ja.JapaneseTokenizer.<init>(JapaneseTokenizer.java:129)
org.apache.lucene.analysis.ja.JapaneseAnalyzer.createComponents(JapaneseAnalyzer.java:110)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.AnalyzerWrapper.createComponents(AnalyzerWrapper.java:120)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:193)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.checkVersions(AnalysisRegistry.java:776)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
3/10 snapshots sharing following 34 elements
java.base@24/java.nio.DirectByteBuffer.putShort(DirectByteBuffer.java:697)
app/org.apache.lucene.analysis.common@10.2.1/org.apache.lucene.analysis.morph.ConnectionCosts.<init>(ConnectionCosts.java:54)
org.apache.lucene.analysis.ja.dict.ConnectionCosts.<init>(ConnectionCosts.java:56)
org.apache.lucene.analysis.ja.dict.ConnectionCosts.<init>(ConnectionCosts.java:52)
org.apache.lucene.analysis.ja.dict.ConnectionCosts$SingletonHolder.<clinit>(ConnectionCosts.java:75)
org.apache.lucene.analysis.ja.dict.ConnectionCosts.getInstance(ConnectionCosts.java:67)
org.apache.lucene.analysis.ja.JapaneseTokenizer.<init>(JapaneseTokenizer.java:181)
org.apache.lucene.analysis.ja.JapaneseTokenizer.<init>(JapaneseTokenizer.java:129)
org.apache.lucene.analysis.ja.JapaneseAnalyzer.createComponents(JapaneseAnalyzer.java:110)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.AnalyzerWrapper.createComponents(AnalyzerWrapper.java:120)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:193)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.checkVersions(AnalysisRegistry.java:776)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
unique snapshot
app/org.apache.lucene.analysis.common@10.2.1/org.apache.lucene.analysis.morph.ConnectionCosts.<init>(ConnectionCosts.java:52)
org.apache.lucene.analysis.ja.dict.ConnectionCosts.<init>(ConnectionCosts.java:56)
org.apache.lucene.analysis.ja.dict.ConnectionCosts.<init>(ConnectionCosts.java:52)
org.apache.lucene.analysis.ja.dict.ConnectionCosts$SingletonHolder.<clinit>(ConnectionCosts.java:75)
org.apache.lucene.analysis.ja.dict.ConnectionCosts.getInstance(ConnectionCosts.java:67)
org.apache.lucene.analysis.ja.JapaneseTokenizer.<init>(JapaneseTokenizer.java:181)
org.apache.lucene.analysis.ja.JapaneseTokenizer.<init>(JapaneseTokenizer.java:129)
org.apache.lucene.analysis.ja.JapaneseAnalyzer.createComponents(JapaneseAnalyzer.java:110)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.AnalyzerWrapper.createComponents(AnalyzerWrapper.java:120)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:193)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.checkVersions(AnalysisRegistry.java:776)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
Constructing a Galician analyzer
100.0% [cpu=26.6%, other=73.4%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[es-es-search-7c899b6966-mnqnb][clusterApplierService#updateTask][T#1]'
2/10 snapshots sharing following 27 elements
app/org.apache.lucene.analysis.common@10.2.1/org.apache.lucene.analysis.gl.GalicianStemFilter.<init>(GalicianStemFilter.java:34)
app/org.apache.lucene.analysis.common@10.2.1/org.apache.lucene.analysis.gl.GalicianAnalyzer.createComponents(GalicianAnalyzer.java:115)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.AnalyzerWrapper.createComponents(AnalyzerWrapper.java:120)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:193)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.checkVersions(AnalysisRegistry.java:776)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
2/10 snapshots sharing following 43 elements
java.base@24/java.lang.ClassLoader.defineClass1(Native Method)
java.base@24/java.lang.ClassLoader.defineClass(ClassLoader.java:962)
java.base@24/java.lang.ClassLoader.defineClass(ClassLoader.java:1039)
java.base@24/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:164)
java.base@24/jdk.internal.loader.Loader.findClassInModuleOrNull(Loader.java:539)
java.base@24/jdk.internal.loader.Loader.loadClass(Loader.java:487)
java.base@24/java.lang.ClassLoader.loadClass(ClassLoader.java:490)
com.ibm.icu@77.1/com.ibm.icu.text.RuleBasedBreakIterator$BreakCache.<init>(RuleBasedBreakIterator.java:1872)
com.ibm.icu@77.1/com.ibm.icu.text.RuleBasedBreakIterator.<init>(RuleBasedBreakIterator.java:294)
com.ibm.icu@77.1/com.ibm.icu.text.RuleBasedBreakIterator.getInstanceFromCompiledRules(RuleBasedBreakIterator.java:127)
com.ibm.icu@77.1/com.ibm.icu.text.RuleBasedBreakIterator.getInstanceFromCompiledRules(RuleBasedBreakIterator.java:104)
com.ibm.icu@77.1/com.ibm.icu.text.BreakIteratorFactory.createBreakInstance(BreakIteratorFactory.java:164)
com.ibm.icu@77.1/com.ibm.icu.text.BreakIteratorFactory.createBreakIterator(BreakIteratorFactory.java:72)
com.ibm.icu@77.1/com.ibm.icu.text.BreakIterator.getBreakInstance(BreakIterator.java:902)
com.ibm.icu@77.1/com.ibm.icu.text.BreakIterator.getWordInstance(BreakIterator.java:628)
org.apache.lucene.analysis.icu@10.2.1/org.apache.lucene.analysis.icu.segmentation.DefaultICUTokenizerConfig.<clinit>(DefaultICUTokenizerConfig.java:73)
org.apache.lucene.analysis.icu@10.2.1/org.apache.lucene.analysis.icu.segmentation.ICUTokenizer.<init>(ICUTokenizer.java:70)
org.elasticsearch.analysis.icu@9.1.0/org.elasticsearch.plugin.analysis.icu.IcuAnalyzerProvider$1.createComponents(IcuAnalyzerProvider.java:58)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.AnalyzerWrapper.createComponents(AnalyzerWrapper.java:120)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:193)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.checkVersions(AnalysisRegistry.java:776)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
3/10 snapshots sharing following 22 elements
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
2/10 snapshots sharing following 31 elements
java.base@24/java.lang.ClassLoader.defineClass1(Native Method)
java.base@24/java.lang.ClassLoader.defineClass(ClassLoader.java:962)
java.base@24/java.lang.ClassLoader.defineClass(ClassLoader.java:1039)
java.base@24/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:164)
java.base@24/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:735)
java.base@24/jdk.internal.loader.BuiltinClassLoader.findClassInModuleOrNull(BuiltinClassLoader.java:678)
java.base@24/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:604)
java.base@24/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:578)
java.base@24/java.lang.ClassLoader.loadClass(ClassLoader.java:490)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.similarity.SimilarityService.lambda$static$1(SimilarityService.java:47)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.similarity.SimilarityService$$Lambda/0x000000001d81f3f0.apply(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.similarity.SimilarityService.<init>(SimilarityService.java:109)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:520)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
unique snapshot
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.Security.lambda$onIndexModule$23(Security.java:1611)
org.elasticsearch.security@9.1.0/org.elasticsearch.xpack.security.Security$$Lambda/0x000000001d6af970.apply(Unknown Source)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexService.<init>(IndexService.java:281)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:552)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
Constructing a Chinese analyzer
100.0% [cpu=31.7%, other=68.3%] (500ms out of 500ms) cpu usage by thread 'elasticsearch[es-es-index-67546d88c4-c8fgz][clusterApplierService#updateTask][T#1]'
2/10 snapshots sharing following 48 elements
java.base@24/java.util.zip.Inflater.inflateBytesBytes(Native Method)
java.base@24/java.util.zip.Inflater.inflate(Inflater.java:350)
java.base@24/java.util.zip.InflaterInputStream.read(InflaterInputStream.java:178)
java.base@24/java.io.FilterInputStream.read(FilterInputStream.java:119)
java.base@24/java.io.ObjectInputStream$PeekInputStream.read(ObjectInputStream.java:2795)
java.base@24/java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2811)
java.base@24/java.io.ObjectInputStream$BlockDataInputStream.readInt(ObjectInputStream.java:3334)
java.base@24/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2022)
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1641)
java.base@24/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2068)
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1641)
java.base@24/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2068)
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1641)
java.base@24/java.io.ObjectInputStream.readObject(ObjectInputStream.java:495)
java.base@24/java.io.ObjectInputStream.readObject(ObjectInputStream.java:453)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.loadFromObjectInputStream(WordDictionary.java:156)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.load(WordDictionary.java:137)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.getInstance(WordDictionary.java:81)
org.apache.lucene.analysis.cn.smart.hhmm.HHMMSegmenter.<clinit>(HHMMSegmenter.java:31)
org.apache.lucene.analysis.cn.smart.WordSegmenter.<init>(WordSegmenter.java:32)
org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer.<init>(HMMChineseTokenizer.java:45)
org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer.<init>(HMMChineseTokenizer.java:50)
org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer.createComponents(SmartChineseAnalyzer.java:125)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.AnalyzerWrapper.createComponents(AnalyzerWrapper.java:120)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:193)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.checkVersions(AnalysisRegistry.java:776)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
5/10 snapshots sharing following 40 elements
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1641)
java.base@24/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2068)
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1641)
java.base@24/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2068)
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1641)
java.base@24/java.io.ObjectInputStream.readObject(ObjectInputStream.java:495)
java.base@24/java.io.ObjectInputStream.readObject(ObjectInputStream.java:453)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.loadFromObjectInputStream(WordDictionary.java:156)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.load(WordDictionary.java:137)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.getInstance(WordDictionary.java:81)
org.apache.lucene.analysis.cn.smart.hhmm.HHMMSegmenter.<clinit>(HHMMSegmenter.java:31)
org.apache.lucene.analysis.cn.smart.WordSegmenter.<init>(WordSegmenter.java:32)
org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer.<init>(HMMChineseTokenizer.java:45)
org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer.<init>(HMMChineseTokenizer.java:50)
org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer.createComponents(SmartChineseAnalyzer.java:125)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.AnalyzerWrapper.createComponents(AnalyzerWrapper.java:120)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:193)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.checkVersions(AnalysisRegistry.java:776)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
2/10 snapshots sharing following 39 elements
java.base@24/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2068)
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1641)
java.base@24/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2068)
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1641)
java.base@24/java.io.ObjectInputStream.readObject(ObjectInputStream.java:495)
java.base@24/java.io.ObjectInputStream.readObject(ObjectInputStream.java:453)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.loadFromObjectInputStream(WordDictionary.java:156)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.load(WordDictionary.java:137)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.getInstance(WordDictionary.java:81)
org.apache.lucene.analysis.cn.smart.hhmm.HHMMSegmenter.<clinit>(HHMMSegmenter.java:31)
org.apache.lucene.analysis.cn.smart.WordSegmenter.<init>(WordSegmenter.java:32)
org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer.<init>(HMMChineseTokenizer.java:45)
org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer.<init>(HMMChineseTokenizer.java:50)
org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer.createComponents(SmartChineseAnalyzer.java:125)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.AnalyzerWrapper.createComponents(AnalyzerWrapper.java:120)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:193)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.checkVersions(AnalysisRegistry.java:776)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
unique snapshot
java.base@24/java.util.zip.Inflater.inflateBytesBytes(Native Method)
java.base@24/java.util.zip.Inflater.inflate(Inflater.java:350)
java.base@24/java.util.zip.InflaterInputStream.read(InflaterInputStream.java:178)
java.base@24/java.util.zip.InflaterInputStream.read(InflaterInputStream.java:125)
java.base@24/java.io.FilterInputStream.read(FilterInputStream.java:71)
java.base@24/java.io.ObjectInputStream$PeekInputStream.peek(ObjectInputStream.java:2773)
java.base@24/java.io.ObjectInputStream$BlockDataInputStream.peek(ObjectInputStream.java:3102)
java.base@24/java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:3112)
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1604)
java.base@24/java.io.ObjectInputStream.readArray(ObjectInputStream.java:2068)
java.base@24/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1641)
java.base@24/java.io.ObjectInputStream.readObject(ObjectInputStream.java:495)
java.base@24/java.io.ObjectInputStream.readObject(ObjectInputStream.java:453)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.loadFromObjectInputStream(WordDictionary.java:156)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.load(WordDictionary.java:137)
org.apache.lucene.analysis.cn.smart.hhmm.WordDictionary.getInstance(WordDictionary.java:81)
org.apache.lucene.analysis.cn.smart.hhmm.HHMMSegmenter.<clinit>(HHMMSegmenter.java:31)
org.apache.lucene.analysis.cn.smart.WordSegmenter.<init>(WordSegmenter.java:32)
org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer.<init>(HMMChineseTokenizer.java:45)
org.apache.lucene.analysis.cn.smart.HMMChineseTokenizer.<init>(HMMChineseTokenizer.java:50)
org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer.createComponents(SmartChineseAnalyzer.java:125)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.AnalyzerWrapper.createComponents(AnalyzerWrapper.java:120)
app/org.apache.lucene.core@10.2.1/org.apache.lucene.analysis.Analyzer.tokenStream(Analyzer.java:193)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.checkVersions(AnalysisRegistry.java:776)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.produceAnalyzer(AnalysisRegistry.java:732)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:620)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.analysis.AnalysisRegistry.build(AnalysisRegistry.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.index.IndexModule.newIndexService(IndexModule.java:518)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndexService(IndicesService.java:790)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:678)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:200)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.createIndicesAndUpdateShards(IndicesClusterStateService.java:606)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.doApplyClusterState(IndicesClusterStateService.java:322)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:276)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:572)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:558)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:531)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:460)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:159)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)
app/org.elasticsearch.server@9.1.0/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)
java.base@24/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)
java.base@24/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)
java.base@24/java.lang.Thread.runWith(Thread.java:1460)
java.base@24/java.lang.Thread.run(Thread.java:1447)
This will hold up subsequent cluster state updates, blocking shard allocation etc. Can we arrange things so that we don't need to build these things during cluster state application? In principle I can't see a good reason why we need these things to exist until we start to recover the first shard of each of these indices.