diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index dceafe753e60e..c3d46b10d92d3 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -22,8 +22,10 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoriesMetrics; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -111,14 +113,15 @@ public static final class Repository { private final RepositoriesMetrics repositoriesMetrics; public AzureRepository( - final ProjectId projectId, + @Nullable final ProjectId projectId, final RepositoryMetadata metadata, final NamedXContentRegistry namedXContentRegistry, final AzureStorageService storageService, final ClusterService clusterService, final BigArrays bigArrays, final RecoverySettings recoverySettings, - final RepositoriesMetrics repositoriesMetrics + final RepositoriesMetrics repositoriesMetrics, + final SnapshotMetrics snapshotMetrics ) { super( projectId, @@ -128,7 +131,8 @@ public AzureRepository( bigArrays, recoverySettings, buildBasePath(metadata), - buildLocation(metadata) + buildLocation(metadata), + snapshotMetrics ); this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings()); this.storageService = storageService; diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index 62d45cb399bfc..8a3194b23d907 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -24,6 +24,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; @@ -60,7 +61,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap(AzureRepository.TYPE, (projectId, metadata) -> { AzureStorageService storageService = azureStoreService.get(); @@ -73,7 +75,8 @@ public Map getRepositories( clusterService, bigArrays, recoverySettings, - repositoriesMetrics + repositoriesMetrics, + snapshotMetrics ); }); } diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java index 5f56e321d8f52..4f6c757f49d75 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoriesMetrics; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -46,7 +47,8 @@ private AzureRepository azureRepository(Settings settings) { BlobStoreTestUtil.mockClusterService(), MockBigArrays.NON_RECYCLING_INSTANCE, new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), - RepositoriesMetrics.NOOP + RepositoriesMetrics.NOOP, + SnapshotMetrics.NOOP ); assertThat(azureRepository.getProjectId(), equalTo(projectId)); assertThat(azureRepository.getBlobStore(), is(nullValue())); diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 0ee1f8073f6cb..3591a608ff71e 100644 --- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -272,7 +273,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( GoogleCloudStorageRepository.TYPE, @@ -284,7 +286,8 @@ public Map getRepositories( clusterService, bigArrays, recoverySettings, - new GcsRepositoryStatsCollector() + new GcsRepositoryStatsCollector(), + snapshotMetrics ) { @Override protected GoogleCloudStorageBlobStore createBlobStore() { diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java index 97b781255bbb6..89548513c4ff1 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java @@ -21,6 +21,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Arrays; @@ -53,7 +54,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( GoogleCloudStorageRepository.TYPE, @@ -65,7 +67,8 @@ public Map getRepositories( clusterService, bigArrays, recoverySettings, - new GcsRepositoryStatsCollector(clusterService.threadPool(), metadata, repositoriesMetrics) + new GcsRepositoryStatsCollector(clusterService.threadPool(), metadata, repositoriesMetrics), + snapshotMetrics ) ); } diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index 91973186bbe39..0a731e8710979 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -21,9 +21,11 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -89,14 +91,15 @@ class GoogleCloudStorageRepository extends MeteredBlobStoreRepository { private final GcsRepositoryStatsCollector statsCollector; GoogleCloudStorageRepository( - final ProjectId projectId, + @Nullable final ProjectId projectId, final RepositoryMetadata metadata, final NamedXContentRegistry namedXContentRegistry, final GoogleCloudStorageService storageService, final ClusterService clusterService, final BigArrays bigArrays, final RecoverySettings recoverySettings, - final GcsRepositoryStatsCollector statsCollector + final GcsRepositoryStatsCollector statsCollector, + final SnapshotMetrics snapshotMetrics ) { super( projectId, @@ -106,7 +109,8 @@ class GoogleCloudStorageRepository extends MeteredBlobStoreRepository { bigArrays, recoverySettings, buildBasePath(metadata), - buildLocation(metadata) + buildLocation(metadata), + snapshotMetrics ); this.storageService = storageService; this.chunkSize = getSetting(CHUNK_SIZE, metadata); diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java index 96750968d898f..7b9ef189b7459 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePluginTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -64,7 +65,8 @@ public void testRepositoryProjectId() { BlobStoreTestUtil.mockClusterService(), MockBigArrays.NON_RECYCLING_INSTANCE, new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), - mock(GcsRepositoryStatsCollector.class) + mock(GcsRepositoryStatsCollector.class), + SnapshotMetrics.NOOP ); assertThat(repository.getProjectId(), equalTo(projectId)); } diff --git a/modules/repository-s3/qa/third-party/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/modules/repository-s3/qa/third-party/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java index 9dbabe16538b4..f6b37b3f34c89 100644 --- a/modules/repository-s3/qa/third-party/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/modules/repository-s3/qa/third-party/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase; import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.fixtures.minio.MinioTestContainer; @@ -155,7 +156,8 @@ public long absoluteTimeInMillis() { ClusterServiceUtils.createClusterService(threadpool), BigArrays.NON_RECYCLING_INSTANCE, new RecoverySettings(node().settings(), node().injector().getInstance(ClusterService.class).getClusterSettings()), - S3RepositoriesMetrics.NOOP + S3RepositoriesMetrics.NOOP, + SnapshotMetrics.NOOP ) ) { repository.start(); diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 2c363177f92ec..e431c7e25d250 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -52,6 +52,7 @@ import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryStats; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase; @@ -591,7 +592,8 @@ protected S3Repository createRepository( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - S3RepositoriesMetrics s3RepositoriesMetrics + S3RepositoriesMetrics s3RepositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return new S3Repository( projectId, @@ -601,7 +603,8 @@ protected S3Repository createRepository( clusterService, bigArrays, recoverySettings, - s3RepositoriesMetrics + s3RepositoriesMetrics, + snapshotMetrics ) { @Override diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 6befdeb2c068a..83a2c9d21d87c 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.ListenableFuture; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -39,6 +40,7 @@ import org.elasticsearch.repositories.FinalizeSnapshotContext; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotsService; @@ -276,14 +278,15 @@ class S3Repository extends MeteredBlobStoreRepository { * Constructs an s3 backed repository */ S3Repository( - final ProjectId projectId, + @Nullable final ProjectId projectId, final RepositoryMetadata metadata, final NamedXContentRegistry namedXContentRegistry, final S3Service service, final ClusterService clusterService, final BigArrays bigArrays, final RecoverySettings recoverySettings, - final S3RepositoriesMetrics s3RepositoriesMetrics + final S3RepositoriesMetrics s3RepositoriesMetrics, + final SnapshotMetrics snapshotMetrics ) { super( projectId, @@ -293,7 +296,8 @@ class S3Repository extends MeteredBlobStoreRepository { bigArrays, recoverySettings, buildBasePath(metadata), - buildLocation(metadata) + buildLocation(metadata), + snapshotMetrics ); this.service = service; this.s3RepositoriesMetrics = s3RepositoriesMetrics; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index ab836040efa9a..da81543515c4e 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -29,6 +29,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -65,7 +66,8 @@ protected S3Repository createRepository( final ClusterService clusterService, final BigArrays bigArrays, final RecoverySettings recoverySettings, - final S3RepositoriesMetrics s3RepositoriesMetrics + final S3RepositoriesMetrics s3RepositoriesMetrics, + final SnapshotMetrics snapshotMetrics ) { return new S3Repository( projectId, @@ -75,7 +77,8 @@ protected S3Repository createRepository( clusterService, bigArrays, recoverySettings, - s3RepositoriesMetrics + s3RepositoriesMetrics, + snapshotMetrics ); } @@ -113,7 +116,8 @@ public Map getRepositories( final ClusterService clusterService, final BigArrays bigArrays, final RecoverySettings recoverySettings, - final RepositoriesMetrics repositoriesMetrics + final RepositoriesMetrics repositoriesMetrics, + final SnapshotMetrics snapshotMetrics ) { final S3RepositoriesMetrics s3RepositoriesMetrics = new S3RepositoriesMetrics(repositoriesMetrics); return Collections.singletonMap( @@ -125,7 +129,8 @@ public Map getRepositories( clusterService, bigArrays, recoverySettings, - s3RepositoriesMetrics + s3RepositoriesMetrics, + snapshotMetrics ) ); } diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java index d41631a79739f..b238e42f76242 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; @@ -178,7 +179,8 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) { BlobStoreTestUtil.mockClusterService(), MockBigArrays.NON_RECYCLING_INSTANCE, new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), - S3RepositoriesMetrics.NOOP + S3RepositoriesMetrics.NOOP, + SnapshotMetrics.NOOP ); assertThat(s3Repository.getProjectId(), equalTo(ProjectId.DEFAULT)); return s3Repository; diff --git a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java index 821b761aa663d..1ca70376a7917 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java @@ -20,6 +20,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.url.URLRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -49,7 +50,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( URLRepository.TYPE, @@ -61,7 +63,8 @@ public Map getRepositories( clusterService, bigArrays, recoverySettings, - httpClientFactory.updateAndGet(factory -> factory == null ? new URLHttpClient.Factory() : factory) + httpClientFactory.updateAndGet(factory -> factory == null ? new URLHttpClient.Factory() : factory), + snapshotMetrics ) ); } diff --git a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java index b2ba97cdd4a1c..43ed19e73d7f8 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java @@ -25,9 +25,11 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.URIPattern; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Nullable; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -93,16 +95,17 @@ public class URLRepository extends BlobStoreRepository { * Constructs a read-only URL-based repository */ public URLRepository( - ProjectId projectId, + @Nullable ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - URLHttpClient.Factory httpClientFactory + URLHttpClient.Factory httpClientFactory, + SnapshotMetrics snapshotMetrics ) { - super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY, snapshotMetrics); if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(environment.settings()) == false) { throw new RepositoryException(metadata.name(), "missing url"); diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java index 34c360550a11b..11c10cf4a8dee 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -45,7 +46,8 @@ private URLRepository createRepository(Settings baseSettings, RepositoryMetadata BlobStoreTestUtil.mockClusterService(), MockBigArrays.NON_RECYCLING_INSTANCE, new RecoverySettings(baseSettings, new ClusterSettings(baseSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), - mock(URLHttpClient.Factory.class) + mock(URLHttpClient.Factory.class), + SnapshotMetrics.NOOP ); assertThat(repository.getProjectId(), equalTo(projectId)); return repository; diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java index 9d80a270c7242..4ed6aa5d4164f 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java @@ -19,6 +19,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Collections; @@ -67,7 +68,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( "hdfs", @@ -78,7 +80,8 @@ public Map getRepositories( namedXContentRegistry, clusterService, bigArrays, - recoverySettings + recoverySettings, + snapshotMetrics ) ); } diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java index de01da575b8ba..88188497ef90e 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java @@ -25,12 +25,14 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -57,15 +59,16 @@ public final class HdfsRepository extends BlobStoreRepository { private final String pathSetting; public HdfsRepository( - ProjectId projectId, + @Nullable ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, BigArrays bigArrays, - RecoverySettings recoverySettings + RecoverySettings recoverySettings, + SnapshotMetrics snapshotMetrics ) { - super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY, snapshotMetrics); this.environment = environment; this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null); diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java index d0b84021e704c..3326f4b41033a 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.fixtures.hdfs.HdfsClientThreadLeakFilter; @@ -126,7 +127,8 @@ public void testRepositoryProjectId() { NamedXContentRegistry.EMPTY, BlobStoreTestUtil.mockClusterService(), MockBigArrays.NON_RECYCLING_INSTANCE, - new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), + SnapshotMetrics.NOOP ); assertThat(repository.getProjectId(), equalTo(projectId)); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java index 06e4d06fcee0e..758b67b4af486 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsServiceIT.java @@ -35,6 +35,7 @@ import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.ShardGeneration; import org.elasticsearch.repositories.ShardSnapshotInfo; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.SnapshotException; @@ -76,7 +77,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( TYPE, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java index 62967fc2d035f..a560063bdeae8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/InvalidRepositoryIT.java @@ -57,9 +57,10 @@ public UnstableRepository( NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, BigArrays bigArrays, - RecoverySettings recoverySettings + RecoverySettings recoverySettings, + SnapshotMetrics snapshotMetrics ) { - super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings, snapshotMetrics); List unstableNodes = UNSTABLE_NODES.get(metadata.settings()); if (unstableNodes.contains(clusterService.getNodeName())) { throw new RepositoryException(metadata.name(), "Failed to create repository: current node is not stable"); @@ -74,7 +75,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( TYPE, @@ -85,7 +87,8 @@ public Map getRepositories( namedXContentRegistry, clusterService, bigArrays, - recoverySettings + recoverySettings, + snapshotMetrics ) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/SnapshotMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/SnapshotMetricsIT.java new file mode 100644 index 0000000000000..91cf76c6f07e4 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/SnapshotMetricsIT.java @@ -0,0 +1,521 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories; + +import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; +import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.telemetry.InstrumentType; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.RecordingMeterRegistry; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.ThreadPool; +import org.hamcrest.Matcher; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CyclicBarrier; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import static org.elasticsearch.threadpool.ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.not; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) +public class SnapshotMetricsIT extends AbstractSnapshotIntegTestCase { + + private static final String REQUIRE_NODE_NAME_SETTING = IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name"; + + @Override + protected Collection> nodePlugins() { + return Stream.concat(super.nodePlugins().stream(), Stream.of(TestTelemetryPlugin.class, MockTransportService.TestPlugin.class)) + .toList(); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + // Make sanity checking duration histograms possible + .put(ESTIMATED_TIME_INTERVAL_SETTING.getKey(), "0s") + .build(); + } + + public void testSnapshotAPMMetrics() throws Exception { + final String indexName = randomIdentifier(); + final int numShards = randomIntBetween(1, 10); + final int numReplicas = randomIntBetween(0, 1); + createIndex(indexName, numShards, numReplicas); + + indexRandom(true, indexName, randomIntBetween(100, 300)); + + final IndicesStatsResponse indicesStats = indicesAdmin().prepareStats(indexName).get(); + final IndexStats indexStats = indicesStats.getIndex(indexName); + long totalSizeInBytes = 0; + for (ShardStats shard : indexStats.getShards()) { + totalSizeInBytes += shard.getStats().getStore().sizeInBytes(); + } + logger.info("--> total shards size: {} bytes", totalSizeInBytes); + + final String repositoryName = randomIdentifier(); + + // we want to ensure some throttling, but not so much that it makes the test excessively slow. + final int shardSizeMultipleToEnsureThrottling = 2; + createRepository( + repositoryName, + "mock", + randomRepositorySettings().put( + BlobStoreRepository.MAX_SNAPSHOT_BYTES_PER_SEC.getKey(), + ByteSizeValue.ofBytes(totalSizeInBytes * shardSizeMultipleToEnsureThrottling) + ) + .put( + BlobStoreRepository.MAX_RESTORE_BYTES_PER_SEC.getKey(), + ByteSizeValue.ofBytes(totalSizeInBytes * shardSizeMultipleToEnsureThrottling) + ) + ); + + // Block the snapshot to test "snapshot shards in progress" + blockAllDataNodes(repositoryName); + final String snapshotName = randomIdentifier(); + final long beforeCreateSnapshotNanos = System.nanoTime(); + try { + clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, snapshotName) + .setIndices(indexName) + .setWaitForCompletion(false) + .get(); + + waitForBlockOnAnyDataNode(repositoryName); + collectMetrics(); + assertShardsInProgressMetricIs(hasItem(greaterThan(0L))); + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOTS_STARTED), equalTo(1L)); + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOTS_COMPLETED), equalTo(0L)); + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_SHARDS_STARTED), greaterThan(0L)); + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_SHARDS_COMPLETED), equalTo(0L)); + } finally { + unblockAllDataNodes(repositoryName); + } + + // wait for snapshot to finish to test the other metrics + awaitNumberOfSnapshotsInProgress(0); + final TimeValue snapshotElapsedTime = TimeValue.timeValueNanos(System.nanoTime() - beforeCreateSnapshotNanos); + collectMetrics(); + + // sanity check blobs, bytes and throttling metrics + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_BLOBS_UPLOADED), greaterThan(0L)); + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_BYTES_UPLOADED), greaterThan(0L)); + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_CREATE_THROTTLE_DURATION), greaterThan(0L)); + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_RESTORE_THROTTLE_DURATION), equalTo(0L)); + + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOTS_STARTED), equalTo(1L)); + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOTS_COMPLETED), equalTo(1L)); + + // Sanity check shard duration observations + assertDoubleHistogramMetrics(SnapshotMetrics.SNAPSHOT_SHARDS_DURATION, hasSize(numShards)); + assertDoubleHistogramMetrics(SnapshotMetrics.SNAPSHOT_SHARDS_DURATION, everyItem(lessThan(snapshotElapsedTime.secondsFrac()))); + + // Sanity check snapshot observations + assertDoubleHistogramMetrics(SnapshotMetrics.SNAPSHOT_DURATION, hasSize(1)); + assertDoubleHistogramMetrics(SnapshotMetrics.SNAPSHOT_DURATION, everyItem(lessThan(snapshotElapsedTime.secondsFrac()))); + + // Work out the maximum amount of concurrency per node + final ThreadPool tp = internalCluster().getDataNodeInstance(ThreadPool.class); + final int snapshotThreadPoolSize = tp.info(ThreadPool.Names.SNAPSHOT).getMax(); + final int maximumPerNodeConcurrency = Math.max(snapshotThreadPoolSize, numShards); + + // sanity check duration values + final long upperBoundTimeSpentOnSnapshotThingsMillis = internalCluster().numDataNodes() * maximumPerNodeConcurrency + * snapshotElapsedTime.millis(); + assertThat( + getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_UPLOAD_DURATION), + allOf(greaterThan(0L), lessThan(upperBoundTimeSpentOnSnapshotThingsMillis)) + ); + assertThat( + getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_UPLOAD_READ_DURATION), + allOf(greaterThan(0L), lessThan(upperBoundTimeSpentOnSnapshotThingsMillis)) + ); + + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_SHARDS_STARTED), equalTo((long) numShards)); + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_SHARDS_COMPLETED), equalTo((long) numShards)); + + assertShardsInProgressMetricIs(everyItem(equalTo(0L))); + + // Restore the snapshot + clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, snapshotName) + .setIndices(indexName) + .setWaitForCompletion(true) + .setRenamePattern("(.+)") + .setRenameReplacement("restored-$1") + .get(); + collectMetrics(); + + // assert we throttled on restore + assertThat(getTotalClusterLongCounterValue(SnapshotMetrics.SNAPSHOT_RESTORE_THROTTLE_DURATION), greaterThan(0L)); + + // assert appropriate attributes are present + final Map expectedAttrs = Map.of( + "project_id", + ProjectId.DEFAULT.id(), + "repo_name", + repositoryName, + "repo_type", + "mock" + ); + final Map expectedAttrsWithShardStage = Maps.copyMapWithAddedEntry( + expectedAttrs, + "stage", + IndexShardSnapshotStatus.Stage.DONE.name() + ); + final Map expectedAttrsWithSnapshotState = Maps.copyMapWithAddedEntry( + expectedAttrs, + "state", + SnapshotState.SUCCESS.name() + ); + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOTS_STARTED, expectedAttrs); + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOTS_COMPLETED, expectedAttrsWithSnapshotState); + assertMetricsHaveAttributes(InstrumentType.DOUBLE_HISTOGRAM, SnapshotMetrics.SNAPSHOT_DURATION, expectedAttrsWithSnapshotState); + + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_SHARDS_STARTED, expectedAttrs); + assertMetricsHaveAttributes(InstrumentType.LONG_GAUGE, SnapshotMetrics.SNAPSHOT_SHARDS_IN_PROGRESS, expectedAttrs); + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_SHARDS_COMPLETED, expectedAttrsWithShardStage); + assertMetricsHaveAttributes(InstrumentType.DOUBLE_HISTOGRAM, SnapshotMetrics.SNAPSHOT_SHARDS_DURATION, expectedAttrsWithShardStage); + + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_RESTORE_THROTTLE_DURATION, expectedAttrs); + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_CREATE_THROTTLE_DURATION, expectedAttrs); + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_UPLOAD_READ_DURATION, expectedAttrs); + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_UPLOAD_DURATION, expectedAttrs); + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_BYTES_UPLOADED, expectedAttrs); + assertMetricsHaveAttributes(InstrumentType.LONG_COUNTER, SnapshotMetrics.SNAPSHOT_BLOBS_UPLOADED, expectedAttrs); + } + + public void testByStateCounts_InitAndQueuedShards() throws Exception { + final String indexName = randomIdentifier(); + final int numShards = randomIntBetween(2, 10); + final int numReplicas = randomIntBetween(0, 1); + createIndex(indexName, numShards, numReplicas); + + indexRandom(true, indexName, randomIntBetween(100, 300)); + + final String repositoryName = randomIdentifier(); + createRepository(repositoryName, "mock"); + // Block repo reads so we can queue snapshots + blockAllDataNodes(repositoryName); + + final String snapshotName = randomIdentifier(); + try { + clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, snapshotName) + .setIndices(indexName) + .setWaitForCompletion(false) + .get(); + + waitForBlockOnAnyDataNode(repositoryName); + + // Should be {numShards} in INIT state, and 1 STARTED snapshot + Map shardStates = getShardStates(); + assertThat(shardStates.get(SnapshotsInProgress.ShardState.INIT), equalTo((long) numShards)); + Map snapshotStates = getSnapshotStates(); + assertThat(snapshotStates.get(SnapshotsInProgress.State.STARTED), equalTo(1L)); + + // Queue up another snapshot + clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, randomIdentifier()) + .setIndices(indexName) + .setWaitForCompletion(false) + .get(); + + // Should be {numShards} in QUEUED and INIT states, and 2 STARTED snapshots + shardStates = getShardStates(); + assertThat(shardStates.get(SnapshotsInProgress.ShardState.INIT), equalTo((long) numShards)); + assertThat(shardStates.get(SnapshotsInProgress.ShardState.QUEUED), equalTo((long) numShards)); + snapshotStates = getSnapshotStates(); + assertThat(snapshotStates.get(SnapshotsInProgress.State.STARTED), equalTo(2L)); + } finally { + unblockAllDataNodes(repositoryName); + } + + // All statuses should return to zero when the snapshots complete + awaitNumberOfSnapshotsInProgress(0); + getShardStates().forEach((key, value) -> assertThat(value, equalTo(0L))); + getSnapshotStates().forEach((key, value) -> assertThat(value, equalTo(0L))); + + // Ensure all common attributes are present + assertMetricsHaveAttributes( + InstrumentType.LONG_GAUGE, + SnapshotMetrics.SNAPSHOT_SHARDS_BY_STATE, + Map.of("project_id", ProjectId.DEFAULT.id(), "repo_name", repositoryName, "repo_type", "mock") + ); + assertMetricsHaveAttributes( + InstrumentType.LONG_GAUGE, + SnapshotMetrics.SNAPSHOTS_BY_STATE, + Map.of("project_id", ProjectId.DEFAULT.id(), "repo_name", repositoryName, "repo_type", "mock") + ); + } + + public void testByStateCounts_PausedForRemovalShards() throws Exception { + final String indexName = randomIdentifier(); + final int numShards = randomIntBetween(2, 10); + final int numReplicas = randomIntBetween(0, 1); + + final String nodeForRemoval = internalCluster().startDataOnlyNode(); + + createIndex( + indexName, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) + .put(REQUIRE_NODE_NAME_SETTING, nodeForRemoval) + .build() + ); + indexRandom(true, indexName, randomIntBetween(100, 300)); + + final String repositoryName = randomIdentifier(); + createRepository(repositoryName, "mock"); + + // block the node to be removed + blockNodeOnAnyFiles(repositoryName, nodeForRemoval); + + final ClusterService clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); + try { + // Kick off a snapshot + clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, randomIdentifier()) + .setIndices(indexName) + .setWaitForCompletion(false) + .get(); + + // Wait till we're blocked + waitForBlock(nodeForRemoval, repositoryName); + + // Put shutdown metadata + putShutdownForRemovalMetadata(nodeForRemoval, clusterService); + } finally { + unblockAllDataNodes(repositoryName); + } + + // Wait for snapshot to be paused + safeAwait(createSnapshotPausedListener(clusterService, repositoryName, indexName, numShards)); + + final Map shardStates = getShardStates(); + assertThat(shardStates.get(SnapshotsInProgress.ShardState.PAUSED_FOR_NODE_REMOVAL), equalTo((long) numShards)); + final Map snapshotStates = getSnapshotStates(); + assertThat(snapshotStates.get(SnapshotsInProgress.State.STARTED), equalTo(1L)); + + // clear shutdown metadata to allow snapshot to complete + clearShutdownMetadata(clusterService); + + // All statuses should return to zero when the snapshot completes + awaitNumberOfSnapshotsInProgress(0); + getShardStates().forEach((key, value) -> assertThat(value, equalTo(0L))); + getSnapshotStates().forEach((key, value) -> assertThat(value, equalTo(0L))); + + // Ensure all common attributes are present + assertMetricsHaveAttributes( + InstrumentType.LONG_GAUGE, + SnapshotMetrics.SNAPSHOT_SHARDS_BY_STATE, + Map.of("project_id", ProjectId.DEFAULT.id(), "repo_name", repositoryName, "repo_type", "mock") + ); + assertMetricsHaveAttributes( + InstrumentType.LONG_GAUGE, + SnapshotMetrics.SNAPSHOTS_BY_STATE, + Map.of("project_id", ProjectId.DEFAULT.id(), "repo_name", repositoryName, "repo_type", "mock") + ); + } + + public void testByStateCounts_WaitingShards() throws Exception { + final String indexName = randomIdentifier(); + final String boundNode = internalCluster().startDataOnlyNode(); + final String destinationNode = internalCluster().startDataOnlyNode(); + + // Create with single shard so we can reliably delay relocation + createIndex( + indexName, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(REQUIRE_NODE_NAME_SETTING, boundNode) + .build() + ); + indexRandom(true, indexName, randomIntBetween(100, 300)); + + final String repositoryName = randomIdentifier(); + createRepository(repositoryName, "mock"); + + final MockTransportService transportService = MockTransportService.getInstance(destinationNode); + final CyclicBarrier handoffRequestBarrier = new CyclicBarrier(2); + transportService.addRequestHandlingBehavior( + PeerRecoveryTargetService.Actions.HANDOFF_PRIMARY_CONTEXT, + (handler, request, channel, task) -> { + safeAwait(handoffRequestBarrier); + safeAwait(handoffRequestBarrier); + handler.messageReceived(request, channel, task); + } + ); + + // Force the index to move to another node + client().admin() + .indices() + .prepareUpdateSettings(indexName) + .setSettings(Settings.builder().put(REQUIRE_NODE_NAME_SETTING, destinationNode).build()) + .get(); + + // Wait for hand-off request to be blocked (the shard should be relocating now) + safeAwait(handoffRequestBarrier); + + // Kick off a snapshot + clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, randomIdentifier()) + .setIndices(indexName) + .setWaitForCompletion(false) + .get(); + + // Wait till we see a shard in WAITING state + createSnapshotInStateListener(clusterService(), repositoryName, indexName, 1, SnapshotsInProgress.ShardState.WAITING); + + // Metrics should have 1 WAITING shard and 1 STARTED snapshot + final Map shardStates = getShardStates(); + assertThat(shardStates.get(SnapshotsInProgress.ShardState.WAITING), equalTo(1L)); + final Map snapshotStates = getSnapshotStates(); + assertThat(snapshotStates.get(SnapshotsInProgress.State.STARTED), equalTo(1L)); + + // allow the relocation to complete + safeAwait(handoffRequestBarrier); + + // All statuses should return to zero when the snapshot completes + awaitNumberOfSnapshotsInProgress(0); + getShardStates().forEach((key, value) -> assertThat(value, equalTo(0L))); + getSnapshotStates().forEach((key, value) -> assertThat(value, equalTo(0L))); + + // Ensure all common attributes are present + assertMetricsHaveAttributes( + InstrumentType.LONG_GAUGE, + SnapshotMetrics.SNAPSHOT_SHARDS_BY_STATE, + Map.of("project_id", ProjectId.DEFAULT.id(), "repo_name", repositoryName, "repo_type", "mock") + ); + assertMetricsHaveAttributes( + InstrumentType.LONG_GAUGE, + SnapshotMetrics.SNAPSHOTS_BY_STATE, + Map.of("project_id", ProjectId.DEFAULT.id(), "repo_name", repositoryName, "repo_type", "mock") + ); + } + + private Map getShardStates() { + collectMetrics(); + + return allTestTelemetryPlugins().flatMap(testTelemetryPlugin -> { + final List longGaugeMeasurement = testTelemetryPlugin.getLongGaugeMeasurement( + SnapshotMetrics.SNAPSHOT_SHARDS_BY_STATE + ); + final Map shardStates = new HashMap<>(); + // last one in wins + for (Measurement measurement : longGaugeMeasurement) { + shardStates.put( + SnapshotsInProgress.ShardState.valueOf(measurement.attributes().get("state").toString()), + measurement.getLong() + ); + } + return shardStates.entrySet().stream(); + }).collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue, Long::sum)); + } + + private Map getSnapshotStates() { + collectMetrics(); + + return allTestTelemetryPlugins().flatMap(testTelemetryPlugin -> { + final List longGaugeMeasurement = testTelemetryPlugin.getLongGaugeMeasurement(SnapshotMetrics.SNAPSHOTS_BY_STATE); + final Map shardStates = new HashMap<>(); + // last one in wins + for (Measurement measurement : longGaugeMeasurement) { + shardStates.put(SnapshotsInProgress.State.valueOf(measurement.attributes().get("state").toString()), measurement.getLong()); + } + return shardStates.entrySet().stream(); + }).collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue, Long::sum)); + } + + private static void assertMetricsHaveAttributes( + InstrumentType instrumentType, + String metricName, + Map expectedAttributes + ) { + final List clusterMeasurements = getClusterMeasurements(instrumentType, metricName); + assertThat(clusterMeasurements, not(empty())); + clusterMeasurements.forEach(recordingMetric -> { + for (Map.Entry entry : expectedAttributes.entrySet()) { + assertThat(recordingMetric.attributes(), hasEntry(entry.getKey(), entry.getValue())); + } + }); + } + + private static List getClusterMeasurements(InstrumentType instrumentType, String metricName) { + return allTestTelemetryPlugins().flatMap( + testTelemetryPlugin -> ((RecordingMeterRegistry) testTelemetryPlugin.getTelemetryProvider(Settings.EMPTY).getMeterRegistry()) + .getRecorder() + .getMeasurements(instrumentType, metricName) + .stream() + ).toList(); + } + + private static void assertDoubleHistogramMetrics(String metricName, Matcher> matcher) { + final List values = allTestTelemetryPlugins().flatMap(testTelemetryPlugin -> { + final List doubleHistogramMeasurement = testTelemetryPlugin.getDoubleHistogramMeasurement(metricName); + return doubleHistogramMeasurement.stream().map(Measurement::getDouble); + }).toList(); + assertThat(values, matcher); + } + + private static void assertShardsInProgressMetricIs(Matcher> matcher) { + final List values = allTestTelemetryPlugins().map(testTelemetryPlugin -> { + final List longGaugeMeasurement = testTelemetryPlugin.getLongGaugeMeasurement( + SnapshotMetrics.SNAPSHOT_SHARDS_IN_PROGRESS + ); + return longGaugeMeasurement.getLast().getLong(); + }).toList(); + assertThat(values, matcher); + } + + private static void collectMetrics() { + allTestTelemetryPlugins().forEach(TestTelemetryPlugin::collect); + } + + private long getTotalClusterLongCounterValue(String metricName) { + return allTestTelemetryPlugins().flatMap(testTelemetryPlugin -> testTelemetryPlugin.getLongCounterMeasurement(metricName).stream()) + .mapToLong(Measurement::getLong) + .sum(); + } + + private static Stream allTestTelemetryPlugins() { + return StreamSupport.stream(internalCluster().getDataOrMasterNodeInstances(PluginsService.class).spliterator(), false) + .flatMap(pluginsService -> pluginsService.filterPlugins(TestTelemetryPlugin.class)); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java index fb119caa25cb6..51ec2d12b90c5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryOperationPurposeIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -97,7 +98,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( ASSERTING_REPO_TYPE, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java index 480a079eb8b5c..b232a34ae4d3a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.mockstore.MockRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -183,9 +184,10 @@ public CountingMockRepository( final NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, BigArrays bigArrays, - RecoverySettings recoverySettings + RecoverySettings recoverySettings, + SnapshotMetrics snapshotMetrics ) { - super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings, snapshotMetrics); } @Override @@ -214,7 +216,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( TYPE, @@ -225,7 +228,8 @@ public Map getRepositories( namedXContentRegistry, clusterService, bigArrays, - recoverySettings + recoverySettings, + snapshotMetrics ) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryThrottlingStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositorySnapshotStatsIT.java similarity index 61% rename from server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryThrottlingStatsIT.java rename to server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositorySnapshotStatsIT.java index b0563bc86b122..e7a1360ead1c2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryThrottlingStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositorySnapshotStatsIT.java @@ -21,18 +21,33 @@ import java.util.Collections; +import static org.elasticsearch.threadpool.ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; @ESIntegTestCase.ClusterScope(numDataNodes = 0, scope = ESIntegTestCase.Scope.TEST) -public class RepositoryThrottlingStatsIT extends AbstractSnapshotIntegTestCase { +public class RepositorySnapshotStatsIT extends AbstractSnapshotIntegTestCase { - public void testRepositoryThrottlingStats() throws Exception { + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + // Make upload time more accurate + .put(ESTIMATED_TIME_INTERVAL_SETTING.getKey(), "0s") + .build(); + } + + public void testRepositorySnapshotStats() { logger.info("--> starting a node"); internalCluster().startNode(); logger.info("--> create index"); - createIndexWithRandomDocs("test-idx", 100); + final int numberOfShards = randomIntBetween(2, 6); + createIndex("test-idx", numberOfShards, 0); + ensureGreen(); + indexRandomDocs("test-idx", 100); IndicesStatsResponse indicesStats = indicesAdmin().prepareStats("test-idx").get(); IndexStats indexStats = indicesStats.getIndex("test-idx"); @@ -70,9 +85,18 @@ public void testRepositoryThrottlingStats() throws Exception { NodesStatsResponse response = clusterAdmin().prepareNodesStats().setRepositoryStats(true).get(); RepositoriesStats stats = response.getNodes().get(0).getRepositoriesStats(); - assertTrue(stats.getRepositoryThrottlingStats().containsKey("test-repo")); - assertTrue(stats.getRepositoryThrottlingStats().get("test-repo").totalWriteThrottledNanos() > 0); - assertTrue(stats.getRepositoryThrottlingStats().get("test-repo").totalReadThrottledNanos() > 0); - + // These are just broad sanity checks on the values. There are more detailed checks in SnapshotMetricsIT + assertTrue(stats.getRepositorySnapshotStats().containsKey("test-repo")); + RepositoriesStats.SnapshotStats snapshotStats = stats.getRepositorySnapshotStats().get("test-repo"); + assertThat(snapshotStats.totalWriteThrottledNanos(), greaterThan(0L)); + assertThat(snapshotStats.totalReadThrottledNanos(), greaterThan(0L)); + assertThat(snapshotStats.shardSnapshotsStarted(), equalTo((long) numberOfShards)); + assertThat(snapshotStats.shardSnapshotsCompleted(), equalTo((long) numberOfShards)); + assertThat(snapshotStats.shardSnapshotsInProgress(), equalTo(0L)); + assertThat(snapshotStats.numberOfBlobsUploaded(), greaterThan(0L)); + assertThat(snapshotStats.numberOfBytesUploaded(), greaterThan(0L)); + assertThat(snapshotStats.totalUploadTimeInMillis(), greaterThan(0L)); + assertThat(snapshotStats.totalUploadReadTimeInMillis(), greaterThan(0L)); + assertThat(snapshotStats.totalUploadReadTimeInMillis(), lessThanOrEqualTo(snapshotStats.totalUploadTimeInMillis())); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java index 5db77fa1f0f42..953cddba0ab7a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java @@ -639,7 +639,7 @@ public void testDynamicRestoreThrottling() throws Exception { assertBusy(() -> { long restorePause = 0L; for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) { - restorePause += repositoriesService.repository("test-repo").getRestoreThrottleTimeInNanos(); + restorePause += repositoriesService.repository("test-repo").getSnapshotStats().totalReadThrottledNanos(); } assertThat(restorePause, greaterThan(TimeValue.timeValueSeconds(randomIntBetween(1, 5)).nanos())); assertFalse(restoreSnapshotResponse.isDone()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java index c1d9977993b49..c57fc25635b6c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java @@ -22,12 +22,9 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.SubscribableListener; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.NodesShutdownMetadata; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.service.ClusterService; @@ -46,12 +43,10 @@ import org.elasticsearch.test.transport.MockTransportService; import java.util.Collection; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import static org.elasticsearch.snapshots.SnapshotShutdownProgressTracker.SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING; import static org.hamcrest.Matchers.containsString; @@ -659,41 +654,6 @@ && switch (shardEntry.getValue().state()) { resetMockLog(); } - private static SubscribableListener createSnapshotPausedListener( - ClusterService clusterService, - String repoName, - String indexName, - int numShards - ) { - return ClusterServiceUtils.addTemporaryStateListener(clusterService, state -> { - final var entriesForRepo = SnapshotsInProgress.get(state).forRepo(repoName); - if (entriesForRepo.isEmpty()) { - // it's (just about) possible for the data node to apply the initial snapshot state, start on the first shard snapshot, and - // hit the IO block, before the master even applies this cluster state, in which case we simply retry: - return false; - } - assertThat(entriesForRepo, hasSize(1)); - final var shardSnapshotStatuses = entriesForRepo.iterator() - .next() - .shards() - .entrySet() - .stream() - .flatMap(e -> e.getKey().getIndexName().equals(indexName) ? Stream.of(e.getValue()) : Stream.of()) - .toList(); - assertThat(shardSnapshotStatuses, hasSize(numShards)); - for (var shardStatus : shardSnapshotStatuses) { - assertThat( - shardStatus.state(), - oneOf(SnapshotsInProgress.ShardState.INIT, SnapshotsInProgress.ShardState.PAUSED_FOR_NODE_REMOVAL) - ); - if (shardStatus.state() == SnapshotsInProgress.ShardState.INIT) { - return false; - } - } - return true; - }); - } - private static void addUnassignedShardsWatcher(ClusterService clusterService, String indexName) { ClusterServiceUtils.addTemporaryStateListener(clusterService, state -> { final var indexRoutingTable = state.routingTable().index(indexName); @@ -705,98 +665,4 @@ private static void addUnassignedShardsWatcher(ClusterService clusterService, St return false; }); } - - private static void putShutdownForRemovalMetadata(String nodeName, ClusterService clusterService) { - safeAwait((ActionListener listener) -> putShutdownForRemovalMetadata(clusterService, nodeName, listener)); - } - - private static void flushMasterQueue(ClusterService clusterService, ActionListener listener) { - clusterService.submitUnbatchedStateUpdateTask("flush queue", new ClusterStateUpdateTask(Priority.LANGUID) { - @Override - public ClusterState execute(ClusterState currentState) { - return currentState; - } - - @Override - public void onFailure(Exception e) { - fail(e); - } - - @Override - public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { - listener.onResponse(null); - } - }); - } - - private static void putShutdownForRemovalMetadata(ClusterService clusterService, String nodeName, ActionListener listener) { - // not testing REPLACE just because it requires us to specify the replacement node - final var shutdownType = randomFrom(SingleNodeShutdownMetadata.Type.REMOVE, SingleNodeShutdownMetadata.Type.SIGTERM); - final var shutdownMetadata = SingleNodeShutdownMetadata.builder() - .setType(shutdownType) - .setStartedAtMillis(clusterService.threadPool().absoluteTimeInMillis()) - .setReason("test"); - switch (shutdownType) { - case SIGTERM -> shutdownMetadata.setGracePeriod(TimeValue.timeValueSeconds(60)); - } - SubscribableListener - - .newForked(l -> putShutdownMetadata(clusterService, shutdownMetadata, nodeName, l)) - .andThen(l -> flushMasterQueue(clusterService, l)) - .addListener(listener); - } - - private static void putShutdownMetadata( - ClusterService clusterService, - SingleNodeShutdownMetadata.Builder shutdownMetadataBuilder, - String nodeName, - ActionListener listener - ) { - clusterService.submitUnbatchedStateUpdateTask("mark node for removal", new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { - final var node = currentState.nodes().resolveNode(nodeName); - return currentState.copyAndUpdateMetadata( - mdb -> mdb.putCustom( - NodesShutdownMetadata.TYPE, - new NodesShutdownMetadata( - Map.of( - node.getId(), - shutdownMetadataBuilder.setNodeId(node.getId()).setNodeEphemeralId(node.getEphemeralId()).build() - ) - ) - ) - ); - } - - @Override - public void onFailure(Exception e) { - fail(e); - } - - @Override - public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { - listener.onResponse(null); - } - }); - } - - private static void clearShutdownMetadata(ClusterService clusterService) { - safeAwait(listener -> clusterService.submitUnbatchedStateUpdateTask("remove restart marker", new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { - return currentState.copyAndUpdateMetadata(mdb -> mdb.putCustom(NodesShutdownMetadata.TYPE, NodesShutdownMetadata.EMPTY)); - } - - @Override - public void onFailure(Exception e) { - fail(e); - } - - @Override - public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { - listener.onResponse(null); - } - })); - } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java index 80ade63624675..ff74e5b0a86cf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotThrottlingIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.core.Tuple; import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.RepositoriesStats; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.MockLog; @@ -59,8 +60,9 @@ private Tuple testThrottledRepository(String maxSnapshotBytesPerSec, long snapshotPause = 0L; long restorePause = 0L; for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) { - snapshotPause += repositoriesService.repository("test-repo").getSnapshotThrottleTimeInNanos(); - restorePause += repositoriesService.repository("test-repo").getRestoreThrottleTimeInNanos(); + final RepositoriesStats.SnapshotStats snapshotStats = repositoriesService.repository("test-repo").getSnapshotStats(); + snapshotPause += snapshotStats.totalWriteThrottledNanos(); + restorePause += snapshotStats.totalReadThrottledNanos(); } cluster().wipeIndices("test2-idx"); logger.warn("--> tested throttled repository with snapshot pause [{}] and restore pause [{}]", snapshotPause, restorePause); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java index 7c00a93fa495b..9cf8163a4661a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceDoubleFinalizationIT.java @@ -35,6 +35,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.mockstore.BlobStoreWrapper; import org.elasticsearch.test.ESIntegTestCase; @@ -209,7 +210,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( REPO_TYPE, diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index ae0ccecf15ed7..0a8967bda8ebe 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -340,6 +340,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_FIXED_INDEX_LIKE = def(9_119_0_00); public static final TransportVersion LOOKUP_JOIN_CCS = def(9_120_0_00); public static final TransportVersion NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO = def(9_121_0_00); + public static final TransportVersion EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO = def(9_122_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java index c745199627026..bf4f8afd302a3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java @@ -68,8 +68,8 @@ public SnapshotIndexShardStatus(StreamInput in) throws IOException { default -> throw new IllegalArgumentException("Unknown stage type " + indexShardStatus.getStage()); }; this.stats = new SnapshotStats( - indexShardStatus.getStartTime(), - indexShardStatus.getTotalTime(), + indexShardStatus.getStartTimeMillis(), + indexShardStatus.getTotalTimeMillis(), indexShardStatus.getIncrementalFileCount(), indexShardStatus.getTotalFileCount(), indexShardStatus.getProcessedFileCount(), diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 155de1c889320..9012d685b25f7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -58,6 +59,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.repositories.ProjectRepo.PROJECT_REPO_SERIALIZER; @@ -69,6 +71,11 @@ public class SnapshotsInProgress extends AbstractNamedDiffable implement private static final Logger logger = LogManager.getLogger(SnapshotsInProgress.class); + private static final Tuple, Map> NO_SNAPSHOTS_IN_PROGRESS_STATS = Tuple.tuple( + Arrays.stream(State.values()).collect(Collectors.toUnmodifiableMap(v -> v, v -> 0)), + Arrays.stream(ShardState.values()).collect(Collectors.toUnmodifiableMap(v -> v, v -> 0)) + ); + public static final SnapshotsInProgress EMPTY = new SnapshotsInProgress(Map.of(), Set.of()); public static final String TYPE = "snapshots"; @@ -178,6 +185,22 @@ public List forRepo(ProjectId projectId, String repository) { return forRepo(new ProjectRepo(projectId, repository)); } + /** + * Calculate snapshot and shard state summaries for this repository + * + * @param projectId The project ID + * @param repository The repository name + * @return A tuple containing the snapshot and shard stat summaries + */ + public Tuple, Map> shardStateSummaryForRepository(ProjectId projectId, String repository) { + ByRepo byRepo = entries.get(new ProjectRepo(projectId, repository)); + if (byRepo != null) { + return byRepo.calculateStateSummaries(); + } else { + return NO_SNAPSHOTS_IN_PROGRESS_STATS; + } + } + /** * Returns the list of snapshots in the specified repository. */ @@ -1895,6 +1918,31 @@ private ByRepo(List entries) { this.entries = List.copyOf(entries); } + /** + * Calculate summaries of how many shards and snapshots are in each shard/snapshot state + * + * @return a {@link Tuple} containing the snapshot and shard state summaries respectively + */ + public Tuple, Map> calculateStateSummaries() { + final int[] snapshotCounts = new int[State.values().length]; + final int[] shardCounts = new int[ShardState.values().length]; + for (Entry entry : entries) { + snapshotCounts[entry.state().ordinal()]++; + if (entry.isClone()) { + // Can't get shards for clone entry + continue; + } + for (ShardSnapshotStatus shardSnapshotStatus : entry.shards().values()) { + shardCounts[shardSnapshotStatus.state().ordinal()]++; + } + } + final Map snapshotStates = Arrays.stream(State.values()) + .collect(Collectors.toUnmodifiableMap(state -> state, state -> snapshotCounts[state.ordinal()])); + final Map shardStates = Arrays.stream(ShardState.values()) + .collect(Collectors.toUnmodifiableMap(shardState -> shardState, state -> shardCounts[state.ordinal()])); + return Tuple.tuple(snapshotStates, shardStates); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeCollection(entries); diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java index 6aa6a5e498789..23e63d5447088 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java @@ -88,8 +88,8 @@ public enum AbortStatus { private final AtomicReference stage; private final AtomicReference generation; private final AtomicReference shardSnapshotResult; // only set in stage DONE - private long startTime; - private long totalTime; + private long startTimeMillis; + private long totalTimeMillis; private int incrementalFileCount; private int totalFileCount; private int processedFileCount; @@ -102,8 +102,8 @@ public enum AbortStatus { private IndexShardSnapshotStatus( final Stage stage, - final long startTime, - final long totalTime, + final long startTimeMillis, + final long totalTimeMillis, final int incrementalFileCount, final int totalFileCount, final int processedFileCount, @@ -117,8 +117,8 @@ private IndexShardSnapshotStatus( this.stage = new AtomicReference<>(Objects.requireNonNull(stage)); this.generation = new AtomicReference<>(generation); this.shardSnapshotResult = new AtomicReference<>(); - this.startTime = startTime; - this.totalTime = totalTime; + this.startTimeMillis = startTimeMillis; + this.totalTimeMillis = totalTimeMillis; this.incrementalFileCount = incrementalFileCount; this.totalFileCount = totalFileCount; this.processedFileCount = processedFileCount; @@ -130,14 +130,14 @@ private IndexShardSnapshotStatus( } public synchronized Copy moveToStarted( - final long startTime, + final long startTimeMillis, final int incrementalFileCount, final int totalFileCount, final long incrementalSize, final long totalSize ) { if (stage.compareAndSet(Stage.INIT, Stage.STARTED)) { - this.startTime = startTime; + this.startTimeMillis = startTimeMillis; this.incrementalFileCount = incrementalFileCount; this.totalFileCount = totalFileCount; this.incrementalSize = incrementalSize; @@ -172,11 +172,11 @@ public synchronized Copy moveToFinalize() { }; } - public synchronized void moveToDone(final long endTime, final ShardSnapshotResult shardSnapshotResult) { + public synchronized void moveToDone(final long endTimeMillis, final ShardSnapshotResult shardSnapshotResult) { assert shardSnapshotResult != null; assert shardSnapshotResult.getGeneration() != null; if (stage.compareAndSet(Stage.FINALIZE, Stage.DONE)) { - this.totalTime = Math.max(0L, endTime - startTime); + this.totalTimeMillis = Math.max(0L, endTimeMillis - startTimeMillis); this.shardSnapshotResult.set(shardSnapshotResult); this.generation.set(shardSnapshotResult.getGeneration()); } else { @@ -191,6 +191,10 @@ public Stage getStage() { return stage.get(); } + public long getTotalTimeMillis() { + return totalTimeMillis; + } + public void addAbortListener(ActionListener listener) { abortListeners.addListener(listener); } @@ -221,7 +225,7 @@ private synchronized void abortAndMoveToStageIfNotCompleted( public synchronized SnapshotsInProgress.ShardState moveToUnsuccessful(final Stage newStage, final String failure, final long endTime) { assert newStage == Stage.PAUSED || newStage == Stage.FAILURE : newStage; if (newStage == Stage.PAUSED && stage.compareAndSet(Stage.PAUSING, Stage.PAUSED)) { - this.totalTime = Math.max(0L, endTime - startTime); + this.totalTimeMillis = Math.max(0L, endTime - startTimeMillis); this.failure = failure; return SnapshotsInProgress.ShardState.PAUSED_FOR_NODE_REMOVAL; } @@ -233,7 +237,7 @@ public synchronized SnapshotsInProgress.ShardState moveToUnsuccessful(final Stag public synchronized void moveToFailed(final long endTime, final String failure) { if (stage.getAndSet(Stage.FAILURE) != Stage.FAILURE) { abortListeners.onResponse(AbortStatus.NO_ABORT); - this.totalTime = Math.max(0L, endTime - startTime); + this.totalTimeMillis = Math.max(0L, endTime - startTimeMillis); this.failure = failure; } } @@ -293,8 +297,8 @@ public void updateStatusDescription(String statusString) { public synchronized IndexShardSnapshotStatus.Copy asCopy() { return new IndexShardSnapshotStatus.Copy( stage.get(), - startTime, - totalTime, + startTimeMillis, + totalTimeMillis, incrementalFileCount, totalFileCount, processedFileCount, @@ -350,8 +354,8 @@ public static IndexShardSnapshotStatus.Copy newDone( public static class Copy { private final Stage stage; - private final long startTime; - private final long totalTime; + private final long startTimeMillis; + private final long totalTimeMillis; private final int incrementalFileCount; private final int totalFileCount; private final int processedFileCount; @@ -375,8 +379,8 @@ public Copy( final String statusDescription ) { this.stage = stage; - this.startTime = startTime; - this.totalTime = totalTime; + this.startTimeMillis = startTime; + this.totalTimeMillis = totalTime; this.incrementalFileCount = incrementalFileCount; this.totalFileCount = totalFileCount; this.processedFileCount = processedFileCount; @@ -391,12 +395,12 @@ public Stage getStage() { return stage; } - public long getStartTime() { - return startTime; + public long getStartTimeMillis() { + return startTimeMillis; } - public long getTotalTime() { - return totalTime; + public long getTotalTimeMillis() { + return totalTimeMillis; } public int getIncrementalFileCount() { @@ -436,10 +440,10 @@ public String toString() { return "index shard snapshot status (" + "stage=" + stage - + ", startTime=" - + startTime - + ", totalTime=" - + totalTime + + ", startTimeMillis=" + + startTimeMillis + + ", totalTimeMillis=" + + totalTimeMillis + ", incrementalFileCount=" + incrementalFileCount + ", totalFileCount=" @@ -466,10 +470,10 @@ public String toString() { return "index shard snapshot status (" + "stage=" + stage - + ", startTime=" - + startTime - + ", totalTime=" - + totalTime + + ", startTimeMillis=" + + startTimeMillis + + ", totalTimeMillis=" + + totalTimeMillis + ", incrementalFileCount=" + incrementalFileCount + ", totalFileCount=" diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index bb28ed4a8aff5..66a92e9c0c172 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -188,6 +188,7 @@ import org.elasticsearch.readiness.ReadinessService; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.ReservedProjectStateHandler; import org.elasticsearch.reservedstate.ReservedStateHandlerProvider; @@ -730,6 +731,7 @@ private void construct( BigArrays bigArrays = serviceProvider.newBigArrays(pluginsService, pageCacheRecycler, circuitBreakerService); final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); + final SnapshotMetrics snapshotMetrics = new SnapshotMetrics(telemetryProvider.getMeterRegistry()); RepositoriesModule repositoriesModule = new RepositoriesModule( environment, pluginsService.filterPlugins(RepositoryPlugin.class).toList(), @@ -739,7 +741,8 @@ private void construct( bigArrays, xContentRegistry, recoverySettings, - telemetryProvider + telemetryProvider, + snapshotMetrics ); RepositoriesService repositoriesService = repositoriesModule.getRepositoryService(); final SetOnce rerouteServiceReference = new SetOnce<>(); @@ -1119,7 +1122,8 @@ public Map queryFields() { transportService, actionModule.getActionFilters(), systemIndices, - projectResolver.supportsMultipleProjects() + projectResolver.supportsMultipleProjects(), + snapshotMetrics ); SnapshotShardsService snapshotShardsService = new SnapshotShardsService( settings, diff --git a/server/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java b/server/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java index aa1587118f166..97ce00cc4c375 100644 --- a/server/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java @@ -16,6 +16,7 @@ import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -42,7 +43,8 @@ default Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.emptyMap(); } diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index b0a55d1f66411..abafef470cb8c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -26,6 +26,7 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import java.io.IOException; import java.util.Collection; @@ -97,16 +98,6 @@ public void deleteSnapshots( in.deleteSnapshots(snapshotIds, repositoryDataGeneration, minimumNodeVersion, repositoryDataUpdateListener, onCompletion); } - @Override - public long getSnapshotThrottleTimeInNanos() { - return in.getSnapshotThrottleTimeInNanos(); - } - - @Override - public long getRestoreThrottleTimeInNanos() { - return in.getRestoreThrottleTimeInNanos(); - } - @Override public String startVerification() { return in.startVerification(); @@ -175,6 +166,16 @@ public void awaitIdle() { in.awaitIdle(); } + @Override + public LongWithAttributes getShardSnapshotsInProgress() { + return in.getShardSnapshotsInProgress(); + } + + @Override + public RepositoriesStats.SnapshotStats getSnapshotStats() { + return in.getSnapshotStats(); + } + @Override public Lifecycle.State lifecycleState() { return in.lifecycleState(); diff --git a/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java b/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java index 37612065ebe58..9bdaeb0933243 100644 --- a/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/InvalidRepository.java @@ -25,6 +25,7 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import java.io.IOException; import java.util.Collection; @@ -106,16 +107,6 @@ public void deleteSnapshots( repositoryDataUpdateListener.onFailure(createCreationException()); } - @Override - public long getSnapshotThrottleTimeInNanos() { - throw createCreationException(); - } - - @Override - public long getRestoreThrottleTimeInNanos() { - throw createCreationException(); - } - @Override public String startVerification() { throw createCreationException(); @@ -180,6 +171,16 @@ public void awaitIdle() { } + @Override + public LongWithAttributes getShardSnapshotsInProgress() { + return null; + } + + @Override + public RepositoriesStats.SnapshotStats getSnapshotStats() { + throw createCreationException(); + } + @Override protected void doStart() { diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java index 22adf929a3a08..610b02eb7c0a1 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java @@ -48,7 +48,8 @@ public RepositoriesModule( BigArrays bigArrays, NamedXContentRegistry namedXContentRegistry, RecoverySettings recoverySettings, - TelemetryProvider telemetryProvider + TelemetryProvider telemetryProvider, + SnapshotMetrics snapshotMetrics ) { final RepositoriesMetrics repositoriesMetrics = new RepositoriesMetrics(telemetryProvider.getMeterRegistry()); Map factories = new HashMap<>(); @@ -72,7 +73,8 @@ public RepositoriesModule( clusterService, bigArrays, recoverySettings, - repositoriesMetrics + repositoriesMetrics, + snapshotMetrics ); for (Map.Entry entry : newRepoTypes.entrySet()) { if (factories.put(entry.getKey(), entry.getValue()) != null) { @@ -135,7 +137,8 @@ public RepositoriesModule( internalRepositoryTypes, threadPool, client, - preRestoreChecks + preRestoreChecks, + snapshotMetrics ); } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index f11232771d48f..cd1d072fcf1ea 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -58,6 +58,7 @@ import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -132,7 +133,8 @@ public RepositoriesService( Map internalTypesRegistry, ThreadPool threadPool, NodeClient client, - List> preRestoreChecks + List> preRestoreChecks, + SnapshotMetrics snapshotMetrics ) { this.typesRegistry = typesRegistry; this.internalTypesRegistry = internalTypesRegistry; @@ -152,6 +154,7 @@ public RepositoriesService( threadPool.relativeTimeInMillisSupplier() ); this.preRestoreChecks = preRestoreChecks; + snapshotMetrics.createSnapshotShardsInProgressMetric(this::getShardSnapshotsInProgress); } /** @@ -902,13 +905,7 @@ public List repositoriesStats() { public RepositoriesStats getRepositoriesThrottlingStats() { return new RepositoriesStats( - getRepositories().stream() - .collect( - Collectors.toMap( - r -> r.getMetadata().name(), - r -> new RepositoriesStats.ThrottlingStats(r.getRestoreThrottleTimeInNanos(), r.getSnapshotThrottleTimeInNanos()) - ) - ) + getRepositories().stream().collect(Collectors.toMap(r -> r.getMetadata().name(), Repository::getSnapshotStats)) ); } @@ -1075,6 +1072,15 @@ public Repository createNonProjectRepository(RepositoryMetadata repositoryMetada return createRepository(null, repositoryMetadata, typesRegistry, RepositoriesService::throwRepositoryTypeDoesNotExists); } + private Collection getShardSnapshotsInProgress() { + return repositories.values() + .stream() + .flatMap(repositories -> repositories.values().stream()) + .map(Repository::getShardSnapshotsInProgress) + .filter(Objects::nonNull) + .toList(); + } + private static Repository throwRepositoryTypeDoesNotExists(ProjectId projectId, RepositoryMetadata repositoryMetadata) { throw new RepositoryException( repositoryMetadata.name(), diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStats.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStats.java index 418fd0ee626c7..f5dec24bd550e 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesStats.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesStats.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.ToXContentObject; @@ -26,41 +27,73 @@ public class RepositoriesStats implements Writeable, ToXContentFragment { - private final Map repositoryThrottlingStats; + private final Map repositorySnapshotStats; public RepositoriesStats(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { - repositoryThrottlingStats = in.readMap(ThrottlingStats::new); + repositorySnapshotStats = in.readMap(SnapshotStats::readFrom); } else { - repositoryThrottlingStats = new HashMap<>(); + repositorySnapshotStats = new HashMap<>(); } } - public RepositoriesStats(Map repositoryThrottlingStats) { - this.repositoryThrottlingStats = new HashMap<>(repositoryThrottlingStats); + public RepositoriesStats(Map repositorySnapshotStats) { + this.repositorySnapshotStats = new HashMap<>(repositorySnapshotStats); } @Override public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { - out.writeMap(repositoryThrottlingStats, StreamOutput::writeWriteable); + out.writeMap(repositorySnapshotStats, StreamOutput::writeWriteable); } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("repositories", repositoryThrottlingStats); + builder.field("repositories", repositorySnapshotStats); return builder; } - public Map getRepositoryThrottlingStats() { - return Collections.unmodifiableMap(repositoryThrottlingStats); + public Map getRepositorySnapshotStats() { + return Collections.unmodifiableMap(repositorySnapshotStats); } - public record ThrottlingStats(long totalReadThrottledNanos, long totalWriteThrottledNanos) implements ToXContentObject, Writeable { + public record SnapshotStats( + long shardSnapshotsStarted, + long shardSnapshotsCompleted, + long shardSnapshotsInProgress, + long totalReadThrottledNanos, + long totalWriteThrottledNanos, + long numberOfBlobsUploaded, + long numberOfBytesUploaded, + long totalUploadTimeInMillis, + long totalUploadReadTimeInMillis + ) implements ToXContentObject, Writeable { - ThrottlingStats(StreamInput in) throws IOException { - this(in.readVLong(), in.readVLong()); + public static final SnapshotStats ZERO = new SnapshotStats(0, 0); + + public static SnapshotStats readFrom(StreamInput in) throws IOException { + final long totalReadThrottledNanos = in.readVLong(); + final long totalWriteThrottledNanos = in.readVLong(); + if (in.getTransportVersion().onOrAfter(TransportVersions.EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO)) { + return new SnapshotStats( + in.readVLong(), + in.readVLong(), + in.readVLong(), + totalReadThrottledNanos, + totalWriteThrottledNanos, + in.readVLong(), + in.readVLong(), + in.readVLong(), + in.readVLong() + ); + } else { + return new SnapshotStats(totalReadThrottledNanos, totalWriteThrottledNanos); + } + } + + public SnapshotStats(long totalReadThrottledNanos, long totalWriteThrottledNanos) { + this(0, 0, 0, totalReadThrottledNanos, totalWriteThrottledNanos, 0, 0, 0, 0); } @Override @@ -72,6 +105,21 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field("total_read_throttled_time_nanos", totalReadThrottledNanos); builder.field("total_write_throttled_time_nanos", totalWriteThrottledNanos); + builder.field("shard_snapshots_started", shardSnapshotsStarted); + builder.field("shard_snapshots_completed", shardSnapshotsCompleted); + builder.field("shard_snapshots_in_progress", shardSnapshotsInProgress); + builder.field("uploaded_blobs", numberOfBlobsUploaded); + builder.humanReadableField("uploaded_size_in_bytes", "uploaded_size", ByteSizeValue.ofBytes(numberOfBytesUploaded)); + builder.humanReadableField( + "total_upload_time_in_millis", + "total_upload_time", + TimeValue.timeValueMillis(totalUploadTimeInMillis) + ); + builder.humanReadableField( + "total_read_time_in_millis", + "total_read_time", + TimeValue.timeValueMillis(totalUploadReadTimeInMillis) + ); builder.endObject(); return builder; } @@ -80,6 +128,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public void writeTo(StreamOutput out) throws IOException { out.writeVLong(totalReadThrottledNanos); out.writeVLong(totalWriteThrottledNanos); + if (out.getTransportVersion().onOrAfter(TransportVersions.EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO)) { + out.writeVLong(shardSnapshotsStarted); + out.writeVLong(shardSnapshotsCompleted); + out.writeVLong(shardSnapshotsInProgress); + out.writeVLong(numberOfBlobsUploaded); + out.writeVLong(numberOfBytesUploaded); + out.writeVLong(totalUploadTimeInMillis); + out.writeVLong(totalUploadReadTimeInMillis); + } } } } diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index e98c3ffb16532..35dd7f1b00626 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -26,6 +26,7 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -205,16 +206,6 @@ void deleteSnapshots( Runnable onCompletion ); - /** - * Returns snapshot throttle time in nanoseconds - */ - long getSnapshotThrottleTimeInNanos(); - - /** - * Returns restore throttle time in nanoseconds - */ - long getRestoreThrottleTimeInNanos(); - /** * Returns stats on the repository usage */ @@ -353,4 +344,14 @@ default Set getUsageFeatures() { static boolean assertSnapshotMetaThread() { return ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SNAPSHOT_META); } + + /** + * Get the current count of snapshots in progress + * + * @return The current number of shard snapshots in progress metric value, or null if this repository doesn't track that + */ + @Nullable + LongWithAttributes getShardSnapshotsInProgress(); + + RepositoriesStats.SnapshotStats getSnapshotStats(); } diff --git a/server/src/main/java/org/elasticsearch/repositories/SnapshotMetrics.java b/server/src/main/java/org/elasticsearch/repositories/SnapshotMetrics.java new file mode 100644 index 0000000000000..9281851d4e6b9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/repositories/SnapshotMetrics.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories; + +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.telemetry.metric.DoubleHistogram; +import org.elasticsearch.telemetry.metric.LongCounter; +import org.elasticsearch.telemetry.metric.LongWithAttributes; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +import java.util.Collection; +import java.util.Map; +import java.util.function.Supplier; + +public record SnapshotMetrics( + LongCounter snapshotsStartedCounter, + LongCounter snapshotsCompletedCounter, + DoubleHistogram snapshotsDurationHistogram, + LongCounter shardsStartedCounter, + LongCounter shardsCompletedCounter, + DoubleHistogram shardsDurationHistogram, + LongCounter blobsUploadedCounter, + LongCounter bytesUploadedCounter, + LongCounter uploadDurationCounter, + LongCounter uploadReadDurationCounter, + LongCounter createThrottleDurationCounter, + LongCounter restoreThrottleDurationCounter, + MeterRegistry meterRegistry +) { + + public static final SnapshotMetrics NOOP = new SnapshotMetrics(MeterRegistry.NOOP); + + public static final String SNAPSHOTS_STARTED = "es.repositories.snapshots.started.total"; + public static final String SNAPSHOTS_COMPLETED = "es.repositories.snapshots.completed.total"; + public static final String SNAPSHOTS_BY_STATE = "es.repositories.snapshots.by_state.current"; + public static final String SNAPSHOT_DURATION = "es.repositories.snapshots.duration.histogram"; + public static final String SNAPSHOT_SHARDS_STARTED = "es.repositories.snapshots.shards.started.total"; + public static final String SNAPSHOT_SHARDS_COMPLETED = "es.repositories.snapshots.shards.completed.total"; + public static final String SNAPSHOT_SHARDS_IN_PROGRESS = "es.repositories.snapshots.shards.current"; + public static final String SNAPSHOT_SHARDS_BY_STATE = "es.repositories.snapshots.shards.by_state.current"; + public static final String SNAPSHOT_SHARDS_DURATION = "es.repositories.snapshots.shards.duration.histogram"; + public static final String SNAPSHOT_BLOBS_UPLOADED = "es.repositories.snapshots.blobs.uploaded.total"; + public static final String SNAPSHOT_BYTES_UPLOADED = "es.repositories.snapshots.upload.bytes.total"; + public static final String SNAPSHOT_UPLOAD_DURATION = "es.repositories.snapshots.upload.upload_time.total"; + public static final String SNAPSHOT_UPLOAD_READ_DURATION = "es.repositories.snapshots.upload.read_time.total"; + public static final String SNAPSHOT_CREATE_THROTTLE_DURATION = "es.repositories.snapshots.create_throttling.time.total"; + public static final String SNAPSHOT_RESTORE_THROTTLE_DURATION = "es.repositories.snapshots.restore_throttling.time.total"; + + public SnapshotMetrics(MeterRegistry meterRegistry) { + this( + meterRegistry.registerLongCounter(SNAPSHOTS_STARTED, "snapshots started", "unit"), + meterRegistry.registerLongCounter(SNAPSHOTS_COMPLETED, "snapshots completed", "unit"), + // We use seconds rather than milliseconds due to the limitations of the default bucket boundaries + // see https://www.elastic.co/docs/reference/apm/agents/java/config-metrics#config-custom-metrics-histogram-boundaries + meterRegistry.registerDoubleHistogram(SNAPSHOT_DURATION, "snapshots duration", "s"), + meterRegistry.registerLongCounter(SNAPSHOT_SHARDS_STARTED, "shard snapshots started", "unit"), + meterRegistry.registerLongCounter(SNAPSHOT_SHARDS_COMPLETED, "shard snapshots completed", "unit"), + // We use seconds rather than milliseconds due to the limitations of the default bucket boundaries + // see https://www.elastic.co/docs/reference/apm/agents/java/config-metrics#config-custom-metrics-histogram-boundaries + meterRegistry.registerDoubleHistogram(SNAPSHOT_SHARDS_DURATION, "shard snapshots duration", "s"), + meterRegistry.registerLongCounter(SNAPSHOT_BLOBS_UPLOADED, "snapshot blobs uploaded", "unit"), + meterRegistry.registerLongCounter(SNAPSHOT_BYTES_UPLOADED, "snapshot bytes uploaded", "bytes"), + meterRegistry.registerLongCounter(SNAPSHOT_UPLOAD_DURATION, "snapshot upload duration", "ms"), + meterRegistry.registerLongCounter(SNAPSHOT_UPLOAD_READ_DURATION, "time spent in read() calls when snapshotting", "ms"), + meterRegistry.registerLongCounter(SNAPSHOT_CREATE_THROTTLE_DURATION, "time throttled in snapshot create", "bytes"), + meterRegistry.registerLongCounter(SNAPSHOT_RESTORE_THROTTLE_DURATION, "time throttled in snapshot restore", "bytes"), + meterRegistry + ); + } + + public void createSnapshotShardsInProgressMetric(Supplier> shardSnapshotsInProgressObserver) { + meterRegistry.registerLongsGauge( + SNAPSHOT_SHARDS_IN_PROGRESS, + "shard snapshots in progress", + "unit", + shardSnapshotsInProgressObserver + ); + } + + public void createSnapshotShardsByStateMetric(Supplier> shardSnapshotsByStatusObserver) { + meterRegistry.registerLongsGauge(SNAPSHOT_SHARDS_BY_STATE, "snapshotting shards by state", "unit", shardSnapshotsByStatusObserver); + } + + public void createSnapshotsByStateMetric(Supplier> snapshotsByStatusObserver) { + meterRegistry.registerLongsGauge(SNAPSHOTS_BY_STATE, "snapshots by state", "unit", snapshotsByStatusObserver); + } + + public static Map createAttributesMap(ProjectId projectId, RepositoryMetadata meta) { + assert projectId != null : "Project ID should always be set"; + return Map.of("project_id", projectId.id(), "repo_type", meta.type(), "repo_name", meta.name()); + } +} diff --git a/server/src/main/java/org/elasticsearch/repositories/SnapshotShardContext.java b/server/src/main/java/org/elasticsearch/repositories/SnapshotShardContext.java index 85692603b1e14..42c91f8b7edf6 100644 --- a/server/src/main/java/org/elasticsearch/repositories/SnapshotShardContext.java +++ b/server/src/main/java/org/elasticsearch/repositories/SnapshotShardContext.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexCommit; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -67,7 +68,8 @@ public SnapshotShardContext( final long snapshotStartTime, ActionListener listener ) { - super(commitRef.closingBefore(listener)); + super(new SubscribableListener<>()); + addListener(commitRef.closingBefore(listener)); this.store = store; this.mapperService = mapperService; this.snapshotId = snapshotId; @@ -131,4 +133,8 @@ public Releasable withCommitRef() { throw new IndexShardSnapshotFailedException(store.shardId(), "Store got closed concurrently"); } } + + public void addListener(ActionListener listener) { + ((SubscribableListener) this.delegate).addListener(listener); + } } diff --git a/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java b/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java index 9750666c8c8a9..d77f7836d20bb 100644 --- a/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/UnknownTypeRepository.java @@ -25,6 +25,7 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import java.io.IOException; import java.util.Collection; @@ -104,16 +105,6 @@ public void deleteSnapshots( repositoryDataUpdateListener.onFailure(createUnknownTypeException()); } - @Override - public long getSnapshotThrottleTimeInNanos() { - throw createUnknownTypeException(); - } - - @Override - public long getRestoreThrottleTimeInNanos() { - throw createUnknownTypeException(); - } - @Override public String startVerification() { throw createUnknownTypeException(); @@ -178,6 +169,16 @@ public void awaitIdle() { } + @Override + public LongWithAttributes getShardSnapshotsInProgress() { + return null; + } + + @Override + public RepositoriesStats.SnapshotStats getSnapshotStats() { + throw createUnknownTypeException(); + } + @Override protected void doStart() { diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index dea02908d215f..54ff971eb2425 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -73,7 +73,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; -import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; @@ -111,6 +110,7 @@ import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.IndexMetaDataGenerations; import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.RepositoriesStats; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryData.SnapshotDetails; @@ -122,6 +122,7 @@ import org.elasticsearch.repositories.ShardGeneration; import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.ShardSnapshotResult; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.SnapshotShardContext; import org.elasticsearch.snapshots.AbortedSnapshotException; import org.elasticsearch.snapshots.PausedSnapshotException; @@ -131,6 +132,7 @@ import org.elasticsearch.snapshots.SnapshotMissingException; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.LeakTracker; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -198,6 +200,7 @@ private class ShutdownLogger { private static final Logger shutdownLogger = LogManager.getLogger(ShutdownLogger.class); } + @Nullable private final ProjectId projectId; protected volatile RepositoryMetadata metadata; @@ -357,14 +360,12 @@ public static String getRepositoryDataBlobName(long repositoryGeneration) { private final boolean cacheRepositoryData; + private final BlobStoreSnapshotMetrics blobStoreSnapshotMetrics; + private volatile RateLimiter snapshotRateLimiter; private volatile RateLimiter restoreRateLimiter; - private final CounterMetric snapshotRateLimitingTimeInNanos = new CounterMetric(); - - private final CounterMetric restoreRateLimitingTimeInNanos = new CounterMetric(); - public static final ChecksumBlobStoreFormat GLOBAL_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( "metadata", METADATA_NAME_FORMAT, @@ -498,13 +499,14 @@ public static String getRepositoryDataBlobName(long repositoryGeneration) { */ @SuppressWarnings("this-escape") protected BlobStoreRepository( - final ProjectId projectId, + @Nullable final ProjectId projectId, final RepositoryMetadata metadata, final NamedXContentRegistry namedXContentRegistry, final ClusterService clusterService, final BigArrays bigArrays, final RecoverySettings recoverySettings, - final BlobPath basePath + final BlobPath basePath, + final SnapshotMetrics snapshotMetrics ) { this.projectId = projectId; this.metadata = metadata; @@ -538,6 +540,7 @@ protected BlobStoreRepository( threadPool.info(ThreadPool.Names.SNAPSHOT).getMax(), threadPool.executor(ThreadPool.Names.SNAPSHOT) ); + this.blobStoreSnapshotMetrics = new BlobStoreSnapshotMetrics(projectId, metadata, snapshotMetrics); } @Override @@ -2200,16 +2203,6 @@ RateLimiter getRestoreRateLimiter() { ); } - @Override - public long getSnapshotThrottleTimeInNanos() { - return snapshotRateLimitingTimeInNanos.count(); - } - - @Override - public long getRestoreThrottleTimeInNanos() { - return restoreRateLimitingTimeInNanos.count(); - } - private void assertSnapshotOrStatelessPermittedThreadPool() { // The Stateless plugin adds custom thread pools for object store operations assert ThreadPool.assertCurrentThreadPool( @@ -3233,6 +3226,8 @@ public void snapshotShard(SnapshotShardContext context) { } private void doSnapshotShard(SnapshotShardContext context) { + blobStoreSnapshotMetrics.shardSnapshotStarted(); + context.addListener(ActionListener.running(() -> blobStoreSnapshotMetrics.shardSnapshotCompleted(context.status()))); if (isReadOnly()) { context.onFailure(new RepositoryException(metadata.name(), "cannot snapshot shard on a readonly repository")); return; @@ -3472,8 +3467,8 @@ private void doSnapshotShard(SnapshotShardContext context) { final BlobStoreIndexShardSnapshot blobStoreIndexShardSnapshot = new BlobStoreIndexShardSnapshot( snapshotId.getName(), indexCommitPointFiles, - lastSnapshotStatus.getStartTime(), - threadPool.absoluteTimeInMillis() - lastSnapshotStatus.getStartTime(), + lastSnapshotStatus.getStartTimeMillis(), + threadPool.absoluteTimeInMillis() - lastSnapshotStatus.getStartTimeMillis(), lastSnapshotStatus.getIncrementalFileCount(), lastSnapshotStatus.getIncrementalSize() ); @@ -3772,16 +3767,16 @@ private static InputStream maybeRateLimit( /** * Wrap the restore rate limiter (controlled by the repository setting `max_restore_bytes_per_sec` and the cluster setting * `indices.recovery.max_bytes_per_sec`) around the given stream. Any throttling is reported to the given listener and not otherwise - * recorded in the value returned by {@link BlobStoreRepository#getRestoreThrottleTimeInNanos}. + * recorded in the value returned by {@link RepositoriesStats.SnapshotStats#totalReadThrottledNanos()}. */ public InputStream maybeRateLimitRestores(InputStream stream) { - return maybeRateLimitRestores(stream, restoreRateLimitingTimeInNanos::inc); + return maybeRateLimitRestores(stream, blobStoreSnapshotMetrics::incrementRestoreRateLimitingTimeInNanos); } /** * Wrap the restore rate limiter (controlled by the repository setting `max_restore_bytes_per_sec` and the cluster setting * `indices.recovery.max_bytes_per_sec`) around the given stream. Any throttling is recorded in the value returned by {@link - * BlobStoreRepository#getRestoreThrottleTimeInNanos}. + * RepositoriesStats.SnapshotStats#totalReadThrottledNanos()}. */ public InputStream maybeRateLimitRestores(InputStream stream, RateLimitingInputStream.Listener throttleListener) { return maybeRateLimit( @@ -3793,17 +3788,17 @@ public InputStream maybeRateLimitRestores(InputStream stream, RateLimitingInputS /** * Wrap the snapshot rate limiter around the given stream. Any throttling is recorded in the value returned by - * {@link BlobStoreRepository#getSnapshotThrottleTimeInNanos()}. Note that speed is throttled by the repository setting + * {@link RepositoriesStats.SnapshotStats#totalWriteThrottledNanos()}. Note that speed is throttled by the repository setting * `max_snapshot_bytes_per_sec` and, if recovery node bandwidth settings have been set, additionally by the * `indices.recovery.max_bytes_per_sec` speed. */ public InputStream maybeRateLimitSnapshots(InputStream stream) { - return maybeRateLimitSnapshots(stream, snapshotRateLimitingTimeInNanos::inc); + return maybeRateLimitSnapshots(stream, blobStoreSnapshotMetrics::incrementSnapshotRateLimitingTimeInNanos); } /** * Wrap the snapshot rate limiter around the given stream. Any throttling is recorded in the value returned by - * {@link BlobStoreRepository#getSnapshotThrottleTimeInNanos()}. Note that speed is throttled by the repository setting + * {@link RepositoriesStats.SnapshotStats#totalWriteThrottledNanos()}. Note that speed is throttled by the repository setting * `max_snapshot_bytes_per_sec` and, if recovery node bandwidth settings have been set, additionally by the * `indices.recovery.max_bytes_per_sec` speed. */ @@ -4135,13 +4130,19 @@ protected void snapshotFile(SnapshotShardContext context, FileInfo fileInfo) thr @Override public int read() throws IOException { checkAborted(); - return super.read(); + final long beforeReadMillis = threadPool.rawRelativeTimeInMillis(); + int value = super.read(); + blobStoreSnapshotMetrics.incrementUploadReadTime(threadPool.rawRelativeTimeInMillis() - beforeReadMillis); + return value; } @Override public int read(byte[] b, int off, int len) throws IOException { checkAborted(); - return super.read(b, off, len); + final long beforeReadMillis = threadPool.rawRelativeTimeInMillis(); + int amountRead = super.read(b, off, len); + blobStoreSnapshotMetrics.incrementUploadReadTime(threadPool.rawRelativeTimeInMillis() - beforeReadMillis); + return amountRead; } private void checkAborted() { @@ -4150,17 +4151,20 @@ private void checkAborted() { }; final String partName = fileInfo.partName(i); logger.trace("[{}] Writing [{}] to [{}]", metadata.name(), partName, shardContainer.path()); - final long startMS = threadPool.relativeTimeInMillis(); + final long startMillis = threadPool.relativeTimeInMillis(); shardContainer.writeBlob(OperationPurpose.SNAPSHOT_DATA, partName, inputStream, partBytes, false); + final long uploadTimeInMillis = threadPool.relativeTimeInMillis() - startMillis; + blobStoreSnapshotMetrics.incrementCountersForPartUpload(partBytes, uploadTimeInMillis); logger.trace( "[{}] Writing [{}] of size [{}b] to [{}] took [{}ms]", metadata.name(), partName, partBytes, shardContainer.path(), - threadPool.relativeTimeInMillis() - startMS + uploadTimeInMillis ); } + blobStoreSnapshotMetrics.incrementNumberOfBlobsUploaded(); Store.verify(indexInput); snapshotStatus.addProcessedFile(fileInfo.length()); } catch (Exception t) { @@ -4234,4 +4238,14 @@ public final Set getUsageFeatures() { protected Set getExtraUsageFeatures() { return Set.of(); } + + @Override + public LongWithAttributes getShardSnapshotsInProgress() { + return blobStoreSnapshotMetrics.getShardSnapshotsInProgress(); + } + + @Override + public RepositoriesStats.SnapshotStats getSnapshotStats() { + return blobStoreSnapshotMetrics.getSnapshotStats(); + } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreSnapshotMetrics.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreSnapshotMetrics.java new file mode 100644 index 0000000000000..3ccf7e658f1e6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreSnapshotMetrics.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.blobstore; + +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.common.metrics.CounterMetric; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; +import org.elasticsearch.repositories.RepositoriesStats; +import org.elasticsearch.repositories.SnapshotMetrics; +import org.elasticsearch.telemetry.metric.LongWithAttributes; + +import java.util.Map; + +public class BlobStoreSnapshotMetrics { + + private final SnapshotMetrics snapshotMetrics; + private final CounterMetric shardSnapshotsInProgress = new CounterMetric(); + private final CounterMetric snapshotRateLimitingTimeInNanos = new CounterMetric(); + private final CounterMetric restoreRateLimitingTimeInNanos = new CounterMetric(); + private final CounterMetric numberOfBlobsUploaded = new CounterMetric(); + private final CounterMetric numberOfBytesUploaded = new CounterMetric(); + private final CounterMetric uploadTimeInMillis = new CounterMetric(); + private final CounterMetric uploadReadTimeInNanos = new CounterMetric(); + private final CounterMetric numberOfShardSnapshotsStarted = new CounterMetric(); + private final CounterMetric numberOfShardSnapshotsCompleted = new CounterMetric(); + private final Map metricAttributes; + + public BlobStoreSnapshotMetrics(@Nullable ProjectId projectId, RepositoryMetadata repositoryMetadata, SnapshotMetrics snapshotMetrics) { + if (projectId != null) { + this.snapshotMetrics = snapshotMetrics; + metricAttributes = SnapshotMetrics.createAttributesMap(projectId, repositoryMetadata); + } else { + // Project ID should only be null for the stateless main blobstore, which is not used for snapshots + this.snapshotMetrics = SnapshotMetrics.NOOP; + this.metricAttributes = Map.of(); + } + } + + public void incrementSnapshotRateLimitingTimeInNanos(long throttleTimeNanos) { + snapshotMetrics.createThrottleDurationCounter().incrementBy(throttleTimeNanos, metricAttributes); + snapshotRateLimitingTimeInNanos.inc(throttleTimeNanos); + } + + public void incrementRestoreRateLimitingTimeInNanos(long throttleTimeNanos) { + snapshotMetrics.restoreThrottleDurationCounter().incrementBy(throttleTimeNanos, metricAttributes); + restoreRateLimitingTimeInNanos.inc(throttleTimeNanos); + } + + public void incrementCountersForPartUpload(long partSizeInBytes, long partWriteTimeMillis) { + snapshotMetrics.bytesUploadedCounter().incrementBy(partSizeInBytes, metricAttributes); + snapshotMetrics.uploadDurationCounter().incrementBy(partWriteTimeMillis, metricAttributes); + numberOfBytesUploaded.inc(partSizeInBytes); + uploadTimeInMillis.inc(partWriteTimeMillis); + } + + public void incrementNumberOfBlobsUploaded() { + snapshotMetrics.blobsUploadedCounter().incrementBy(1, metricAttributes); + numberOfBlobsUploaded.inc(); + } + + public void shardSnapshotStarted() { + snapshotMetrics.shardsStartedCounter().incrementBy(1, metricAttributes); + numberOfShardSnapshotsStarted.inc(); + shardSnapshotsInProgress.inc(); + } + + public void shardSnapshotCompleted(IndexShardSnapshotStatus status) { + final Map attrsWithStage = Maps.copyMapWithAddedEntry(metricAttributes, "stage", status.getStage().name()); + snapshotMetrics.shardsCompletedCounter().incrementBy(1, attrsWithStage); + snapshotMetrics.shardsDurationHistogram().record(status.getTotalTimeMillis() / 1_000d, attrsWithStage); + numberOfShardSnapshotsCompleted.inc(); + shardSnapshotsInProgress.dec(); + } + + public void incrementUploadReadTime(long readTimeInMillis) { + snapshotMetrics.uploadReadDurationCounter().incrementBy(readTimeInMillis, metricAttributes); + uploadReadTimeInNanos.inc(readTimeInMillis); + } + + public LongWithAttributes getShardSnapshotsInProgress() { + return new LongWithAttributes(shardSnapshotsInProgress.count(), metricAttributes); + } + + public RepositoriesStats.SnapshotStats getSnapshotStats() { + return new RepositoriesStats.SnapshotStats( + numberOfShardSnapshotsStarted.count(), + numberOfShardSnapshotsCompleted.count(), + shardSnapshotsInProgress.count(), + restoreRateLimitingTimeInNanos.count(), + snapshotRateLimitingTimeInNanos.count(), + numberOfBlobsUploaded.count(), + numberOfBytesUploaded.count(), + uploadTimeInMillis.count(), + uploadReadTimeInNanos.count() + ); + } +} diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java index aa7fae749c328..3b226875de7ef 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/MeteredBlobStoreRepository.java @@ -15,9 +15,11 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoryInfo; import org.elasticsearch.repositories.RepositoryStatsSnapshot; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -27,16 +29,17 @@ public abstract class MeteredBlobStoreRepository extends BlobStoreRepository { private final RepositoryInfo repositoryInfo; public MeteredBlobStoreRepository( - ProjectId projectId, + @Nullable ProjectId projectId, RepositoryMetadata metadata, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, BlobPath basePath, - Map location + Map location, + SnapshotMetrics snapshotMetrics ) { - super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath, snapshotMetrics); ThreadPool threadPool = clusterService.getClusterApplierService().threadPool(); this.repositoryInfo = new RepositoryInfo( UUIDs.randomBase64UUID(), diff --git a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java index 97e2c0b2de44b..15b5e9a7d480e 100644 --- a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java @@ -21,9 +21,11 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -76,7 +78,7 @@ public class FsRepository extends BlobStoreRepository { * Constructs a shared file system repository. */ public FsRepository( - ProjectId projectId, + @Nullable ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, @@ -84,7 +86,23 @@ public FsRepository( BigArrays bigArrays, RecoverySettings recoverySettings ) { - super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); + this(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings, SnapshotMetrics.NOOP); + } + + /** + * Constructs a shared file system repository. + */ + public FsRepository( + @Nullable ProjectId projectId, + RepositoryMetadata metadata, + Environment environment, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + BigArrays bigArrays, + RecoverySettings recoverySettings, + SnapshotMetrics snapshotMetrics + ) { + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY, snapshotMetrics); this.environment = environment; String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings()); if (location.isEmpty()) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 68eb8bd8cf2cd..bfe0e9f7895eb 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -101,7 +101,9 @@ import org.elasticsearch.repositories.ShardGeneration; import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.ShardSnapshotResult; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.tasks.Task; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -200,10 +202,14 @@ public final class SnapshotsService extends AbstractLifecycleComponent implement private final boolean serializeProjectMetadata; + private final SnapshotMetrics snapshotMetrics; + private final MasterServiceTaskQueue masterServiceTaskQueue; private final ShardSnapshotUpdateCompletionHandler shardSnapshotUpdateCompletionHandler; + private CachedSnapshotStateMetrics cachedSnapshotStateMetrics; + /** * Setting that specifies the maximum number of allowed concurrent snapshot create and delete operations in the * cluster state. The number of concurrent operations in a cluster state is defined as the sum of @@ -228,7 +234,8 @@ public SnapshotsService( TransportService transportService, ActionFilters actionFilters, SystemIndices systemIndices, - boolean serializeProjectMetadata + boolean serializeProjectMetadata, + SnapshotMetrics snapshotMetrics ) { this.clusterService = clusterService; this.rerouteService = rerouteService; @@ -236,6 +243,9 @@ public SnapshotsService( this.repositoriesService = repositoriesService; this.threadPool = transportService.getThreadPool(); this.transportService = transportService; + this.snapshotMetrics = snapshotMetrics; + snapshotMetrics.createSnapshotShardsByStateMetric(this::getShardsByState); + snapshotMetrics.createSnapshotsByStateMetric(this::getSnapshotsByState); // The constructor of UpdateSnapshotStatusAction will register itself to the TransportService. this.updateSnapshotStatusHandler = new UpdateSnapshotStatusAction(transportService, clusterService, threadPool, actionFilters); @@ -1647,6 +1657,14 @@ protected void doRun() { @Override public void onResponse(List> actionListeners) { completeListenersIgnoringException(actionListeners, snapshotInfo); + final Map attributesWithState = Maps.copyMapWithAddedEntry( + SnapshotMetrics.createAttributesMap(snapshot.getProjectId(), repo.getMetadata()), + "state", + snapshotInfo.state().name() + ); + snapshotMetrics.snapshotsCompletedCounter().incrementBy(1, attributesWithState); + snapshotMetrics.snapshotsDurationHistogram() + .record((snapshotInfo.endTime() - snapshotInfo.startTime()) / 1_000.0, attributesWithState); logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state()); } @@ -4427,6 +4445,11 @@ private SnapshotsInProgress createSnapshot( final var res = snapshotsInProgress.withAddedEntry(newEntry); taskContext.success(() -> { logger.info("snapshot [{}] started", snapshot); + final Map attributes = SnapshotMetrics.createAttributesMap( + snapshot.getProjectId(), + repository.getMetadata() + ); + snapshotMetrics.snapshotsStartedCounter().incrementBy(1, attributes); createSnapshotTask.listener.onResponse(snapshot); if (newEntry.state().completed()) { endSnapshot(newEntry, currentState.metadata(), createSnapshotTask.repositoryData); @@ -4436,6 +4459,61 @@ private SnapshotsInProgress createSnapshot( } } + private Collection getShardsByState() { + final ClusterState currentState = clusterService.state(); + // Only the master should report on shards-by-state + if (currentState.nodes().isLocalNodeElectedMaster() == false) { + return List.of(); + } + return recalculateIfStale(currentState).shardStateMetrics(); + } + + private Collection getSnapshotsByState() { + final ClusterState currentState = clusterService.state(); + // Only the master should report on snapshots-by-state + if (currentState.nodes().isLocalNodeElectedMaster() == false) { + return List.of(); + } + return recalculateIfStale(currentState).snapshotStateMetrics(); + } + + private CachedSnapshotStateMetrics recalculateIfStale(ClusterState currentState) { + if (cachedSnapshotStateMetrics == null || cachedSnapshotStateMetrics.isStale(currentState)) { + cachedSnapshotStateMetrics = recalculateSnapshotStats(currentState); + } + return cachedSnapshotStateMetrics; + } + + private CachedSnapshotStateMetrics recalculateSnapshotStats(ClusterState currentState) { + final SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.get(currentState); + final List snapshotStateMetrics = new ArrayList<>(); + final List shardStateMetrics = new ArrayList<>(); + + currentState.metadata().projects().forEach((projectId, project) -> { + final RepositoriesMetadata repositoriesMetadata = RepositoriesMetadata.get(project); + if (repositoriesMetadata != null) { + for (RepositoryMetadata repository : repositoriesMetadata.repositories()) { + final Tuple, Map> stateSummaries = snapshotsInProgress + .shardStateSummaryForRepository(projectId, repository.name()); + final Map attributesMap = SnapshotMetrics.createAttributesMap(projectId, repository); + stateSummaries.v1() + .forEach( + (snapshotState, count) -> snapshotStateMetrics.add( + new LongWithAttributes(count, Maps.copyMapWithAddedEntry(attributesMap, "state", snapshotState.name())) + ) + ); + stateSummaries.v2() + .forEach( + (shardState, count) -> shardStateMetrics.add( + new LongWithAttributes(count, Maps.copyMapWithAddedEntry(attributesMap, "state", shardState.name())) + ) + ); + } + } + }); + return new CachedSnapshotStateMetrics(currentState, snapshotStateMetrics, shardStateMetrics); + } + private record UpdateNodeIdsForRemovalTask() implements ClusterStateTaskListener { @Override public void onFailure(Exception e) { @@ -4467,4 +4545,38 @@ private static boolean supportsNodeRemovalTracking(ClusterState clusterState) { } private final MasterServiceTaskQueue updateNodeIdsToRemoveQueue; + + /** + * A cached copy of the snapshot and shard state metrics + */ + private record CachedSnapshotStateMetrics( + String clusterStateId, + int snapshotsInProgressIdentityHashcode, + Collection snapshotStateMetrics, + Collection shardStateMetrics + ) { + CachedSnapshotStateMetrics( + ClusterState sourceState, + Collection snapshotStateMetrics, + Collection shardStateMetrics + ) { + this( + sourceState.stateUUID(), + System.identityHashCode(SnapshotsInProgress.get(sourceState)), + snapshotStateMetrics, + shardStateMetrics + ); + } + + /** + * Are these metrics stale? + * + * @param currentClusterState The current cluster state + * @return true if these metrics were calculated from a prior cluster state and need to be recalculated, false otherwise + */ + public boolean isStale(ClusterState currentClusterState) { + return (Objects.equals(clusterStateId, currentClusterState.stateUUID()) == false + && System.identityHashCode(SnapshotsInProgress.get(currentClusterState)) != snapshotsInProgressIdentityHashcode); + } + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index e8bbe412cbbab..8a89ce1048872 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -78,6 +78,7 @@ import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.threadpool.ThreadPoolStats; import org.elasticsearch.transport.TransportActionStats; @@ -468,11 +469,13 @@ public void testSerialization() throws IOException { assertNotSame(scriptCacheStats, deserializedScriptCacheStats); } - RepositoriesStats repoThrottlingStats = deserializedNodeStats.getRepositoriesStats(); - assertTrue(repoThrottlingStats.getRepositoryThrottlingStats().containsKey("test-repository")); - assertEquals(100, repoThrottlingStats.getRepositoryThrottlingStats().get("test-repository").totalReadThrottledNanos()); - assertEquals(200, repoThrottlingStats.getRepositoryThrottlingStats().get("test-repository").totalWriteThrottledNanos()); - + RepositoriesStats repoSnapshotStats = deserializedNodeStats.getRepositoriesStats(); + assertTrue(repoSnapshotStats.getRepositorySnapshotStats().containsKey("test-repository")); + RepositoriesStats.SnapshotStats expectedSnapshotStats = nodeStats.getRepositoriesStats() + .getRepositorySnapshotStats() + .get("test-repository"); + RepositoriesStats.SnapshotStats actualSnapshotStats = repoSnapshotStats.getRepositorySnapshotStats().get("test-repository"); + assertEquals(XContentTestUtils.convertToMap(expectedSnapshotStats), XContentTestUtils.convertToMap(actualSnapshotStats)); } } } @@ -1069,7 +1072,20 @@ public static NodeStats createNodeStats() { ); } RepositoriesStats repositoriesStats = new RepositoriesStats( - Map.of("test-repository", new RepositoriesStats.ThrottlingStats(100, 200)) + Map.of( + "test-repository", + new RepositoriesStats.SnapshotStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong() + ) + ) ); NodeAllocationStats nodeAllocationStats = new NodeAllocationStats( randomIntBetween(0, 10000), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java index bc45f6b7b3c70..b1728aeddadaf 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -151,7 +152,8 @@ public Repository create(ProjectId projectId, RepositoryMetadata metadata) { Map.of("fs", fsFactory), threadPool, mock(NodeClient.class), - null + null, + SnapshotMetrics.NOOP ) ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusActionTests.java index 09bf072c7225d..275687a615187 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusActionTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.ShardGeneration; import org.elasticsearch.repositories.ShardSnapshotResult; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.tasks.CancellableTask; @@ -74,7 +75,8 @@ public void initializeComponents() throws Exception { Map.of(), threadPool, nodeClient, - List.of() + List.of(), + SnapshotMetrics.NOOP ); action = new TransportSnapshotsStatusAction( transportService, diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index 464d9fe092d1c..bc60cb9699a53 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -47,6 +47,7 @@ import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; import org.elasticsearch.indices.recovery.SnapshotFilesProvider; import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; @@ -549,7 +550,8 @@ private IndicesClusterStateService createIndicesClusterStateService( Collections.emptyMap(), threadPool, client, - List.of() + List.of(), + SnapshotMetrics.NOOP ); final PeerRecoveryTargetService recoveryTargetService = new PeerRecoveryTargetService( client, diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java index 7f6885e7a977f..9fb575f5a6b79 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java @@ -72,7 +72,8 @@ public void testCanRegisterTwoRepositoriesWithDifferentTypes() { eq(clusterService), eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class) + any(RepositoriesMetrics.class), + any(SnapshotMetrics.class) ) ).thenReturn(Collections.singletonMap("type1", factory)); when( @@ -82,7 +83,8 @@ public void testCanRegisterTwoRepositoriesWithDifferentTypes() { eq(clusterService), eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class) + any(RepositoriesMetrics.class), + any(SnapshotMetrics.class) ) ).thenReturn(Collections.singletonMap("type2", factory)); @@ -96,7 +98,8 @@ public void testCanRegisterTwoRepositoriesWithDifferentTypes() { MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, recoverySettings, - TelemetryProvider.NOOP + TelemetryProvider.NOOP, + SnapshotMetrics.NOOP ); } @@ -108,7 +111,8 @@ public void testCannotRegisterTwoRepositoriesWithSameTypes() { eq(clusterService), eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class) + any(RepositoriesMetrics.class), + any(SnapshotMetrics.class) ) ).thenReturn(Collections.singletonMap("type1", factory)); when( @@ -118,7 +122,8 @@ public void testCannotRegisterTwoRepositoriesWithSameTypes() { eq(clusterService), eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class) + any(RepositoriesMetrics.class), + any(SnapshotMetrics.class) ) ).thenReturn(Collections.singletonMap("type1", factory)); @@ -133,7 +138,8 @@ public void testCannotRegisterTwoRepositoriesWithSameTypes() { MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, recoverySettings, - TelemetryProvider.NOOP + TelemetryProvider.NOOP, + SnapshotMetrics.NOOP ) ); @@ -159,7 +165,8 @@ public void testCannotRegisterTwoInternalRepositoriesWithSameTypes() { MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, recoverySettings, - TelemetryProvider.NOOP + TelemetryProvider.NOOP, + SnapshotMetrics.NOOP ) ); @@ -174,7 +181,8 @@ public void testCannotRegisterNormalAndInternalRepositoriesWithSameTypes() { eq(clusterService), eq(MockBigArrays.NON_RECYCLING_INSTANCE), eq(recoverySettings), - any(RepositoriesMetrics.class) + any(RepositoriesMetrics.class), + any(SnapshotMetrics.class) ) ).thenReturn(Collections.singletonMap("type1", factory)); when(plugin2.getInternalRepositories(environment, contentRegistry, clusterService, recoverySettings)).thenReturn( @@ -192,7 +200,8 @@ public void testCannotRegisterNormalAndInternalRepositoriesWithSameTypes() { MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, recoverySettings, - TelemetryProvider.NOOP + TelemetryProvider.NOOP, + SnapshotMetrics.NOOP ) ); diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index da6d6ff78a989..1617001019ec3 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -53,6 +53,7 @@ import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; @@ -155,7 +156,8 @@ public void setUp() throws Exception { typesRegistry, threadPool, client, - List.of() + List.of(), + SnapshotMetrics.NOOP ); clusterService.start(); @@ -377,8 +379,8 @@ public void testRepositoriesThrottlingStats() { var clusterState = createClusterStateWithRepo(repoName, TestRepository.TYPE); repositoriesService.applyClusterState(new ClusterChangedEvent("put test repository", clusterState, emptyState())); RepositoriesStats throttlingStats = repositoriesService.getRepositoriesThrottlingStats(); - assertTrue(throttlingStats.getRepositoryThrottlingStats().containsKey(repoName)); - assertNotNull(throttlingStats.getRepositoryThrottlingStats().get(repoName)); + assertTrue(throttlingStats.getRepositorySnapshotStats().containsKey(repoName)); + assertNotNull(throttlingStats.getRepositorySnapshotStats().get(repoName)); } // InvalidRepository is created when current node is non-master node and failed to create repository by applying cluster state from @@ -700,16 +702,6 @@ public void deleteSnapshots( repositoryDataUpdateListener.onFailure(new UnsupportedOperationException()); } - @Override - public long getSnapshotThrottleTimeInNanos() { - return 0; - } - - @Override - public long getRestoreThrottleTimeInNanos() { - return 0; - } - @Override public String startVerification() { return null; @@ -771,6 +763,16 @@ public void cloneShardSnapshot( @Override public void awaitIdle() {} + @Override + public LongWithAttributes getShardSnapshotsInProgress() { + return null; + } + + @Override + public RepositoriesStats.SnapshotStats getSnapshotStats() { + return RepositoriesStats.SnapshotStats.ZERO; + } + @Override public Lifecycle.State lifecycleState() { return null; @@ -832,7 +834,8 @@ private MeteredRepositoryTypeA(ProjectId projectId, RepositoryMetadata metadata, MockBigArrays.NON_RECYCLING_INSTANCE, mock(RecoverySettings.class), BlobPath.EMPTY, - Map.of("bucket", "bucket-a") + Map.of("bucket", "bucket-a"), + SnapshotMetrics.NOOP ); } @@ -860,7 +863,8 @@ private MeteredRepositoryTypeB(ProjectId projectId, RepositoryMetadata metadata, MockBigArrays.NON_RECYCLING_INSTANCE, mock(RecoverySettings.class), BlobPath.EMPTY, - Map.of("bucket", "bucket-b") + Map.of("bucket", "bucket-b"), + SnapshotMetrics.NOOP ); } diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java index de1979cfcf5d2..0d8011830dcec 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryDeleteThrottlingTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -71,7 +72,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( TEST_REPO_TYPE, diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 81f4f743d821a..7558850a0f5a5 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -174,6 +174,7 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.VerifyNodeRepositoryAction; import org.elasticsearch.repositories.VerifyNodeRepositoryCoordinationAction; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; @@ -2411,7 +2412,8 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { emptyMap(), threadPool, client, - List.of() + List.of(), + SnapshotMetrics.NOOP ); final ActionFilters actionFilters = new ActionFilters(emptySet()); snapshotsService = new SnapshotsService( @@ -2423,7 +2425,8 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { transportService, actionFilters, EmptySystemIndices.INSTANCE, - false + false, + SnapshotMetrics.NOOP ); nodeEnv = new NodeEnvironment(settings, environment); final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); diff --git a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java index c553f0f10062a..f7cfb79be4893 100644 --- a/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java +++ b/test/external-modules/latency-simulating-directory/src/internalClusterTest/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepositoryTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -61,7 +62,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( REPO_TYPE, diff --git a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java index a4126c41c0925..dbe648e49ee03 100644 --- a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java +++ b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingBlobStoreRepository.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.blobstore.support.FilterBlobContainer; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Nullable; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.fs.FsRepository; @@ -31,7 +32,7 @@ class LatencySimulatingBlobStoreRepository extends FsRepository { private final Runnable simulator; protected LatencySimulatingBlobStoreRepository( - ProjectId projectId, + @Nullable ProjectId projectId, RepositoryMetadata metadata, Environment env, NamedXContentRegistry namedXContentRegistry, diff --git a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingRepositoryPlugin.java b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingRepositoryPlugin.java index b60c965a66537..e4e5626017945 100644 --- a/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingRepositoryPlugin.java +++ b/test/external-modules/latency-simulating-directory/src/main/java/org/elasticsearch/test/simulatedlatencyrepo/LatencySimulatingRepositoryPlugin.java @@ -18,6 +18,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.Map; @@ -37,7 +38,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( TYPE, diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index a3e8f6e84600f..d7322cb337f7f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -22,6 +22,7 @@ import org.elasticsearch.repositories.FinalizeSnapshotContext; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.IndexMetaDataGenerations; +import org.elasticsearch.repositories.RepositoriesStats; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryShardId; @@ -31,6 +32,7 @@ import org.elasticsearch.repositories.SnapshotShardContext; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import java.util.Collection; import java.util.Collections; @@ -124,16 +126,6 @@ public void deleteSnapshots( repositoryDataUpdateListener.onFailure(new UnsupportedOperationException()); } - @Override - public long getSnapshotThrottleTimeInNanos() { - return 0; - } - - @Override - public long getRestoreThrottleTimeInNanos() { - return 0; - } - @Override public String startVerification() { return null; @@ -175,4 +167,14 @@ public void cloneShardSnapshot( throw new UnsupportedOperationException("Unsupported for restore-only repository"); } + + @Override + public LongWithAttributes getShardSnapshotsInProgress() { + return null; + } + + @Override + public RepositoriesStats.SnapshotStats getSnapshotStats() { + return RepositoriesStats.SnapshotStats.ZERO; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index ead163eaf26a4..5b912e3bb3527 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -17,17 +17,21 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.metadata.NodesShutdownMetadata; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; @@ -35,6 +39,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.plugins.Plugin; @@ -52,6 +57,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.snapshots.mockstore.MockRepository; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.threadpool.ThreadPool; @@ -80,6 +86,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.stream.Stream; import java.util.stream.StreamSupport; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.READONLY_SETTING_KEY; @@ -91,6 +98,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase { @@ -701,6 +709,148 @@ protected List createNSnapshots(String repoName, int count) throws Excep return createNSnapshots(logger, repoName, count); } + protected static void putShutdownForRemovalMetadata(String nodeName, ClusterService clusterService) { + safeAwait((ActionListener listener) -> putShutdownForRemovalMetadata(clusterService, nodeName, listener)); + } + + protected static void flushMasterQueue(ClusterService clusterService, ActionListener listener) { + clusterService.submitUnbatchedStateUpdateTask("flush queue", new ClusterStateUpdateTask(Priority.LANGUID) { + @Override + public ClusterState execute(ClusterState currentState) { + return currentState; + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { + listener.onResponse(null); + } + }); + } + + protected static void putShutdownForRemovalMetadata(ClusterService clusterService, String nodeName, ActionListener listener) { + // not testing REPLACE just because it requires us to specify the replacement node + final var shutdownType = randomFrom(SingleNodeShutdownMetadata.Type.REMOVE, SingleNodeShutdownMetadata.Type.SIGTERM); + final var shutdownMetadata = SingleNodeShutdownMetadata.builder() + .setType(shutdownType) + .setStartedAtMillis(clusterService.threadPool().absoluteTimeInMillis()) + .setReason("test"); + switch (shutdownType) { + case SIGTERM -> shutdownMetadata.setGracePeriod(TimeValue.timeValueSeconds(60)); + } + SubscribableListener + + .newForked(l -> putShutdownMetadata(clusterService, shutdownMetadata, nodeName, l)) + .andThen(l -> flushMasterQueue(clusterService, l)) + .addListener(listener); + } + + protected static void putShutdownMetadata( + ClusterService clusterService, + SingleNodeShutdownMetadata.Builder shutdownMetadataBuilder, + String nodeName, + ActionListener listener + ) { + clusterService.submitUnbatchedStateUpdateTask("mark node for removal", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + final var node = currentState.nodes().resolveNode(nodeName); + return currentState.copyAndUpdateMetadata( + mdb -> mdb.putCustom( + NodesShutdownMetadata.TYPE, + new NodesShutdownMetadata( + Map.of( + node.getId(), + shutdownMetadataBuilder.setNodeId(node.getId()).setNodeEphemeralId(node.getEphemeralId()).build() + ) + ) + ) + ); + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { + listener.onResponse(null); + } + }); + } + + protected static void clearShutdownMetadata(ClusterService clusterService) { + safeAwait(listener -> clusterService.submitUnbatchedStateUpdateTask("remove restart marker", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + return currentState.copyAndUpdateMetadata(mdb -> mdb.putCustom(NodesShutdownMetadata.TYPE, NodesShutdownMetadata.EMPTY)); + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + + @Override + public void clusterStateProcessed(ClusterState initialState, ClusterState newState) { + listener.onResponse(null); + } + })); + } + + protected static SubscribableListener createSnapshotInStateListener( + ClusterService clusterService, + String repoName, + String indexName, + int numShards, + SnapshotsInProgress.ShardState shardState + ) { + return ClusterServiceUtils.addTemporaryStateListener(clusterService, state -> { + final var entriesForRepo = SnapshotsInProgress.get(state).forRepo(repoName); + if (entriesForRepo.isEmpty()) { + // it's (just about) possible for the data node to apply the initial snapshot state, start on the first shard snapshot, and + // hit the IO block, before the master even applies this cluster state, in which case we simply retry: + return false; + } + assertThat(entriesForRepo, hasSize(1)); + final var shardSnapshotStatuses = entriesForRepo.iterator() + .next() + .shards() + .entrySet() + .stream() + .flatMap(e -> e.getKey().getIndexName().equals(indexName) ? Stream.of(e.getValue()) : Stream.of()) + .toList(); + assertThat(shardSnapshotStatuses, hasSize(numShards)); + for (var shardStatus : shardSnapshotStatuses) { + assertThat(shardStatus.state(), oneOf(SnapshotsInProgress.ShardState.INIT, shardState)); + if (shardStatus.state() == SnapshotsInProgress.ShardState.INIT) { + return false; + } + } + return true; + }); + } + + protected static SubscribableListener createSnapshotPausedListener( + ClusterService clusterService, + String repoName, + String indexName, + int numShards + ) { + return createSnapshotInStateListener( + clusterService, + repoName, + indexName, + numShards, + SnapshotsInProgress.ShardState.PAUSED_FOR_NODE_REMOVAL + ); + } + public static List createNSnapshots(Logger logger, String repoName, int count) throws Exception { final PlainActionFuture> allSnapshotsDone = new PlainActionFuture<>(); final ActionListener snapshotsListener = new GroupedActionListener<>(count, allSnapshotsDone); diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java index 252091bb45d82..6088dc00efc26 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java @@ -34,12 +34,14 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.PathUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -86,7 +88,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( "mock", @@ -97,7 +100,8 @@ public Map getRepositories( namedXContentRegistry, clusterService, bigArrays, - recoverySettings + recoverySettings, + snapshotMetrics ) ); } @@ -185,13 +189,14 @@ public long getFailureCount() { private volatile boolean failOnDeleteContainer = false; public MockRepository( - ProjectId projectId, + @Nullable ProjectId projectId, RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, BigArrays bigArrays, - RecoverySettings recoverySettings + RecoverySettings recoverySettings, + SnapshotMetrics snapshotMetrics ) { super( projectId, @@ -200,7 +205,8 @@ public MockRepository( namedXContentRegistry, clusterService, bigArrays, - recoverySettings + recoverySettings, + snapshotMetrics ); randomControlIOExceptionRate = metadata.settings().getAsDouble("random_control_io_exception_rate", 0.0); randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index 6c56724a6f20a..b264c38737268 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -317,7 +317,7 @@ public void testRateLimitingIsEmployed() throws Exception { startRestore(clusterService, restoreService, restoreRequest).actionGet(); if (followerRateLimiting) { - assertTrue(repositories.stream().anyMatch(cr -> cr.getRestoreThrottleTimeInNanos() > 0)); + assertTrue(repositories.stream().anyMatch(cr -> cr.getSnapshotStats().totalReadThrottledNanos() > 0)); } else { assertTrue(restoreSources.stream().anyMatch(cr -> cr.getThrottleTime() > 0)); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index f63501bcada10..a0b3be170294c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -73,6 +73,7 @@ import org.elasticsearch.repositories.FinalizeSnapshotContext; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.IndexMetaDataGenerations; +import org.elasticsearch.repositories.RepositoriesStats; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryShardId; @@ -86,6 +87,7 @@ import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.RemoteClusterService; @@ -389,16 +391,6 @@ public void deleteSnapshots( repositoryDataUpdateListener.onFailure(new UnsupportedOperationException("Unsupported for repository of type: " + TYPE)); } - @Override - public long getSnapshotThrottleTimeInNanos() { - throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); - } - - @Override - public long getRestoreThrottleTimeInNanos() { - return throttledTime.count(); - } - @Override public String startVerification() { throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); @@ -618,6 +610,16 @@ public void cloneShardSnapshot( @Override public void awaitIdle() {} + @Override + public LongWithAttributes getShardSnapshotsInProgress() { + return null; + } + + @Override + public RepositoriesStats.SnapshotStats getSnapshotStats() { + return new RepositoriesStats.SnapshotStats(throttledTime.count(), 0); + } + private void updateMappings( RemoteClusterClient leaderClient, Index leaderIndex, diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java index 6074b81d68608..5de93cc173faa 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.slice.SliceBuilder; @@ -89,7 +90,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap("source", SourceOnlySnapshotRepository.newRepositoryFactory()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 23ca86d48bd1e..bb0a21a6afd2a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -80,6 +80,7 @@ import org.elasticsearch.protocol.xpack.XPackUsageRequest; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.snapshots.sourceonly.SourceOnlySnapshotRepository; @@ -441,7 +442,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap("source", SourceOnlySnapshotRepository.newRepositoryFactory()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index 98a0c04830561..117141708ed43 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -81,6 +81,7 @@ import org.elasticsearch.plugins.interceptor.RestServerActionPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestHeaderDefinition; @@ -512,14 +513,31 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { HashMap repositories = new HashMap<>( - super.getRepositories(env, namedXContentRegistry, clusterService, bigArrays, recoverySettings, repositoriesMetrics) + super.getRepositories( + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings, + repositoriesMetrics, + snapshotMetrics + ) ); filterPlugins(RepositoryPlugin.class).forEach( r -> repositories.putAll( - r.getRepositories(env, namedXContentRegistry, clusterService, bigArrays, recoverySettings, RepositoriesMetrics.NOOP) + r.getRepositories( + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings, + RepositoriesMetrics.NOOP, + SnapshotMetrics.NOOP + ) ) ); return repositories; diff --git a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java index 614036d3792ca..5030956c8e063 100644 --- a/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java +++ b/x-pack/plugin/old-lucene-versions/src/internalClusterTest/java/org/elasticsearch/xpack/lucene/bwc/AbstractArchiveTestCase.java @@ -25,6 +25,7 @@ import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; import org.elasticsearch.snapshots.SnapshotId; @@ -63,7 +64,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( FAKE_VERSIONS_TYPE, diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 16543a30238b3..0fe63df106862 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -496,7 +496,7 @@ public void testMaxRestoreBytesPerSecIsUsed() throws Exception { final IndicesService service = internalCluster().getInstance(IndicesService.class, node); if (service != null && service.hasIndex(restoredIndex)) { assertThat( - getRepositoryOnNode(repositoryName, node).getRestoreThrottleTimeInNanos(), + getRepositoryOnNode(repositoryName, node).getSnapshotStats().totalReadThrottledNanos(), useRateLimits ? greaterThan(0L) : equalTo(0L) ); } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java index 518ff2354f498..3e71b161489a1 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotState; @@ -377,7 +378,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( TYPE, @@ -388,7 +390,8 @@ public Map getRepositories( namedXContentRegistry, clusterService, bigArrays, - recoverySettings + recoverySettings, + snapshotMetrics ) ); } @@ -405,9 +408,10 @@ public CustomMockRepository( NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, BigArrays bigArrays, - RecoverySettings recoverySettings + RecoverySettings recoverySettings, + SnapshotMetrics snapshotMetrics ) { - super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings); + super(projectId, metadata, environment, namedXContentRegistry, clusterService, bigArrays, recoverySettings, snapshotMetrics); } private void unlockRestore() { diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java index dcf77d06e323a..d3f1407c04136 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase; import org.elasticsearch.repositories.fs.FsRepository; @@ -450,7 +451,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( "tracking", diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java index e0674c0151f65..69e66110aeab5 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase; import org.elasticsearch.repositories.fs.FsRepository; @@ -244,7 +245,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( "test-fs", diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMHealthBlockedSnapshotIT.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMHealthBlockedSnapshotIT.java index 219dd20f4e620..5348ddbcbc655 100644 --- a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMHealthBlockedSnapshotIT.java +++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMHealthBlockedSnapshotIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.SnapshotShardContext; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; @@ -116,7 +117,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( TestDelayedRepo.TYPE, diff --git a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java index 5837354cd5bb2..7c9eb5652e974 100644 --- a/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java +++ b/x-pack/plugin/slm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMStatDisruptionIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.repositories.FinalizeSnapshotContext; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.SnapshotShardContext; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; @@ -134,7 +135,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( TestDelayedRepo.TYPE, @@ -206,7 +208,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( TestRestartBeforeListenersRepo.TYPE, diff --git a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java index 6f73efb2fa095..1eda191f60b35 100644 --- a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java +++ b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/SnapshotBasedIndexRecoveryIT.java @@ -57,6 +57,7 @@ import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.search.SearchHit; @@ -148,7 +149,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( FAULTY_TYPE, diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java index 40d1c12af9061..7ec2bab1588c5 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisFailureIT.java @@ -40,6 +40,7 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.repositories.RepositoryVerificationException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; @@ -592,7 +593,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( DISRUPTABLE_REPO_TYPE, @@ -603,7 +605,8 @@ public Map getRepositories( clusterService, bigArrays, recoverySettings, - BlobPath.EMPTY + BlobPath.EMPTY, + snapshotMetrics ) ); } @@ -620,9 +623,10 @@ static class DisruptableRepository extends BlobStoreRepository { ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - BlobPath basePath + BlobPath basePath, + SnapshotMetrics snapshotMetrics ) { - super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath, snapshotMetrics); } void setBlobStore(BlobStore blobStore) { diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java index a65716a63f6d2..f93c4018b6232 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/analyze/RepositoryAnalysisSuccessIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; @@ -184,7 +185,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Map.of( ASSERTING_REPO_TYPE, @@ -195,7 +197,8 @@ public Map getRepositories( clusterService, bigArrays, recoverySettings, - buildBlobPath(metadata.settings()) + buildBlobPath(metadata.settings()), + snapshotMetrics ) ); } @@ -225,9 +228,10 @@ static class AssertingRepository extends BlobStoreRepository { ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - BlobPath basePath + BlobPath basePath, + SnapshotMetrics snapshotMetrics ) { - super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); + super(projectId, metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath, snapshotMetrics); } void setBlobStore(BlobStore blobStore) { diff --git a/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java b/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java index 1175b6b7ea299..8a045a7614799 100644 --- a/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java +++ b/x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.SnapshotMetrics; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotState; @@ -260,7 +261,8 @@ public Map getRepositories( ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings, - RepositoriesMetrics repositoriesMetrics + RepositoriesMetrics repositoriesMetrics, + SnapshotMetrics snapshotMetrics ) { return Collections.singletonMap( "verifyaccess-fs",