diff --git a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/EmbeddedPrometheusBootstrapConfiguration.java b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/EmbeddedPrometheusBootstrapConfiguration.java deleted file mode 100644 index 9ada9b698..000000000 --- a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/EmbeddedPrometheusBootstrapConfiguration.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2019 Playtika - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.netflix.kayenta.configuration; - -import static com.playtika.test.common.utils.ContainerUtils.containerLogsConsumer; - -import com.netflix.kayenta.utils.EnvironmentUtils; -import java.time.Duration; -import java.util.LinkedHashMap; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.ConfigurableEnvironment; -import org.testcontainers.Testcontainers; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; -import org.testcontainers.containers.wait.strategy.WaitStrategy; -import org.testcontainers.utility.MountableFile; - -@Slf4j -@Configuration -public class EmbeddedPrometheusBootstrapConfiguration { - - // Exposes host machine port to be used by prometheus container for scraping metrics from - // /prometehus endpoint - // See for more details: - // https://www.testcontainers.org/features/networking/#exposing-host-ports-to-the-container - static { - Testcontainers.exposeHostPorts(8081); - } - - private static final int PORT = 9090; - - @Bean(name = "prometheusWaitStrategy") - public WaitStrategy prometheusWaitStrategy() { - return new HttpWaitStrategy().forPath("/status").forPort(PORT).forStatusCode(200); - } - - @Bean(name = "prometheus", destroyMethod = "stop") - public GenericContainer prometheus( - ConfigurableEnvironment environment, WaitStrategy prometheusWaitStrategy) { - - GenericContainer container = - new GenericContainer("prom/prometheus:v2.10.0") - .withLogConsumer(containerLogsConsumer(log)) - .withExposedPorts(PORT) - .withCopyFileToContainer( - MountableFile.forClasspathResource("/external/prometheus/prometheus.yml"), - "/etc/prometheus/prometheus.yml") - .waitingFor(prometheusWaitStrategy) - .withStartupTimeout(Duration.ofSeconds(30)); - container.start(); - Map env = registerEnvironment(environment, container.getMappedPort(PORT)); - log.info("Started Prometheus server. Connection details: {}", env); - return container; - } - - static Map registerEnvironment(ConfigurableEnvironment environment, int port) { - Map map = new LinkedHashMap<>(); - map.put("embedded.prometheus.port", port); - EnvironmentUtils.registerPropertySource("embeddedPrometheusInfo", environment, map); - return map; - } -} diff --git a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/MetricsReportingConfiguration.java b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/MetricsReportingConfiguration.java index 0da08cfc1..5fbf1c2b8 100644 --- a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/MetricsReportingConfiguration.java +++ b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/MetricsReportingConfiguration.java @@ -24,10 +24,10 @@ import com.netflix.spectator.api.Registry; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.config.MeterFilter; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; @EnableConfigurationProperties(CanaryAnalysisCasesConfigurationProperties.class) @Configuration @@ -56,9 +56,8 @@ public MetricsGenerator metricsGenerator( @Bean public StandaloneCanaryAnalysisSteps canaryAnalysisSteps( - @Value("${server.port}") int serverPort, - CanaryAnalysisCasesConfigurationProperties configuration) { - return new StandaloneCanaryAnalysisSteps(serverPort, configuration); + Environment environment, CanaryAnalysisCasesConfigurationProperties configuration) { + return new StandaloneCanaryAnalysisSteps(environment, configuration); } @Bean diff --git a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/TestPrometheusConfiguration.java b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/TestPrometheusConfiguration.java new file mode 100644 index 000000000..84d8ea08a --- /dev/null +++ b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/configuration/TestPrometheusConfiguration.java @@ -0,0 +1,131 @@ +package com.netflix.kayenta.configuration; + +import static com.playtika.test.common.utils.ContainerUtils.containerLogsConsumer; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.ApplicationListener; +import org.springframework.context.annotation.Bean; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.Environment; +import org.testcontainers.Testcontainers; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.containers.wait.strategy.WaitStrategy; +import org.testcontainers.utility.MountableFile; + +@Slf4j +@TestConfiguration +@RequiredArgsConstructor +public class TestPrometheusConfiguration { + + private static final int PROMETHEUS_INTERNAL_PORT = 9090; + private final Environment environment; + + @Bean + public WaitStrategy prometheusWaitStrategy() { + return new HttpWaitStrategy() + .forPath("/status") + .forPort(PROMETHEUS_INTERNAL_PORT) + .forStatusCode(200); + } + + private GenericContainer prometheusContainer; + + @Bean + public ApplicationListener prometheus( + ConfigurableEnvironment env, WaitStrategy prometheusWaitStrategy) { + return event -> { + int managementPort = waitForManagementPort(); + exposeManagementPort(managementPort); + + File prometheusConfigFile = createPrometheusConfigFile(managementPort); + + prometheusContainer = + new GenericContainer<>("prom/prometheus:v2.10.0") + .withLogConsumer(containerLogsConsumer(log)) + .withExposedPorts(PROMETHEUS_INTERNAL_PORT) + .withCopyFileToContainer( + MountableFile.forHostPath(prometheusConfigFile.getAbsolutePath()), + "/etc/prometheus/prometheus.yml") + .waitingFor(prometheusWaitStrategy) + .withStartupTimeout(Duration.ofSeconds(30)); + + prometheusContainer.start(); + + Map prometheusEnv = + registerEnvironment(env, prometheusContainer.getMappedPort(PROMETHEUS_INTERNAL_PORT)); + log.info("Started Prometheus server. Connection details: {}", prometheusEnv); + }; + } + + private int waitForManagementPort() { + int retries = 30; // wait up to 30 seconds + while (retries-- > 0) { + String managementPortStr = environment.getProperty("local.management.port"); + if (managementPortStr != null) { + return Integer.parseInt(managementPortStr); + } + try { + Thread.sleep(1000); // wait 1 second + } catch (InterruptedException ignored) { + } + } + throw new IllegalStateException( + "Property 'local.management.port' not available after waiting!"); + } + + private int getManagementPort() { + String managementPortStr = environment.getProperty("local.management.port"); + if (managementPortStr == null) { + throw new IllegalStateException( + "Property 'local.management.port' not available yet! Maybe server not started?"); + } + return Integer.parseInt(managementPortStr); + } + + private void exposeManagementPort(int managementPort) { + log.info("Exposing management port {} to Testcontainers", managementPort); + Testcontainers.exposeHostPorts(managementPort); + } + + private File createPrometheusConfigFile(int managementPort) { + try { + File tempFile = File.createTempFile("prometheus", ".yml"); + try (FileWriter writer = new FileWriter(tempFile)) { + writer.write( + String.format( + """ + global: + scrape_interval: 1s + + scrape_configs: + - job_name: kayenta-test-metrics + metrics_path: /prometheus + static_configs: + - targets: ['host.testcontainers.internal:%d'] + """, + managementPort)); + } + return tempFile; + } catch (IOException e) { + throw new RuntimeException("Failed to create prometheus.yml dynamically", e); + } + } + + private Map registerEnvironment(ConfigurableEnvironment environment, int port) { + Map map = new LinkedHashMap<>(); + map.put("embedded.prometheus.port", port); + com.netflix.kayenta.utils.EnvironmentUtils.registerPropertySource( + "embeddedPrometheusInfo", environment, map); + return map; + } +} diff --git a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/steps/StandaloneCanaryAnalysisSteps.java b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/steps/StandaloneCanaryAnalysisSteps.java index dc3a52d51..e602e8a91 100644 --- a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/steps/StandaloneCanaryAnalysisSteps.java +++ b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/steps/StandaloneCanaryAnalysisSteps.java @@ -31,14 +31,27 @@ import java.util.Arrays; import java.util.concurrent.TimeUnit; import lombok.RequiredArgsConstructor; +import org.springframework.core.env.Environment; import org.springframework.http.HttpStatus; @RequiredArgsConstructor public class StandaloneCanaryAnalysisSteps { - private final int serverPort; + private final Environment environment; private final CanaryAnalysisCasesConfigurationProperties cases; + private Integer serverPort; + + private int getServerPort() { + if (serverPort != null) { + return serverPort; + } + + serverPort = environment.getProperty("local.server.port", Integer.class); + + return serverPort; + } + public String createCanaryAnalysis( String caseName, String metricsAccountName, @@ -50,7 +63,7 @@ public String createCanaryAnalysis( ValidatableResponse createAnalysisResponse = given() - .port(serverPort) + .port(getServerPort()) .header("Content-Type", "application/json") .queryParam("metricsAccountName", metricsAccountName) .queryParam("storageAccountName", storageAccountName) @@ -85,7 +98,7 @@ public ValidatableResponse waitUntilCanaryAnalysisCompleted(String canaryAnalysi public ValidatableResponse getCanaryAnalysisExecution(String canaryAnalysisExecutionId) { return given() - .port(serverPort) + .port(getServerPort()) .get("/standalone_canary_analysis/" + canaryAnalysisExecutionId) .then(); } diff --git a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/BaseIntegrationTest.java b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/BaseIntegrationTest.java index c4a0362d9..790f321c0 100644 --- a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/BaseIntegrationTest.java +++ b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/BaseIntegrationTest.java @@ -17,25 +17,124 @@ import com.netflix.kayenta.Main; import com.netflix.kayenta.configuration.MetricsReportingConfiguration; +import com.netflix.kayenta.configuration.TestPrometheusConfiguration; +import com.netflix.kayenta.prometheus.config.PrometheusConfigurationProperties; +import com.netflix.kayenta.prometheus.config.PrometheusManagedAccount; +import com.netflix.kayenta.prometheus.config.PrometheusResponseConverter; +import com.netflix.kayenta.prometheus.service.PrometheusRemoteService; +import com.netflix.kayenta.retrofit.config.RetrofitClientFactory; +import com.netflix.kayenta.security.AccountCredentials; +import com.netflix.kayenta.security.AccountCredentialsRepository; +import java.io.IOException; +import java.util.Set; +import okhttp3.OkHttpClient; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.actuate.metrics.AutoConfigureMetrics; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.core.env.Environment; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit.jupiter.SpringExtension; @AutoConfigureMetrics @SpringBootTest( classes = {MetricsReportingConfiguration.class, Main.class}, - webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, value = "spring.application.name=kayenta") @ExtendWith(SpringExtension.class) @ActiveProfiles({"base", "prometheus", "graphite", "cases"}) +@Import(TestPrometheusConfiguration.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class BaseIntegrationTest { - @Value("${management.server.port}") - protected int managementPort; + @Autowired protected Environment environment; - @Value("${server.port}") - protected int serverPort; + private Integer managementPort; + + private Integer serverPort; + + @Autowired private AccountCredentialsRepository accountCredentialsRepository; + + @Autowired PrometheusResponseConverter prometheusConverter; + @Autowired PrometheusConfigurationProperties prometheusConfigurationProperties; + @Autowired RetrofitClientFactory retrofitClientFactory; + @Autowired OkHttpClient okHttpClient; + + private boolean setupDone = false; + + @BeforeAll + public void setupPrometheusAccounts() throws InterruptedException { + if (setupDone) { + return; + } + + int retries = 30; // wait up to 30 seconds + String prometheusPortStr = null; + + while (retries-- > 0) { + prometheusPortStr = environment.getProperty("embedded.prometheus.port"); + if (prometheusPortStr != null) { + break; + } + Thread.sleep(1000); + } + + if (prometheusPortStr == null) { + throw new IllegalStateException("embedded.prometheus.port not set even after waiting!"); + } + Set accountCredentialsSet = + accountCredentialsRepository.getAllOf(AccountCredentials.Type.METRICS_STORE); + String dynamicEndpoint = "http://localhost:" + prometheusPortStr; + + accountCredentialsSet.stream() + .filter(credentials -> credentials instanceof PrometheusManagedAccount) + .map(PrometheusManagedAccount.class::cast) + .forEach( + prometheusManagedAccount -> { + prometheusManagedAccount.getEndpoint().setBaseUrl(dynamicEndpoint); + PrometheusRemoteService prometheusRemoteService; + try { + prometheusRemoteService = + retrofitClientFactory.createClient( + PrometheusRemoteService.class, + prometheusConverter, + prometheusManagedAccount.getEndpoint(), + okHttpClient, + prometheusManagedAccount.getUsername(), + prometheusManagedAccount.getPassword(), + prometheusManagedAccount.getUsernamePasswordFile(), + prometheusManagedAccount.getBearerToken()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + prometheusManagedAccount.setPrometheusRemoteService(prometheusRemoteService); + }); + + Thread.sleep(60000); // 1 minute + setupDone = true; + } + + protected int getManagementPort() { + if (managementPort != null) { + return managementPort; + } + + managementPort = environment.getProperty("local.management.port", Integer.class); + + return managementPort; + } + + protected int getServerPort() { + if (serverPort != null) { + return serverPort; + } + + serverPort = environment.getProperty("local.server.port", Integer.class); + + return serverPort; + } } diff --git a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/ManagementTest.java b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/ManagementTest.java index 2afed8f52..899c846ec 100644 --- a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/ManagementTest.java +++ b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/ManagementTest.java @@ -19,17 +19,34 @@ import static io.restassured.RestAssured.given; import static org.hamcrest.CoreMatchers.is; +import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; +@Slf4j public class ManagementTest extends BaseIntegrationTest { - @Value("${embedded.prometheus.port}") - int prometheusPort; - @Test - public void prometheusTargetsAreAllReportingUp() { + public void prometheusTargetsAreAllReportingUp() throws InterruptedException { + int retries = 30; // wait up to 30 seconds + String prometheusPortStr = null; + + while (retries-- > 0) { + prometheusPortStr = environment.getProperty("embedded.prometheus.port"); + if (prometheusPortStr != null) { + break; + } + Thread.sleep(1000); + } + + if (prometheusPortStr == null) { + throw new IllegalStateException("embedded.prometheus.port not set even after waiting!"); + } + + int prometheusPort = Integer.parseInt(prometheusPortStr); + + System.out.println("Prometheus Port: " + prometheusPort); + awaitThirtySecondsUntil( () -> given() @@ -48,7 +65,7 @@ public void healthIsUp() { awaitThirtySecondsUntil( () -> given() - .port(managementPort) + .port(getManagementPort()) .get("/health") .prettyPeek() .then() diff --git a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/SwaggerTest.java b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/SwaggerTest.java index e15ddf4a0..55de8faed 100644 --- a/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/SwaggerTest.java +++ b/kayenta-integration-tests/src/test/java/com/netflix/kayenta/tests/SwaggerTest.java @@ -24,7 +24,7 @@ public class SwaggerTest extends BaseIntegrationTest { @Test public void swaggerUiIsPresent() { RestAssured.given() - .port(serverPort) + .port(getServerPort()) .get("/swagger-ui/index.html") .prettyPeek() .then() diff --git a/kayenta-integration-tests/src/test/resources/META-INF/spring.factories b/kayenta-integration-tests/src/test/resources/META-INF/spring.factories index 405816da7..400ca2d2e 100644 --- a/kayenta-integration-tests/src/test/resources/META-INF/spring.factories +++ b/kayenta-integration-tests/src/test/resources/META-INF/spring.factories @@ -1,3 +1,2 @@ org.springframework.cloud.bootstrap.BootstrapConfiguration=\ -com.netflix.kayenta.configuration.EmbeddedPrometheusBootstrapConfiguration,\ com.netflix.kayenta.configuration.EmbeddedGraphiteBootstrapConfiguration \ No newline at end of file diff --git a/kayenta-integration-tests/src/test/resources/application-base.yml b/kayenta-integration-tests/src/test/resources/application-base.yml index 9122ef57b..12430109e 100644 --- a/kayenta-integration-tests/src/test/resources/application-base.yml +++ b/kayenta-integration-tests/src/test/resources/application-base.yml @@ -28,8 +28,8 @@ kayenta: s3: enabled: true -server: - port: 8080 +#server: +# port: 8080 # Does it make sense to publish all Kayenta metrics to all datasources in integraion tests? @@ -48,7 +48,7 @@ management: exposure.include: '*' endpoint.health.show-details: always server: - port: 8081 + port: 0 metrics: export: graphite: diff --git a/kayenta-integration-tests/src/test/resources/application-prometheus.yml b/kayenta-integration-tests/src/test/resources/application-prometheus.yml index 25b762839..924037655 100644 --- a/kayenta-integration-tests/src/test/resources/application-prometheus.yml +++ b/kayenta-integration-tests/src/test/resources/application-prometheus.yml @@ -3,6 +3,7 @@ kayenta: enabled: true health: enabled: true + initial-delay: PT1M accounts: - name: prometheus-account endpoint: diff --git a/kayenta-prometheus/src/test/resources/application-prometheusHealth.yml b/kayenta-prometheus/src/test/resources/application-prometheusHealth.yml index 25b762839..924037655 100644 --- a/kayenta-prometheus/src/test/resources/application-prometheusHealth.yml +++ b/kayenta-prometheus/src/test/resources/application-prometheusHealth.yml @@ -3,6 +3,7 @@ kayenta: enabled: true health: enabled: true + initial-delay: PT1M accounts: - name: prometheus-account endpoint: