[FLINK-39359][docs] Introduce integration test cases to ensure that the generated REST-API related documentation stays consistent with the code. (#27884)

Co-authored-by: 1996fanrui <1996fanrui@gmail.com>
Co-authored-by: Sergey Nuyanzin <snuyanzin@gmail.com>

Author: 3 people

flink-docs/src/main/java/org/apache/flink/docs/rest/RuntimeOpenApiSpecGenerator.java

@@ -36,6 +36,8 @@
  */
 public class RuntimeOpenApiSpecGenerator {
 
+    public static final String RUNTIME_OPEN_API_TITLE = "Flink JobManager REST API";
+
     /**
      * Generates the Runtime REST API OpenAPI spec.
      *
@@ -51,7 +53,7 @@ public static void main(String[] args) throws IOException, ConfigurationExceptio
                 continue;
             }
             createDocumentationFile(
-                    "Flink JobManager REST API",
+                    RUNTIME_OPEN_API_TITLE,
                     new DocumentingDispatcherRestEndpoint(),
                     apiVersion,
                     Paths.get(

flink-docs/src/main/java/org/apache/flink/docs/rest/SqlGatewayOpenApiSpecGenerator.java

@@ -36,6 +36,8 @@
  */
 public class SqlGatewayOpenApiSpecGenerator {
 
+    public static final String SQL_GATEWAY_OPEN_API_TITLE = "Flink SQL Gateway REST API";
+
     /**
      * Generates the Sql Gateway REST API OpenAPI spec.
      *
@@ -51,7 +53,7 @@ public static void main(String[] args) throws IOException, ConfigurationExceptio
                 continue;
             }
             createDocumentationFile(
-                    "Flink SQL Gateway REST API",
+                    SQL_GATEWAY_OPEN_API_TITLE,
                     new DocumentingSqlGatewayRestEndpoint(),
                     apiVersion,
                     Paths.get(

flink-docs/src/main/java/org/apache/flink/docs/util/Utils.java

@@ -18,6 +18,8 @@
 
 package org.apache.flink.docs.util;
 
+import java.io.File;
+
 /** Contains various shared utility functions. */
 public enum Utils {
     ;
@@ -34,4 +36,15 @@ public static String escapeCharacters(String value) {
                 .replaceAll(">", ">")
                 .replaceAll(TEMPORARY_PLACEHOLDER, "<wbr>");
     }
+
+    public static String getProjectRootDir() {
+        final String dirFromProperty = System.getProperty("rootDir");
+        if (dirFromProperty != null) {
+            return dirFromProperty;
+        }
+
+        // to make this work in the IDE use a default fallback
+        final String currentDir = System.getProperty("user.dir");
+        return new File(currentDir).getParent();
+    }
 }

flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java

@@ -49,6 +49,7 @@
 import java.util.Map;
 
 import static org.apache.flink.configuration.description.TextElement.text;
+import static org.apache.flink.docs.util.Utils.getProjectRootDir;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatException;
 import static org.assertj.core.api.Assertions.assertThatNoException;
@@ -768,15 +769,4 @@ void testVerifyClassAnnotation() {
                                 ConfigOptionsDocGenerator.verifyClassAnnotation(
                                         InternalOptions.class));
     }
-
-    static String getProjectRootDir() {
-        final String dirFromProperty = System.getProperty("rootDir");
-        if (dirFromProperty != null) {
-            return dirFromProperty;
-        }
-
-        // to make this work in the IDE use a default fallback
-        final String currentDir = System.getProperty("user.dir");
-        return new File(currentDir).getParent();
-    }
 }

flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocsCompletenessITCase.java

@@ -22,6 +22,7 @@
 import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.docs.util.ConfigurationOptionLocator;
 import org.apache.flink.docs.util.OptionWithMetaInfo;
+import org.apache.flink.docs.util.Utils;
 
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -232,7 +233,7 @@ private static void compareDocumentedAndExistingOptions(
     }
 
     private static Map<String, List<DocumentedOption>> parseDocumentedOptions() throws IOException {
-        final String rootDir = ConfigOptionsDocGeneratorTest.getProjectRootDir();
+        final String rootDir = Utils.getProjectRootDir();
 
         Path includeFolder =
                 Paths.get(rootDir, "docs", "layouts", "shortcodes", "generated").toAbsolutePath();
@@ -284,7 +285,7 @@ private static Collection<DocumentedOption> parseDocumentedOptionsFromFile(Path
 
     private static Map<String, List<ExistingOption>> findExistingOptions(
             Predicate<OptionWithMetaInfo> predicate) throws Exception {
-        final String rootDir = ConfigOptionsDocGeneratorTest.getProjectRootDir();
+        final String rootDir = Utils.getProjectRootDir();
 
         final Collection<ExistingOption> existingOptions = new ArrayList<>();
         new ConfigurationOptionLocator()

flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsYamlSpecTest.java

@@ -29,6 +29,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.apache.flink.docs.util.Utils.getProjectRootDir;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
@@ -44,7 +45,7 @@ void testNoKeyPrefixOfOtherKey() throws Exception {
         final Collection<String> allOptions = new ArrayList<>();
         new ConfigurationOptionLocator()
                 .discoverOptionsAndApply(
-                        Paths.get(ConfigOptionsDocGeneratorTest.getProjectRootDir()),
+                        Paths.get(getProjectRootDir()),
                         (aClass, optionWithMetaInfos) -> {
                             optionWithMetaInfos.stream()
                                     .filter(o -> o.field.getAnnotation(Deprecated.class) == null)

flink-docs/src/test/java/org/apache/flink/docs/rest/RuntimeOpenRestAPIDocsCompletenessITCase.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.docs.rest;
+
+import org.apache.flink.docs.util.Utils;
+import org.apache.flink.runtime.rest.util.DocumentingDispatcherRestEndpoint;
+import org.apache.flink.runtime.rest.versioning.RuntimeRestAPIVersion;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.apache.flink.docs.rest.RuntimeOpenApiSpecGenerator.RUNTIME_OPEN_API_TITLE;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Verifies that generated Runtime REST and Open API docs (HTML + OpenAPI yml) match committed
+ * files.
+ *
+ * <p>This acts as a freshness check to ensure the documentation stays in sync with the code.
+ */
+class RuntimeOpenRestAPIDocsCompletenessITCase {
+
+    @Test
+    void testRuntimeRestApiDocsUpToDate(@TempDir Path tempDir) throws Exception {
+        final DocumentingDispatcherRestEndpoint endpoint = new DocumentingDispatcherRestEndpoint();
+        for (final RuntimeRestAPIVersion apiVersion : RuntimeRestAPIVersion.values()) {
+            if (apiVersion == RuntimeRestAPIVersion.V0) {
+                continue;
+            }
+            final String version = apiVersion.getURLVersionPrefix();
+            String targetHtmlName = String.format("rest_%s_dispatcher.html", version);
+            final Path pathOfGeneratedHtml = tempDir.resolve(targetHtmlName);
+            final Path pathOfCommittedHtml = getPathOfCommittedHtml(targetHtmlName);
+            RestAPIDocGenerator.createHtmlFile(endpoint, apiVersion, pathOfGeneratedHtml);
+            assertThat(pathOfCommittedHtml)
+                    .as("Missing committed %s file: %s", targetHtmlName, pathOfCommittedHtml)
+                    .exists()
+                    .as(
+                            "Committed `%s` file is out of date. Please regenerate docs under `flink-docs` module based on `README.md`.",
+                            targetHtmlName)
+                    .hasSameTextualContentAs(pathOfGeneratedHtml);
+
+            String targetYmlName = String.format("rest_%s_dispatcher.yml", version);
+            final Path pathOfGeneratedYml = tempDir.resolve(targetYmlName);
+            final Path pathOfCommittedYml = getPathOfCommittedYaml(targetYmlName);
+            OpenApiSpecGenerator.createDocumentationFile(
+                    RUNTIME_OPEN_API_TITLE, endpoint, apiVersion, pathOfGeneratedYml);
+            assertThat(pathOfCommittedYml)
+                    .as("Missing committed %s file: %s", targetYmlName, pathOfCommittedYml)
+                    .exists()
+                    .as(
+                            "Committed `%s` file is out of date. Please regenerate docs under `flink-docs` module based on `README.md`.",
+                            targetYmlName)
+                    .hasSameTextualContentAs(pathOfGeneratedYml);
+        }
+    }
+
+    static Path getPathOfCommittedHtml(String fileName) {
+        final String rootDir = Utils.getProjectRootDir();
+        return Paths.get(rootDir, "docs", "layouts", "shortcodes", "generated", fileName)
+                .toAbsolutePath();
+    }
+
+    static Path getPathOfCommittedYaml(String fileName) {
+        final String rootDir = Utils.getProjectRootDir();
+        return Paths.get(rootDir, "docs", "static", "generated", fileName).toAbsolutePath();
+    }
+}

flink-docs/src/test/java/org/apache/flink/docs/rest/SqlGatewayOpenRestAPIDocsCompletenessITCase.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.docs.rest;
+
+import org.apache.flink.table.gateway.rest.util.DocumentingSqlGatewayRestEndpoint;
+import org.apache.flink.table.gateway.rest.util.SqlGatewayRestAPIVersion;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+
+import static org.apache.flink.docs.rest.RuntimeOpenRestAPIDocsCompletenessITCase.getPathOfCommittedHtml;
+import static org.apache.flink.docs.rest.RuntimeOpenRestAPIDocsCompletenessITCase.getPathOfCommittedYaml;
+import static org.apache.flink.docs.rest.SqlGatewayOpenApiSpecGenerator.SQL_GATEWAY_OPEN_API_TITLE;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Verifies that generated SQL Gateway REST and Open API docs (HTML + OpenAPI yml) match committed
+ * files.
+ *
+ * <p>This acts as a freshness check to ensure the documentation stays in sync with the code.
+ */
+class SqlGatewayOpenRestAPIDocsCompletenessITCase {
+
+    @Test
+    void testSqlGatewayRestApiDocsUpToDate(@TempDir Path tempDir) throws Exception {
+        final DocumentingSqlGatewayRestEndpoint endpoint = new DocumentingSqlGatewayRestEndpoint();
+        for (final SqlGatewayRestAPIVersion apiVersion : SqlGatewayRestAPIVersion.values()) {
+            if (apiVersion == SqlGatewayRestAPIVersion.V0) {
+                continue;
+            }
+            final String version = apiVersion.getURLVersionPrefix();
+            String targetHtmlName = String.format("rest_%s_sql_gateway.html", version);
+            final Path pathOfGeneratedHtml = tempDir.resolve(targetHtmlName);
+            final Path pathOfCommittedHtml = getPathOfCommittedHtml(targetHtmlName);
+            RestAPIDocGenerator.createHtmlFile(endpoint, apiVersion, pathOfGeneratedHtml);
+            assertThat(pathOfCommittedHtml)
+                    .as("Missing committed %s file: %s", targetHtmlName, pathOfCommittedHtml)
+                    .exists()
+                    .as(
+                            "Committed `%s` file is out of date. Please regenerate docs under `flink-docs` module based on `README.md`.",
+                            targetHtmlName)
+                    .hasSameTextualContentAs(pathOfGeneratedHtml);
+
+            String targetYmlName = String.format("rest_%s_sql_gateway.yml", version);
+            final Path pathOfGeneratedYml = tempDir.resolve(targetYmlName);
+            final Path pathOfCommittedYml = getPathOfCommittedYaml(targetYmlName);
+            OpenApiSpecGenerator.createDocumentationFile(
+                    SQL_GATEWAY_OPEN_API_TITLE, endpoint, apiVersion, pathOfGeneratedYml);
+            assertThat(pathOfCommittedYml)
+                    .as("Missing committed %s file: %s", targetYmlName, pathOfCommittedYml)
+                    .exists()
+                    .as(
+                            "Committed `%s` file is out of date. Please regenerate docs under `flink-docs` module based on `README.md`.",
+                            targetYmlName)
+                    .hasSameTextualContentAs(pathOfGeneratedYml);
+        }
+    }
+}