Run unit tests w/ drogon in separate processes

run_unit_tests.sh

@@ -44,9 +44,10 @@ if [ "$RUN_GPU_TESTS" == "1" ]; then
 fi
 
 compress_logs() {
-    tar -czf test_logs.tar.gz ${TEST_LOG}
+    tar -czf test_logs.tar.gz ${TEST_LOG} drogon_singleton_test_log_*.log
     rm -rf ${TEST_LOG}
     rm -rf tmp.log
+    #rm -rf drogon_singleton_test_log_*.log
 } 
 
 generate_coverage_report() {
@@ -108,6 +109,41 @@ if [ "$RUN_TESTS" == "1" ] ; then
     fi
     grep -a " ms \| ms)" ${TEST_LOG} > linux_tests_summary.log
     echo "Tests completed:" `grep -a " ms \| ms)" ${TEST_LOG} | grep " OK " | wc -l`
+
+    if [ $failed -eq 0 ]; then
+        echo "All main tests passed. Running drogon singleton tests..."
+
+        run_additional_drogon_singleton_tests() {
+            failed_drogon_singleton=0
+            echo "Running ConfigChangeStressTest suite..."
+            bazel test --jobs=$JOBS ${debug_bazel_flags} ${SHARED_OPTIONS} --test_summary=detailed --test_output=streamed --test_filter="ConfigChangeStressTest.GetMetricsDuringLoad_DROGON" --test_env=DROGON_RESTART=1 //src:ovms_test > drogon_singleton_test_log_1.log 2>&1 || failed_drogon_singleton=1 &
+            pid1=$!
+
+            echo "Running StressCapiConfigChanges suite..."
+            bazel test --jobs=$JOBS ${debug_bazel_flags} ${SHARED_OPTIONS} --test_summary=detailed --test_output=streamed --test_filter="StressCapiConfigChanges.GetMetricsDuringLoad_DROGON" --test_env=DROGON_RESTART=1 //src:ovms_test > drogon_singleton_test_log_2.log 2>&1 || failed_drogon_singleton=1 &
+            pid2=$!
+
+            echo "Running Server suite..."
+            bazel test --jobs=$JOBS ${debug_bazel_flags} ${SHARED_OPTIONS} --test_summary=detailed --test_output=streamed --test_filter="Server.CAPIAliveGrpcNotHttpYes_DROGON" --test_env=DROGON_RESTART=1 //src:ovms_test > drogon_singleton_test_log_3.log 2>&1 || failed_drogon_singleton=1 &
+            pid3=$!
+
+            wait $pid1
+            wait $pid2
+            wait $pid3
+
+            if [ $failed_drogon_singleton -ne 0 ]; then
+                echo "Some drogon singleton tests failed:"
+                cat drogon_singleton_test_log_*.txt
+                return 1
+            else
+                echo "All drogon singleton tests passed successfully."
+                return 0
+            fi
+        }
+
+        run_additional_drogon_singleton_tests || exit 1
+    fi
+
     compress_logs
     exit $failed
 fi

src/test/c_api_stress_tests.cpp

@@ -14,6 +14,7 @@
 // limitations under the License.
 //*****************************************************************************
 #include <regex>
+#include <cstdlib>
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
@@ -160,11 +161,11 @@ TEST_F(StressCapiConfigChanges, KFSAddNewVersionDuringPredictLoad) {
         requiredLoadResults,
         allowedLoadResults);
 }
-#if (USE_DROGON == 0)
-TEST_F(StressCapiConfigChanges, GetMetricsDuringLoad) {
-#else
-TEST_F(StressCapiConfigChanges, DISABLED_GetMetricsDuringLoad) {
-#endif
+// Workaround because we cannot start http server multiple times https://github.com/drogonframework/drogon/issues/2210
+TEST_F(StressCapiConfigChanges, GetMetricsDuringLoad_DROGON) {
+    if (!std::getenv("DROGON_RESTART")) {
+        GTEST_SKIP() << "Run with DROGON_RESTART to enable this test";
+    }
     bool performWholeConfigReload = false;                        // we just need to have all model versions rechecked
     std::set<StatusCode> requiredLoadResults = {StatusCode::OK};  // we expect full continuity of operation
     std::set<StatusCode> allowedLoadResults = {};

src/test/ensemble_config_change_stress.cpp

@@ -181,12 +181,11 @@ TEST_F(StressPipelineConfigChanges, KFSAddNewVersionDuringPredictLoad) {
         requiredLoadResults,
         allowedLoadResults);
 }
-// Disabled because we cannot start http server multiple times https://github.com/drogonframework/drogon/issues/2210
-#if (USE_DROGON == 0)
-TEST_F(ConfigChangeStressTest, GetMetricsDuringLoad) {
-#else
-TEST_F(ConfigChangeStressTest, DISABLED_GetMetricsDuringLoad) {
-#endif
+// Workaround because we cannot start http server multiple times https://github.com/drogonframework/drogon/issues/2210
+TEST_F(ConfigChangeStressTest, GetMetricsDuringLoad_DROGON) {
+    if (!std::getenv("DROGON_RESTART")) {
+        GTEST_SKIP() << "Run with DROGON_RESTART to enable this test";
+    }
     bool performWholeConfigReload = false;                        // we just need to have all model versions rechecked
     std::set<StatusCode> requiredLoadResults = {StatusCode::OK};  // we expect full continuity of operation
     std::set<StatusCode> allowedLoadResults = {};

src/test/server_test.cpp

@@ -481,8 +481,10 @@ TEST(Server, CAPIAliveGrpcNotHttpNot) {
     requestServerAlive(portOldDefault.c_str(), grpc::StatusCode::UNAVAILABLE, false);
     requestRestServerAlive(typicalRestDefault.c_str(), httplib::StatusCode::NotFound_404, false);
 }
-TEST(Server, CAPIAliveGrpcNotHttpYes) {
-    GTEST_SKIP() << "Until we have a way to launch all tests restarting drogon";  // TODO @dkalinow to enable drogon tests
+TEST(Server, CAPIAliveGrpcNotHttpYes_DROGON) {
+    if (!std::getenv("DROGON_RESTART")) {
+        GTEST_SKIP() << "Run with DROGON_RESTART to enable this test";
+    }
     std::string port = "9000";
     randomizeAndEnsureFree(port);
     char* argv[] = {

src/test/stress_test_utils.hpp

@@ -1154,15 +1154,12 @@ class ConfigChangeStressTest : public TestWithTempDir {
         ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsNew(&modelsSettings));
         randomizeAndEnsureFrees(port, restPort);
         ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcPort(serverSettings, std::stoi(port)));
-#if (USE_DROGON == 0)                                                                                  // when jusing drogon we cannot start rest server multiple times within the same process
-        ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestPort(serverSettings, std::stoi(restPort)));  // required for metrics  - but disabled because drogon http server cannot be restarted
-#endif
-        // ideally we would want to have emptyConfigWithMetrics
-#if (USE_DROGON == 0)
-        ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsSetConfigPath(modelsSettings, getGenericFullPathForSrcTest("/ovms/src/test/configs/emptyConfigWithMetrics.json").c_str()));  // the content of config json is irrelevant - we just need server to be ready for C-API use in mediapipe
-#else
-        ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsSetConfigPath(modelsSettings, getGenericFullPathForSrcTest("/ovms/src/test/configs/emptyConfig.json").c_str()));  // the content of config json is irrelevant - we just need server to be ready for C-API use in mediapipe
-#endif
+        if (std::getenv("DROGON_RESTART")) {                                                                                                                                        // when using drogon we cannot start rest server multiple times within the same process
+            ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestPort(serverSettings, std::stoi(restPort)));                                                                           // required for metrics  - but disabled because drogon http server cannot be restarted
+            ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsSetConfigPath(modelsSettings, getGenericFullPathForSrcTest("/ovms/src/test/configs/emptyConfigWithMetrics.json").c_str()));  // the content of config json is irrelevant - we just need server to be ready for C-API use in mediapipe
+        } else {
+            ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsSetConfigPath(modelsSettings, getGenericFullPathForSrcTest("/ovms/src/test/configs/emptyConfig.json").c_str()));  // the content of config json is irrelevant - we just need server to be ready for C-API use in mediapipe
+        }
         ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetFileSystemPollWaitSeconds(serverSettings, 0));  // set to 0 to reload only through test and avoid races
         ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&cserver));
         ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, serverSettings, modelsSettings));