Fuzz fail new

fuzz/cli11_app_fuzz.cpp

@@ -17,9 +17,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     std::string parseString(reinterpret_cast<const char *>(Data), Size);
 
     CLI::FuzzApp fuzzdata;
-    CLI::FuzzApp fuzzdata2;
+
     auto app = fuzzdata.generateApp();
-    auto app2 = fuzzdata2.generateApp();
     std::size_t pstring_start{0};
     try {
         pstring_start = fuzzdata.add_custom_options(app.get(), parseString);
@@ -40,17 +39,21 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
         // this just indicates we caught an error known by CLI
         return 0;  // Non-zero return values are reserved for future use.
     }
-    // should be able to write the config to a file and read from it again
-    std::string configOut = app->config_to_str();
-    app->clear();
-    std::stringstream out(configOut);
-    if(pstring_start > 0) {
-        fuzzdata2.add_custom_options(app2.get(), parseString);
-    }
-    app2->parse_from_stream(out);
-    auto result = fuzzdata2.compare(fuzzdata);
-    if(!result) {
-        throw CLI::ValidationError("fuzzer", "file input results don't match parse results");
+    if(fuzzdata.support_config_file_only()) {
+        CLI::FuzzApp fuzzdata2;
+        auto app2 = fuzzdata2.generateApp();
+        // should be able to write the config to a file and read from it again
+        std::string configOut = app->config_to_str();
+        std::stringstream out(configOut);
+        if(pstring_start > 0) {
+            fuzzdata2.add_custom_options(app2.get(), parseString);
+        }
+        app2->parse_from_stream(out);
+        auto result = fuzzdata2.compare(fuzzdata);
+        if(!result) {
+            throw CLI::ValidationError("fuzzer", "file input results don't match parse results");
+        }
     }
+
     return 0;
 }

fuzz/fuzzApp.cpp

@@ -6,6 +6,7 @@
 
 #include "fuzzApp.hpp"
 #include <algorithm>
+#include <iostream>
 
 namespace CLI {
 /*
@@ -151,7 +152,27 @@ std::shared_ptr<CLI::App> FuzzApp::generateApp() {
     return fApp;
 }
 
-bool FuzzApp::compare(const FuzzApp &other) const {
+static void print_string_comparison(const std::string &s1,
+                                    const std::string &s2,
+                                    const std::string &prefix,
+                                    const std::string &s1name,
+                                    const std::string &s2name) {
+    for(size_t jj = 0; jj < (std::max)(s1.size(), s2.size()); ++jj) {
+        if(jj >= s1.size()) {
+            std::cout << prefix << ":" << s1name << "[" << jj << "] = [empty], " << s2name << "[" << jj
+                      << "]=" << static_cast<int>(s2[jj]) << '\n';
+        } else if(jj >= s2.size()) {
+            std::cout << prefix << ":" << s1name << "[" << jj << "]=" << static_cast<int>(s1[jj]) << ", " << s2name
+                      << "[" << jj << "]=[empty] \n";
+        } else if(s1[jj] != s2[jj]) {
+            std::cout << "-->" << prefix << ":" << s1name << "[" << jj << "]=" << static_cast<int>(s1[jj]) << ", "
+                      << s2name << "[" << jj << "]=" << static_cast<int>(s2[jj]) << '\n';
+        } else {
+            std::cout << prefix << ":" << s1name << "[" << jj << "]=" << static_cast<int>(s1[jj]) << '\n';
+        }
+    }
+}
+bool FuzzApp::compare(const FuzzApp &other, bool print_error) const {
     if(val32 != other.val32) {
         return false;
     }
@@ -291,6 +312,20 @@ bool FuzzApp::compare(const FuzzApp &other) const {
         std::vector<std::string> res = vstrD;
         std::reverse(res.begin(), res.end());
         if(res != other.vstrD) {
+            if(print_error) {
+                if(res.size() != other.vstrD.size()) {
+                    std::cout << "size is different vstrD.size()=" << res.size()
+                              << " other.vstrD.size=" << other.vstrD.size() << '\n';
+                } else {
+                    for(size_t ii = 0; ii < res.size(); ++ii) {
+                        print_string_comparison(res[ii],
+                                                other.vstrD[ii],
+                                                std::string("string[") + std::to_string(ii) + ']',
+                                                "vstrD",
+                                                "other.vstrD");
+                    }
+                }
+            }
             return false;
         }
     }
@@ -311,17 +346,28 @@ bool FuzzApp::compare(const FuzzApp &other) const {
         return false;
     }
     for(std::size_t ii = 0; ii < custom_string_options.size(); ++ii) {
-        if(*custom_string_options[ii] != *other.custom_string_options[ii]) {
-            return false;
+        if(custom_string_options[ii]->first != other.custom_string_options[ii]->first) {
+            if(custom_string_options[ii]->second) {
+                if(print_error) {
+                    print_string_comparison(custom_string_options[ii]->first,
+                                            other.custom_string_options[ii]->first,
+                                            std::string("custom_string[") + std::to_string(ii) + ']',
+                                            "c1",
+                                            "other.c1");
+                }
+                return false;
+            }
         }
     }
     // now test custom vector_options
     if(custom_vector_options.size() != other.custom_vector_options.size()) {
         return false;
     }
     for(std::size_t ii = 0; ii < custom_vector_options.size(); ++ii) {
-        if(*custom_vector_options[ii] != *other.custom_vector_options[ii]) {
-            return false;
+        if(custom_vector_options[ii]->first != other.custom_vector_options[ii]->first) {
+            if(custom_vector_options[ii]->second) {
+                return false;
+            }
         }
     }
     return true;
@@ -447,11 +493,17 @@ std::size_t FuzzApp::add_custom_options(CLI::App *app, const std::string &descri
                 break;
             }
             std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
-            custom_string_options.push_back(std::make_shared<std::string>());
-            auto *opt = app->add_option(name, *(custom_string_options.back()));
+            custom_string_options.push_back(std::make_shared<std::pair<std::string, bool>>("", true));
+            auto *opt = app->add_option(name, custom_string_options.back()->first);
             if(header_close > current_index + 19) {
                 std::string attributes = description_string.substr(current_index + 8, header_close - 8 - current_index);
                 modify_option(opt, attributes);
+                if(!opt->get_configurable()) {
+                    custom_string_options.back()->second = false;
+                    if(opt->get_required()) {
+                        non_config_required = true;
+                    }
+                }
             }
 
             current_index = end_option + 9;
@@ -465,12 +517,18 @@ std::size_t FuzzApp::add_custom_options(CLI::App *app, const std::string &descri
                 break;
             }
             std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
-            custom_string_options.push_back(std::make_shared<std::string>());
-            auto *opt = app->add_option(name, *(custom_string_options.back()));
+            custom_string_options.push_back(std::make_shared<std::pair<std::string, bool>>("", true));
+            auto *opt = app->add_option(name, custom_string_options.back()->first);
 
             if(header_close > current_index + 17) {
                 std::string attributes = description_string.substr(current_index + 6, header_close - 6 - current_index);
                 modify_option(opt, attributes);
+                if(!opt->get_configurable()) {
+                    custom_string_options.back()->second = false;
+                    if(opt->get_required()) {
+                        non_config_required = true;
+                    }
+                }
             }
             current_index = end_option + 7;
         } else if(description_string.compare(current_index, 7, "<vector") == 0) {
@@ -483,11 +541,18 @@ std::size_t FuzzApp::add_custom_options(CLI::App *app, const std::string &descri
                 break;
             }
             std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
-            custom_vector_options.push_back(std::make_shared<std::vector<std::string>>());
-            auto *opt = app->add_option(name, *(custom_vector_options.back()));
+            custom_vector_options.push_back(std::make_shared<std::pair<std::vector<std::string>, bool>>());
+            custom_vector_options.back()->second = true;
+            auto *opt = app->add_option(name, custom_vector_options.back()->first);
             if(header_close > current_index + 19) {
                 std::string attributes = description_string.substr(current_index + 8, header_close - 8 - current_index);
                 modify_option(opt, attributes);
+                if(!opt->get_configurable()) {
+                    custom_vector_options.back()->second = false;
+                    if(opt->get_required()) {
+                        non_config_required = true;
+                    }
+                }
             }
             current_index = end_option + 9;
         } else {

fuzz/fuzzApp.hpp

@@ -57,12 +57,13 @@ class FuzzApp {
     /** generate a fuzzing application with a bunch of different interfaces*/
     std::shared_ptr<CLI::App> generateApp();
     /** compare two fuzz apps for equality*/
-    CLI11_NODISCARD bool compare(const FuzzApp &other) const;
+    CLI11_NODISCARD bool compare(const FuzzApp &other, bool print_error = false) const;
     /** generate additional options based on a string config*/
     std::size_t add_custom_options(CLI::App *app, const std::string &description_string);
     /** modify an option based on string*/
-    void modify_option(CLI::Option *opt, const std::string &modifier);
+    static void modify_option(CLI::Option *opt, const std::string &modifier);
 
+    CLI11_NODISCARD bool support_config_file_only() const { return !non_config_required; }
     int32_t val32{0};
     int16_t val16{0};
     int8_t val8{0};
@@ -121,7 +122,10 @@ class FuzzApp {
     std::vector<std::string> vstrF{};
     std::string mergeBuffer{};
     std::vector<std::string> validator_strings{};
-    std::vector<std::shared_ptr<std::string>> custom_string_options{};
-    std::vector<std::shared_ptr<std::vector<std::string>>> custom_vector_options{};
+    std::vector<std::shared_ptr<std::pair<std::string, bool>>> custom_string_options{};
+    std::vector<std::shared_ptr<std::pair<std::vector<std::string>, bool>>> custom_vector_options{};
+
+  private:
+    bool non_config_required{false};
 };
 }  // namespace CLI

include/CLI/App.hpp

@@ -1322,6 +1322,9 @@ class App {
     /// Fill in a single config option
     bool _parse_single_config(const ConfigItem &item, std::size_t level = 0);
 
+    /// @brief store the results for a flag like option
+    bool _add_flag_like_result(Option *op, const ConfigItem &item, const std::vector<std::string> &inputs);
+
     /// Parse "one" argument (some may eat more than one), delegate to parent if fails, add to missing if missing
     /// from main return false if the parse has failed and needs to return to parent
     bool _parse_single(std::vector<std::string> &args, bool &positional_only);

include/CLI/StringTools.hpp

@@ -251,7 +251,7 @@ CLI11_INLINE std::string add_escaped_characters(const std::string &str);
 CLI11_INLINE std::string remove_escaped_characters(const std::string &str);
 
 /// generate a string with all non printable characters escaped to hex codes
-CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape);
+CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape, bool force = false);
 
 CLI11_INLINE bool is_binary_escaped_string(const std::string &escaped_string);