init

Author: mzegla

src/llm/io_processing/partial_json_builder.cpp

@@ -185,9 +185,21 @@ Document PartialJsonBuilder::add(const std::string& chunk) {
             }
         } else {
             if (c == '"') {
-                if (it != buffer.begin() && *(it - 1) == '\\') {
-                    continue;
+                // Count consecutive backslashes before the quote to determine if the quote is escaped
+                int backslashCount = 0;
+                auto tmpIt = it;
+                while (tmpIt != buffer.begin()) {
+                    --tmpIt;
+                    if (*tmpIt == '\\') {
+                        ++backslashCount;
+                    } else {
+                        break;
+                    }
+                }
+                if (backslashCount % 2 == 1) {
+                    continue;  // Quote is escaped, we can move on
                 } else {
+                    // Quote is not escaped, so we close the string/key
                     if (state == IteratorState::PROCESSING_KEY) {
                         // We processed a key, now we expect a colon
                         state = IteratorState::AWAITING_COLON;

src/test/llm/output_parsers/partial_json_builder_test.cpp

@@ -262,6 +262,23 @@ TEST_F(PartialJsonBuilderTest, escapedCharactersSanityCheck) {
     ASSERT_EQ(parsedJson["arguments"].GetString(), std::string("{\"impl\": \"TYPE MQTT_Config AS\\n    VAR\\n  \\\"txt\\\"      BrokerIP : STRING := '127.0.0.1';\"}"));
 }
 
+TEST_F(PartialJsonBuilderTest, tmptest) {
+    std::string targetJson = R"({"name": "set_status", "arguments": "{\"status\": \"CONNECTED,\\"})";
+    PartialJsonBuilder builder;
+    rapidjson::Document parsedJson;
+    for (size_t i = 0; i < targetJson.size(); ++i) {
+        std::string partialInput(1, targetJson[i]);
+        parsedJson = builder.add(partialInput);
+    }
+    ASSERT_TRUE(parsedJson.IsObject());
+    ASSERT_TRUE(parsedJson.HasMember("name"));
+    ASSERT_TRUE(parsedJson["name"].IsString());
+    ASSERT_EQ(parsedJson["name"].GetString(), std::string("set_status"));
+    ASSERT_TRUE(parsedJson.HasMember("arguments"));
+    ASSERT_TRUE(parsedJson["arguments"].IsString());
+    ASSERT_EQ(parsedJson["arguments"].GetString(), std::string("{\"status\": \"CONNECTED,\\"));
+}
+
 TEST_F(PartialJsonBuilderTest, complexJsonIncrementalParsingSanityCheck) {
     std::string targetJson = R"(