Allow partial processing of XML, JSON, and multipart request body

headers/modsecurity/transaction.h

@@ -397,7 +397,7 @@ class Transaction : public TransactionAnchoredVariables, public TransactionSecMa
     bool addArgument(const std::string& orig, const std::string& key,
         const std::string& value, size_t offset);
     bool extractArguments(const std::string &orig, const std::string& buf,
-        size_t offset);
+        size_t offset, bool partial_processing_enabled = false);
 
     const char *getResponseBody() const;
     size_t getResponseBodyLength();
@@ -645,6 +645,11 @@ class Transaction : public TransactionAnchoredVariables, public TransactionSecMa
      * the web server (connector) log.
      */
     void *m_logCbData;
+
+    /**
+     * Whether the request body was bigger than RequestBodyLimit.
+     */
+    bool m_requestBodyLimitExceeded;
 };
 
 

src/request_body_processor/json.cc

@@ -29,7 +29,8 @@ namespace RequestBodyProcessor {
 static const double json_depth_limit_default = 10000.0;
 static const char* json_depth_limit_exceeded_msg = ". Parsing depth limit exceeded";
 
-JSON::JSON(Transaction *transaction) : m_transaction(transaction),
+JSON::JSON(Transaction *transaction)
+    : m_transaction(transaction),
     m_handle(NULL),
     m_current_key(""),
     m_max_depth(json_depth_limit_default),
@@ -68,8 +69,6 @@ JSON::JSON(Transaction *transaction) : m_transaction(transaction),
      * TODO: make UTF8 validation optional, as it depends on Content-Encoding
      */
     m_handle = yajl_alloc(&callbacks, NULL, this);
-
-    yajl_config(m_handle, yajl_allow_partial_values, 0);
 }
 
 
@@ -83,7 +82,8 @@ JSON::~JSON() {
 }
 
 
-bool JSON::init() {
+bool JSON::init(unsigned int allow_partial_values) {
+    yajl_config(m_handle, yajl_allow_partial_values, allow_partial_values);
     return true;
 }
 

src/request_body_processor/json.h

@@ -60,7 +60,7 @@ class JSON {
     explicit JSON(Transaction *transaction);
     ~JSON();
 
-    static bool init();
+    bool init(unsigned int allow_partial_values = 0);
     bool processChunk(const char *buf, unsigned int size, std::string *err);
     bool complete(std::string *err);
 

src/request_body_processor/multipart.cc

@@ -121,6 +121,7 @@ Multipart::Multipart(const std::string &header, Transaction *transaction)
     m_reserve{0},
     m_seen_data(0),
     m_is_complete(0),
+    m_allow_partial(false),
     m_flag_error(0),
     m_flag_data_before(0),
     m_flag_data_after(0),
@@ -1151,53 +1152,108 @@ int Multipart::multipart_complete(std::string *error) {
              * processed yet) in the buffer.
              */
             if (m_buf_contains_line) {
-                if (((unsigned int)(MULTIPART_BUF_SIZE - m_bufleft)
-                        == (4 + m_boundary.size()))
+                /*
+                 * Note that the buffer may end with the final boundary followed by only CR,
+                 * coming from the [CRLF epilogue], when allow_process_partial == 1 (which is
+                 * set when SecRequestBodyLimitAction is ProcessPartial and the request body
+                 * length exceeds SecRequestBodyLimit).
+                 *
+                 * The following definitions are copied from RFC 2046:
+                 *
+                 * dash-boundary := "--" boundary
+                 *
+                 * delimiter := CRLF dash-boundary
+                 *
+                 * close-delimiter := delimiter "--"
+                 *
+                 * multipart-body := [preamble CRLF]
+                 *                   dash-boundary transport-padding CRLF
+                 *                   body-part *encapsulation
+                 *                   close-delimiter transport-padding
+                 *                   [CRLF epilogue]
+                 */
+                unsigned int buf_data_len = (unsigned int)(MULTIPART_BUF_SIZE - m_bufleft);
+                if ((buf_data_len >= 2 + m_boundary.size())
                     && (*(m_buf) == '-')
                     && (*(m_buf + 1) == '-')
                     && (strncmp(m_buf + 2, m_boundary.c_str(),
-                        m_boundary.size()) == 0)
-                    && (*(m_buf + 2 + m_boundary.size()) == '-')
-                    && (*(m_buf + 2 + m_boundary.size() + 1) == '-')) {
-                    // these next two checks may result in repeating work from earlier in this fn
-                    // ignore the duplication for now to minimize refactoring
-                    if ((m_crlf_state_buf_end == 2) && (m_flag_lf_line != 1)) {
-                        m_flag_lf_line = 1;
-                        m_transaction->m_variableMultipartLFLine.set(std::to_string(m_flag_lf_line),
-                            m_transaction->m_variableOffset);
-                        m_transaction->m_variableMultipartCrlfLFLines.set(std::to_string(m_flag_crlf_line && m_flag_lf_line),
-                            m_transaction->m_variableOffset);
-                        if (m_flag_crlf_line && m_flag_lf_line) {
-                            ms_dbg_a(m_transaction, 4, "Multipart: Warning: mixed line endings used (CRLF/LF).");
-                        } else if (m_flag_lf_line) {
-                            ms_dbg_a(m_transaction, 4, "Multipart: Warning: incorrect line endings used (LF).");
+                        m_boundary.size()) == 0)) {
+                    if ((buf_data_len >= 2 + m_boundary.size() + 2)
+                            && (*(m_buf + 2 + m_boundary.size()) == '-')
+                        && (*(m_buf + 2 + m_boundary.size() + 1) == '-')) {
+                        /* If body fits in limit and ends with final boundary plus just CR, reject it. */
+                        if ( (m_allow_partial == 0)
+                            && (buf_data_len == 2 + m_boundary.size() + 2 + 1)
+                            && (*(m_buf + 2 + m_boundary.size() + 2) == '\r') ) {
+                            ms_dbg_a(m_transaction, 1,
+                                "Multipart: Invalid epilogue after final boundary.");
+                            error->assign("Multipart: Invalid epilogue after final boundary.");
+                            return false;
+                        }
+                        // these next two checks may result in repeating work from earlier in this fn
+                        // ignore the duplication for now to minimize refactoring
+                        if ((m_crlf_state_buf_end == 2) && (m_flag_lf_line != 1)) {
+                            m_flag_lf_line = 1;
+                            m_transaction->m_variableMultipartLFLine.set(std::to_string(m_flag_lf_line),
+                                m_transaction->m_variableOffset);
+                            m_transaction->m_variableMultipartCrlfLFLines.set(std::to_string(m_flag_crlf_line && m_flag_lf_line),
+                                m_transaction->m_variableOffset);
+                            if (m_flag_crlf_line && m_flag_lf_line) {
+                                ms_dbg_a(m_transaction, 4, "Multipart: Warning: mixed line endings used (CRLF/LF).");
+                            } else if (m_flag_lf_line) {
+                                ms_dbg_a(m_transaction, 4, "Multipart: Warning: incorrect line endings used (LF).");
+                            }
+                            m_transaction->m_variableMultipartStrictError.set(
+                                std::to_string(m_flag_lf_line) , m_transaction->m_variableOffset);
+                        }
+                        if ((m_mpp_substate_part_data_read == 0) && (m_flag_invalid_part != 1)) {
+                            // it looks like the final boundary, but it's where part data should begin
+                            m_flag_invalid_part = 1;
+                            ms_dbg_a(m_transaction, 3, "Multipart: Invalid part (data contains final boundary)");
+                            m_transaction->m_variableMultipartStrictError.set(
+                                std::to_string(m_flag_invalid_part) , m_transaction->m_variableOffset);
+                            m_transaction->m_variableMultipartInvalidPart.set(std::to_string(m_flag_invalid_part),
+                                m_transaction->m_variableOffset);
+                            ms_dbg_a(m_transaction, 4, "Multipart: Warning: invalid part parsing.");
                         }
-                        m_transaction->m_variableMultipartStrictError.set(
-                            std::to_string(m_flag_lf_line) , m_transaction->m_variableOffset);
-		    }
-                    if ((m_mpp_substate_part_data_read == 0) && (m_flag_invalid_part != 1)) {
-                        // it looks like the final boundary, but it's where part data should begin
-                        m_flag_invalid_part = 1;
-                        ms_dbg_a(m_transaction, 3, "Multipart: Invalid part (data contains final boundary)");
-                        m_transaction->m_variableMultipartStrictError.set(
-                            std::to_string(m_flag_invalid_part) , m_transaction->m_variableOffset);
-                        m_transaction->m_variableMultipartInvalidPart.set(std::to_string(m_flag_invalid_part),
-                            m_transaction->m_variableOffset);
-                        ms_dbg_a(m_transaction, 4, "Multipart: Warning: invalid part parsing.");
-                    }
 
-                    /* Looks like the final boundary - process it. */
-                    if (process_boundary(1 /* final */) < 0) {
-                        m_flag_error = 1;
-                        return -1;
-                    }
+                        /* Looks like the final boundary - process it. */
+                        if (process_boundary(1 /* final */) < 0) {
+                            m_flag_error = 1;
+                            return -1;
+                        }
 
-                    /* The payload is complete after all. */
-                    m_is_complete = 1;
+                        /* The payload is complete after all. */
+                        m_is_complete = 1;
+                    } else if (m_allow_partial) {
+                        if (buf_data_len >= 2 + m_boundary.size() + 1) {
+                            if (*(m_buf + 2 + m_boundary.size()) == '-') {
+                                if ((buf_data_len >= 2 + m_boundary.size() + 2)
+                                    && (*(m_buf + 2 + m_boundary.size() + 1) != '-')) {
+                                    ms_dbg_a(m_transaction, 1,
+                                        "Multipart: Invalid final boundary.");
+                                    error->assign("Multipart: Invalid final boundary.");
+                                    return false;
+                                }
+                            } else if ((*(m_buf + 2 + m_boundary.size()) != '\r') 
+                                || ((buf_data_len >= 2 + m_boundary.size() + 2)
+                                    && (*(m_buf + 2 + m_boundary.size() + 1) != '\n'))) {
+                                ms_dbg_a(m_transaction, 1,
+                                    "Multipart: Invalid boundary.");
+                                error->assign("Multipart: Invalid boundary.");
+                                return false;
+                            }
+                        }
+                        /* process it as a non-final boundary to avoid building a new part. */
+                        if (process_boundary(0) < 0) {
+                            m_flag_error = 1;
+                            return -1;
+                        }
+                    }
                 }
             }
 
-            if (m_is_complete == 0) {
+            if (m_is_complete == 0 && !m_allow_partial) {
                 ms_dbg_a(m_transaction, 1,
                     "Multipart: Final boundary missing.");
                 error->assign("Multipart: Final boundary missing.");
@@ -1563,7 +1619,7 @@ bool Multipart::process(const std::string& data, std::string *error,
                         m_boundary.size()) == 0)) {
                     if (m_crlf_state_buf_end == 2) {
                         m_flag_lf_line = 1;
-		    }
+                    }
                     if ((m_mpp_substate_part_data_read == 0) && (m_boundary_count > 0)) {
                         /* string matches our boundary, but it's where part data should begin */
                         m_flag_invalid_part = 1;

src/request_body_processor/multipart.h

@@ -228,6 +228,7 @@ class Multipart {
 
     int m_seen_data;
     int m_is_complete;
+    bool m_allow_partial;
 
     int m_flag_error;
     int m_flag_data_before;

src/request_body_processor/xml.cc

@@ -150,7 +150,7 @@ extern "C" {
 }
 
 XML::XML(Transaction *transaction)
-    : m_transaction(transaction) {
+    : m_transaction(transaction), m_require_well_formed(false) {
     m_data.doc = NULL;
     m_data.parsing_ctx = NULL;
     m_data.sax_handler = NULL;
@@ -171,7 +171,8 @@ XML::~XML() {
     }
 }
 
-bool XML::init() {
+bool XML::init(bool require_well_formed) {
+    m_require_well_formed = require_well_formed;
     //xmlParserInputBufferCreateFilenameFunc entity;
     if (m_transaction->m_rules->m_secXMLExternalEntity
         == RulesSetProperties::TrueConfigBoolean) {
@@ -280,7 +281,7 @@ bool XML::processChunk(const char *buf, unsigned int size,
         != RulesSetProperties::OnlyArgsConfigXMLParseXmlIntoArgs) {
         xmlParseChunk(m_data.parsing_ctx, buf, size, 0);
         m_data.xml_parser_state->parsing_ctx_arg = m_data.parsing_ctx_arg;
-        if (m_data.parsing_ctx->wellFormed != 1) {
+        if (m_require_well_formed && m_data.parsing_ctx->wellFormed != 1) {
             error->assign("XML: Failed to parse document.");
             ms_dbg_a(m_transaction, 4, "XML: Failed to parse document.");
             return false;
@@ -296,7 +297,7 @@ bool XML::processChunk(const char *buf, unsigned int size,
               == RulesSetProperties::TrueConfigXMLParseXmlIntoArgs)
         ) {
         xmlParseChunk(m_data.parsing_ctx_arg, buf, size, 0);
-        if (m_data.parsing_ctx_arg->wellFormed != 1) {
+        if (m_require_well_formed && m_data.parsing_ctx_arg->wellFormed != 1) {
             error->assign("XML: Failed to parse document for ARGS.");
             ms_dbg_a(m_transaction, 4, "XML: Failed to parse document for ARGS.");
             return false;
@@ -326,7 +327,7 @@ bool XML::complete(std::string *error) {
             ms_dbg_a(m_transaction, 4, "XML: Parsing complete (well_formed " \
                 + std::to_string(m_data.well_formed) + ").");
 
-            if (m_data.well_formed != 1) {
+            if (m_require_well_formed && m_data.well_formed != 1) {
                 error->assign("XML: Failed to parse document.");
                 ms_dbg_a(m_transaction, 4, "XML: Failed to parse document.");
                 return false;

src/request_body_processor/xml.h

@@ -87,7 +87,7 @@ class XML {
  public:
     explicit XML(Transaction *transaction);
     ~XML();
-    bool init();
+    bool init(bool require_well_formed = true);
     bool processChunk(const char *buf, unsigned int size, std::string *err);
     bool complete(std::string *err);
     static xmlParserInputBufferPtr unloadExternalEntity(const char *URI,
@@ -98,6 +98,7 @@ class XML {
  private:
     Transaction *m_transaction;
     std::string m_header;
+    bool m_require_well_formed;
 };
 
 #endif