Fixed flush behavior

Author: andsel

logstash-core/src/main/java/org/logstash/common/BufferedTokenizer.java

@@ -84,7 +84,7 @@ public void append(String data) {
         }
 
         public String flush() {
-            return accumulator.toString();
+            return accumulator.substring(currentIdx);
         }
 
         @Override

logstash-core/src/main/java/org/logstash/common/BufferedTokenizerExt.java

@@ -201,7 +201,20 @@ private RubyString toEncodedRubyString(ThreadContext context, String input) {
     @JRubyMethod
     public IRubyObject flush(final ThreadContext context) {
         String s = tokenizer.flush();
-        return RubyUtil.toRubyObject(s);
+
+        // create new RubyString with the last data specified encoding, if exists
+        if (encodingName != null) {
+            return toEncodedRubyString(context, s);
+        } else {
+            // When used with TCP input it could be that on socket connection the flush method
+            // is invoked while no invocation of extract, leaving the encoding name unassigned.
+            // In such case also the headToken must be empty
+            if (!s.isEmpty()) {
+                throw new IllegalStateException("invoked flush with unassigned encoding but not empty head token, this shouldn't happen");
+            }
+            return RubyUtil.toRubyObject(s);
+        }
+
 //        final IRubyObject buffer = RubyUtil.toRubyObject(headToken.toString());
 //        headToken = new StringBuilder();
 //        inputSize = 0;