[Test] added test to verify buffer full error is notified not only on the first token

Author: andsel

logstash-core/src/test/java/org/logstash/common/BufferedTokenizerWithSizeLimitTest.java

@@ -22,6 +22,7 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Iterator;
 import java.util.List;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -109,4 +110,22 @@ public void giveMultipleSegmentsThatGeneratesMultipleBufferFullErrorsThenIsAbleT
         List<String> tokens = toList(sut.extract("ccc\nddd\n"));
         assertEquals(List.of("ccccc", "ddd"), tokens);
     }
+
+    @Test
+    public void givenFragmentThatHasTheSecondTokenOverrunsSizeLimitThenAnErrorIsThrown() {
+        Iterable<String> tokensIterable = sut.extract("aaaa\nbbbbbbbbbbb\nccc\n");
+        Iterator<String> tokensIterator = tokensIterable.iterator();
+
+        // first token length = 4, it's ok
+        assertEquals("aaaa", tokensIterator.next());
+
+        // second token is an overrun, length = 11
+        Exception exception = assertThrows(IllegalStateException.class, () -> {
+            tokensIterator.next();
+        });
+        assertThat(exception.getMessage(), containsString("input buffer full"));
+
+        // third token resumes
+        assertEquals("ccc", tokensIterator.next());
+    }
 }