20
20
21
21
import java .util .Iterator ;
22
22
import java .util .NoSuchElementException ;
23
- import java .util .function .IntPredicate ;
24
23
25
24
public class BufferedTokenizer {
26
25
@@ -42,25 +41,29 @@ public boolean hasNext() {
42
41
43
42
static class DataSplitter implements Iterator <String > {
44
43
private final String separator ;
45
- private final IntPredicate sizeChecker ;
46
44
private final StringBuilder accumulator = new StringBuilder ();
47
45
private int currentIdx = 0 ;
48
46
private boolean dropNextPartialFragments = false ;
47
+ private final int sizeLimit ;
49
48
49
+ /**
50
+ * @param separator
51
+ * is the token separator string.
52
+ * */
50
53
DataSplitter (String separator ) {
51
54
this .separator = separator ;
52
- this .sizeChecker = value -> false ;
55
+ this .sizeLimit = Integer . MIN_VALUE ;
53
56
}
54
57
55
58
/**
56
59
* @param separator
57
60
* is the token separator string.
58
- * @param sizeChecker
59
- * function that verifies if token size is bigger then a limit
61
+ * @param sizeLimit
62
+ * maximum token size length.
60
63
* */
61
- DataSplitter (String separator , IntPredicate sizeChecker ) {
64
+ DataSplitter (String separator , int sizeLimit ) {
62
65
this .separator = separator ;
63
- this .sizeChecker = sizeChecker ;
66
+ this .sizeLimit = sizeLimit ;
64
67
}
65
68
66
69
@ Override
@@ -71,7 +74,7 @@ public boolean hasNext() {
71
74
cleanupAccumulator ();
72
75
// if it has a remaining bigger than the admitted size, then it start drop other next fragments that
73
76
// doesn't contain any separator
74
- if (sizeChecker . test ( accumulator .length ()) ) {
77
+ if (sizeLimit != Integer . MIN_VALUE && accumulator .length () > sizeLimit ) {
75
78
dropNextPartialFragments = true ;
76
79
}
77
80
return false ;
@@ -90,8 +93,8 @@ public String next() {
90
93
} else {
91
94
String token = accumulator .substring (currentIdx , nextIdx );
92
95
currentIdx = nextIdx + separator .length ();
93
- if (sizeChecker . test ( token .length ()) ) {
94
- throw new IllegalStateException ("input buffer full, consumed token which exceeded the sizeLimit " ); // TODO + sizeLimit
96
+ if (sizeLimit != Integer . MIN_VALUE && token .length () > sizeLimit ) {
97
+ throw new IllegalStateException ("input buffer full, consumed token which exceeded the sizeLimit " + sizeLimit );
95
98
}
96
99
return token ;
97
100
}
@@ -134,7 +137,7 @@ public BufferedTokenizer(String separator, int sizeLimit) {
134
137
throw new IllegalArgumentException ("Size limit must be positive" );
135
138
}
136
139
137
- this .dataSplitter = new DataSplitter (separator , tokenSize -> tokenSize > sizeLimit );
140
+ this .dataSplitter = new DataSplitter (separator , sizeLimit );
138
141
this .iterable = () -> dataSplitter ;
139
142
}
140
143
0 commit comments