Fix reading past end to consistently return NULL

Author: thekid

ChangeLog.md

@@ -3,6 +3,10 @@ Tokenize log
 
 ## ?.?.? / ????-??-??
 
+## 9.0.1 / 2020-10-09
+
+* Fixed reading past end to consistently return `NULL` - @thekid
+
 ## 9.0.0 / 2019-11-30
 
 * Dropped support for PHP 5.6, see xp-framework/rfc#334 - @thekid

src/main/php/text/StreamTokenizer.class.php

@@ -69,12 +69,13 @@ public function pushBack($str) {
    */
   public function nextToken($delimiters= null) {
     if (empty($this->_stack)) {
-    
+      if (false === $this->_buf) return null;
+
       // Read until we have either find a delimiter or until we have 
       // consumed the entire content.
       do {
         $offset= strcspn($this->_buf, $delimiters ? $delimiters : $this->delimiters);
-        if ($offset < strlen($this->_buf)- 1 || !$this->_src->available()) break;
+        if ($offset < strlen($this->_buf) - 1 || !$this->_src->available()) break;
         $this->_buf.= $this->_src->read();
       } while (true);
 

src/main/php/text/StringTokenizer.class.php

@@ -74,6 +74,8 @@ public function pushBack($str) {
    */
   public function nextToken($delimiters= null) {
     if (empty($this->_stack)) {
+      if ($this->_ofs >= $this->_len) return null;
+
       $offset= strcspn($this->source, $delimiters ? $delimiters : $this->delimiters, $this->_ofs);
       if (!$this->returnDelims || $offset > 0) $this->_stack[]= substr($this->source, $this->_ofs, $offset);
       if ($this->returnDelims && $this->_ofs + $offset < $this->_len) {

src/main/php/text/TextTokenizer.class.php

@@ -67,12 +67,13 @@ public function pushBack($str) {
    */
   public function nextToken($delimiters= null) {
     if (empty($this->_stack)) {
-    
+      if (false === $this->_buf) return null;
+
       // Read until we have either find a delimiter or until we have 
       // consumed the entire content.
       do {
         $offset= strcspn($this->_buf, $delimiters ? $delimiters : $this->delimiters);
-        if ($offset < strlen($this->_buf)- 1) break;
+        if ($offset < strlen($this->_buf) - 1) break;
         if (null === ($buf= $this->source->read())) {
           break;
         }

src/test/php/text/unittest/AbstractTokenizerTest.class.php

@@ -224,4 +224,13 @@ public function performance() {
       $token= $t->nextToken();
     }
   }
+
+  #[Test]
+  public function reading_past_end_returns_null() {
+    $t= $this->tokenizerInstance('Test', "\n", false);
+    while ($t->hasMoreTokens()) {
+      $t->nextToken();
+    }
+    $this->assertEquals([null, null], [$t->nextToken(), $t->nextToken()]);
+  }
 }