refactor: introduced LexerCommons and wip: fixing test cases

Author: cybercoder-naj

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt

@@ -1,12 +1,11 @@
 package io.github.cybercodernaj.parkour.lexer
 
 import io.github.cybercodernaj.parkour.lexer.internal.Lexer
-import io.github.cybercodernaj.parkour.exceptions.LexicalException
 
 /**
  * A helper class to create the [Lexer].
  * Contains functions to be used as part of the [lexer] DSL.
- * Each property's default value is detailed in [Lexer.Defaults].
+ * Each property's default value is detailed in [LexerCommons].
  *
  * @author Nishant Aanjaney Jalan
  * @since 0.2.0
@@ -24,11 +23,10 @@ class LexerBuilder internal constructor() {
    * }
    * ```
    *
-   * @see Lexer.Defaults.ignorePattern
    * @author Nishant Aanjaney Jalan
    * @since 0.2.0
    */
-  var ignorePattern: Regex = Lexer.Defaults.ignorePattern
+  var ignorePattern: Regex? = null
 
   /**
    * When the lexer identifies a [singleLineComments] pattern, it will skip to the next line
@@ -42,11 +40,10 @@ class LexerBuilder internal constructor() {
    * }
    * ```
    *
-   * @see Lexer.Defaults.singleLineComments
    * @author Nishant Aanjaney Jalan
    * @since 0.2.0
    */
-  var singleLineComments: Regex? = Lexer.Defaults.singleLineComments
+  var singleLineComments: Regex? = null
 
   /**
    * There are two parts to [multilineComments]: the starting and the ending pattern.
@@ -62,11 +59,10 @@ class LexerBuilder internal constructor() {
    * }
    * ```
    *
-   * @see Lexer.Defaults.multilineComments
    * @author Nishant Aanjaney Jalan
    * @since 0.2.0
    */
-  var multilineComments: Pair<Regex, Regex>? = Lexer.Defaults.multilineComments
+  var multilineComments: Pair<Regex, Regex>? = null
 
   /**
    * The regex pattern that defines the rules for identifiers.
@@ -82,11 +78,11 @@ class LexerBuilder internal constructor() {
    * }
    * ```
    *
-   * @see Lexer.Defaults.identifiers
+   * @see LexerCommons.identifiers
    * @author Nishant Aanjaney Jalan
    * @since 0.2.0
    */
-  var identifiers: Regex = Lexer.Defaults.identifiers
+  var identifiers: Regex? = null
 
   private val _hardKeywords: MutableList<String> = mutableListOf()
   internal val hardKeywords: List<String> get() = _hardKeywords
@@ -100,6 +96,7 @@ class LexerBuilder internal constructor() {
   /**
    * The regex that detects and extracts integer literals from the string.
    * This should usually only consider strict integers.
+   * This configuration is required only if your integer pattern is different from [LexerCommons.integerLiteral]
    *
    * ### Usage
    *
@@ -109,15 +106,16 @@ class LexerBuilder internal constructor() {
    * }
    * ```
    *
-   * @see Lexer.Defaults.integerLiteral
+   * @see LexerCommons.integerLiteral
    * @author Nishant Aanjaney Jalan
    * @since 0.2.0
    */
-  var integerLiteral: Regex = Lexer.Defaults.integerLiteral
+  var integerLiteral: Regex = LexerCommons.integerLiteral
 
   /**
    * The regex that detects and extracts floating point literals from the string.
    * This should usually only consider numbers that have a decimal point.
+   * This configuration is required only if your floating pattern is different from [LexerCommons.floatingLiteral]
    *
    * ### Usage
    *
@@ -127,29 +125,30 @@ class LexerBuilder internal constructor() {
    * }
    * ```
    *
-   * @see Lexer.Defaults.floatingLiteral
+   * @see LexerCommons.floatingLiteral
    * @author Nishant Aanjaney Jalan
    * @since 0.2.0
    */
-  var floatingLiteral: Regex = Lexer.Defaults.floatingLiteral
+  var floatingLiteral: Regex = LexerCommons.floatingLiteral
 
   /**
    * The enclosing strings that should denote the start and end of single-line strings.
    * The lexer will throw a [LexicalException] when a string literal is not terminated in the same line.
+   * This configuration is required only if your string defining patterns are different from [LexerCommons.floatingLiteral]
    *
    * ### Usage
    *
    * ```kt
    * val myLexer = lexer {
-   *   floatingLiteral = Regex("[-+]?[0-9_]*\.[0-9_]+(?:[eE][-+]?[0-9_]+)?")
+   *   singleLineString = setOf("'", "\"")
    * }
    * ```
    *
-   * @see Lexer.Defaults.singleLineString
+   * @see LexerCommons.singleLineString
    * @author Nishant Aanjaney Jalan
    * @since 0.2.0
    */
-  var singleLineString: Set<String> = Lexer.Defaults.singleLineString
+  var singleLineString: Set<String> = LexerCommons.singleLineString
 
   /**
    * Hard keywords are a characters and symbols that give a particular meaning to a program.

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerCommons.kt

@@ -0,0 +1,15 @@
+package io.github.cybercodernaj.parkour.lexer
+
+/**
+ * A list of common patterns and lists of items that most programming languages and
+ * data serialization formats.
+ *
+ * @author Nishant Aanjaney Jalan
+ * @since 0.2.0
+ */
+object LexerCommons {
+  val identifiers = Regex("""[a-zA-Z_]\w*""")
+  val integerLiteral = Regex("""[-+]?[0-9_]+""")
+  val floatingLiteral = Regex("""[-+]?[0-9_]*\.[0-9_]+(?:[eE][-+]?[0-9_]+)?""")
+  val singleLineString: Set<String> = setOf("\"", "\'")
+}

core/src/main/kotlin/io/github/cybercodernaj/parkour/exceptions/LexicalException.kt renamed to core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexicalException.kt

@@ -1,4 +1,4 @@
-package io.github.cybercodernaj.parkour.exceptions
+package io.github.cybercodernaj.parkour.lexer
 
 /**
  * This is thrown when there is an error when trying to tokenize a string.

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt

@@ -1,8 +1,9 @@
 package io.github.cybercodernaj.parkour.lexer.internal
 
 import io.github.cybercodernaj.parkour.datasource.TextSource
-import io.github.cybercodernaj.parkour.exceptions.LexicalException
 import io.github.cybercodernaj.parkour.lexer.LexerBuilder
+import io.github.cybercodernaj.parkour.lexer.LexerCommons
+import io.github.cybercodernaj.parkour.lexer.LexicalException
 import io.github.cybercodernaj.parkour.utils.Position
 
 /**
@@ -12,42 +13,18 @@ import io.github.cybercodernaj.parkour.utils.Position
  * @since 0.1.0
  */
 class Lexer internal constructor(
-  private val ignorePattern: Regex = Defaults.ignorePattern,
-  private val singleLineComments: Regex? = Defaults.singleLineComments,
-  private val multilineComments: Pair<Regex, Regex>? = Defaults.multilineComments,
-  private val identifiers: Regex = Defaults.identifiers,
+  private val ignorePattern: Regex? = Regex("""\s+"""),
+  private val singleLineComments: Regex? = null,
+  private val multilineComments: Pair<Regex, Regex>? = null,
+  private val identifiers: Regex? = Regex("""[a-zA-Z_]\w*"""),
   private val hardKeywords: List<String> = emptyList(),
   private val operators: List<String> = emptyList(),
   private val separators: List<String> = emptyList(),
-  private val integerLiteral: Regex? = Defaults.integerLiteral,
-  private val floatingLiteral: Regex? = Defaults.floatingLiteral,
-  private val singleLineString: Set<String> = Defaults.singleLineString,
-  private val escapeSequences: List<Pair<Regex, (String) -> Char>> = Defaults.escapeSequences,
+  private val integerLiteral: Regex? = LexerCommons.integerLiteral,
+  private val floatingLiteral: Regex? = LexerCommons.floatingLiteral,
+  private val singleLineString: Set<String> = LexerCommons.singleLineString,
+  private val escapeSequences: List<Pair<Regex, (String) -> Char>> = listOf(),
 ) {
-  /**
-   * A list of common patterns and lists of items that most programming languages and
-   * data serialization formats.
-   *
-   * @author Nishant Aanjaney Jalan
-   * @since 0.2.0
-   */
-  object Defaults {
-    /**
-     * The lexer will skip over any strings that match this regex.
-     * This acts like a token separator.
-     *
-     * @author Nishant Aanjaney Jalan
-     * @since 0.2.0
-     */
-    val ignorePattern = Regex("""\s+""")
-    val singleLineComments: Regex? = null
-    val multilineComments: Pair<Regex, Regex>? = null
-    val identifiers = Regex("""[a-zA-Z_]\w*""")
-    val integerLiteral = Regex("""[-+]?[0-9_]+""")
-    val floatingLiteral = Regex("""[-+]?[0-9_]*\.[0-9_]+(?:[eE][-+]?[0-9_]+)?""")
-    val singleLineString: Set<String> = setOf("\"", "\'")
-    val escapeSequences: List<Pair<Regex, (String) -> Char>> = emptyList() // TODO fill this list
-  }
 
   private var position = Position(0, 0)
 
@@ -62,9 +39,6 @@ class Lexer internal constructor(
 
   private var insideMultilineComment = false
 
-  private val _separators = separators.sortedByDescending(String::length)
-  private val _operators = operators.sortedByDescending(String::length)
-
   /**
    * Fetches the next [Token] from the source
    *
@@ -115,10 +89,11 @@ class Lexer internal constructor(
         }
       }
 
-      (position pointsAt ignorePattern)
-        ?.let { match ->
-          position = position.copy(col = match.range.last + 1)
-        }
+      val ignoreMatch = position pointsAt ignorePattern
+      if (ignoreMatch != null) {
+        position = position.copy(col = ignoreMatch.range.last + 1)
+        continue
+      }
 
       (position pointsAtSome hardKeywords)
         ?.let { keyword ->
@@ -133,14 +108,14 @@ class Lexer internal constructor(
           position = token.end + 1
         }
 
-      (position pointsAtSome _operators)
+      (position pointsAtSome operators)
         ?.let { keyword ->
           val end = position.copy(col = position.col + keyword.length - 1)
           tokenStream.addOperator(position, end)
           position = end + 1
         }
 
-      (position pointsAtSome _separators)
+      (position pointsAtSome separators)
         ?.let { keyword ->
           val end = position.copy(col = position.col + keyword.length - 1)
           tokenStream.addSeparator(position, end)
@@ -300,5 +275,11 @@ class Lexer internal constructor(
   private infix fun Position.pointsAt(pattern: String): Boolean {
     return currentLine.startsWith(pattern, startIndex = this.col)
   }
+
+  fun resetPosition() {
+    position = Position(0, 0)
+    tokenIndex = 0
+    tokenStream = emptyList()
+  }
 }
 

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilderTest.kt

@@ -4,20 +4,34 @@ import io.github.cybercodernaj.parkour.datasource.StringSource
 import io.github.cybercodernaj.parkour.lexer.internal.Token
 import io.github.cybercodernaj.parkour.testutils.assertTokens
 import io.github.cybercodernaj.parkour.utils.Position
+import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 
 class LexerBuilderTest {
+  private val myLexer = lexer {
+    ignorePattern = Regex("[. ]+")
+    singleLineComments = Regex("//")
+    multilineComments = Regex("/\\*") to Regex("\\*/")
+    identifiers = Regex("[a-z][a-zA-Z0-9]+")
+
+    hardKeywords("val", "var")
+    separators("(", ")", "<", ">", ",", ".")
+    hardKeywords("val")
+    operators("*", "**", "/", "+", "-", "=", "==")
+  }
+
   @Test
   fun `initialise a default lexer`() {
     lexer {}
   }
 
+  @BeforeEach
+  fun setupEach() {
+    myLexer.resetPosition()
+  }
+
   @Test
   fun `sets ignore patterns`() {
-    val myLexer = lexer {
-      ignorePattern = Regex("\\.")
-    }
-
     myLexer.source = StringSource("hi.hello")
     assertTokens(
       myLexer,
@@ -30,10 +44,6 @@ class LexerBuilderTest {
 
   @Test
   fun `sets single line comments`() {
-    val myLexer = lexer {
-      singleLineComments = Regex("//")
-    }
-
     myLexer.source = StringSource("hi // hello\nhru")
     assertTokens(
       myLexer,
@@ -46,10 +56,6 @@ class LexerBuilderTest {
 
   @Test
   fun `sets multiline comments`() {
-    val myLexer = lexer {
-      multilineComments = Regex("/\\*") to Regex("\\*/")
-    }
-
     myLexer.source = StringSource("hi /* hello */\nhru")
     assertTokens(
       myLexer,
@@ -62,10 +68,6 @@ class LexerBuilderTest {
 
   @Test
   fun `sets identifiers rule`() {
-    val myLexer = lexer {
-      identifiers = Regex("[a-z][a-zA-Z0-9]+")
-    }
-
     myLexer.source = StringSource("hi value")
     assertTokens(
       myLexer,
@@ -78,10 +80,6 @@ class LexerBuilderTest {
 
   @Test
   fun `sets hard keywords`() {
-    val myLexer = lexer {
-      hardKeywords("val", "var")
-    }
-
     myLexer.source = StringSource("val c")
     assertTokens(
       myLexer,
@@ -94,11 +92,6 @@ class LexerBuilderTest {
 
   @Test
   fun `set separators`() {
-    val myLexer = lexer {
-      separators("(", ")", "<", ">", ",", ".")
-    }
-
-
     myLexer.source = StringSource("Array<List<Set<Int>>>")
 
     assertTokens(
@@ -119,11 +112,6 @@ class LexerBuilderTest {
 
   @Test
   fun `set operators`() {
-    val myLexer = lexer {
-      hardKeywords("val")
-      operators("*", "**", "/", "//", "+", "-", "=", "==")
-    }
-
     myLexer.source = StringSource("val diff = new - old")
 
     assertTokens(

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/internal/LexerIdentifierTest.kt

@@ -1,7 +1,7 @@
 package io.github.cybercodernaj.parkour.lexer.internal
 
 import io.github.cybercodernaj.parkour.datasource.StringSource
-import io.github.cybercodernaj.parkour.exceptions.LexicalException
+import io.github.cybercodernaj.parkour.lexer.LexicalException
 import io.github.cybercodernaj.parkour.utils.Position
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.fail

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/internal/LexerLiteralTest.kt

@@ -1,7 +1,7 @@
 package io.github.cybercodernaj.parkour.lexer.internal
 
 import io.github.cybercodernaj.parkour.datasource.StringSource
-import io.github.cybercodernaj.parkour.exceptions.LexicalException
+import io.github.cybercodernaj.parkour.lexer.LexicalException
 import io.github.cybercodernaj.parkour.testutils.assertTokens
 import io.github.cybercodernaj.parkour.utils.Position
 import org.junit.jupiter.api.Test