refactor: hard keywords, operators and separators

Author: cybercoder-naj

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/Lexer.kt

@@ -1,6 +1,7 @@
 package io.github.cybercodernaj.parkour.lexer
 
-import arrow.core.*
+import arrow.core.None
+import arrow.core.Option
 import arrow.core.raise.OptionRaise
 import arrow.core.raise.option
 import io.github.cybercodernaj.parkour.datasource.TextSource
@@ -38,9 +39,9 @@ class Lexer(
   private val singleLineComments: Regex? = null,
   private val multilineComments: Pair<Regex, Regex>? = null,
   private val identifiers: Regex = Regex("""[a-zA-Z_]\w*"""),
-  private val hardKeywords: Set<String> = emptySet(),
-  private val operators: Set<String> = emptySet(),
-  private val separators: Set<String> = emptySet(),
+  val hardKeywords: Set<String> = emptySet(),
+  val operators: Set<String> = emptySet(),
+  val separators: Set<String> = emptySet(),
   private val literals: Literals = Literals()
 ) {
   private var position: Position = Position(0, 0)
@@ -68,15 +69,21 @@ class Lexer(
 
   private var insideMultilineComment = false
 
-  private val _hardKeywords = StringTrieMap<Kind>()
+  private val definitions = StringTrieMap<Kind>()
 
   private enum class Kind {
     KEYWORD, OPERATOR, SEPARATOR
   }
 
   init {
     hardKeywords.forEach {
-      _hardKeywords[it] = Kind.KEYWORD
+      definitions[it] = Kind.KEYWORD
+    }
+    operators.forEach {
+      definitions[it] = Kind.OPERATOR
+    }
+    separators.forEach {
+      definitions[it] = Kind.SEPARATOR
     }
   }
 
@@ -99,7 +106,9 @@ class Lexer(
     // If the file is fully read, then return EOF
     currentLine.onNone { return Token.EOF }
 
-    val winner = listOf(identifiers()).firstNotNullOfOrNull { it.getOrNull() }
+    val winner = listOf(definitions(), identifiers())
+      .mapNotNull { it.getOrNull() }
+      .maxByOrNull { it.size }
 
     if (winner != null) {
       position = position.copy(col = winner.end.col + 1)
@@ -122,6 +131,23 @@ class Lexer(
     }
   }
 
+  private fun definitions(): Option<Token> = option {
+    val line = currentLine.bind().substring(position.col)
+    val kind = definitions.getLongest(line)
+    val token = kind?.let { (k, length) ->
+      val content = line.substring(0, length)
+      val end = position + (length - 1)
+      when (k) {
+        Kind.KEYWORD -> Token.Keyword(content, position, end)
+        Kind.OPERATOR -> Token.Operator(content, position, end)
+        Kind.SEPARATOR -> Token.Separator(content, position, end)
+      }
+    }
+    if (token == null)
+      raise(None)
+    token
+  }
+
   private fun identifiers(): Option<Token.Identifier> = option {
     val match = startsWith(identifiers)
 
@@ -147,9 +173,6 @@ class Lexer(
   }
 }
 
-private fun <A> Option<A>.getOrThrow(cause: () -> Exception): A {
-  return getOrNull() ?: throw cause()
-}
 
 //  private fun updateTokenStream() {
 //    fetchNextLine()

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/StringTrieMap.kt

@@ -24,6 +24,29 @@ internal class StringTrieMap<V> {
     }
     return if (node.terminal) node.value else null
   }
+
+  /**
+   * The [content] will walk down the trie to find a terminal value with the highest depth.
+   *
+   * @param content the line to find the longest match
+   * @return the terminal value and the depth of trie traversed. Null if not found
+   */
+  fun getLongest(content: String): Pair<V, Int>? {
+    var node = root
+    var candidate: V? = null
+    var depth = 0
+    for (elem in content) {
+      if (elem !in node.children) {
+        return candidate?.let { it to depth }
+      }
+      node = node.children[elem]!!
+      depth++
+      if (node.terminal) {
+        candidate = node.value!!
+      }
+    }
+    return candidate?.let { it to depth }
+  }
 }
 
 private data class TrieNode<T, U>(

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/Token.kt

@@ -23,7 +23,7 @@ internal sealed class Token(val value: Any, val start: Position, val end: Positi
   data object EOF : Token(Any(), Position(-1, -1), Position(-1, -1))
 
   val size: Int
-    get() = this.end - this.start
+    get() = this.end - this.start + 1
 
   override fun equals(other: Any?): Boolean {
     if (this === other) return true

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/LexerKeywordTest.kt

@@ -1,6 +1,7 @@
 package io.github.cybercodernaj.parkour.lexer
 
 import io.github.cybercodernaj.parkour.datasource.StringSource
+import io.github.cybercodernaj.parkour.testutils.assertTokens
 import io.github.cybercodernaj.parkour.utils.Position
 import org.junit.jupiter.api.Test
 import kotlin.test.assertEquals
@@ -25,29 +26,31 @@ class LexerKeywordTest {
   }
 
   @Test
-  fun `returns 2 keywords in same line`() {
-    lexer.source = StringSource("val class")
+  fun `returns 3 keywords in same line`() {
+    lexer.source = StringSource("val class var")
 
-    val token = lexer.nextToken()
-    assertEquals(
-      Token.Keyword(
-        value = "val",
-        start = Position(0, 0),
-        end = Position(0, 2),
-        soft = false
-      ),
-      token
-    )
-
-    val token2 = lexer.nextToken()
-    assertEquals(
-      Token.Keyword(
-        value = "class",
-        start = Position(0, 4),
-        end = Position(0, 8),
-        soft = false
-      ),
-      token2
+    assertTokens(
+      lexer,
+      listOf(
+        Token.Keyword(
+          value = "val",
+          start = Position(0, 0),
+          end = Position(0, 2),
+          soft = false
+        ),
+        Token.Keyword(
+          value = "class",
+          start = Position(0, 4),
+          end = Position(0, 8),
+          soft = false
+        ),
+        Token.Keyword(
+          value = "var",
+          start = Position(0, 10),
+          end = Position(0, 12),
+          soft = false
+        ),
+      )
     )
   }
 
@@ -106,4 +109,25 @@ class LexerKeywordTest {
     val token3 = lexer.nextToken()
     assertEquals(Token.EOF, token3)
   }
+
+  @Test
+  fun `differentiate between keyword and identifier`() {
+    lexer.source = StringSource("val value")
+
+    assertTokens(
+      lexer,
+      listOf(
+        Token.Keyword(
+          value = "val",
+          start = Position(0, 0),
+          end = Position(0, 2)
+        ),
+        Token.Identifier(
+          value = "value",
+          start = Position(0, 4),
+          end = Position(0, 8)
+        )
+      )
+    )
+  }
 }