feat: added operators and separators in lexer dsl

Author: cybercoder-naj

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt

@@ -26,6 +26,12 @@ class LexerBuilder internal constructor() {
   private val _hardKeywords: MutableList<String> = mutableListOf()
   internal val hardKeywords: List<String> get() = _hardKeywords
 
+  private val _separators: MutableList<String> = mutableListOf()
+  internal val separators: List<String> get() = _separators
+
+  private val _operators: MutableList<String> = mutableListOf()
+  internal val operators: List<String> get() = _operators
+
   /**
    * The lexer will skip over any strings that match this regex.
    * This acts like a token separator.
@@ -138,6 +144,44 @@ class LexerBuilder internal constructor() {
   fun hardKeywords(vararg keywords: String) {
     this._hardKeywords.addAll(keywords)
   }
+
+  /**
+   * Separators are characters and symbols that act like delimiters to separate other meaningful elements. (Default: [])
+   *
+   * ### Usage
+   *
+   * ```kt
+   * val myLexer = lexer {
+   *   separators(":", ",", "(", ")")
+   * }
+   * ```
+   *
+   * @param separators a variable number of separators
+   * @author Nishant Aanjaney Jalan
+   * @since 0.2.0
+   */
+  fun separators(vararg separators: String) {
+    this._separators.addAll(separators)
+  }
+
+  /**
+   * Operators are characters and symbols that may perform arithmetic or logical operations. (Default: [])
+   *
+   * ### Usage
+   *
+   * ```kt
+   * val myLexer = lexer {
+   *   operators("+", "-", "||", "<=")
+   * }
+   * ```
+   *
+   * @param operators a variable number of operators
+   * @author Nishant Aanjaney Jalan
+   * @since 0.2.0
+   */
+  fun operators(vararg operators: String) {
+    this._operators.addAll(operators)
+  }
 }
 
 /**
@@ -167,6 +211,8 @@ fun lexer(init: LexerBuilder.() -> Unit): Lexer {
     singleLineComments = builder.singleLineComments,
     multilineComments = builder.multilineComments,
     identifiers = builder.identifiers,
-    hardKeywords = builder.hardKeywords.sortedByDescending(String::length)
+    hardKeywords = builder.hardKeywords.sortedByDescending(String::length),
+    operators = builder.operators.sortedByDescending(String::length),
+    separators = builder.separators.sortedByDescending(String::length),
   )
 }

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt

@@ -49,8 +49,8 @@ class Lexer internal constructor(
   private val multilineComments: Pair<Regex, Regex>? = Defaults.multilineComments,
   private val identifiers: Regex = Defaults.identifiers,
   private val hardKeywords: List<String> = emptyList(),
-  private val operators: Set<String> = emptySet(),
-  private val separators: Set<String> = emptySet(),
+  private val operators: List<String> = emptyList(),
+  private val separators: List<String> = emptyList(),
   private val integerLiteral: Regex? = Defaults.integerLiterals,
   private val floatingLiteral: Regex? = Defaults.floatingLiterals,
   private val singleLineString: Set<String> = Defaults.singleLineString,

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilderTest.kt

@@ -91,5 +91,76 @@ class LexerBuilderTest {
       )
     )
   }
+
+  @Test
+  fun `set separators`() {
+    val myLexer = lexer {
+      separators("(", ")", "<", ">", ",", ".")
+    }
+
+
+    myLexer.source = StringSource("Array<List<Set<Int>>>")
+
+    assertTokens(
+      myLexer, listOf(
+        Token.Identifier("Array", start = Position(0, 0), end = Position(0, 4)),
+        Token.Separator("<", start = Position(0, 5), end = Position(0, 5)),
+        Token.Identifier("List", start = Position(0, 6), end = Position(0, 9)),
+        Token.Separator("<", start = Position(0, 10), end = Position(0, 10)),
+        Token.Identifier("Set", start = Position(0, 11), end = Position(0, 13)),
+        Token.Separator("<", start = Position(0, 14), end = Position(0, 14)),
+        Token.Identifier("Int", start = Position(0, 15), end = Position(0, 17)),
+        Token.Separator(">", start = Position(0, 18), end = Position(0, 18)),
+        Token.Separator(">", start = Position(0, 19), end = Position(0, 19)),
+        Token.Separator(">", start = Position(0, 20), end = Position(0, 20)),
+      )
+    )
+  }
+
+  @Test
+  fun `set operators`() {
+    val myLexer = lexer {
+      hardKeywords("val")
+      operators("*", "**", "/", "//", "+", "-", "=", "==")
+    }
+
+    myLexer.source = StringSource("val diff = new - old")
+
+    assertTokens(
+      myLexer, listOf(
+        Token.Keyword(
+          value = "val",
+          start = Position(0, 0),
+          end = Position(0, 2),
+          soft = false
+        ),
+        Token.Identifier(
+          value = "diff",
+          start = Position(0, 4),
+          end = Position(0, 7)
+        ),
+        Token.Operator(
+          value = "=",
+          start = Position(0, 9),
+          end = Position(0, 9)
+        ),
+        Token.Identifier(
+          value = "new",
+          start = Position(0, 11),
+          end = Position(0, 13)
+        ),
+        Token.Operator(
+          value = "-",
+          start = Position(0, 15),
+          end = Position(0, 15)
+        ),
+        Token.Identifier(
+          value = "old",
+          start = Position(0, 17),
+          end = Position(0, 19)
+        )
+      )
+    )
+  }
 }
 

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/internal/LexerLiteralTest.kt

@@ -11,7 +11,7 @@ import kotlin.test.fail
 class LexerLiteralTest {
   private val lexer = Lexer(
     hardKeywords = listOf("char"),
-    operators = setOf("="),
+    operators = listOf("="),
     escapeSequences = listOf(
       Regex("""\\f""") to { _ -> 'f' },
       Regex("""\\n""") to { _ -> '\n' },

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/internal/LexerOperatorTest.kt

@@ -8,7 +8,7 @@ import org.junit.jupiter.api.Test
 class LexerOperatorTest {
   private val lexer = Lexer(
     hardKeywords = listOf("val", "var"),
-    operators = setOf("*", "**", "/", "//", "+", "-", "=", "==")
+    operators = listOf("**", "*", "//", "==", "/", "+", "-", "=")
   )
 
   @Test

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/internal/LexerSeparatorTest.kt

@@ -6,7 +6,7 @@ import io.github.cybercodernaj.parkour.utils.Position
 import org.junit.jupiter.api.Test
 
 class LexerSeparatorTest {
-  private val lexer = Lexer(separators = setOf("(", ")", "<", ">", ",", "."))
+  private val lexer = Lexer(separators = listOf("(", ")", "<", ">", ",", "."))
 
   @Test
   fun `returns a series of valid tokens`() {

docs/core/io.github.cybercodernaj.parkour.lexer/-lexer-builder/hard-keywords.html

@@ -63,7 +63,7 @@
   <div class="cover ">
     <h1 class="cover"><span>hard</span><wbr><span><span>Keywords</span></span></h1>
   </div>
-  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">fun </span><a href="hard-keywords.html"><span class="token function">hardKeywords</span></a><span class="token punctuation">(</span><span class="parameters "><span class="parameter "><span class="token keyword">vararg </span>keywords<span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-string/index.html">String</a></span></span><span class="token punctuation">)</span><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt#L138">source</a>)</span></span></div><p class="paragraph">Hard keywords are a characters and symbols that give a particular meaning to a program. They may not be used as identifiers. (Default: [])</p><h3 class="">Usage</h3><div class="sample-container"><pre><code class="block lang-kt" theme="idea">val myLexer = lexer {<br>  hardKeywords("class", "this", "fun", "keywords")<br>}</code></pre><span class="top-right-position"><span class="copy-icon"></span><div class="copy-popup-wrapper popup-to-left"><span class="copy-popup-icon"></span><span>Content copied to clipboard</span></div></span></div><span class="kdoc-tag"><h4 class="">Author</h4><p class="paragraph">Nishant Aanjaney Jalan</p></span><span class="kdoc-tag"><h4 class="">Since</h4><p class="paragraph">0.2.0</p></span><h4 class="">Parameters</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><u><span><span>keywords</span></span></u></div></span></div><div><div class="title"><p class="paragraph">a variable number of hard keywords</p></div></div></div></div></div></div></div>
+  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">fun </span><a href="hard-keywords.html"><span class="token function">hardKeywords</span></a><span class="token punctuation">(</span><span class="parameters "><span class="parameter "><span class="token keyword">vararg </span>keywords<span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-string/index.html">String</a></span></span><span class="token punctuation">)</span><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt#L144">source</a>)</span></span></div><p class="paragraph">Hard keywords are a characters and symbols that give a particular meaning to a program. They may not be used as identifiers. (Default: [])</p><h3 class="">Usage</h3><div class="sample-container"><pre><code class="block lang-kt" theme="idea">val myLexer = lexer {<br>  hardKeywords("class", "this", "fun", "keywords")<br>}</code></pre><span class="top-right-position"><span class="copy-icon"></span><div class="copy-popup-wrapper popup-to-left"><span class="copy-popup-icon"></span><span>Content copied to clipboard</span></div></span></div><span class="kdoc-tag"><h4 class="">Author</h4><p class="paragraph">Nishant Aanjaney Jalan</p></span><span class="kdoc-tag"><h4 class="">Since</h4><p class="paragraph">0.2.0</p></span><h4 class="">Parameters</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><u><span><span>keywords</span></span></u></div></span></div><div><div class="title"><p class="paragraph">a variable number of hard keywords</p></div></div></div></div></div></div></div>
 </div>
       <div class="footer">
         <span class="go-to-top-icon"><a href="#content" id="go-to-top-link"></a></span><span>© 2024 Nishant Aanjaney Jalan</span><span class="pull-right"><span>Generated by </span><a href="https://github.com/Kotlin/dokka"><span>dokka</span><span class="padded-icon"></span></a></span>

docs/core/io.github.cybercodernaj.parkour.lexer/-lexer-builder/identifiers.html

@@ -63,7 +63,7 @@
   <div class="cover ">
     <h1 class="cover"><span><span>identifiers</span></span></h1>
   </div>
-  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">fun </span><a href="identifiers.html"><span class="token function">identifiers</span></a><span class="token punctuation">(</span><span class="parameters "><span class="parameter ">identifiers<span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a></span></span><span class="token punctuation">)</span><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt#L117">source</a>)</span></span></div><p class="paragraph">Supply the regex pattern that defines the rules for identifiers. Identifiers are parts of the program that are named by the user; for instance, the field name or class name.</p><h3 class="">Usage</h3><div class="sample-container"><pre><code class="block lang-kt" theme="idea">val myLexer = lexer {<br>  // If all identifiers must be at least 2 characters and start with a lowercase english alphabet.<br>  identifiers(Regex("[a-z][a-zA-Z0-9]+"))<br>}</code></pre><span class="top-right-position"><span class="copy-icon"></span><div class="copy-popup-wrapper popup-to-left"><span class="copy-popup-icon"></span><span>Content copied to clipboard</span></div></span></div><span class="kdoc-tag"><h4 class="">Author</h4><p class="paragraph">Nishant Aanjaney Jalan</p></span><span class="kdoc-tag"><h4 class="">Since</h4><p class="paragraph">0.2.0</p></span><h4 class="">Parameters</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><u><span><span>identifiers</span></span></u></div></span></div><div><div class="title"><p class="paragraph">regex defining the rules of the naming identifiers</p></div></div></div></div></div><h4 class="">See also</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><a href="../../io.github.cybercodernaj.parkour.lexer.internal/-lexer/-defaults/identifiers.html"><span>Lexer.</span><wbr><span>Defaults.</span><wbr><span><span>identifiers</span></span></a></div></span></div><div></div></div></div></div></div></div>
+  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">fun </span><a href="identifiers.html"><span class="token function">identifiers</span></a><span class="token punctuation">(</span><span class="parameters "><span class="parameter ">identifiers<span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a></span></span><span class="token punctuation">)</span><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt#L123">source</a>)</span></span></div><p class="paragraph">Supply the regex pattern that defines the rules for identifiers. Identifiers are parts of the program that are named by the user; for instance, the field name or class name.</p><h3 class="">Usage</h3><div class="sample-container"><pre><code class="block lang-kt" theme="idea">val myLexer = lexer {<br>  // If all identifiers must be at least 2 characters and start with a lowercase english alphabet.<br>  identifiers(Regex("[a-z][a-zA-Z0-9]+"))<br>}</code></pre><span class="top-right-position"><span class="copy-icon"></span><div class="copy-popup-wrapper popup-to-left"><span class="copy-popup-icon"></span><span>Content copied to clipboard</span></div></span></div><span class="kdoc-tag"><h4 class="">Author</h4><p class="paragraph">Nishant Aanjaney Jalan</p></span><span class="kdoc-tag"><h4 class="">Since</h4><p class="paragraph">0.2.0</p></span><h4 class="">Parameters</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><u><span><span>identifiers</span></span></u></div></span></div><div><div class="title"><p class="paragraph">regex defining the rules of the naming identifiers</p></div></div></div></div></div><h4 class="">See also</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><a href="../../io.github.cybercodernaj.parkour.lexer.internal/-lexer/-defaults/identifiers.html"><span>Lexer.</span><wbr><span>Defaults.</span><wbr><span><span>identifiers</span></span></a></div></span></div><div></div></div></div></div></div></div>
 </div>
       <div class="footer">
         <span class="go-to-top-icon"><a href="#content" id="go-to-top-link"></a></span><span>© 2024 Nishant Aanjaney Jalan</span><span class="pull-right"><span>Generated by </span><a href="https://github.com/Kotlin/dokka"><span>dokka</span><span class="padded-icon"></span></a></span>