Skip to content

Commit 2dfe152

Browse files
cybercoder-najcybercoder-naj
committed
feat: revamped Token sealed class and introduced integer literals
1 parent dc53056 commit 2dfe152

File tree

9 files changed

+149
-59
lines changed

9 files changed

+149
-59
lines changed

‎src/main/kotlin/io/github/cybercodernaj/parkour/lexer/Lexer.kt

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,12 @@ class Lexer(
119119
position = end + 1
120120
}
121121

122+
tryLiterals()
123+
?.let { token ->
124+
tokenStream.add(token)
125+
position = token.end + 1
126+
}
127+
122128
(position startsWith _operators)
123129
?.let { keyword ->
124130
val end = position.copy(col = position.col + keyword.length - 1)
@@ -148,6 +154,32 @@ class Lexer(
148154
adjustPositionIfNeeded()
149155
}
150156

157+
private fun tryLiterals(): Token.Literal? {
158+
// (position pointsAt literals.floatingLiteral)
159+
// ?.let { match ->
160+
// if (match.value.isBlank())
161+
// return null
162+
//
163+
// val end = position.copy(col = match.range.last)
164+
// match.value.toDoubleOrNull()?.let { value ->
165+
// return Token.Literal.FloatLiteral(value, position, end)
166+
// } ?: throw LexicalException("Double regex is badly formed.")
167+
// }
168+
169+
(position pointsAt literals.integerLiteral)
170+
?.let { match ->
171+
if (match.value.isBlank())
172+
return null
173+
174+
val end = position.copy(col = match.range.last)
175+
match.value.toLongOrNull()?.let { value ->
176+
return Token.Literal.IntLiteral(value, position, end)
177+
} ?: throw LexicalException("Int regex is badly formed.")
178+
}
179+
180+
return null
181+
}
182+
151183
private fun fetchNextLine() {
152184
_currentLine = source.fetchLine(position.line)
153185
}

‎src/main/kotlin/io/github/cybercodernaj/parkour/lexer/Literals.kt

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ package io.github.cybercodernaj.parkour.lexer
55
* @author Nishant Aanjaney Jalan
66
* @since 0.1.0
77
*/
8-
class Literals {
9-
10-
}
8+
class Literals(
9+
internal val integerLiteral: Regex = Regex("""-?[0-9_]+[lL]?"""),
10+
internal val floatingLiteral: Regex = Regex("""[+-]?[0-9_]*(\.[0-9_]+)?([eE][-+]?[0-9_]+)?[fF]?"""),
11+
internal val characterEnclosure: String = "'",
12+
internal val stringEnclosure: String = "\"",
13+
internal val booleanLiteral: Regex = Regex("true|false"),
14+
internal val escapeSequences: Map<String, Int> = mapOf(),
15+
)

‎src/main/kotlin/io/github/cybercodernaj/parkour/lexer/Token.kt

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,48 @@ import io.github.cybercodernaj.parkour.utils.Position
88
* @author Nishant Aanjaney Jalan
99
* @since 0.1.0
1010
*/
11-
internal sealed class Token {
12-
data class Identifier(val name: String, val start: Position, val end: Position) : Token()
13-
data class Keyword(val name: String, val start: Position, val end: Position, val soft: Boolean = false) : Token()
14-
data class Operator(val name: String, val start: Position, val end: Position) : Token()
15-
data class Separator(val name: String, val start: Position, val end: Position) : Token()
16-
data object EOF : Token()
11+
internal sealed class Token(open val value: Any?, open val start: Position?, open val end: Position?) {
12+
class Identifier(override val value: String, start: Position, end: Position) : Token(value, start, end)
13+
14+
class Keyword(override val value: String, start: Position, end: Position, val soft: Boolean = false) : Token(value, start, end)
15+
16+
class Operator(override val value: String, start: Position, end: Position) : Token(value, start, end)
17+
18+
class Separator(override val value: String, start: Position, end: Position) : Token(value, start, end)
19+
20+
sealed class Literal(
21+
override val value: Any,
22+
override val start: Position,
23+
override val end: Position
24+
) : Token(value, start, end) {
25+
class IntLiteral(override val value: Long, start: Position, end: Position) : Literal(value, start, end)
26+
class FloatLiteral(override val value: Double, start: Position, end: Position) : Literal(value, start, end)
27+
class StringLiteral(override val value: String, start: Position, end: Position) : Literal(value, start, end)
28+
class CharacterLiteral(override val value: Char, start: Position, end: Position) : Literal(value, start, end)
29+
class BooleanLiteral(override val value: Boolean, start: Position, end: Position) : Literal(value, start, end)
30+
}
31+
32+
data object EOF : Token(null, null, null)
33+
34+
override fun equals(other: Any?): Boolean {
35+
if (this === other) return true
36+
if (other !is Token) return false
37+
38+
if (value != other.value) return false
39+
if (start != other.start) return false
40+
if (end != other.end) return false
41+
42+
return true
43+
}
44+
45+
override fun hashCode(): Int {
46+
var result = value?.hashCode() ?: 0
47+
result = 31 * result + (start?.hashCode() ?: 0)
48+
result = 31 * result + (end?.hashCode() ?: 0)
49+
return result
50+
}
51+
52+
override fun toString(): String {
53+
return "${this::class.simpleName}(value=$value, start=$start, end=$end)"
54+
}
1755
}

‎src/test/kotlin/io/github/cybercodernaj/parkour/lexer/LexerCommentsTest.kt

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ class LexerCommentsTest {
2626
val token = lexer.nextToken()
2727
assertEquals(
2828
Token.Identifier(
29-
name = "age",
29+
value = "age",
3030
start = Position(0, 0),
3131
end = Position(0, 2)
3232
),
@@ -41,7 +41,7 @@ class LexerCommentsTest {
4141
val token = lexer.nextToken()
4242
assertEquals(
4343
Token.Identifier(
44-
name = "age",
44+
value = "age",
4545
start = Position(0, 0),
4646
end = Position(0, 2)
4747
),
@@ -51,7 +51,7 @@ class LexerCommentsTest {
5151
val token2 = lexer.nextToken()
5252
assertEquals(
5353
Token.Identifier(
54-
name = "hello",
54+
value = "hello",
5555
start = Position(1, 0),
5656
end = Position(1, 4)
5757
),
@@ -66,7 +66,7 @@ class LexerCommentsTest {
6666
val token = lexer.nextToken()
6767
assertEquals(
6868
Token.Identifier(
69-
name = "age",
69+
value = "age",
7070
start = Position(0, 0),
7171
end = Position(0, 2)
7272
),
@@ -76,7 +76,7 @@ class LexerCommentsTest {
7676
val token2 = lexer.nextToken()
7777
assertEquals(
7878
Token.Identifier(
79-
name = "hello",
79+
value = "hello",
8080
start = Position(1, 0),
8181
end = Position(1, 4)
8282
),
@@ -99,7 +99,7 @@ class LexerCommentsTest {
9999
val token = lexer.nextToken()
100100
assertEquals(
101101
Token.Identifier(
102-
name = "age",
102+
value = "age",
103103
start = Position(0, 0),
104104
end = Position(0, 2)
105105
),
@@ -109,7 +109,7 @@ class LexerCommentsTest {
109109
val token2 = lexer.nextToken()
110110
assertEquals(
111111
Token.Identifier(
112-
name = "hello",
112+
value = "hello",
113113
start = Position(6, 0),
114114
end = Position(6, 4)
115115
),
@@ -131,7 +131,7 @@ class LexerCommentsTest {
131131
val token = lexer.nextToken()
132132
assertEquals(
133133
Token.Identifier(
134-
name = "age",
134+
value = "age",
135135
start = Position(0, 0),
136136
end = Position(0, 2)
137137
),

‎src/test/kotlin/io/github/cybercodernaj/parkour/lexer/LexerIdentifierTest.kt

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ class LexerIdentifierTest {
2626
val token = lexer.nextToken()
2727
assertEquals(
2828
Token.Identifier(
29-
name = "name",
29+
value = "name",
3030
start = Position(0, 0),
3131
end = Position(0, 3)
3232
),
@@ -41,7 +41,7 @@ class LexerIdentifierTest {
4141
val token = lexer.nextToken()
4242
assertEquals(
4343
Token.Identifier(
44-
name = "name",
44+
value = "name",
4545
start = Position(0, 0),
4646
end = Position(0, 3)
4747
),
@@ -51,7 +51,7 @@ class LexerIdentifierTest {
5151
val token2 = lexer.nextToken()
5252
assertEquals(
5353
Token.Identifier(
54-
name = "age",
54+
value = "age",
5555
start = Position(0, 5),
5656
end = Position(0, 7)
5757
),
@@ -61,7 +61,7 @@ class LexerIdentifierTest {
6161
val token3 = lexer.nextToken()
6262
assertEquals(
6363
Token.Identifier(
64-
name = "hello",
64+
value = "hello",
6565
start = Position(0, 9),
6666
end = Position(0, 13)
6767
),
@@ -76,7 +76,7 @@ class LexerIdentifierTest {
7676
val token = lexer.nextToken()
7777
assertEquals(
7878
Token.Identifier(
79-
name = "name",
79+
value = "name",
8080
start = Position(0, 0),
8181
end = Position(0, 3)
8282
),
@@ -105,7 +105,7 @@ class LexerIdentifierTest {
105105
val token = lexer.nextToken()
106106
assertEquals(
107107
Token.Identifier(
108-
name = "name",
108+
value = "name",
109109
start = Position(0, 0),
110110
end = Position(0, 3)
111111
),
@@ -115,7 +115,7 @@ class LexerIdentifierTest {
115115
val token2 = lexer.nextToken()
116116
assertEquals(
117117
Token.Identifier(
118-
name = "name2",
118+
value = "name2",
119119
start = Position(1, 2),
120120
end = Position(1, 6)
121121
),
@@ -125,7 +125,7 @@ class LexerIdentifierTest {
125125
val token3 = lexer.nextToken()
126126
assertEquals(
127127
Token.Identifier(
128-
name = "name3",
128+
value = "name3",
129129
start = Position(2, 2),
130130
end = Position(2, 6)
131131
),

‎src/test/kotlin/io/github/cybercodernaj/parkour/lexer/LexerKeywordTest.kt

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ class LexerKeywordTest {
1515
val token = lexer.nextToken()
1616
assertEquals(
1717
Token.Keyword(
18-
name = "val",
18+
value = "val",
1919
start = Position(0, 0),
2020
end = Position(0, 2),
2121
soft = false
@@ -31,7 +31,7 @@ class LexerKeywordTest {
3131
val token = lexer.nextToken()
3232
assertEquals(
3333
Token.Keyword(
34-
name = "val",
34+
value = "val",
3535
start = Position(0, 0),
3636
end = Position(0, 2),
3737
soft = false
@@ -42,7 +42,7 @@ class LexerKeywordTest {
4242
val token2 = lexer.nextToken()
4343
assertEquals(
4444
Token.Keyword(
45-
name = "class",
45+
value = "class",
4646
start = Position(0, 4),
4747
end = Position(0, 8),
4848
soft = false
@@ -58,7 +58,7 @@ class LexerKeywordTest {
5858
val token = lexer.nextToken()
5959
assertEquals(
6060
Token.Keyword(
61-
name = "val",
61+
value = "val",
6262
start = Position(0, 0),
6363
end = Position(0, 2),
6464
soft = false
@@ -69,7 +69,7 @@ class LexerKeywordTest {
6969
val token2 = lexer.nextToken()
7070
assertEquals(
7171
Token.Keyword(
72-
name = "class",
72+
value = "class",
7373
start = Position(1, 1),
7474
end = Position(1, 5),
7575
soft = false
@@ -85,7 +85,7 @@ class LexerKeywordTest {
8585
val token = lexer.nextToken()
8686
assertEquals(
8787
Token.Keyword(
88-
name = "val",
88+
value = "val",
8989
start = Position(0, 0),
9090
end = Position(0, 2),
9191
soft = false
@@ -96,7 +96,7 @@ class LexerKeywordTest {
9696
val token2 = lexer.nextToken()
9797
assertEquals(
9898
Token.Identifier(
99-
name = "age",
99+
value = "age",
100100
start = Position(0, 4),
101101
end = Position(0, 6)
102102
),

‎src/test/kotlin/io/github/cybercodernaj/parkour/lexer/LexerLiteralTest.kt

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package io.github.cybercodernaj.parkour.lexer
2+
3+
import io.github.cybercodernaj.parkour.datasource.StringSource
4+
import io.github.cybercodernaj.parkour.utils.Position
5+
import org.junit.jupiter.api.Test
6+
import kotlin.test.assertEquals
7+
8+
class LexerLiteralTest {
9+
private val lexer = Lexer()
10+
11+
@Test
12+
fun `returns an int number`() {
13+
lexer.source = StringSource("12345")
14+
15+
val token = lexer.nextToken()
16+
assertEquals(Token.Literal.IntLiteral(12345L, Position(0, 0), Position(0, 4)), token)
17+
}
18+
}

0 commit comments

Comments
 (0)