progress

Author: psteinroe

.claude/settings.local.json

@@ -0,0 +1,11 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(grep:*)",
+      "Bash(rg:*)",
+      "Bash(cargo test:*)",
+      "Bash(cargo run:*)"
+    ],
+    "deny": []
+  }
+}

Cargo.lock

@@ -47,6 +47,15 @@ impl From<String> for MessageAndDescription {
     }
 }
 
+impl From<&str> for MessageAndDescription {
+    fn from(description: &str) -> Self {
+        Self {
+            message: markup! { {description} }.to_owned(),
+            description: description.into(),
+        }
+    }
+}
+
 impl From<MarkupBuf> for MessageAndDescription {
     fn from(message: MarkupBuf) -> Self {
         let description = markup_to_string(&message);

crates/pgt_diagnostics/src/display/message.rs

@@ -12,7 +12,9 @@ version              = "0.0.0"
 
 
 [dependencies]
+pgt_diagnostics.workspace       = true
 pgt_lexer_new_codegen.workspace = true
+pgt_text_size.workspace         = true
 pgt_tokenizer.workspace         = true
 
 [dev-dependencies]

crates/pgt_lexer_new/Cargo.toml

@@ -0,0 +1,44 @@
+use pgt_lexer_new::{SyntaxKind, lex};
+
+fn main() {
+    let sql = "SELECT id, name FROM users WHERE active = true;";
+    let lexed = lex(sql);
+
+    println!("Total tokens: {}", lexed.len());
+    println!("\nToken details:");
+
+    // Iterate over tokens
+    for (idx, kind) in lexed.tokens().enumerate() {
+        // Skip whitespace for cleaner output
+        if matches!(
+            kind,
+            SyntaxKind::SPACE | SyntaxKind::TAB | SyntaxKind::NEWLINE
+        ) {
+            continue;
+        }
+
+        let range = lexed.range(idx);
+        let text = lexed.text(idx);
+
+        println!("  [{:3}] {:?} @ {:?} = {:?}", idx, kind, range, text);
+    }
+
+    // Check for errors
+    let errors = lexed.errors();
+    if !errors.is_empty() {
+        println!("\nLexing errors:");
+        for error in errors {
+            println!("  Error at {:?}: {}", error.span, error.message);
+        }
+    } else {
+        println!("\nNo lexing errors found.");
+    }
+
+    // Example: Find all identifiers
+    println!("\nIdentifiers found:");
+    for (idx, kind) in lexed.tokens().enumerate() {
+        if kind == SyntaxKind::IDENT {
+            println!("  - {} at {:?}", lexed.text(idx), lexed.range(idx));
+        }
+    }
+}

crates/pgt_lexer_new/examples/basic_usage.rs

@@ -0,0 +1,41 @@
+use pgt_diagnostics::{Diagnostic, MessageAndDescription};
+use pgt_text_size::TextRange;
+
+/// A specialized diagnostic for lex errors.
+#[derive(Clone, Debug, Diagnostic, PartialEq)]
+#[diagnostic(category = "syntax", severity = Error)]
+pub struct LexError {
+    /// The location where the error is occurred
+    #[location(span)]
+    pub span: TextRange,
+    #[message]
+    #[description]
+    pub message: MessageAndDescription,
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::lex;
+
+    #[test]
+    fn finds_lex_errors() {
+        // Test with unterminated block comment
+        let input = "/* unterminated comment";
+        let lexed = lex(input);
+        let errors = lexed.errors();
+
+        // Should have error for unterminated block comment
+        assert!(!errors.is_empty());
+        assert!(errors[0].message.to_string().contains("Missing trailing"));
+        assert!(errors[0].span.start() < errors[0].span.end());
+
+        // Test with unterminated string
+        let input2 = "SELECT 'unterminated string";
+        let lexed2 = lex(input2);
+        let errors2 = lexed2.errors();
+
+        // Should have error for unterminated string
+        assert!(!errors2.is_empty());
+        assert!(errors2[0].message.to_string().contains("Missing trailing"));
+    }
+}

crates/pgt_lexer_new/src/diagnostics.rs

@@ -31,20 +31,20 @@ impl<'a> LexedStr<'a> {
         conv.finalize_with_eof()
     }
 
-    pub(crate) fn len(&self) -> usize {
+    pub fn len(&self) -> usize {
         self.kind.len() - 1
     }
 
-    pub(crate) fn kind(&self, i: usize) -> SyntaxKind {
+    pub fn kind(&self, i: usize) -> SyntaxKind {
         assert!(i < self.len());
         self.kind[i]
     }
 
-    pub(crate) fn text(&self, i: usize) -> &str {
+    pub fn text(&self, i: usize) -> &str {
         self.range_text(i..i + 1)
     }
 
-    pub(crate) fn range_text(&self, r: ops::Range<usize>) -> &str {
+    pub fn range_text(&self, r: ops::Range<usize>) -> &str {
         assert!(r.start < r.end && r.end <= self.len());
         let lo = self.start[r.start] as usize;
         let hi = self.start[r.end] as usize;

crates/pgt_lexer_new/src/lexed_str.rs

@@ -1,4 +1,161 @@
 mod codegen;
+mod diagnostics;
 mod lexed_str;
 
+use diagnostics::LexError;
+use lexed_str::LexedStr;
+use pgt_text_size::TextRange;
+
 pub use crate::codegen::syntax_kind::SyntaxKind;
+
+/// Result of lexing a string, providing access to tokens and diagnostics
+pub struct Lexed<'a> {
+    inner: LexedStr<'a>,
+}
+
+impl<'a> Lexed<'a> {
+    /// Returns the number of tokens (excluding EOF)
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+
+    /// Returns true if there are no tokens
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Returns an iterator over token kinds
+    pub fn tokens(&self) -> impl Iterator<Item = SyntaxKind> + '_ {
+        (0..self.len()).map(move |i| self.inner.kind(i))
+    }
+
+    /// Returns the kind of token at the given index
+    pub fn kind(&self, idx: usize) -> SyntaxKind {
+        self.inner.kind(idx)
+    }
+
+    /// Returns the text range of token at the given index
+    pub fn range(&self, idx: usize) -> TextRange {
+        let range = self.inner.text_range(idx);
+        TextRange::new(
+            range.start.try_into().unwrap(),
+            range.end.try_into().unwrap(),
+        )
+    }
+
+    /// Returns the text of token at the given index
+    pub fn text(&self, idx: usize) -> &str {
+        self.inner.text(idx)
+    }
+
+    /// Returns all lexing errors with their text ranges
+    pub fn errors(&self) -> Vec<LexError> {
+        self.inner
+            .errors()
+            .map(|(i, msg)| {
+                let range = self.inner.text_range(i);
+                LexError {
+                    message: msg.into(),
+                    span: TextRange::new(
+                        range.start.try_into().unwrap(),
+                        range.end.try_into().unwrap(),
+                    ),
+                }
+            })
+            .collect()
+    }
+}
+
+/// Lex the input string into tokens and diagnostics
+pub fn lex(input: &str) -> Lexed {
+    Lexed {
+        inner: LexedStr::new(input),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basic_lexing() {
+        let input = "SELECT * FROM users WHERE id = 1;";
+        let lexed = lex(input);
+
+        // Check we have tokens
+        assert!(!lexed.is_empty());
+
+        // Iterate over tokens and collect identifiers
+        let mut identifiers = Vec::new();
+        for (idx, kind) in lexed.tokens().enumerate() {
+            if kind == SyntaxKind::IDENT {
+                identifiers.push((lexed.text(idx), lexed.range(idx)));
+            }
+        }
+
+        // Should find at least "users" and "id" as identifiers
+        assert!(identifiers.len() >= 2);
+    }
+
+    #[test]
+    fn test_lexing_with_errors() {
+        let input = "SELECT 'unterminated string";
+        let lexed = lex(input);
+
+        // Should have tokens
+        assert!(!lexed.is_empty());
+
+        // Should have an error for unterminated string
+        let errors = lexed.errors();
+        assert!(!errors.is_empty());
+        // Check the error message exists
+        assert!(!errors[0].message.to_string().is_empty());
+    }
+
+    #[test]
+    fn test_token_ranges() {
+        let input = "SELECT id";
+        let lexed = lex(input);
+
+        // First token should be a keyword (SELECT gets parsed as a keyword)
+        let _first_kind = lexed.kind(0);
+        assert_eq!(u32::from(lexed.range(0).start()), 0);
+        assert_eq!(u32::from(lexed.range(0).end()), 6);
+        assert_eq!(lexed.text(0), "SELECT");
+
+        // Find the id token
+        for (idx, kind) in lexed.tokens().enumerate() {
+            if kind == SyntaxKind::IDENT && lexed.text(idx) == "id" {
+                assert_eq!(u32::from(lexed.range(idx).start()), 7);
+                assert_eq!(u32::from(lexed.range(idx).end()), 9);
+            }
+        }
+    }
+
+    #[test]
+    fn test_empty_input() {
+        let input = "";
+        let lexed = lex(input);
+        assert!(lexed.is_empty());
+        assert_eq!(lexed.len(), 0);
+    }
+
+    #[test]
+    fn test_whitespace_handling() {
+        let input = "  SELECT  \n  id  ";
+        let lexed = lex(input);
+
+        // Collect non-whitespace tokens
+        let mut non_whitespace = Vec::new();
+        for (idx, kind) in lexed.tokens().enumerate() {
+            if !matches!(
+                kind,
+                SyntaxKind::SPACE | SyntaxKind::TAB | SyntaxKind::NEWLINE
+            ) {
+                non_whitespace.push(lexed.text(idx));
+            }
+        }
+
+        assert_eq!(non_whitespace.len(), 2); // SELECT and id
+    }
+}