progress

Author: psteinroe

Cargo.lock

@@ -71,8 +71,8 @@ pgt_diagnostics_categories = { path = "./crates/pgt_diagnostics_categories", ver
 pgt_diagnostics_macros     = { path = "./crates/pgt_diagnostics_macros", version = "0.0.0" }
 pgt_flags                  = { path = "./crates/pgt_flags", version = "0.0.0" }
 pgt_fs                     = { path = "./crates/pgt_fs", version = "0.0.0" }
-pgt_lexer_new              = { path = "./crates/pgt_lexer_new", version = "0.0.0" }
-pgt_lexer_new_codegen      = { path = "./crates/pgt_lexer_new_codegen", version = "0.0.0" }
+pgt_lexer                  = { path = "./crates/pgt_lexer", version = "0.0.0" }
+pgt_lexer_codegen          = { path = "./crates/pgt_lexer_codegen", version = "0.0.0" }
 pgt_lsp                    = { path = "./crates/pgt_lsp", version = "0.0.0" }
 pgt_markup                 = { path = "./crates/pgt_markup", version = "0.0.0" }
 pgt_query_ext              = { path = "./crates/pgt_query_ext", version = "0.0.0" }

Cargo.toml

@@ -6,16 +6,16 @@ edition.workspace    = true
 homepage.workspace   = true
 keywords.workspace   = true
 license.workspace    = true
-name                 = "pgt_lexer_new"
+name                 = "pgt_lexer"
 repository.workspace = true
 version              = "0.0.0"
 
 
 [dependencies]
-pgt_diagnostics.workspace       = true
-pgt_lexer_new_codegen.workspace = true
-pgt_text_size.workspace         = true
-pgt_tokenizer.workspace         = true
+pgt_diagnostics.workspace   = true
+pgt_lexer_codegen.workspace = true
+pgt_text_size.workspace     = true
+pgt_tokenizer.workspace     = true
 
 [dev-dependencies]
 insta.workspace = true

crates/pgt_lexer_new/Cargo.toml renamed to crates/pgt_lexer/Cargo.toml

@@ -0,0 +1 @@
+pgt_lexer_codegen::syntax_kind_codegen!();

crates/pgt_lexer_new/README.md renamed to crates/pgt_lexer/README.md

@@ -6,7 +6,7 @@ edition.workspace    = true
 homepage.workspace   = true
 keywords.workspace   = true
 license.workspace    = true
-name                 = "pgt_lexer_new_codegen"
+name                 = "pgt_lexer_codegen"
 repository.workspace = true
 version              = "0.0.0"
 

crates/pgt_lexer_new/src/codegen/mod.rs renamed to crates/pgt_lexer/src/codegen/mod.rs

@@ -13,7 +13,7 @@ version              = "0.0.0"
 
 [dependencies]
 pgt_diagnostics         = { workspace = true }
-pgt_lexer_new.workspace = true
+pgt_lexer.workspace     = true
 pgt_query_ext.workspace = true
 pgt_text_size.workspace = true
 regex.workspace         = true

crates/pgt_lexer/src/codegen/syntax_kind.rs

@@ -1,5 +1,5 @@
 use pgt_diagnostics::{Diagnostic, MessageAndDescription};
-use pgt_lexer_new::{LexDiagnostic, Lexed};
+use pgt_lexer::{LexDiagnostic, Lexed};
 use pgt_text_size::TextRange;
 
 use crate::splitter::SplitError;

crates/pgt_lexer_new/src/lexed.rs renamed to crates/pgt_lexer/src/lexed.rs

@@ -5,7 +5,7 @@ pub mod diagnostics;
 mod splitter;
 
 use diagnostics::SplitDiagnostic;
-use pgt_lexer_new::Lexer;
+use pgt_lexer::Lexer;
 use pgt_text_size::TextRange;
 use splitter::{Splitter, source};
 
@@ -42,7 +42,7 @@ pub fn split(sql: &str) -> SplitResult {
 mod tests {
     use diagnostics::SplitDiagnostic;
     use ntest::timeout;
-    use pgt_lexer_new::SyntaxKind;
+    use pgt_lexer::SyntaxKind;
     use pgt_text_size::TextRange;
 
     use super::*;

crates/pgt_lexer_new/src/lexer.rs renamed to crates/pgt_lexer/src/lexer.rs

@@ -5,7 +5,7 @@ mod dml;
 
 pub use common::source;
 
-use pgt_lexer_new::{Lexed, SyntaxKind};
+use pgt_lexer::{Lexed, SyntaxKind};
 use pgt_text_size::TextRange;
 
 pub struct SplitResult {
@@ -39,7 +39,6 @@ pub struct Splitter<'a> {
 
 impl<'a> Splitter<'a> {
     pub fn new(lexed: &'a Lexed<'a>) -> Self {
-        println!("{:?}", lexed.tokens().collect::<Vec<_>>());
         Self {
             lexed,
             current_pos: 0,

crates/pgt_lexer_new/src/lib.rs renamed to crates/pgt_lexer/src/lib.rs

@@ -1,5 +1,5 @@
 use super::TRIVIA_TOKENS;
-use pgt_lexer_new::SyntaxKind;
+use pgt_lexer::SyntaxKind;
 
 use super::{
     Splitter,

crates/pgt_lexer_new_codegen/Cargo.toml renamed to crates/pgt_lexer_codegen/Cargo.toml

@@ -1,4 +1,4 @@
-use pgt_lexer_new::SyntaxKind;
+use pgt_lexer::SyntaxKind;
 
 // All tokens listed here must be explicitly handled in the `unknown` function to ensure that we do
 // not break in the middle of another statement that contains a statement start token.

crates/pgt_lexer_new_codegen/README.md renamed to crates/pgt_lexer_codegen/README.md

@@ -1,4 +1,4 @@
-use pgt_lexer_new::SyntaxKind;
+use pgt_lexer::SyntaxKind;
 
 use super::{Splitter, common::unknown};
 

crates/pgt_lexer_new_codegen/build.rs renamed to crates/pgt_lexer_codegen/build.rs

@@ -1,4 +1,4 @@
-use pgt_lexer_new::SyntaxKind;
+use pgt_lexer::SyntaxKind;
 
 use super::{
     Splitter,

crates/pgt_lexer_new_codegen/postgres/17-6.1.0/kwlist.h renamed to crates/pgt_lexer_codegen/postgres/17-6.1.0/kwlist.h

@@ -25,7 +25,7 @@ pgt_configuration         = { workspace = true }
 pgt_console               = { workspace = true }
 pgt_diagnostics           = { workspace = true }
 pgt_fs                    = { workspace = true, features = ["serde"] }
-pgt_lexer_new             = { workspace = true }
+pgt_lexer                 = { workspace = true }
 pgt_query_ext             = { workspace = true }
 pgt_schema_cache          = { workspace = true }
 pgt_statement_splitter    = { workspace = true }

crates/pgt_lexer_new_codegen/src/keywords.rs renamed to crates/pgt_lexer_codegen/src/keywords.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use dashmap::DashMap;
-use pgt_lexer_new::SyntaxKind;
+use pgt_lexer::SyntaxKind;
 
 use super::statement_identifier::StatementId;
 
@@ -29,7 +29,7 @@ impl AnnotationStore {
             return existing;
         }
 
-        let lexed = pgt_lexer_new::lex(content);
+        let lexed = pgt_lexer::lex(content);
 
         let mut ends_with_semicolon = false;
 

crates/pgt_lexer_new_codegen/src/lib.rs renamed to crates/pgt_lexer_codegen/src/lib.rs

@@ -445,9 +445,7 @@ fn get_affected(content: &str, range: TextRange) -> &str {
 
 #[cfg(test)]
 mod tests {
-
     use super::*;
-    use pgt_diagnostics::Diagnostic;
     use pgt_text_size::TextRange;
 
     use crate::workspace::{ChangeFileParams, ChangeParams};