refactor(ecma/lexer): split lexer of parser

Author: bvanjoi

Cargo.lock

@@ -0,0 +1,66 @@
+[package]
+authors       = ["강동윤 <kdy1997.dev@gmail.com>"]
+description   = "Feature-complete es2019 lexer."
+documentation = "https://rustdoc.swc.rs/swc_ecma_lexer/"
+edition       = { workspace = true }
+include       = ["Cargo.toml", "src/**/*.rs", "examples/**/*.rs"]
+license       = { workspace = true }
+name          = "swc_ecma_lexer"
+repository    = { workspace = true }
+version       = "11.1.2"
+
+  [package.metadata.docs.rs]
+  all-features = true
+  rustdoc-args = ["--cfg", "docsrs"]
+
+[lib]
+bench = false
+
+[features]
+# Used for debugging
+debug         = ["tracing-spans"]
+default       = ["typescript", "stacker"]
+tracing-spans = []
+typescript    = []
+verify        = ["swc_ecma_visit"]
+
+[dependencies]
+arrayvec    = { workspace = true }
+bitflags    = { workspace = true }
+either      = { workspace = true }
+num-bigint  = { workspace = true }
+num-traits  = { workspace = true }
+rustc-hash  = { workspace = true }
+serde       = { workspace = true, features = ["derive"] }
+smallvec    = { workspace = true }
+smartstring = { workspace = true }
+tracing     = { workspace = true }
+typed-arena = { workspace = true }
+
+new_debug_unreachable = { workspace = true }
+phf                   = { workspace = true, features = ["macros"] }
+swc_atoms             = { version = "5.0.0", path = "../swc_atoms" }
+swc_common            = { version = "8.1.0", path = "../swc_common" }
+swc_ecma_ast          = { version = "8.1.2", path = "../swc_ecma_ast" }
+swc_ecma_visit        = { version = "8.0.0", path = "../swc_ecma_visit", optional = true }
+
+[target.'cfg(not(any(target_arch = "wasm32", target_arch = "arm")))'.dependencies]
+stacker = { version = "0.1.15", optional = true }
+
+[dev-dependencies]
+criterion         = { workspace = true }
+pretty_assertions = { workspace = true }
+serde_json        = { workspace = true }
+walkdir           = { workspace = true }
+
+codspeed-criterion-compat = { workspace = true }
+swc_ecma_ast = { version = "8.1.2", path = "../swc_ecma_ast", features = [
+  "serde-impl",
+] }
+swc_ecma_visit = { version = "8.0.0", path = "../swc_ecma_visit" }
+swc_malloc = { version = "1.2.2", path = "../swc_malloc" }
+testing = { version = "9.0.0", path = "../testing" }
+
+[[bench]]
+harness = false
+name    = "lexer"

crates/swc_ecma_lexer/Cargo.toml

@@ -1,8 +1,8 @@
 extern crate swc_malloc;
 
 use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
-use swc_common::FileName;
-use swc_ecma_parser::{lexer::Lexer, StringInput, Syntax, TsSyntax};
+use swc_common::{input::StringInput, FileName};
+use swc_ecma_lexer::{lexer::Lexer, Syntax, TsSyntax};
 
 fn bench_module(b: &mut Bencher, syntax: Syntax, src: &'static str) {
     let _ = ::testing::run_test(false, |cm, _| {
@@ -21,66 +21,74 @@ fn bench_module(b: &mut Bencher, syntax: Syntax, src: &'static str) {
 fn bench_files(c: &mut Criterion) {
     c.bench_function("es/lexer/colors", |b| {
         // Copied from ratel-rust
-        bench_module(b, Default::default(), include_str!("../colors.js"))
+        bench_module(
+            b,
+            Default::default(),
+            include_str!("../../swc_ecma_parser/colors.js"),
+        )
     });
 
     c.bench_function("es/lexer/angular", |b| {
         bench_module(
             b,
             Default::default(),
-            include_str!("./files/angular-1.2.5.js"),
+            include_str!("../../swc_ecma_parser/benches/files/angular-1.2.5.js"),
         )
     });
 
     c.bench_function("es/lexer/backbone", |b| {
         bench_module(
             b,
             Default::default(),
-            include_str!("./files/backbone-1.1.0.js"),
+            include_str!("../../swc_ecma_parser/benches/files/backbone-1.1.0.js"),
         )
     });
 
     c.bench_function("es/lexer/jquery", |b| {
         bench_module(
             b,
             Default::default(),
-            include_str!("./files/jquery-1.9.1.js"),
+            include_str!("../../swc_ecma_parser/benches/files/jquery-1.9.1.js"),
         )
     });
 
     c.bench_function("es/lexer/jquery mobile", |b| {
         bench_module(
             b,
             Default::default(),
-            include_str!("./files/jquery.mobile-1.4.2.js"),
+            include_str!("../../swc_ecma_parser/benches/files/jquery.mobile-1.4.2.js"),
         )
     });
     c.bench_function("es/lexer/mootools", |b| {
         bench_module(
             b,
             Default::default(),
-            include_str!("./files/mootools-1.4.5.js"),
+            include_str!("../../swc_ecma_parser/benches/files/mootools-1.4.5.js"),
         )
     });
 
     c.bench_function("es/lexer/underscore", |b| {
         bench_module(
             b,
             Default::default(),
-            include_str!("./files/underscore-1.5.2.js"),
+            include_str!("../../swc_ecma_parser/benches/files/underscore-1.5.2.js"),
         )
     });
 
     c.bench_function("es/lexer/three", |b| {
         bench_module(
             b,
             Default::default(),
-            include_str!("./files/three-0.138.3.js"),
+            include_str!("../../swc_ecma_parser/benches/files/three-0.138.3.js"),
         )
     });
 
     c.bench_function("es/lexer/yui", |b| {
-        bench_module(b, Default::default(), include_str!("./files/yui-3.12.0.js"))
+        bench_module(
+            b,
+            Default::default(),
+            include_str!("../../swc_ecma_parser/benches/files/yui-3.12.0.js"),
+        )
     });
 
     c.bench_function("es/lexer/cal-com", |b| {
@@ -90,12 +98,16 @@ fn bench_files(c: &mut Criterion) {
                 tsx: true,
                 ..Default::default()
             }),
-            include_str!("./files/cal.com.tsx"),
+            include_str!("../../swc_ecma_parser/benches/files/cal.com.tsx"),
         )
     });
 
     c.bench_function("es/lexer/typescript", |b| {
-        bench_module(b, Default::default(), include_str!("./files/typescript.js"))
+        bench_module(
+            b,
+            Default::default(),
+            include_str!("../../swc_ecma_parser/benches/files/typescript.js"),
+        )
     });
 }
 

crates/swc_ecma_parser/benches/lexer.rs renamed to crates/swc_ecma_lexer/benches/lexer.rs

@@ -24,7 +24,7 @@ impl Spanned for Error {
 
 impl Error {
     #[cold]
-    pub(crate) fn new(span: Span, error: SyntaxError) -> Self {
+    pub fn new(span: Span, error: SyntaxError) -> Self {
         Self {
             error: Box::new((span, error)),
         }