From 3d8c9c809542665df13d13aea599a77b4c8a5fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 12 Jul 2024 01:18:21 +0200 Subject: [PATCH 01/21] feat: Add native SourceMap support --- Cargo.lock | 13 +++++-- core/Cargo.toml | 1 + core/error.rs | 2 +- core/ops_builtin_v8.rs | 13 ++++--- core/runtime/jsruntime.rs | 5 +++ core/source_map.rs | 74 +++++++++++++++++++++++++++++++++++++++ foo.js | 6 ++++ 7 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 foo.js diff --git a/Cargo.lock b/Cargo.lock index f90c91978..b06ed10ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,6 +149,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64-simd" version = "0.7.0" @@ -519,7 +525,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f340bc18a3c351cb4af21e56a8af13cdcc08ff7616799429c745bce26fea6385" dependencies = [ "anyhow", - "base64", + "base64 0.21.7", "deno_media_type", "deno_terminal", "dprint-swc-ext", @@ -557,6 +563,7 @@ name = "deno_core" version = "0.293.0" dependencies = [ "anyhow", + "base64 0.22.1", "bencher", "bincode", "bit-set", @@ -737,7 +744,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63dd7b57f9b33b1741fa631c9522eb35d43e96dcca4a6a91d5e4ca7c93acdc1" dependencies = [ - "base64", + "base64 0.21.7", "http-body-util", "hyper", "hyper-util", @@ -2275,7 +2282,7 @@ version = "0.183.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b7b7de90ff41560bf021acda3fb16fb0f4f5885aeb44b6b7e638b563124d087" dependencies = [ - "base64", + "base64 0.21.7", "dashmap", "indexmap", "once_cell", diff --git a/core/Cargo.toml b/core/Cargo.toml index f0453fba8..ce25a10aa 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -24,6 +24,7 @@ snapshot_flags_eager_parse = [] [dependencies] anyhow.workspace = true +base64 = "0.22.1" bincode.workspace = true bit-set.workspace = true bit-vec.workspace = true diff --git a/core/error.rs b/core/error.rs index 9b5340886..16abf6fd8 100644 --- a/core/error.rs +++ b/core/error.rs @@ -249,7 +249,7 @@ impl JsStackFrame { let c = message.get_start_column() as u32 + 1; let state = JsRuntime::state_from(scope); let mut source_mapper = state.source_mapper.borrow_mut(); - match source_mapper.apply_source_map(&f, l, c) { + match source_mapper.apply_source_map(scope, &f, l, c) { SourceMapApplication::Unchanged => Some(JsStackFrame::from_location( Some(f), Some(l.into()), diff --git a/core/ops_builtin_v8.rs b/core/ops_builtin_v8.rs index e02fe131e..ecd740e67 100644 --- a/core/ops_builtin_v8.rs +++ b/core/ops_builtin_v8.rs @@ -1047,9 +1047,10 @@ fn write_line_and_col_to_ret_buf( // 2: mapped line, column, and file name. new line, column, and file name are in // ret_buf. retrieve file name by calling `op_apply_source_map_filename` // immediately after this op returns. -#[op2(fast)] +#[op2] #[smi] pub fn op_apply_source_map( + scope: &mut v8::HandleScope, state: &JsRuntimeState, #[string] file_name: &str, #[smi] line_number: u32, @@ -1060,8 +1061,12 @@ pub fn op_apply_source_map( return Err(type_error("retBuf must be 8 bytes")); } let mut source_mapper = state.source_mapper.borrow_mut(); - let application = - source_mapper.apply_source_map(file_name, line_number, column_number); + let application = source_mapper.apply_source_map( + scope, + file_name, + line_number, + column_number, + ); match application { SourceMapApplication::Unchanged => Ok(0), SourceMapApplication::LineAndColumn { @@ -1137,7 +1142,7 @@ pub fn op_current_user_call_site( let application = js_runtime_state .source_mapper .borrow_mut() - .apply_source_map(&file_name, line_number, column_number); + .apply_source_map(scope, &file_name, line_number, column_number); match application { SourceMapApplication::Unchanged => { diff --git a/core/runtime/jsruntime.rs b/core/runtime/jsruntime.rs index ce658acae..430dae7e8 100644 --- a/core/runtime/jsruntime.rs +++ b/core/runtime/jsruntime.rs @@ -922,6 +922,11 @@ impl JsRuntime { } } + state_rc + .source_mapper + .borrow_mut() + .set_module_map(module_map.clone()); + // SAFETY: Set the module map slot in the context unsafe { context.set_aligned_pointer_in_embedder_data( diff --git a/core/source_map.rs b/core/source_map.rs index 396e21370..84bffe960 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -2,7 +2,11 @@ //! This mod provides functions to remap a `JsError` based on a source map. +use crate::modules::ModuleMap; use crate::resolve_url; +use crate::RequestedModuleType; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; pub use sourcemap::SourceMap; use std::borrow::Cow; use std::collections::HashMap; @@ -62,6 +66,7 @@ pub struct SourceMapper { // This is not the right place for this, but it's the easiest way to make // op_apply_source_map a fast op. This stashing should happen in #[op2]. pub(crate) stashed_file_name: Option, + pub(crate) maybe_module_map: Option>, } impl SourceMapper { @@ -72,13 +77,61 @@ impl SourceMapper { ext_source_maps: Default::default(), getter, stashed_file_name: Default::default(), + maybe_module_map: None, } } + pub fn set_module_map(&mut self, module_map: Rc) { + assert!(self.maybe_module_map.replace(module_map).is_none()); + } + pub fn has_user_sources(&self) -> bool { self.getter.is_some() } + pub fn apply_source_map_from_module_map( + &mut self, + scope: &mut v8::HandleScope, + file_name: &str, + line_number: u32, + column_number: u32, + ) -> Option { + let module_map = self.maybe_module_map.as_ref()?; + let id = module_map.get_id(file_name, RequestedModuleType::None)?; + + let module_handle = module_map.get_handle(id).unwrap(); + let module = v8::Local::new(scope, module_handle); + let unbound_module_script = module.get_unbound_module_script(scope); + let maybe_source_mapping_url = + unbound_module_script.get_source_mapping_url(scope); + + // TODO(bartlomieju): decide which one should be checked first + let maybe_source_url = unbound_module_script + .get_source_url(scope) + .to_rust_string_lossy(scope); + eprintln!("maybe source url {}", maybe_source_url); + + if !maybe_source_mapping_url.is_string() { + return None; + } + + let source_map_string = + maybe_source_mapping_url.to_rust_string_lossy(scope); + eprintln!("maybe source mapping url {}", source_map_string); + + let b64 = + source_map_string.strip_prefix("data:application/json;base64,")?; + let decoded_b64 = BASE64_STANDARD.decode(b64).ok()?; + let source_map = SourceMap::from_slice(&decoded_b64).ok()?; + + Some(Self::get_application( + &source_map, + file_name, + line_number, + column_number, + )) + } + /// Apply a source map to the passed location. If there is no source map for /// this location, or if the location remains unchanged after mapping, the /// changed values are returned. @@ -86,6 +139,7 @@ impl SourceMapper { /// Line and column numbers are 1-based. pub fn apply_source_map( &mut self, + scope: &mut v8::HandleScope, file_name: &str, line_number: u32, column_number: u32, @@ -94,6 +148,17 @@ impl SourceMapper { let line_number = line_number - 1; let column_number = column_number - 1; + // TODO(bartlomieju): requires scope and should only be called in a fallback op, + // that will access scope if the fast op doesn't return anything. + if let Some(app) = self.apply_source_map_from_module_map( + scope, + file_name, + line_number, + column_number, + ) { + return app; + } + let getter = self.getter.as_ref(); let maybe_source_map = self.maps.entry(file_name.to_owned()).or_insert_with(|| { @@ -110,6 +175,15 @@ impl SourceMapper { return SourceMapApplication::Unchanged; }; + Self::get_application(source_map, file_name, line_number, column_number) + } + + fn get_application( + source_map: &SourceMap, + file_name: &str, + line_number: u32, + column_number: u32, + ) -> SourceMapApplication { let Some(token) = source_map.lookup_token(line_number, column_number) else { return SourceMapApplication::Unchanged; diff --git a/foo.js b/foo.js new file mode 100644 index 000000000..ab149db68 --- /dev/null +++ b/foo.js @@ -0,0 +1,6 @@ +"use strict"; + +throw new Error("Hello world!"); +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiaHR0cDovL2xvY2FsaG9zdDo0NTQ1L3J1bi9pbmxpbmVfanNfc291cmNlX21hcF8yLnRzIl0sInNvdXJjZXNDb250ZW50IjpbIjErMTtcbmludGVyZmFjZSBUZXN0IHtcbiAgaGVsbG86IHN0cmluZztcbn1cblxudGhyb3cgbmV3IEVycm9yKFwiSGVsbG8gd29ybGQhXCIgYXMgdW5rbm93biBhcyBzdHJpbmcpO1xuIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQSxDQUFDLEdBQUMsQ0FBQyxDQUFDO0FBS0osTUFBTSxJQUFJLEtBQUssQ0FBQyxjQUErQixDQUFDLENBQUMifQ== + +asd; From 2905e0d11b6a398e01cad5eaba7d375f3d788a43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 12 Jul 2024 01:22:04 +0200 Subject: [PATCH 02/21] update comment --- core/source_map.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/source_map.rs b/core/source_map.rs index 84bffe960..0be64beb9 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -105,7 +105,8 @@ impl SourceMapper { let maybe_source_mapping_url = unbound_module_script.get_source_mapping_url(scope); - // TODO(bartlomieju): decide which one should be checked first + // TODO(bartlomieju): This should be the last fallback and it's only useful + // for eval - probably for `Deno.core.evalContext()`. let maybe_source_url = unbound_module_script .get_source_url(scope) .to_rust_string_lossy(scope); From 23cdef97ff9027d966cc047c008d683f20b9655c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 12 Jul 2024 02:01:40 +0200 Subject: [PATCH 03/21] prototype source mapping url being non-base64 url --- bar.js | 6 ++++++ bar.js.map | 11 +++++++++++ core/source_map.rs | 41 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 bar.js create mode 100644 bar.js.map diff --git a/bar.js b/bar.js new file mode 100644 index 000000000..3e6128c6c --- /dev/null +++ b/bar.js @@ -0,0 +1,6 @@ +"use strict"; + +throw new Error("Hello world!"); +//# sourceMappingURL=bar.js.map + +asd; diff --git a/bar.js.map b/bar.js.map new file mode 100644 index 000000000..16b12f346 --- /dev/null +++ b/bar.js.map @@ -0,0 +1,11 @@ +{ + "version": 3, + "file": "", + "sourceRoot": "", + "sources": ["http://localhost:4545/run/inline_js_source_map_2.ts"], + "sourcesContent": [ + "1+1;\ninterface Test {\n hello: string;\n}\n\nthrow new Error(\"Hello world!\" as unknown as string);\n" + ], + "names": [], + "mappings": ";AAAA,CAAC,GAAC,CAAC,CAAC;AAKJ,MAAM,IAAI,KAAK,CAAC,cAA+B,CAAC,CAAC" +} diff --git a/core/source_map.rs b/core/source_map.rs index 0be64beb9..7f5f7853f 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -2,7 +2,9 @@ //! This mod provides functions to remap a `JsError` based on a source map. +use crate::module_specifier::ModuleResolutionError; use crate::modules::ModuleMap; +use crate::resolve_import; use crate::resolve_url; use crate::RequestedModuleType; use base64::prelude::BASE64_STANDARD; @@ -40,6 +42,7 @@ where } } +#[derive(Debug)] pub enum SourceMapApplication { /// No mapping was applied, the location is unchanged. Unchanged, @@ -120,10 +123,40 @@ impl SourceMapper { maybe_source_mapping_url.to_rust_string_lossy(scope); eprintln!("maybe source mapping url {}", source_map_string); - let b64 = - source_map_string.strip_prefix("data:application/json;base64,")?; - let decoded_b64 = BASE64_STANDARD.decode(b64).ok()?; - let source_map = SourceMap::from_slice(&decoded_b64).ok()?; + // TODO(bartlomieju): this is a fast path - if it fails, we should try to parse + // the URL (or resolve it from the current file being mapped) and fallback to + // acquiring a source map from that URL. In Deno we might want to apply permissions + // checks for fetching the map. + let source_map = if let Some(b64) = + source_map_string.strip_prefix("data:application/json;base64,") + { + let decoded_b64 = BASE64_STANDARD.decode(b64).ok()?; + eprintln!( + "source map {:?}", + String::from_utf8(decoded_b64.clone()).unwrap() + ); + SourceMap::from_slice(&decoded_b64).ok()? + } else { + let url = match resolve_import(&source_map_string, file_name) { + Ok(url) => Some(url), + Err(err) => match err { + ModuleResolutionError::ImportPrefixMissing(_, _) => { + resolve_import(&format!("./{}", source_map_string), file_name).ok() + } + _ => None, + }, + }; + eprintln!( + "source map url {} {} {:?}", + source_map_string, file_name, url + ); + let url = url?; + if url.scheme() != "file" { + return None; + } + let contents = std::fs::read(url.to_file_path().unwrap()).unwrap(); + SourceMap::from_slice(&contents).ok()? + }; Some(Self::get_application( &source_map, From ec8e4f173e99acd8b5d61e99c786139df250ea1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 12 Jul 2024 03:21:42 +0200 Subject: [PATCH 04/21] start moving source map loading to ModuleLoader --- core/modules/loaders.rs | 4 ++++ core/source_map.rs | 11 ++++++++--- testing/checkin/runner/ts_module_loader.rs | 4 ++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/core/modules/loaders.rs b/core/modules/loaders.rs index edab4a809..507f0c3f7 100644 --- a/core/modules/loaders.rs +++ b/core/modules/loaders.rs @@ -95,6 +95,10 @@ pub trait ModuleLoader { ) -> Pin>> { async {}.boxed_local() } + + fn get_source_map(&self, _path: &str) -> Option> { + None + } } /// Placeholder structure used when creating diff --git a/core/source_map.rs b/core/source_map.rs index 7f5f7853f..9415a45ae 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -147,14 +147,19 @@ impl SourceMapper { }, }; eprintln!( - "source map url {} {} {:?}", - source_map_string, file_name, url + "source map url sourceMappingURL={} file_name={} url={}", + source_map_string, + file_name, + url.as_ref().map(|s| s.as_str()).unwrap_or_default() ); let url = url?; if url.scheme() != "file" { return None; } - let contents = std::fs::read(url.to_file_path().unwrap()).unwrap(); + let contents = module_map + .loader + .borrow() + .get_source_map(&url.to_file_path().ok()?.to_str()?)?; SourceMap::from_slice(&contents).ok()? }; diff --git a/testing/checkin/runner/ts_module_loader.rs b/testing/checkin/runner/ts_module_loader.rs index 2c7e2f606..c5e1616f8 100644 --- a/testing/checkin/runner/ts_module_loader.rs +++ b/testing/checkin/runner/ts_module_loader.rs @@ -156,6 +156,10 @@ impl ModuleLoader for TypescriptModuleLoader { requested_module_type, )) } + + fn get_source_map(&self, path: &str) -> Option> { + std::fs::read(path).ok() + } } pub fn maybe_transpile_source( From a46cd55ed5b9594f73af81653b5e88f4fea518df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 00:53:52 +0200 Subject: [PATCH 05/21] fixes after merge --- core/source_map.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/core/source_map.rs b/core/source_map.rs index cf51e9e9c..41cc20d4d 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -82,10 +82,6 @@ impl SourceMapper { assert!(self.maybe_module_map.replace(module_map).is_none()); } - pub(crate) fn has_user_sources(&self) -> bool { - self.getter.is_some() - } - pub fn apply_source_map_from_module_map( &mut self, scope: &mut v8::HandleScope, @@ -309,11 +305,13 @@ mod tests { use super::*; use crate::ascii_str; + use crate::JsRuntime; use crate::ModuleCodeString; use crate::ModuleLoadResponse; use crate::ModuleSpecifier; use crate::RequestedModuleType; use crate::ResolutionKind; + use crate::RuntimeOptions; struct SourceMapLoaderContent { source_map: Option, @@ -376,19 +374,29 @@ mod tests { }, ); - let mut source_mapper = SourceMapper::new(Rc::new(loader), None); + let loader = Rc::new(loader); + + let mut js_runtime = JsRuntime::new(RuntimeOptions { + module_loader: Some(loader.clone()), + ..Default::default() + }); + let state = JsRuntime::state_from(js_runtime.v8_isolate()); + let scope = &mut js_runtime.handle_scope(); + let mut source_mapper = state.source_mapper.borrow_mut(); // Non-existent file let application = - source_mapper.apply_source_map("file:///doesnt_exist.js", 1, 1); + source_mapper.apply_source_map(scope, "file:///doesnt_exist.js", 1, 1); assert_eq!(application, SourceMapApplication::Unchanged); // File with no source map - let application = source_mapper.apply_source_map("file:///b.js", 1, 1); + let application = + source_mapper.apply_source_map(scope, "file:///b.js", 1, 1); assert_eq!(application, SourceMapApplication::Unchanged); // File with a source map - let application = source_mapper.apply_source_map("file:///a.ts", 1, 21); + let application = + source_mapper.apply_source_map(scope, "file:///a.ts", 1, 21); assert_eq!( application, SourceMapApplication::LineAndColumn { From 0a2821c93b12e3d1fbd21479c7a0b520f23b2ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 00:56:45 +0200 Subject: [PATCH 06/21] rename --- core/source_map.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/source_map.rs b/core/source_map.rs index 41cc20d4d..d7060e694 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -153,7 +153,7 @@ impl SourceMapper { SourceMap::from_slice(&contents).ok()? }; - Some(Self::get_application( + Some(Self::compute_application( &source_map, file_name, line_number, @@ -207,10 +207,10 @@ impl SourceMapper { return SourceMapApplication::Unchanged; }; - Self::get_application(source_map, file_name, line_number, column_number) + Self::compute_application(source_map, file_name, line_number, column_number) } - fn get_application( + fn compute_application( source_map: &SourceMap, file_name: &str, line_number: u32, From 6fc5d1d73dac105cff3830b8824b645b78daf9b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 00:57:45 +0200 Subject: [PATCH 07/21] lint --- core/source_map.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/core/source_map.rs b/core/source_map.rs index d7060e694..152efbd89 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -100,10 +100,10 @@ impl SourceMapper { // TODO(bartlomieju): This should be the last fallback and it's only useful // for eval - probably for `Deno.core.evalContext()`. - let maybe_source_url = unbound_module_script - .get_source_url(scope) - .to_rust_string_lossy(scope); - eprintln!("maybe source url {}", maybe_source_url); + // let maybe_source_url = unbound_module_script + // .get_source_url(scope) + // .to_rust_string_lossy(scope); + // eprintln!("maybe source url {}", maybe_source_url); if !maybe_source_mapping_url.is_string() { return None; @@ -111,7 +111,7 @@ impl SourceMapper { let source_map_string = maybe_source_mapping_url.to_rust_string_lossy(scope); - eprintln!("maybe source mapping url {}", source_map_string); + // eprintln!("maybe source mapping url {}", source_map_string); // TODO(bartlomieju): this is a fast path - if it fails, we should try to parse // the URL (or resolve it from the current file being mapped) and fallback to @@ -121,10 +121,10 @@ impl SourceMapper { source_map_string.strip_prefix("data:application/json;base64,") { let decoded_b64 = BASE64_STANDARD.decode(b64).ok()?; - eprintln!( - "source map {:?}", - String::from_utf8(decoded_b64.clone()).unwrap() - ); + // eprintln!( + // "source map {:?}", + // String::from_utf8(decoded_b64.clone()).unwrap() + // ); SourceMap::from_slice(&decoded_b64).ok()? } else { let url = match resolve_import(&source_map_string, file_name) { @@ -136,12 +136,12 @@ impl SourceMapper { _ => None, }, }; - eprintln!( - "source map url sourceMappingURL={} file_name={} url={}", - source_map_string, - file_name, - url.as_ref().map(|s| s.as_str()).unwrap_or_default() - ); + // eprintln!( + // "source map url sourceMappingURL={} file_name={} url={}", + // source_map_string, + // file_name, + // url.as_ref().map(|s| s.as_str()).unwrap_or_default() + // ); let url = url?; if url.scheme() != "file" { return None; @@ -149,7 +149,7 @@ impl SourceMapper { let contents = module_map .loader .borrow() - .get_source_map(&url.to_file_path().ok()?.to_str()?)?; + .get_source_map(url.to_file_path().ok()?.to_str()?)?; SourceMap::from_slice(&contents).ok()? }; From 0166243cec36a157b44c0ec2ea553f380965d379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 01:26:18 +0200 Subject: [PATCH 08/21] use ModuleCode static for ext_source_maps --- core/extension_set.rs | 2 +- core/runtime/jsruntime.rs | 6 ++++-- core/runtime/snapshot.rs | 2 +- core/source_map.rs | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/extension_set.rs b/core/extension_set.rs index fafa6a6a6..4387d6319 100644 --- a/core/extension_set.rs +++ b/core/extension_set.rs @@ -254,7 +254,7 @@ fn load( if let Some(source_map) = source_map { source_mapper .ext_source_maps - .insert(source.specifier.to_owned(), source_map); + .insert(ModuleName::from_static(source.specifier), source_map); } Ok(source_code) } diff --git a/core/runtime/jsruntime.rs b/core/runtime/jsruntime.rs index 014ddcb79..5e71553ca 100644 --- a/core/runtime/jsruntime.rs +++ b/core/runtime/jsruntime.rs @@ -922,7 +922,9 @@ impl JsRuntime { let mut mapper = state_rc.source_mapper.borrow_mut(); for (key, map) in snapshotted_data.ext_source_maps { - mapper.ext_source_maps.insert(key, map.into()); + mapper + .ext_source_maps + .insert(ModuleName::from_static(key), map.into()); } } @@ -1958,7 +1960,7 @@ impl JsRuntimeForSnapshot { ); let mut ext_source_maps = HashMap::with_capacity(source_maps.len()); for (k, v) in &source_maps { - ext_source_maps.insert(k.clone(), v.as_ref()); + ext_source_maps.insert(k.as_static_str().unwrap(), v.as_ref()); } // Serialize the module map and store its data in the snapshot. diff --git a/core/runtime/snapshot.rs b/core/runtime/snapshot.rs index 66aa8b61e..7fc5d051a 100644 --- a/core/runtime/snapshot.rs +++ b/core/runtime/snapshot.rs @@ -255,7 +255,7 @@ pub(crate) struct SnapshottedData<'snapshot> { pub source_count: usize, pub addl_refs_count: usize, #[serde(borrow)] - pub ext_source_maps: HashMap, + pub ext_source_maps: HashMap<&'snapshot str, &'snapshot [u8]>, #[serde(borrow)] pub external_strings: Vec<&'snapshot [u8]>, } diff --git a/core/source_map.rs b/core/source_map.rs index 72d507d7d..d588edf36 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -7,6 +7,7 @@ use crate::resolve_url; use crate::ModuleLoader; +use crate::ModuleName; pub use sourcemap::SourceMap; use std::borrow::Cow; use std::collections::HashMap; @@ -49,7 +50,7 @@ pub struct SourceMapper { loader: Rc, getter: Option>, - pub(crate) ext_source_maps: HashMap, + pub(crate) ext_source_maps: HashMap, // This is not the right place for this, but it's the easiest way to make // op_apply_source_map a fast op. This stashing should happen in #[op2]. pub(crate) stashed_file_name: Option, From 6bfb77bc15f99815dfeba23a59b5fa09fb044ddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 02:05:38 +0200 Subject: [PATCH 09/21] more cleanup --- core/extension_set.rs | 38 +++++++++++++++++++++------------- core/runtime/jsruntime.rs | 43 ++++++++++++++++++++++++++------------- core/source_map.rs | 22 ++++++++++++++++++-- 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/core/extension_set.rs b/core/extension_set.rs index 4387d6319..7de9ef14f 100644 --- a/core/extension_set.rs +++ b/core/extension_set.rs @@ -14,7 +14,6 @@ use crate::ops::OpCtx; use crate::runtime::ExtensionTranspiler; use crate::runtime::JsRuntimeState; use crate::runtime::OpDriverImpl; -use crate::source_map::SourceMapper; use crate::ExtensionFileSource; use crate::FastString; use crate::GetErrorClassFn; @@ -22,6 +21,7 @@ use crate::ModuleCodeString; use crate::OpDecl; use crate::OpMetricsFactoryFn; use crate::OpState; +use crate::SourceMapData; /// Contribute to the `OpState` from each extension. pub fn setup_op_state(op_state: &mut OpState, extensions: &mut [Extension]) { @@ -241,9 +241,8 @@ impl<'a> IntoIterator for &'a mut LoadedSources { fn load( transpiler: Option<&ExtensionTranspiler>, source: &ExtensionFileSource, - source_mapper: &mut SourceMapper, load_callback: &mut impl FnMut(&ExtensionFileSource), -) -> Result { +) -> Result<(ModuleCodeString, Option), AnyError> { load_callback(source); let mut source_code = source.load()?; let mut source_map = None; @@ -251,21 +250,20 @@ fn load( (source_code, source_map) = transpiler(ModuleName::from_static(source.specifier), source_code)?; } + let mut maybe_source_map = None; if let Some(source_map) = source_map { - source_mapper - .ext_source_maps - .insert(ModuleName::from_static(source.specifier), source_map); + maybe_source_map = Some(source_map); } - Ok(source_code) + Ok((source_code, maybe_source_map)) } -pub fn into_sources( +pub fn into_sources_and_source_maps( transpiler: Option<&ExtensionTranspiler>, extensions: &[Extension], - source_mapper: &mut SourceMapper, mut load_callback: impl FnMut(&ExtensionFileSource), -) -> Result { +) -> Result<(LoadedSources, Vec<(ModuleName, SourceMapData)>), AnyError> { let mut sources = LoadedSources::default(); + let mut source_maps = vec![]; for extension in extensions { if let Some(esm_entry_point) = extension.esm_entry_point { @@ -274,29 +272,41 @@ pub fn into_sources( .push(FastString::from_static(esm_entry_point)); } for file in &*extension.lazy_loaded_esm_files { - let code = load(transpiler, file, source_mapper, &mut load_callback)?; + let (code, maybe_source_map) = + load(transpiler, file, &mut load_callback)?; sources.lazy_esm.push(LoadedSource { source_type: ExtensionSourceType::LazyEsm, specifier: ModuleName::from_static(file.specifier), code, }); + if let Some(source_map) = maybe_source_map { + source_maps.push((ModuleName::from_static(file.specifier), source_map)); + } } for file in &*extension.js_files { - let code = load(transpiler, file, source_mapper, &mut load_callback)?; + let (code, maybe_source_map) = + load(transpiler, file, &mut load_callback)?; sources.js.push(LoadedSource { source_type: ExtensionSourceType::Js, specifier: ModuleName::from_static(file.specifier), code, }); + if let Some(source_map) = maybe_source_map { + source_maps.push((ModuleName::from_static(file.specifier), source_map)); + } } for file in &*extension.esm_files { - let code = load(transpiler, file, source_mapper, &mut load_callback)?; + let (code, maybe_source_map) = + load(transpiler, file, &mut load_callback)?; sources.esm.push(LoadedSource { source_type: ExtensionSourceType::Esm, specifier: ModuleName::from_static(file.specifier), code, }); + if let Some(source_map) = maybe_source_map { + source_maps.push((ModuleName::from_static(file.specifier), source_map)); + } } } - Ok(sources) + Ok((sources, source_maps)) } diff --git a/core/runtime/jsruntime.rs b/core/runtime/jsruntime.rs index 5e71553ca..9f9b0c32e 100644 --- a/core/runtime/jsruntime.rs +++ b/core/runtime/jsruntime.rs @@ -678,17 +678,31 @@ impl JsRuntime { let loader = options .module_loader .unwrap_or_else(|| Rc::new(NoopModuleLoader)); + + // TODO(bartlomieju): defer creation of `SourceMapper` until we can create a `ModuleMap`. + // The problem is that `op_apply_source_map` relies on `source_mapper` being presented in + // the `JsRuntimeState`. Maybe we can make it an `Option` and expect it always is present. + // Alternatively we could have a cheap default impl of `SourceMapper` that we later replace + // with actual impl. #[allow(deprecated)] let mut source_mapper = SourceMapper::new(loader.clone(), options.source_map_getter); - let mut sources = extension_set::into_sources( - options.extension_transpiler.as_deref(), - &extensions, - &mut source_mapper, - |source| { - mark_as_loaded_from_fs_during_snapshot(&mut files_loaded, &source.code) - }, - )?; + + let (mut sources, source_maps) = + extension_set::into_sources_and_source_maps( + options.extension_transpiler.as_deref(), + &extensions, + |source| { + mark_as_loaded_from_fs_during_snapshot( + &mut files_loaded, + &source.code, + ) + }, + )?; + + for (module_name, source_map_data) in source_maps { + source_mapper.add_ext_source_map(module_name, source_map_data); + } // ...now let's set up ` JsRuntimeState`, we'll need to set some fields // later, after `JsRuntime` is all set up... @@ -922,9 +936,7 @@ impl JsRuntime { let mut mapper = state_rc.source_mapper.borrow_mut(); for (key, map) in snapshotted_data.ext_source_maps { - mapper - .ext_source_maps - .insert(ModuleName::from_static(key), map.into()); + mapper.add_ext_source_map(ModuleName::from_static(key), map.into()); } } @@ -1955,9 +1967,12 @@ impl JsRuntimeForSnapshot { } // Borrow the source maps during the snapshot to avoid copies - let source_maps = std::mem::take( - &mut self.inner.state.source_mapper.borrow_mut().ext_source_maps, - ); + let source_maps = self + .inner + .state + .source_mapper + .borrow_mut() + .take_ext_source_maps(); let mut ext_source_maps = HashMap::with_capacity(source_maps.len()); for (k, v) in &source_maps { ext_source_maps.insert(k.as_static_str().unwrap(), v.as_ref()); diff --git a/core/source_map.rs b/core/source_map.rs index d588edf36..4da002797 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -45,12 +45,15 @@ pub enum SourceMapApplication { pub type SourceMapData = Cow<'static, [u8]>; pub struct SourceMapper { + // TODO(bartlomieju): I feel like these two should be cleared when Isolate + // reaches "near heap limit" to free up some space. This needs to be confirmed though. maps: HashMap>, source_lines: HashMap<(String, i64), Option>, - loader: Rc, + loader: Rc, getter: Option>, - pub(crate) ext_source_maps: HashMap, + + ext_source_maps: HashMap, // This is not the right place for this, but it's the easiest way to make // op_apply_source_map a fast op. This stashing should happen in #[op2]. pub(crate) stashed_file_name: Option, @@ -71,6 +74,21 @@ impl SourceMapper { } } + /// Add a source map for particular `ext:` module. + pub(crate) fn add_ext_source_map( + &mut self, + module_name: ModuleName, + source_map_data: SourceMapData, + ) { + self.ext_source_maps.insert(module_name, source_map_data); + } + + pub(crate) fn take_ext_source_maps( + &mut self, + ) -> HashMap { + std::mem::take(&mut self.ext_source_maps) + } + /// Apply a source map to the passed location. If there is no source map for /// this location, or if the location remains unchanged after mapping, the /// changed values are returned. From 224bcf2249c6f699470ce98d7e043ce30fb82c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 02:06:08 +0200 Subject: [PATCH 10/21] remove a TODO comment --- core/runtime/jsruntime.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/runtime/jsruntime.rs b/core/runtime/jsruntime.rs index 9f9b0c32e..bff9c2b9c 100644 --- a/core/runtime/jsruntime.rs +++ b/core/runtime/jsruntime.rs @@ -679,11 +679,6 @@ impl JsRuntime { .module_loader .unwrap_or_else(|| Rc::new(NoopModuleLoader)); - // TODO(bartlomieju): defer creation of `SourceMapper` until we can create a `ModuleMap`. - // The problem is that `op_apply_source_map` relies on `source_mapper` being presented in - // the `JsRuntimeState`. Maybe we can make it an `Option` and expect it always is present. - // Alternatively we could have a cheap default impl of `SourceMapper` that we later replace - // with actual impl. #[allow(deprecated)] let mut source_mapper = SourceMapper::new(loader.clone(), options.source_map_getter); From 7ff9928b9ce4cbb28b46109986d68b2ce011e192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 02:34:04 +0200 Subject: [PATCH 11/21] remove setting up `ModuleMap` for `SourceMapper` --- core/runtime/jsruntime.rs | 5 ----- core/source_map.rs | 16 +++++----------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/core/runtime/jsruntime.rs b/core/runtime/jsruntime.rs index 14d2c137b..bff9c2b9c 100644 --- a/core/runtime/jsruntime.rs +++ b/core/runtime/jsruntime.rs @@ -935,11 +935,6 @@ impl JsRuntime { } } - state_rc - .source_mapper - .borrow_mut() - .set_module_map(module_map.clone()); - // SAFETY: Set the module map slot in the context unsafe { context.set_aligned_pointer_in_embedder_data( diff --git a/core/source_map.rs b/core/source_map.rs index 44d987a22..7cb06b8d7 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -6,9 +6,9 @@ #![allow(deprecated)] use crate::module_specifier::ModuleResolutionError; -use crate::modules::ModuleMap; use crate::resolve_import; use crate::resolve_url; +use crate::runtime::JsRealm; use crate::ModuleLoader; use crate::ModuleName; use crate::RequestedModuleType; @@ -63,7 +63,6 @@ pub struct SourceMapper { // This is not the right place for this, but it's the easiest way to make // op_apply_source_map a fast op. This stashing should happen in #[op2]. pub(crate) stashed_file_name: Option, - pub(crate) maybe_module_map: Option>, } impl SourceMapper { @@ -78,7 +77,6 @@ impl SourceMapper { loader, getter, stashed_file_name: Default::default(), - maybe_module_map: None, } } @@ -97,10 +95,6 @@ impl SourceMapper { std::mem::take(&mut self.ext_source_maps) } - pub fn set_module_map(&mut self, module_map: Rc) { - assert!(self.maybe_module_map.replace(module_map).is_none()); - } - pub fn apply_source_map_from_module_map( &mut self, scope: &mut v8::HandleScope, @@ -108,10 +102,10 @@ impl SourceMapper { line_number: u32, column_number: u32, ) -> Option { - let module_map = self.maybe_module_map.as_ref()?; - let id = module_map.get_id(file_name, RequestedModuleType::None)?; + let module_map_rc = JsRealm::module_map_from(scope); + let id = module_map_rc.get_id(file_name, RequestedModuleType::None)?; - let module_handle = module_map.get_handle(id).unwrap(); + let module_handle = module_map_rc.get_handle(id).unwrap(); let module = v8::Local::new(scope, module_handle); let unbound_module_script = module.get_unbound_module_script(scope); let maybe_source_mapping_url = @@ -165,7 +159,7 @@ impl SourceMapper { if url.scheme() != "file" { return None; } - let contents = module_map + let contents = module_map_rc .loader .borrow() .get_source_map(url.to_file_path().ok()?.to_str()?)?; From 40aff1a8797df59a1b47be7bce8ba6ec4e8da328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 02:59:33 +0200 Subject: [PATCH 12/21] simplify SourceMapper::get_source_line --- core/error.rs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/core/error.rs b/core/error.rs index e7beaaced..9f67e7d36 100644 --- a/core/error.rs +++ b/core/error.rs @@ -354,15 +354,12 @@ impl JsError { { let state = JsRuntime::state_from(scope); let mut source_mapper = state.source_mapper.borrow_mut(); - for (i, frame) in frames.iter().enumerate() { + if let Some(frame) = frames.iter().next() { if let (Some(file_name), Some(line_number)) = (&frame.file_name, frame.line_number) { - if !file_name.trim_start_matches('[').starts_with("ext:") { - source_line = source_mapper.get_source_line(file_name, line_number); - source_line_frame_index = Some(i); - break; - } + source_line = source_mapper.get_source_line(file_name, line_number); + source_line_frame_index = Some(0); } } } @@ -475,16 +472,12 @@ impl JsError { { let state = JsRuntime::state_from(scope); let mut source_mapper = state.source_mapper.borrow_mut(); - for (i, frame) in frames.iter().enumerate() { + if let Some(frame) = frames.iter().next() { if let (Some(file_name), Some(line_number)) = (&frame.file_name, frame.line_number) { - if !file_name.trim_start_matches('[').starts_with("ext:") { - source_line = - source_mapper.get_source_line(file_name, line_number); - source_line_frame_index = Some(i); - break; - } + source_line = source_mapper.get_source_line(file_name, line_number); + source_line_frame_index = Some(0); } } } From 84d6c3fee0cbaeb780fce358adc604e566786f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 03:02:16 +0200 Subject: [PATCH 13/21] lint --- core/error.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/error.rs b/core/error.rs index 9f67e7d36..f2d3d2437 100644 --- a/core/error.rs +++ b/core/error.rs @@ -354,7 +354,7 @@ impl JsError { { let state = JsRuntime::state_from(scope); let mut source_mapper = state.source_mapper.borrow_mut(); - if let Some(frame) = frames.iter().next() { + if let Some(frame) = frames.first() { if let (Some(file_name), Some(line_number)) = (&frame.file_name, frame.line_number) { @@ -472,7 +472,7 @@ impl JsError { { let state = JsRuntime::state_from(scope); let mut source_mapper = state.source_mapper.borrow_mut(); - if let Some(frame) = frames.iter().next() { + if let Some(frame) = frames.first() { if let (Some(file_name), Some(line_number)) = (&frame.file_name, frame.line_number) { From 85a9634f90eda1e3036935560eb127f3c14dfb3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 15:14:27 +0200 Subject: [PATCH 14/21] fix after merge --- core/extension_set.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/core/extension_set.rs b/core/extension_set.rs index 33aa870d2..9f111f574 100644 --- a/core/extension_set.rs +++ b/core/extension_set.rs @@ -262,9 +262,8 @@ pub fn into_sources_and_source_maps( transpiler: Option<&ExtensionTranspiler>, extensions: &[Extension], mut load_callback: impl FnMut(&ExtensionFileSource), -) -> Result<(LoadedSources, Vec<(ModuleName, SourceMapData)>), AnyError> { +) -> Result { let mut sources = LoadedSources::default(); - let mut source_maps = vec![]; for extension in extensions { if let Some(esm_entry_point) = extension.esm_entry_point { @@ -281,9 +280,6 @@ pub fn into_sources_and_source_maps( code, maybe_source_map, }); - if let Some(source_map) = maybe_source_map { - source_maps.push((ModuleName::from_static(file.specifier), source_map)); - } } for file in &*extension.js_files { let (code, maybe_source_map) = @@ -294,9 +290,6 @@ pub fn into_sources_and_source_maps( code, maybe_source_map, }); - if let Some(source_map) = maybe_source_map { - source_maps.push((ModuleName::from_static(file.specifier), source_map)); - } } for file in &*extension.esm_files { let (code, maybe_source_map) = @@ -307,10 +300,7 @@ pub fn into_sources_and_source_maps( code, maybe_source_map, }); - if let Some(source_map) = maybe_source_map { - source_maps.push((ModuleName::from_static(file.specifier), source_map)); - } } } - Ok((sources, source_maps)) + Ok(sources) } From b0bfb5cb7d77573e63cb8132b0a6aaa0eff269cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 22:47:50 +0200 Subject: [PATCH 15/21] simplify error --- core/error.rs | 60 ++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/core/error.rs b/core/error.rs index f2d3d2437..4cd1404ec 100644 --- a/core/error.rs +++ b/core/error.rs @@ -333,7 +333,7 @@ impl JsError { Self::inner_from_v8_exception(scope, exception, Default::default()) } - pub fn from_v8_message<'a>( + pub(crate) fn from_v8_message<'a>( scope: &'a mut v8::HandleScope, msg: v8::Local<'a, v8::Message>, ) -> Self { @@ -343,38 +343,36 @@ impl JsError { let exception_message = msg.get(scope).to_rust_string_lossy(scope); - // Convert them into Vec - let mut frames: Vec = vec![]; - let mut source_line = None; - let mut source_line_frame_index = None; - - if let Some(stack_frame) = JsStackFrame::from_v8_message(scope, msg) { - frames = vec![stack_frame]; - } - { - let state = JsRuntime::state_from(scope); - let mut source_mapper = state.source_mapper.borrow_mut(); - if let Some(frame) = frames.first() { - if let (Some(file_name), Some(line_number)) = - (&frame.file_name, frame.line_number) - { - source_line = source_mapper.get_source_line(file_name, line_number); - source_line_frame_index = Some(0); - } - } - } - - Self { + let mut js_error = Self { name: None, message: None, exception_message, cause: None, - source_line, - source_line_frame_index, - frames, + source_line: None, + source_line_frame_index: None, + frames: vec![], stack: None, aggregated: None, + }; + + let Some(stack_frame) = JsStackFrame::from_v8_message(scope, msg) else { + return js_error; + }; + + let state = JsRuntime::state_from(scope); + let mut source_mapper = state.source_mapper.borrow_mut(); + if let (Some(file_name), Some(line_number)) = + (&stack_frame.file_name, stack_frame.line_number) + { + if !file_name.trim_start_matches('[').starts_with("ext:") { + js_error.source_line = + source_mapper.get_source_line(file_name, line_number); + js_error.source_line_frame_index = Some(0); + } } + + js_error.frames = vec![stack_frame]; + js_error } fn inner_from_v8_exception<'a>( @@ -472,12 +470,16 @@ impl JsError { { let state = JsRuntime::state_from(scope); let mut source_mapper = state.source_mapper.borrow_mut(); - if let Some(frame) = frames.first() { + for (index, frame) in frames.iter().enumerate() { if let (Some(file_name), Some(line_number)) = (&frame.file_name, frame.line_number) { - source_line = source_mapper.get_source_line(file_name, line_number); - source_line_frame_index = Some(0); + if !file_name.trim_start_matches('[').starts_with("ext:") { + source_line = + source_mapper.get_source_line(file_name, line_number); + source_line_frame_index = Some(index); + break; + } } } } From a6c6f763a50aca1b65568a1b33607df92bffa409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 23:11:27 +0200 Subject: [PATCH 16/21] self-review --- core/examples/ts_module_loader.rs | 12 ++++- core/modules/loaders.rs | 27 +++++++++- core/source_map.rs | 62 ++++++++++------------ testing/checkin/runner/ts_module_loader.rs | 12 ++++- 4 files changed, 72 insertions(+), 41 deletions(-) diff --git a/core/examples/ts_module_loader.rs b/core/examples/ts_module_loader.rs index 6e995076a..d075bbc1b 100644 --- a/core/examples/ts_module_loader.rs +++ b/core/examples/ts_module_loader.rs @@ -116,8 +116,16 @@ impl ModuleLoader for TypescriptModuleLoader { ModuleLoadResponse::Sync(load(source_maps, module_specifier)) } - fn get_source_map(&self, specifier: &str) -> Option> { - self.source_maps.borrow().get(specifier).cloned() + fn get_source_map_for_file(&self, file_name: &str) -> Option> { + self.source_maps.borrow().get(file_name).cloned() + } + + fn load_source_map_file( + &self, + source_map_file_name: &str, + _file_name: &str, + ) -> Option> { + std::fs::read(source_map_file_name).ok() } } diff --git a/core/modules/loaders.rs b/core/modules/loaders.rs index 88c8dbb5c..bb9339293 100644 --- a/core/modules/loaders.rs +++ b/core/modules/loaders.rs @@ -98,11 +98,34 @@ pub trait ModuleLoader { /// Returns a source map for given `file_name`. /// - /// This function will soon be deprecated or renamed. - fn get_source_map(&self, _file_name: &str) -> Option> { + /// Some embedders might opt into stripping out inlined source map from the source + /// code before it's loaded into V8. The source mapping logic will first ask + /// embedder to provide a source map for given file, before falling back + /// to looking for source map inlined in the code. + fn get_source_map_for_file(&self, _file_name: &str) -> Option> { None } + /// Returns a source map file. + /// + /// This is fallback API for [`Self::get_source_map_for_file`] that will only be called + /// if a module contains magic comment like `//# sourceMappingURL=foo.js.map`. + /// + /// Embedders might apply additional logic to decide if the source map should be returned. + /// Eg. if a local file tries to link to a remote source map, embedder might opt into + /// refusing to load it. + fn load_source_map_file( + &self, + _source_map_file_name: &str, + _file_name: &str, + ) -> Option> { + None + } + + /// Return an "original" version of the given line for a file. + /// + /// In the future this API might be deprecated in favor of handling it internally + /// in `deno_core` using contents of the source map. fn get_source_mapped_source_line( &self, _file_name: &str, diff --git a/core/source_map.rs b/core/source_map.rs index 7cb06b8d7..8dfe16095 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -111,20 +111,12 @@ impl SourceMapper { let maybe_source_mapping_url = unbound_module_script.get_source_mapping_url(scope); - // TODO(bartlomieju): This should be the last fallback and it's only useful - // for eval - probably for `Deno.core.evalContext()`. - // let maybe_source_url = unbound_module_script - // .get_source_url(scope) - // .to_rust_string_lossy(scope); - // eprintln!("maybe source url {}", maybe_source_url); - if !maybe_source_mapping_url.is_string() { return None; } let source_map_string = maybe_source_mapping_url.to_rust_string_lossy(scope); - // eprintln!("maybe source mapping url {}", source_map_string); // TODO(bartlomieju): this is a fast path - if it fails, we should try to parse // the URL (or resolve it from the current file being mapped) and fallback to @@ -134,10 +126,6 @@ impl SourceMapper { source_map_string.strip_prefix("data:application/json;base64,") { let decoded_b64 = BASE64_STANDARD.decode(b64).ok()?; - // eprintln!( - // "source map {:?}", - // String::from_utf8(decoded_b64.clone()).unwrap() - // ); SourceMap::from_slice(&decoded_b64).ok()? } else { let url = match resolve_import(&source_map_string, file_name) { @@ -149,20 +137,16 @@ impl SourceMapper { _ => None, }, }; - // eprintln!( - // "source map url sourceMappingURL={} file_name={} url={}", - // source_map_string, - // file_name, - // url.as_ref().map(|s| s.as_str()).unwrap_or_default() - // ); let url = url?; if url.scheme() != "file" { return None; } + let source_map_file_name = url.to_file_path().ok()?; + let source_map_file_name = source_map_file_name.to_str()?; let contents = module_map_rc .loader .borrow() - .get_source_map(url.to_file_path().ok()?.to_str()?)?; + .load_source_map_file(source_map_file_name, file_name)?; SourceMap::from_slice(&contents).ok()? }; @@ -190,17 +174,6 @@ impl SourceMapper { let line_number = line_number - 1; let column_number = column_number - 1; - // TODO(bartlomieju): requires scope and should only be called in a fallback op, - // that will access scope if the fast op doesn't return anything. - if let Some(app) = self.apply_source_map_from_module_map( - scope, - file_name, - line_number, - column_number, - ) { - return app; - } - let getter = self.getter.as_ref(); let maybe_source_map = self.maps.entry(file_name.to_owned()).or_insert_with(|| { @@ -209,18 +182,37 @@ impl SourceMapper { SourceMap::from_slice(self.ext_source_maps.get(file_name)?).ok() }) .or_else(|| { - SourceMap::from_slice(&self.loader.get_source_map(file_name)?).ok() + SourceMap::from_slice( + &self.loader.get_source_map_for_file(file_name)?, + ) + .ok() }) .or_else(|| { SourceMap::from_slice(&getter?.get_source_map(file_name)?).ok() }) }); - let Some(source_map) = maybe_source_map.as_ref() else { - return SourceMapApplication::Unchanged; + // If source map is provided externally, return early + if let Some(source_map) = maybe_source_map.as_ref() { + return Self::compute_application( + source_map, + file_name, + line_number, + column_number, + ); }; - Self::compute_application(source_map, file_name, line_number, column_number) + // Finally fallback to using V8 APIs to discover source map inside the source code of the module. + if let Some(app) = self.apply_source_map_from_module_map( + scope, + file_name, + line_number, + column_number, + ) { + return app; + } + + SourceMapApplication::Unchanged } fn compute_application( @@ -355,7 +347,7 @@ mod tests { unreachable!() } - fn get_source_map(&self, file_name: &str) -> Option> { + fn get_source_map_for_file(&self, file_name: &str) -> Option> { let url = Url::parse(file_name).unwrap(); let content = self.map.get(&url)?; content diff --git a/testing/checkin/runner/ts_module_loader.rs b/testing/checkin/runner/ts_module_loader.rs index 7534cd814..27efb0c46 100644 --- a/testing/checkin/runner/ts_module_loader.rs +++ b/testing/checkin/runner/ts_module_loader.rs @@ -143,8 +143,16 @@ impl ModuleLoader for TypescriptModuleLoader { )) } - fn get_source_map(&self, specifier: &str) -> Option> { - self.source_maps.borrow().get(specifier).cloned() + fn get_source_map_for_file(&self, file_name: &str) -> Option> { + self.source_maps.borrow().get(file_name).cloned() + } + + fn load_source_map_file( + &self, + source_map_file_name: &str, + _file_name: &str, + ) -> Option> { + std::fs::read(source_map_file_name).ok() } } From 126eb6bd89ffe33d98bcecb5e58ce1c70702622d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 23:15:55 +0200 Subject: [PATCH 17/21] add a TODO --- core/source_map.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/source_map.rs b/core/source_map.rs index 8dfe16095..ca2447055 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -356,6 +356,14 @@ mod tests { .map(|s| s.to_string().into_bytes()) } + fn load_source_map_file( + &self, + _source_map_file_name: &str, + _file_name: &str, + ) -> Option> { + todo!() + } + fn get_source_mapped_source_line( &self, _file_name: &str, From 4e4036d23b8e1240ef7a90b344abfb9242a22b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 17 Jul 2024 23:39:56 +0200 Subject: [PATCH 18/21] remove base64 --- Cargo.lock | 13 +++---------- core/Cargo.toml | 1 - core/source_map.rs | 17 ++++++++++------- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f4b401f2..0e9d85890 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,12 +149,6 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "base64-simd" version = "0.7.0" @@ -525,7 +519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f340bc18a3c351cb4af21e56a8af13cdcc08ff7616799429c745bce26fea6385" dependencies = [ "anyhow", - "base64 0.21.7", + "base64", "deno_media_type", "deno_terminal", "dprint-swc-ext", @@ -563,7 +557,6 @@ name = "deno_core" version = "0.295.0" dependencies = [ "anyhow", - "base64 0.22.1", "bencher", "bincode", "bit-set", @@ -744,7 +737,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63dd7b57f9b33b1741fa631c9522eb35d43e96dcca4a6a91d5e4ca7c93acdc1" dependencies = [ - "base64 0.21.7", + "base64", "http-body-util", "hyper", "hyper-util", @@ -2282,7 +2275,7 @@ version = "0.183.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b7b7de90ff41560bf021acda3fb16fb0f4f5885aeb44b6b7e638b563124d087" dependencies = [ - "base64 0.21.7", + "base64", "dashmap", "indexmap", "once_cell", diff --git a/core/Cargo.toml b/core/Cargo.toml index 42266a24f..a77688e79 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -24,7 +24,6 @@ snapshot_flags_eager_parse = [] [dependencies] anyhow.workspace = true -base64 = "0.22.1" bincode.workspace = true bit-set.workspace = true bit-vec.workspace = true diff --git a/core/source_map.rs b/core/source_map.rs index ca2447055..7cde90422 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -12,14 +12,15 @@ use crate::runtime::JsRealm; use crate::ModuleLoader; use crate::ModuleName; use crate::RequestedModuleType; -use base64::prelude::BASE64_STANDARD; -use base64::Engine; +use sourcemap::DecodedMap; pub use sourcemap::SourceMap; use std::borrow::Cow; use std::collections::HashMap; use std::rc::Rc; use std::str; +const BASE64_PREFIX: &str = "data:application/json;base64,"; + #[deprecated = "Use `ModuleLoader` methods instead. This trait will be removed in deno_core v0.300.0."] pub trait SourceMapGetter { /// Returns the raw source map file. @@ -122,11 +123,13 @@ impl SourceMapper { // the URL (or resolve it from the current file being mapped) and fallback to // acquiring a source map from that URL. In Deno we might want to apply permissions // checks for fetching the map. - let source_map = if let Some(b64) = - source_map_string.strip_prefix("data:application/json;base64,") - { - let decoded_b64 = BASE64_STANDARD.decode(b64).ok()?; - SourceMap::from_slice(&decoded_b64).ok()? + let source_map = if source_map_string.starts_with(BASE64_PREFIX) { + let DecodedMap::Regular(sm) = + sourcemap::decode_data_url(&source_map_string).ok()? + else { + return None; + }; + sm } else { let url = match resolve_import(&source_map_string, file_name) { Ok(url) => Some(url), From b2533712d65a6e0c0800bed2dc1336ef8ff53a30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 18 Jul 2024 01:08:47 +0200 Subject: [PATCH 19/21] wip - dcore, can't print source_line in JsError --- core/error.rs | 22 +++++++++++++++++++++ core/source_map.rs | 2 ++ fizz.ts | 13 ++++++++++++ testing/checkin/runner/ts_module_loader.rs | 23 ++++++++++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 fizz.ts diff --git a/core/error.rs b/core/error.rs index 4cd1404ec..f529d835c 100644 --- a/core/error.rs +++ b/core/error.rs @@ -540,13 +540,35 @@ impl std::error::Error for JsError {} impl Display for JsError { fn fmt(&self, f: &mut Formatter) -> fmt::Result { + eprintln!("inside js error {}", self.source_line.is_some()); if let Some(stack) = &self.stack { + eprintln!("inside js error stack"); let stack_lines = stack.lines(); if stack_lines.count() > 1 { return write!(f, "{stack}"); } } + write!(f, "{}", self.exception_message)?; + if let Some(source_line) = self.source_line.as_ref() { + writeln!(f, "")?; + write!(f, " {}", source_line)?; + let column_number = self + .source_line_frame_index + .and_then(|i| self.frames.get(i).unwrap().column_number); + if let Some(column_number) = column_number { + let mut s = String::new(); + for _i in 0..(column_number - 1) { + if source_line.chars().nth(_i as usize).unwrap() == '\t' { + s.push('\t'); + } else { + s.push(' '); + } + } + s.push('^'); + write!(f, " {}", s)?; + } + } let location = self.frames.first().and_then(|f| f.maybe_format_location()); if let Some(location) = location { write!(f, "\n at {location}")?; diff --git a/core/source_map.rs b/core/source_map.rs index 7cde90422..b77f39876 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -271,6 +271,7 @@ impl SourceMapper { file_name: &str, line_number: i64, ) -> Option { + eprintln!("get_source_line {} {}", file_name, line_number); if let Some(maybe_source_line) = self.source_lines.get(&(file_name.to_string(), line_number)) { @@ -283,6 +284,7 @@ impl SourceMapper { .get_source_mapped_source_line(file_name, (line_number - 1) as usize) .filter(|s| s.len() <= Self::MAX_SOURCE_LINE_LENGTH) { + eprintln!("line {}", source_line); // Cache and return self.source_lines.insert( (file_name.to_string(), line_number), diff --git a/fizz.ts b/fizz.ts new file mode 100644 index 000000000..1a34dfefa --- /dev/null +++ b/fizz.ts @@ -0,0 +1,13 @@ +interface Foobar { + a: string; + b: number; + c: boolean; +} + +console.log("hello there"); + +function boom(): never { + throw new Error("boom" as string); +} + +boom(); diff --git a/testing/checkin/runner/ts_module_loader.rs b/testing/checkin/runner/ts_module_loader.rs index 27efb0c46..5f245193d 100644 --- a/testing/checkin/runner/ts_module_loader.rs +++ b/testing/checkin/runner/ts_module_loader.rs @@ -154,6 +154,29 @@ impl ModuleLoader for TypescriptModuleLoader { ) -> Option> { std::fs::read(source_map_file_name).ok() } + + fn get_source_mapped_source_line( + &self, + file_name: &str, + line_number: usize, + ) -> Option { + let url = Url::parse(file_name).ok()?; + eprintln!("get_source_mapped_source_line {}", url.as_str()); + if url.scheme() != "file" { + return None; + } + let path = url.to_file_path().unwrap(); + let code = std::fs::read_to_string(&path).ok()?; + eprintln!("code {}", code); + // Do NOT use .lines(): it skips the terminating empty line. + // (due to internally using_terminator() instead of .split()) + let lines: Vec<&str> = code.split('\n').collect(); + if line_number >= lines.len() { + None + } else { + Some(lines[line_number].to_string()) + } + } } pub fn maybe_transpile_source( From a503d80e885f04891d62b2540bcc8fd1e9c76791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 18 Jul 2024 02:17:24 +0200 Subject: [PATCH 20/21] add transpiler api --- a.js | 12 ++++++++ a.js.map | 0 testing/checkin/runner/extensions.rs | 9 +++++- testing/checkin/runner/mod.rs | 2 ++ testing/checkin/runner/ops_fs.rs | 22 +++++++++++++++ testing/checkin/runner/ops_transpiler.rs | 36 ++++++++++++++++++++++++ testing/checkin/runtime/__init.js | 5 ++++ testing/checkin/runtime/fs.ts | 10 +++++++ testing/checkin/runtime/transpiler.ts | 11 ++++++++ testing/ops.d.ts | 6 ++++ testing/tsconfig.json | 2 ++ transpile.js | 30 ++++++++++++++++++++ 12 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 a.js create mode 100644 a.js.map create mode 100644 testing/checkin/runner/ops_fs.rs create mode 100644 testing/checkin/runner/ops_transpiler.rs create mode 100644 testing/checkin/runtime/fs.ts create mode 100644 testing/checkin/runtime/transpiler.ts create mode 100644 transpile.js diff --git a/a.js b/a.js new file mode 100644 index 000000000..b84b9381e --- /dev/null +++ b/a.js @@ -0,0 +1,12 @@ +export function a() { + return "a"; +} +function spawn() { + const b = new Deno.Command(); + b.spawn(); +} +function runSpawn() { + spawn(); +} +runSpawn(); +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImZpbGU6Ly8vYS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLFNBQVM7RUFDZCxPQUFPO0FBQ1Q7QUFFQSxTQUFTO0VBQ1AsTUFBTSxJQUFJLElBQUksS0FBSyxPQUFPO0VBRTFCLEVBQUUsS0FBSztBQUNUO0FBUUEsU0FBUztFQUNQO0FBQ0Y7QUFFQSJ9 \ No newline at end of file diff --git a/a.js.map b/a.js.map new file mode 100644 index 000000000..e69de29bb diff --git a/testing/checkin/runner/extensions.rs b/testing/checkin/runner/extensions.rs index bdabaa56d..5d6578b2c 100644 --- a/testing/checkin/runner/extensions.rs +++ b/testing/checkin/runner/extensions.rs @@ -3,7 +3,9 @@ use crate::checkin::runner::ops; use crate::checkin::runner::ops_async; use crate::checkin::runner::ops_buffer; use crate::checkin::runner::ops_error; +use crate::checkin::runner::ops_fs; use crate::checkin::runner::ops_io; +use crate::checkin::runner::ops_transpiler; use crate::checkin::runner::ops_worker; use crate::checkin::runner::Output; use crate::checkin::runner::TestData; @@ -39,6 +41,8 @@ deno_core::extension!( ops_error::op_error_custom_sync, ops_error::op_error_context_sync, ops_error::op_error_context_async, + ops_fs::op_fs_read_text_file, + ops_fs::op_fs_write_text_file, ops_buffer::op_v8slice_store, ops_buffer::op_v8slice_clone, ops_worker::op_worker_spawn, @@ -47,6 +51,7 @@ deno_core::extension!( ops_worker::op_worker_parent, ops_worker::op_worker_await_close, ops_worker::op_worker_terminate, + ops_transpiler::op_transpile, ], esm_entry_point = "ext:checkin_runtime/__init.js", esm = [ @@ -55,9 +60,11 @@ deno_core::extension!( "checkin:async" = "async.ts", "checkin:console" = "console.ts", "checkin:error" = "error.ts", + "checkin:fs" = "fs.ts", + "checkin:throw" = "throw.ts", "checkin:timers" = "timers.ts", + "checkin:transpiler" = "transpiler.ts", "checkin:worker" = "worker.ts", - "checkin:throw" = "throw.ts", ], state = |state| { state.put(TestData::default()); diff --git a/testing/checkin/runner/mod.rs b/testing/checkin/runner/mod.rs index 98dc529ab..ac63af338 100644 --- a/testing/checkin/runner/mod.rs +++ b/testing/checkin/runner/mod.rs @@ -31,7 +31,9 @@ mod ops; mod ops_async; mod ops_buffer; mod ops_error; +mod ops_fs; mod ops_io; +mod ops_transpiler; mod ops_worker; pub mod snapshot; #[cfg(test)] diff --git a/testing/checkin/runner/ops_fs.rs b/testing/checkin/runner/ops_fs.rs new file mode 100644 index 000000000..9adf5dfda --- /dev/null +++ b/testing/checkin/runner/ops_fs.rs @@ -0,0 +1,22 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_core::error::AnyError; +use deno_core::op2; + +#[op2] +#[string] +pub fn op_fs_read_text_file( + #[string] file_path: &str, +) -> Result { + let content = std::fs::read_to_string(file_path)?; + Ok(content) +} + +#[op2(fast)] +pub fn op_fs_write_text_file( + #[string] file_path: &str, + #[string] content: &str, +) -> Result<(), AnyError> { + std::fs::write(file_path, content)?; + Ok(()) +} diff --git a/testing/checkin/runner/ops_transpiler.rs b/testing/checkin/runner/ops_transpiler.rs new file mode 100644 index 000000000..72c1dd310 --- /dev/null +++ b/testing/checkin/runner/ops_transpiler.rs @@ -0,0 +1,36 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_ast::MediaType; +use deno_ast::ParseParams; +use deno_ast::SourceMapOption; +use deno_ast::SourceTextInfo; +use deno_core::error::AnyError; +use deno_core::op2; +use deno_core::url::Url; + +#[op2] +#[serde] +pub fn op_transpile( + #[string] specifier: &str, + #[string] source: String, +) -> Result<(String, Option), AnyError> { + let media_type = MediaType::from_str(specifier); + + let parsed = deno_ast::parse_module(ParseParams { + specifier: Url::parse(specifier).unwrap(), + text_info: SourceTextInfo::from_string(source), + media_type, + capture_tokens: false, + scope_analysis: false, + maybe_syntax: None, + })?; + let transpiled_source = parsed.transpile(&deno_ast::EmitOptions { + imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, + source_map: SourceMapOption::Inline, + inline_sources: false, + use_decorators_proposal: true, + ..Default::default() + })?; + + Ok((transpiled_source.text, transpiled_source.source_map)) +} diff --git a/testing/checkin/runtime/__init.js b/testing/checkin/runtime/__init.js index 9b4ac1e00..fa595f963 100644 --- a/testing/checkin/runtime/__init.js +++ b/testing/checkin/runtime/__init.js @@ -2,8 +2,10 @@ import * as async from "checkin:async"; import * as console from "checkin:console"; import * as error from "checkin:error"; +import * as fs from "checkin:fs"; import * as timers from "checkin:timers"; import * as worker from "checkin:worker"; +import * as transpiler from "checkin:transpiler"; import * as throw_ from "checkin:throw"; async; error; @@ -15,6 +17,9 @@ globalThis.setInterval = timers.setInterval; globalThis.clearTimeout = timers.clearTimeout; globalThis.clearInterval = timers.clearInterval; globalThis.Worker = worker.Worker; +globalThis.Transpiler = transpiler.Transpiler; +globalThis.readTextFile = fs.readTextFile; +globalThis.writeTextFile = fs.writeTextFile; Deno.core.addMainModuleHandler((module) => { if (onMainModuleCb) onMainModuleCb(module); }); diff --git a/testing/checkin/runtime/fs.ts b/testing/checkin/runtime/fs.ts new file mode 100644 index 000000000..5fd2a84b2 --- /dev/null +++ b/testing/checkin/runtime/fs.ts @@ -0,0 +1,10 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +import { op_fs_read_text_file, op_fs_write_text_file } from "ext:core/ops"; + +export function readTextFile(filePath: string): string { + return op_fs_read_text_file(filePath); +} + +export function writeTextFile(filePath: string, content: string) { + op_fs_write_text_file(filePath, content); +} diff --git a/testing/checkin/runtime/transpiler.ts b/testing/checkin/runtime/transpiler.ts new file mode 100644 index 000000000..4d8728d11 --- /dev/null +++ b/testing/checkin/runtime/transpiler.ts @@ -0,0 +1,11 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +import { op_transpile } from "ext:core/ops"; + +export class Transpiler { + constructor() { + } + + transpile(specifier: string, source: string) { + return op_transpile(specifier, source); + } +} diff --git a/testing/ops.d.ts b/testing/ops.d.ts index c89a661ae..f3da37a22 100644 --- a/testing/ops.d.ts +++ b/testing/ops.d.ts @@ -12,6 +12,12 @@ export function op_async_throw_error_lazy(...any: any[]): any; export function op_error_context_async(...any: any[]): any; export function op_error_context_sync(...any: any[]): any; export function op_error_custom_sync(...any: any[]): any; +export function op_fs_read_text_file(path: string): string; +export function op_fs_write_text_file(path: string, content: string): void; +export function op_transpile( + specifier: string, + source: string, +): [string, string | undefined]; export function op_worker_await_close(...any: any[]): any; export function op_worker_parent(...any: any[]): any; diff --git a/testing/tsconfig.json b/testing/tsconfig.json index 292944138..bd325881a 100644 --- a/testing/tsconfig.json +++ b/testing/tsconfig.json @@ -8,9 +8,11 @@ "checkin:async": ["checkin/runtime/async.ts"], "checkin:console": ["checkin/runtime/console.ts"], "checkin:error": ["checkin/runtime/error.ts"], + "checkin:fs": ["checkin/runtime/fs.ts"], "checkin:testing": ["checkin/runtime/testing.ts"], "checkin:throw": ["checkin/runtime/throw.ts"], "checkin:timers": ["checkin/runtime/timers.ts"], + "checkin:transpiler": ["checkin/runtime/transpiler.ts"], "checkin:worker": ["checkin/runtime/worker.ts"] }, "lib": ["ESNext", "DOM", "ES2023.Array"], diff --git a/transpile.js b/transpile.js new file mode 100644 index 000000000..8dcb63b38 --- /dev/null +++ b/transpile.js @@ -0,0 +1,30 @@ +const transpiler = new Transpiler(); + +const SOURCE = `export function a(): string { + return "a"; +} + +function spawn() { + const b = new Deno.Command(); + + b.spawn(); +} + +interface Foobar { + a: string; + b: number; + c: boolean; +} + +function runSpawn() { + spawn(); +} + +runSpawn();`; + +const SPECIFIER = "file:///a.ts"; +const TRANSPILED_SOURCE_FILE_PATH = "./a.js"; +const SOURCE_MAP_FILE_PATH = "./a.js.map"; +const [transpiledSource, sourceMap] = transpiler.transpile(SPECIFIER, SOURCE); +writeTextFile(TRANSPILED_SOURCE_FILE_PATH, transpiledSource); +writeTextFile(SOURCE_MAP_FILE_PATH, sourceMap); From cc573b4c99aa53bbbde9dc1292d709bcb419d5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 18 Jul 2024 02:22:51 +0200 Subject: [PATCH 21/21] wip upgrade --- Cargo.lock | 119 +++++++++++------------ Cargo.toml | 2 +- testing/checkin/runner/ops_transpiler.rs | 30 ++++-- 3 files changed, 76 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e9d85890..42ba2e997 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,9 +112,9 @@ checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" [[package]] name = "ast_node" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e521452c6bce47ee5a5461c5e5d707212907826de14124962c58fcaf463115e" +checksum = "2ab31376d309dd3bfc9cfb3c11c93ce0e0741bbe0354b20e7f8c60b044730b79" dependencies = [ "proc-macro2", "quote", @@ -514,9 +514,9 @@ dependencies = [ [[package]] name = "deno_ast" -version = "0.35.3" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f340bc18a3c351cb4af21e56a8af13cdcc08ff7616799429c745bce26fea6385" +checksum = "4d08372522975cce97fe0efbe42fea508c76eea4421619de6d63baae32792f7d" dependencies = [ "anyhow", "base64", @@ -548,6 +548,7 @@ dependencies = [ "swc_visit", "swc_visit_macros", "text_lines", + "thiserror", "unicode-width", "url", ] @@ -690,9 +691,9 @@ dependencies = [ [[package]] name = "dprint-swc-ext" -version = "0.15.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bad772f9e49af3a613fcddf1671d1e2e877e0a6d94f2b7162bfea4ac8140bee" +checksum = "5a0d5b63e52434314e3d767c463b1f68c467c31e61d279bc019227016c44e535" dependencies = [ "num-bigint", "rustc-hash", @@ -767,9 +768,9 @@ dependencies = [ [[package]] name = "from_variant" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a0b11eeb173ce52f84ebd943d42e58813a2ebb78a6a3ff0a243b71c5199cd7b" +checksum = "fdc9cc75639b041067353b9bce2450d6847e547276c6fbe4487d7407980e07db" dependencies = [ "proc-macro2", "swc_macros_common", @@ -1859,17 +1860,6 @@ dependencies = [ "v8", ] -[[package]] -name = "sha-1" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha1" version = "0.10.6" @@ -1999,9 +1989,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "string_enum" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6960defec35d15d58331ffb8a315d551634f757fe139c7b3d6063cae88ec90f6" +checksum = "05e383308aebc257e7d7920224fa055c632478d92744eca77f99be8fa1545b90" dependencies = [ "proc-macro2", "quote", @@ -2039,9 +2029,9 @@ dependencies = [ [[package]] name = "swc_atoms" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d9d1941a7d24fc503efa29c53f88dd61e6a15cc371947a75cca3b48d564b5b" +checksum = "bb6567e4e67485b3e7662b486f1565bdae54bd5b9d6b16b2ba1a9babb1e42125" dependencies = [ "hstr", "once_cell", @@ -2065,9 +2055,9 @@ dependencies = [ [[package]] name = "swc_common" -version = "0.33.22" +version = "0.34.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f91d53367db183e55ecaa090c9a2b6e62ddbd1258aa2ac6c6925772eec9e2b" +checksum = "9087befec6b63911f9d2f239e4f91c9b21589c169b86ed2d616944d23cf4a243" dependencies = [ "ast_node", "better_scoped_tls", @@ -2091,9 +2081,9 @@ dependencies = [ [[package]] name = "swc_config" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ada712ac5e28a301683c8af957e8a56deca675cbc376473dd207a527b989efb5" +checksum = "84b67e115ab136fe0eb03558bb0508ca7782eeb446a96d165508c48617e3fd94" dependencies = [ "anyhow", "indexmap", @@ -2105,9 +2095,9 @@ dependencies = [ [[package]] name = "swc_config_macro" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2574f75082322a27d990116cd2a24de52945fc94172b24ca0b3e9e2a6ceb6b" +checksum = "7c5f56139042c1a95b54f5ca48baa0e0172d369bcc9d3d473dad1de36bae8399" dependencies = [ "proc-macro2", "quote", @@ -2117,9 +2107,9 @@ dependencies = [ [[package]] name = "swc_ecma_ast" -version = "0.112.7" +version = "0.115.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bcd97ee367b48444f90416ea56e71d761600f816bcae9df4f99293d1fa36bd5" +checksum = "7be1306930c235435a892104c00c2b5e16231043c085d5a10bd3e7537b15659b" dependencies = [ "bitflags", "is-macro", @@ -2135,9 +2125,9 @@ dependencies = [ [[package]] name = "swc_ecma_codegen" -version = "0.148.15" +version = "0.151.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5206067d7c5034e4e19e71ebce270081594eb726c108d0a0a026f7ecbb6f8abd" +checksum = "f5141a8cb4eb69e090e6aea5d49061b46919be5210f3d084f9d9ad63d30f5cff" dependencies = [ "memchr", "num-bigint", @@ -2154,9 +2144,9 @@ dependencies = [ [[package]] name = "swc_ecma_codegen_macros" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "394b8239424b339a12012ceb18726ed0244fce6bf6345053cb9320b2791dcaa5" +checksum = "090e409af49c8d1a3c13b3aab1ed09dd4eda982207eb3e63c2ad342f072b49c8" dependencies = [ "proc-macro2", "quote", @@ -2166,9 +2156,9 @@ dependencies = [ [[package]] name = "swc_ecma_loader" -version = "0.45.25" +version = "0.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea397a556585ffd78cda6ef420f6fd0ad54320778930eeb876f02041f58bb017" +checksum = "5a9febebf047d1286e7b723fa2758f3229da2c103834f3eaee69833f46692612" dependencies = [ "anyhow", "pathdiff", @@ -2180,9 +2170,9 @@ dependencies = [ [[package]] name = "swc_ecma_parser" -version = "0.143.13" +version = "0.146.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5354a20ab66c2ec5001982271b6e7c750b7fca3409888ab9703ae3d3c845fed4" +checksum = "0a4e0c2e85f12c63b85c805e923079b04d1fb3e25edd069d638eed5f2098de74" dependencies = [ "either", "new_debug_unreachable", @@ -2202,9 +2192,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_base" -version = "0.137.17" +version = "0.140.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b5818db80d8d9fcbc1d3453f1d246a7f56ea708ba136717a84a8caf0977afd" +checksum = "d37dc505c92af56d0f77cf6f31a6ccd37ac40cad1e01ff77277e0b1c70e8f8ff" dependencies = [ "better_scoped_tls", "bitflags", @@ -2225,9 +2215,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_classes" -version = "0.126.16" +version = "0.129.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47af84e64f0216f110839f5552a615d07ed74b45757927f29482700966ab4e97" +checksum = "a3eab5f8179e5b0aedf385eacc2c033691c6d211a7babd1bbbff12cf794a824e" dependencies = [ "swc_atoms", "swc_common", @@ -2239,9 +2229,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_macros" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e309b88f337da54ef7fe4c5b99c2c522927071f797ee6c9fb8b6bf2d100481" +checksum = "500a1dadad1e0e41e417d633b3d6d5de677c9e0d3159b94ba3348436cdb15aab" dependencies = [ "proc-macro2", "quote", @@ -2251,9 +2241,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_proposal" -version = "0.171.20" +version = "0.174.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac54efb40cb374c53323823da2b5f74406b8531b59a2cb689801cbf22d0057db" +checksum = "6df8aa6752cc2fcf3d78ac67827542fb666e52283f2b26802aa058906bb750d3" dependencies = [ "either", "rustc-hash", @@ -2271,16 +2261,16 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_react" -version = "0.183.18" +version = "0.186.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b7b7de90ff41560bf021acda3fb16fb0f4f5885aeb44b6b7e638b563124d087" +checksum = "446da32cac8299973aaf1d37496562bfd0c1e4f3c3ab5d0af6f07f42e8184102" dependencies = [ "base64", "dashmap", "indexmap", "once_cell", "serde", - "sha-1", + "sha1", "string_enum", "swc_atoms", "swc_common", @@ -2295,9 +2285,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_typescript" -version = "0.188.18" +version = "0.191.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "845c3ad4949c2317076e929e42c78dd43868f6c6f820911353731a5bb859e78c" +checksum = "f1ce8af2865449e714ae56dacb6b54b3f6dc4cc25074da4e39b878bd93c5e39c" dependencies = [ "ryu-js", "serde", @@ -2312,14 +2302,15 @@ dependencies = [ [[package]] name = "swc_ecma_utils" -version = "0.127.17" +version = "0.130.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624c19fdbe1807275b16560892cf7a12a9ac3f631fb10ad45aaa3eeac903e6e5" +checksum = "13e62b199454a576c5fdbd7e1bef8ab88a395427456d8a713d994b7d469833aa" dependencies = [ "indexmap", "num_cpus", "once_cell", "rustc-hash", + "ryu-js", "swc_atoms", "swc_common", "swc_ecma_ast", @@ -2330,9 +2321,9 @@ dependencies = [ [[package]] name = "swc_ecma_visit" -version = "0.98.7" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93692bdcdbb63db8f5e10fea5d202b5487cb27eb443aec424f4335c88f9864af" +checksum = "ce0d997f0c9b4e181225f603d161f6757c2a97022258170982cfe005ec69ec92" dependencies = [ "num-bigint", "swc_atoms", @@ -2355,9 +2346,9 @@ dependencies = [ [[package]] name = "swc_macros_common" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50176cfc1cbc8bb22f41c6fe9d1ec53fbe057001219b5954961b8ad0f336fce9" +checksum = "91745f3561057493d2da768437c427c0e979dff7396507ae02f16c981c4a8466" dependencies = [ "proc-macro2", "quote", @@ -2366,9 +2357,9 @@ dependencies = [ [[package]] name = "swc_visit" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0263be55289abfe9c877ffef83d877b5bdfac036ffe2de793f48f5e47e41dbae" +checksum = "043d11fe683dcb934583ead49405c0896a5af5face522e4682c16971ef7871b9" dependencies = [ "either", "swc_visit_macros", @@ -2376,9 +2367,9 @@ dependencies = [ [[package]] name = "swc_visit_macros" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fc817055fe127b4285dc85058596768bfde7537ae37da82c67815557f03e33" +checksum = "4ae9ef18ff8daffa999f729db056d2821cd2f790f3a11e46422d19f46bb193e7" dependencies = [ "Inflector", "proc-macro2", @@ -2658,9 +2649,9 @@ checksum = "b1b6def86329695390197b82c1e244a54a131ceb66c996f2088a3876e2ae083f" [[package]] name = "unicode-id-start" -version = "1.1.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8f73150333cb58412db36f2aca8f2875b013049705cc77b94ded70a1ab1f5da" +checksum = "02aebfa694eccbbbffdd92922c7de136b9fe764396d2f10e21bce1681477cfc1" [[package]] name = "unicode-ident" diff --git a/Cargo.toml b/Cargo.toml index 770d540b9..0db847574 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ serde_v8 = { version = "0.204.0", path = "./serde_v8" } deno_core_testing = { path = "./testing" } v8 = { version = "0.98.1", default-features = false } -deno_ast = { version = "=0.35.3", features = ["transpiling"] } +deno_ast = { version = "=0.40.0", features = ["transpiling"] } deno_unsync = "0.3.10" deno_core_icudata = "0.0.73" diff --git a/testing/checkin/runner/ops_transpiler.rs b/testing/checkin/runner/ops_transpiler.rs index 72c1dd310..d36fcbcf9 100644 --- a/testing/checkin/runner/ops_transpiler.rs +++ b/testing/checkin/runner/ops_transpiler.rs @@ -3,7 +3,6 @@ use deno_ast::MediaType; use deno_ast::ParseParams; use deno_ast::SourceMapOption; -use deno_ast::SourceTextInfo; use deno_core::error::AnyError; use deno_core::op2; use deno_core::url::Url; @@ -18,19 +17,30 @@ pub fn op_transpile( let parsed = deno_ast::parse_module(ParseParams { specifier: Url::parse(specifier).unwrap(), - text_info: SourceTextInfo::from_string(source), + text: source.into(), media_type, capture_tokens: false, scope_analysis: false, maybe_syntax: None, })?; - let transpiled_source = parsed.transpile(&deno_ast::EmitOptions { - imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, - source_map: SourceMapOption::Inline, - inline_sources: false, - use_decorators_proposal: true, - ..Default::default() - })?; + let transpile_result = parsed.transpile( + &deno_ast::TranspileOptions { + use_decorators_proposal: true, + imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, + ..Default::default() + }, + &deno_ast::EmitOptions { + source_map: SourceMapOption::Inline, + inline_sources: false, + ..Default::default() + }, + )?; + let transpiled_source = transpile_result.into_source(); - Ok((transpiled_source.text, transpiled_source.source_map)) + Ok(( + String::from_utf8(transpiled_source.source).unwrap(), + transpiled_source + .source_map + .map(|s| String::from_utf8(s).unwrap()), + )) }