Turbopack: properly replace __dirname and __filename (vercel#78843)

Replace `__dirname` and `__filename` references at compile time

Our current approach of computing `__dirname` at runtime based on the module id is broken in production builds where numeric ids are used.  Arguably it is incorrect in development builds as well as the value we produce is not actually usable as a path.

We already compute a correct __dirname and __filename for our statically evaluated functions, so this uses the same approach which is to use the source path of the module.

Closes PACK-4332

Author: lukesandberg

turbopack/crates/turbopack-core/src/compile_time_info.rs

@@ -1,6 +1,7 @@
 use anyhow::Result;
+use serde::{Deserialize, Serialize};
 use turbo_rcstr::RcStr;
-use turbo_tasks::{FxIndexMap, ResolvedVc, Vc};
+use turbo_tasks::{trace::TraceRawVcs, FxIndexMap, NonLocalValue, ResolvedVc, Vc};
 use turbo_tasks_fs::FileSystemPath;
 
 use crate::environment::Environment;
@@ -200,6 +201,14 @@ impl CompileTimeDefines {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, NonLocalValue)]
+pub enum InputRelativeConstant {
+    // The project relative directory name of the source file
+    DirName,
+    // The project relative file name of the source file.
+    FileName,
+}
+
 #[turbo_tasks::value]
 #[derive(Debug, Clone)]
 pub enum FreeVarReference {
@@ -211,6 +220,7 @@ pub enum FreeVarReference {
     Ident(RcStr),
     Member(RcStr, RcStr),
     Value(CompileTimeDefineValue),
+    InputRelative(InputRelativeConstant),
     Error(RcStr),
 }
 

turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/build-base.ts

@@ -125,10 +125,6 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
         U: relativeURL,
         R: createResolvePathFromModule(r),
         b: getWorkerBlobURL,
-        d:
-          typeof module.id === 'string'
-            ? module.id.replace(/(^|\/)\/+$/, '')
-            : module.id,
       })
     )
   } catch (error) {

turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/runtime/base/dev-base.ts

@@ -129,7 +129,7 @@ const getOrInstantiateModuleFromParent: GetOrInstantiateModuleFromParent<
   })
 }
 
-function getDevWorkerBlobURL(chunks) {
+function getDevWorkerBlobURL(chunks: ChunkPath[]) {
   return getWorkerBlobURL(
     chunks,
     `// noop fns to prevent runtime errors during initialization
@@ -232,10 +232,6 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
           R: createResolvePathFromModule(r),
           b: getDevWorkerBlobURL,
           z: requireStub,
-          d:
-            typeof module.id === 'string'
-              ? module.id.replace(/(^|\/)\/+$/, '')
-              : module.id,
         })
       )
     })

turbopack/crates/turbopack-ecmascript-runtime/js/src/nodejs/runtime.ts

@@ -295,10 +295,6 @@ function instantiateModule(
       R: createResolvePathFromModule(r),
       b: getWorkerBlobURL,
       z: requireStub,
-      __dirname:
-        typeof module.id === 'string'
-          ? module.id.replace(/(^|\/)\/+$/, '')
-          : module.id,
     })
   } catch (error) {
     module.error = error as any

turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-types.d.ts

@@ -112,5 +112,4 @@ interface TurbopackBaseContext<M> {
   U: RelativeURL
   b: GetWorkerBlobURL
   z: CommonJsRequire
-  d: string
 }

turbopack/crates/turbopack-ecmascript/src/analyzer/mod.rs

@@ -602,6 +602,20 @@ impl From<&FreeVarReference> for JsValue {
             FreeVarReference::Error(_) => {
                 JsValue::unknown_empty(false, "compile time injected free var error")
             }
+            FreeVarReference::InputRelative(kind) => {
+                use turbopack_core::compile_time_info::InputRelativeConstant;
+                JsValue::unknown_empty(
+                    false,
+                    match kind {
+                        InputRelativeConstant::DirName => {
+                            "compile time injected free var referencing the directory name"
+                        }
+                        InputRelativeConstant::FileName => {
+                            "compile time injected free var referencing the file name"
+                        }
+                    },
+                )
+            }
         }
     }
 }

turbopack/crates/turbopack-ecmascript/src/chunk/item.rs

@@ -87,11 +87,8 @@ impl EcmascriptChunkItemContent {
 
     #[turbo_tasks::function]
     pub async fn module_factory(&self) -> Result<Vc<Code>> {
-        let mut args = vec![
-            "g: global",
-            // HACK
-            "__dirname",
-        ];
+        // TODO(lukesandberg): find a better way to bind this parameter.
+        let mut args = vec!["g: global"];
         if self.options.async_module.is_some() {
             args.push("a: __turbopack_async_module__");
         }

turbopack/crates/turbopack-ecmascript/src/references/mod.rs

@@ -60,7 +60,7 @@ use turbo_tasks_fs::FileSystemPath;
 use turbopack_core::{
     compile_time_info::{
         CompileTimeInfo, DefineableNameSegment, FreeVarReference, FreeVarReferences,
-        FreeVarReferencesIndividual,
+        FreeVarReferencesIndividual, InputRelativeConstant,
     },
     environment::Rendering,
     error::PrettyPrintError,
@@ -1495,6 +1495,32 @@ async fn compile_time_info_for_module_type(
         .entry(vec![DefineableNameSegment::Name("require".into())])
         .or_insert(require.into());
 
+    let dir_name: RcStr = "__dirname".into();
+    free_var_references
+        .entry(vec![
+            DefineableNameSegment::Name(dir_name.clone()),
+            DefineableNameSegment::TypeOf,
+        ])
+        .or_insert("string".into());
+    free_var_references
+        .entry(vec![DefineableNameSegment::Name(dir_name)])
+        .or_insert(FreeVarReference::InputRelative(
+            InputRelativeConstant::DirName,
+        ));
+    let file_name: RcStr = "__filename".into();
+
+    free_var_references
+        .entry(vec![
+            DefineableNameSegment::Name(file_name.clone()),
+            DefineableNameSegment::TypeOf,
+        ])
+        .or_insert("string".into());
+    free_var_references
+        .entry(vec![DefineableNameSegment::Name(file_name)])
+        .or_insert(FreeVarReference::InputRelative(
+            InputRelativeConstant::FileName,
+        ));
+
     free_var_references.extend(TUBROPACK_RUNTIME_FUNCTION_SHORTCUTS.into_iter().map(
         |(name, shortcut)| {
             (
@@ -2508,7 +2534,6 @@ async fn handle_free_var_reference(
                 errors::failed_to_analyse::ecmascript::FREE_VAR_REFERENCE.to_string(),
             ),
         ),
-
         FreeVarReference::Value(value) => {
             analysis.add_code_gen(ConstantValueCodeGen::new(
                 Value::new(value.clone()),
@@ -2576,6 +2601,17 @@ async fn handle_free_var_reference(
                 ast_path.to_vec().into(),
             ));
         }
+        FreeVarReference::InputRelative(kind) => {
+            let source_path = (*state.source).ident().path();
+            let source_path = match kind {
+                InputRelativeConstant::DirName => source_path.parent(),
+                InputRelativeConstant::FileName => source_path,
+            };
+            analysis.add_code_gen(ConstantValueCodeGen::new(
+                Value::new(as_abs_path(source_path).await?.into()),
+                ast_path.to_vec().into(),
+            ));
+        }
     }
     Ok(true)
 }
@@ -2774,7 +2810,7 @@ async fn analyze_amd_define_with_deps(
 
 /// Used to generate the "root" path to a __filename/__dirname/import.meta.url
 /// reference.
-pub async fn as_abs_path(path: Vc<FileSystemPath>) -> Result<JsValue> {
+pub async fn as_abs_path(path: Vc<FileSystemPath>) -> Result<String> {
     // TODO: This should be updated to generate a real system path on the fly
     // during runtime, so that the generated code is constant between systems
     // but the runtime evaluation can take into account the project's
@@ -2783,8 +2819,8 @@ pub async fn as_abs_path(path: Vc<FileSystemPath>) -> Result<JsValue> {
 }
 
 /// Generates an absolute path usable for `require.resolve()` calls.
-async fn require_resolve(path: Vc<FileSystemPath>) -> Result<JsValue> {
-    Ok(format!("/ROOT/{}", path.await?.path.as_str()).into())
+async fn require_resolve(path: Vc<FileSystemPath>) -> Result<String> {
+    Ok(format!("/ROOT/{}", path.await?.path.as_str()))
 }
 
 async fn early_value_visitor(mut v: JsValue) -> Result<(JsValue, bool)> {
@@ -2908,8 +2944,8 @@ async fn value_visitor_inner(
             }
         }
         JsValue::FreeVar(ref kind) => match &**kind {
-            "__dirname" => as_abs_path(origin.origin_path().parent()).await?,
-            "__filename" => as_abs_path(origin.origin_path()).await?,
+            "__dirname" => as_abs_path(origin.origin_path().parent()).await?.into(),
+            "__filename" => as_abs_path(origin.origin_path()).await?.into(),
 
             "require" => JsValue::unknown_if(
                 ignore,
@@ -2971,7 +3007,11 @@ async fn require_resolve_visitor(
             .primary_sources()
             .await?
             .iter()
-            .map(|&source| async move { require_resolve(source.ident().path()).await })
+            .map(|&source| async move {
+                require_resolve(source.ident().path())
+                    .await
+                    .map(JsValue::from)
+            })
             .try_join()
             .await?;
 

turbopack/crates/turbopack-tests/tests/execution/turbopack/code-gen/dirname-filename/input/index.js

@@ -1,6 +1,8 @@
 it('__dirname and __filename should be set', () => {
-  expect(__dirname).toMatch(
-    /^\[project\]\/.*\/code-gen\/dirname-filename\/input\/index\.js \[test\] \(ecmascript\)$/
+  expect(__dirname).toEqual(
+    '/ROOT/turbopack/crates/turbopack-tests/tests/execution/turbopack/code-gen/dirname-filename/input'
+  )
+  expect(__filename).toEqual(
+    '/ROOT/turbopack/crates/turbopack-tests/tests/execution/turbopack/code-gen/dirname-filename/input/index.js'
   )
-  expect(__filename).toMatch(/dirname-filename\/output\/.*.js$/)
 })

turbopack/crates/turbopack-tests/tests/snapshot/basic-tree-shake/dynamic-import/output/4c35f_tests_snapshot_basic-tree-shake_dynamic-import_input_index_92a5f455.js

@@ -2,7 +2,7 @@
 
 "[project]/turbopack/crates/turbopack-tests/tests/snapshot/basic-tree-shake/dynamic-import/input/lib.js [test] (ecmascript, async loader)": ((__turbopack_context__) => {
 
-var { g: global, __dirname } = __turbopack_context__;
+var { g: global } = __turbopack_context__;
 {
 __turbopack_context__.v((parentImport) => {
     return Promise.all([