[turbopack] Implement improved deobfuscation for free calls and module identifiers.

examples/basic-css/tsconfig.json

@@ -21,6 +21,12 @@
     "strictNullChecks": true,
     "target": "ES2017"
   },
-  "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"],
+  "include": [
+    "next-env.d.ts",
+    ".next/types/**/*.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/dev/types/**/*.ts"
+  ],
   "exclude": ["node_modules", "**/*.test.ts", "**/*.test.tsx"]
 }

examples/with-turbopack/app/page.tsx

@@ -1,3 +1,6 @@
+import { foo } from "./foo";
 export default function Page() {
+  /** @ts-ignore */
+  foo();
   return <h1>Hello, Next.js!</h1>;
 }

examples/with-turbopack/tsconfig.json

@@ -20,6 +20,12 @@
       }
     ]
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts",
+    ".next/dev/types/**/*.ts"
+  ],
   "exclude": ["node_modules"]
 }

packages/next/src/next-devtools/dev-overlay/components/hot-linked-text/index.tsx

@@ -1,59 +1,59 @@
 import React from 'react'
-import {
-  decodeMagicIdentifier,
-  MAGIC_IDENTIFIER_REGEX,
-} from '../../../../shared/lib/magic-identifier'
+import { deobfuscateTextParts } from '../../../../shared/lib/magic-identifier'
 
 const linkRegex = /https?:\/\/[^\s/$.?#].[^\s)'"]*/i
 
-const splitRegexp = new RegExp(`(${MAGIC_IDENTIFIER_REGEX.source}|\\s+)`)
-
 export const HotlinkedText: React.FC<{
   text: string
   matcher?: (text: string) => boolean
 }> = function HotlinkedText(props) {
   const { text, matcher } = props
 
-  const wordsAndWhitespaces = text.split(splitRegexp)
+  // Deobfuscate the entire text first
+  const deobfuscatedParts = deobfuscateTextParts(text)
 
   return (
     <>
-      {wordsAndWhitespaces.map((word, index) => {
-        if (linkRegex.test(word)) {
-          const link = linkRegex.exec(word)!
-          const href = link[0]
-          // If link matcher is present but the link doesn't match, don't turn it into a link
-          if (typeof matcher === 'function' && !matcher(href)) {
-            return word
-          }
-          return (
-            <React.Fragment key={`link-${index}`}>
-              <a href={href} target="_blank" rel="noreferrer noopener">
-                {word}
-              </a>
-            </React.Fragment>
-          )
-        }
-        try {
-          const decodedWord = decodeMagicIdentifier(word)
-          if (decodedWord !== word) {
-            return (
-              <i key={`ident-${index}`}>
-                {'{'}
-                {decodedWord}
-                {'}'}
-              </i>
-            )
-          }
-        } catch (e) {
+      {deobfuscatedParts.map(([type, part], outerIndex) => {
+        if (type === 'raw') {
           return (
-            <i key={`ident-${index}`}>
-              {'{'}
-              {word} (decoding failed: {'' + e}){'}'}
-            </i>
+            part
+              // Split on whitespace and links
+              .split(/(\s+|https?:\/\/[^\s/$.?#].[^\s)'"]*)/)
+              .map((rawPart, index) => {
+                if (linkRegex.test(rawPart)) {
+                  const link = linkRegex.exec(rawPart)!
+                  const href = link[0]
+                  // If link matcher is present but the link doesn't match, don't turn it into a link
+                  if (typeof matcher === 'function' && !matcher(href)) {
+                    return (
+                      <React.Fragment key={`link-${outerIndex}-${index}`}>
+                        {rawPart}
+                      </React.Fragment>
+                    )
+                  }
+                  return (
+                    <React.Fragment key={`link-${outerIndex}-${index}`}>
+                      <a href={href} target="_blank" rel="noreferrer noopener">
+                        {rawPart}
+                      </a>
+                    </React.Fragment>
+                  )
+                } else {
+                  return (
+                    <React.Fragment key={`text-${outerIndex}-${index}`}>
+                      {rawPart}
+                    </React.Fragment>
+                  )
+                }
+              })
           )
+        } else if (type === 'deobfuscated') {
+          // italicize the deobfuscated part
+          return <i key={`ident-${outerIndex}`}>{part}</i>
+        } else {
+          throw new Error(`Unknown text part type: ${type}`)
         }
-        return <React.Fragment key={`text-${index}`}>{word}</React.Fragment>
       })}
     </>
   )

packages/next/src/server/lib/router-utils/setup-dev-bundler.ts

@@ -88,6 +88,7 @@ import {
 import { isParallelRouteSegment } from '../../../shared/lib/segment'
 import { ensureLeadingSlash } from '../../../shared/lib/page-path/ensure-leading-slash'
 import { Lockfile } from '../../../build/lockfile'
+import { deobfuscateText } from '../../../shared/lib/magic-identifier'
 
 export type SetupOpts = {
   renderServer: LazyRenderServerInstance
@@ -1224,6 +1225,9 @@ async function startWatcher(
     err: unknown,
     type?: 'unhandledRejection' | 'uncaughtException' | 'warning' | 'app-dir'
   ) {
+    if (err instanceof Error) {
+      err.message = deobfuscateText(err.message)
+    }
     if (err instanceof ModuleBuildError) {
       // Errors that may come from issues from the user's code
       Log.error(err.message)

packages/next/src/shared/lib/magic-identifier.test.ts

@@ -0,0 +1,178 @@
+import {
+  decodeMagicIdentifier,
+  MAGIC_IDENTIFIER_REGEX,
+  deobfuscateModuleId,
+  removeFreeCallWrapper,
+  deobfuscateText,
+  deobfuscateTextParts,
+} from './magic-identifier'
+
+describe('decodeMagicIdentifier', () => {
+  // Basic decoding tests (ported from Rust)
+  test('decodes module evaluation', () => {
+    expect(decodeMagicIdentifier('__TURBOPACK__module__evaluation__')).toBe(
+      'module evaluation'
+    )
+  })
+
+  test('decodes path with slashes', () => {
+    expect(decodeMagicIdentifier('__TURBOPACK__Hello$2f$World__')).toBe(
+      'Hello/World'
+    )
+  })
+
+  test('decodes emoji', () => {
+    expect(decodeMagicIdentifier('__TURBOPACK__Hello$_1f600$World__')).toBe(
+      'Hello😀World'
+    )
+  })
+
+  test('returns unchanged if not a magic identifier', () => {
+    expect(decodeMagicIdentifier('regular_identifier')).toBe(
+      'regular_identifier'
+    )
+  })
+})
+
+describe('MAGIC_IDENTIFIER_REGEX', () => {
+  test('matches magic identifiers globally', () => {
+    const text =
+      'Hello __TURBOPACK__Hello__World__ and __TURBOPACK__foo$2f$bar__'
+    const matches = text.match(MAGIC_IDENTIFIER_REGEX)
+    expect(matches).toHaveLength(2)
+  })
+})
+
+describe('deobfuscateModuleId', () => {
+  test('replaces [project] with .', () => {
+    expect(
+      deobfuscateModuleId('[project]/examples/with-turbopack/app/foo.ts')
+    ).toBe('./examples/with-turbopack/app/foo.ts')
+  })
+
+  test('removes content in square brackets', () => {
+    expect(
+      deobfuscateModuleId('./examples/with-turbopack/app/foo.ts [app-rsc]')
+    ).toBe('./examples/with-turbopack/app/foo.ts')
+  })
+
+  test('removes content in parentheses', () => {
+    expect(
+      deobfuscateModuleId('./examples/with-turbopack/app/foo.ts (ecmascript)')
+    ).toBe('./examples/with-turbopack/app/foo.ts')
+  })
+
+  test('removes content in angle brackets', () => {
+    expect(
+      deobfuscateModuleId('./examples/with-turbopack/app/foo.ts <locals>')
+    ).toBe('./examples/with-turbopack/app/foo.ts')
+  })
+
+  test('handles combined cleanup', () => {
+    expect(
+      deobfuscateModuleId(
+        '[project]/examples/with-turbopack/app/foo.ts [app-rsc] (ecmascript)'
+      )
+    ).toBe('./examples/with-turbopack/app/foo.ts')
+  })
+
+  test('handles parenthesis in path', () => {
+    expect(
+      deobfuscateModuleId(
+        '[project]/examples/(group)/with-turbopack/app/foo.ts [app-rsc] (ecmascript)'
+      )
+    ).toBe('./examples/(group)/with-turbopack/app/foo.ts')
+  })
+})
+
+describe('removeFreeCallWrapper', () => {
+  test('removes (0, ) wrapper', () => {
+    expect(removeFreeCallWrapper('(0, __TURBOPACK__foo__.bar)')).toBe(
+      '__TURBOPACK__foo__.bar'
+    )
+  })
+
+  test('removes (0 , ) wrapper with spaces', () => {
+    expect(removeFreeCallWrapper('(0 , __TURBOPACK__foo__.bar)')).toBe(
+      '__TURBOPACK__foo__.bar'
+    )
+  })
+
+  test('leaves non-free-call expressions unchanged', () => {
+    expect(removeFreeCallWrapper('(foo, bar)')).toBe('(foo, bar)')
+    expect(removeFreeCallWrapper('foo()')).toBe('foo()')
+  })
+})
+
+describe('deobfuscateText', () => {
+  test('deobfuscates complete error message with imported module', () => {
+    const input =
+      '(0 , __TURBOPACK__imported__module__$5b$project$5d2f$examples$2f$with$2d$turbopack$2f$app$2f$foo$2e$ts__$5b$app$2d$rsc$5d$__$28$ecmascript$29$__.foo) is not a function'
+    const output = deobfuscateText(input)
+    expect(output).toBe(
+      '{imported module ./examples/with-turbopack/app/foo.ts}.foo is not a function'
+    )
+  })
+
+  test('handles multiple magic identifiers', () => {
+    const input =
+      '__TURBOPACK__module__evaluation__ called __TURBOPACK__foo$2f$bar__'
+    const output = deobfuscateText(input)
+    expect(output).toBe('{module evaluation} called {foo/bar}')
+  })
+
+  test('leaves regular text unchanged', () => {
+    const input = 'This is a regular error message'
+    expect(deobfuscateText(input)).toBe(input)
+  })
+})
+
+describe('deobfuscateTextParts', () => {
+  test('returns discriminated parts with raw and deobfuscated text', () => {
+    const input = 'Error in __TURBOPACK__module__evaluation__ at line 10'
+    const output = deobfuscateTextParts(input)
+    expect(output).toEqual([
+      ['raw', 'Error in '],
+      ['deobfuscated', '{module evaluation}'],
+      ['raw', ' at line 10'],
+    ])
+  })
+
+  test('handles multiple magic identifiers with interleaved raw text', () => {
+    const input =
+      '__TURBOPACK__module__evaluation__ called __TURBOPACK__foo$2f$bar__'
+    const output = deobfuscateTextParts(input)
+    expect(output).toEqual([
+      ['deobfuscated', '{module evaluation}'],
+      ['raw', ' called '],
+      ['deobfuscated', '{foo/bar}'],
+    ])
+  })
+
+  test('returns single raw part for text without magic identifiers', () => {
+    const input = 'This is a regular error message'
+    const output = deobfuscateTextParts(input)
+    expect(output).toEqual([['raw', 'This is a regular error message']])
+  })
+
+  test('handles imported module with free call wrapper', () => {
+    const input =
+      '(0 , __TURBOPACK__imported__module__$5b$project$5d2f$examples$2f$with$2d$turbopack$2f$app$2f$foo$2e$ts__$5b$app$2d$rsc$5d$__$28$ecmascript$29$__.foo) is not a function'
+    const output = deobfuscateTextParts(input)
+    expect(output).toEqual([
+      [
+        'deobfuscated',
+        '{imported module ./examples/with-turbopack/app/foo.ts}',
+      ],
+      ['raw', '.foo is not a function'],
+    ])
+  })
+
+  test('produces same result as deobfuscateText when joined', () => {
+    const input =
+      'Error in __TURBOPACK__module__evaluation__ at __TURBOPACK__foo$2f$bar__'
+    const parts = deobfuscateTextParts(input)
+    const joined = parts.map((part) => part[1]).join('')
+    expect(joined).toBe(deobfuscateText(input))
+  })
+})