From 3bb9430b1a498fc0146b4afa9b0c947e2905bd45 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Sat, 8 Nov 2025 14:53:50 -0800 Subject: [PATCH 1/4] Enable the filesystem cache for dev in canary builds --- packages/next/src/server/config-shared.ts | 4 + packages/next/src/server/config.ts | 2 +- ...stem-cache.ts => filesystem-cache.test.ts} | 153 ++++++++++-------- test/e2e/filesystem-cache/next.config.js | 9 +- .../turbo-tasks-backend/src/backend/mod.rs | 21 ++- 5 files changed, 112 insertions(+), 77 deletions(-) rename test/e2e/filesystem-cache/{filesystem-cache.ts => filesystem-cache.test.ts} (65%) diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index fa23ce95f64163..fab9f81736cae5 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -415,6 +415,8 @@ export interface ExperimentalConfig { /** * Enable filesystem cache for the turbopack dev server. + * + * Defaults to `true` in canary releases. */ turbopackFileSystemCacheForDev?: boolean @@ -1532,6 +1534,8 @@ export const defaultConfig = Object.freeze({ proxyClientMaxBodySize: 10_485_760, // 10MB hideLogsAfterAbort: false, mcpServer: true, + turbopackFileSystemCacheForDev: !isStableBuild(), + turbopackFileSystemCacheForBuild: false, }, htmlLimitedBots: undefined, bundlePagesRouterDependencies: false, diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index 8c756a7d32a484..741452c4d89202 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -400,7 +400,7 @@ function assignDefaultsAndValidate( if (isStableBuild()) { // Prevents usage of certain experimental features outside of canary - if (result.experimental?.turbopackFileSystemCacheForBuild) { + if (result.experimental.turbopackFileSystemCacheForBuild) { throw new CanaryOnlyConfigError({ feature: 'experimental.turbopackFileSystemCacheForBuild', }) diff --git a/test/e2e/filesystem-cache/filesystem-cache.ts b/test/e2e/filesystem-cache/filesystem-cache.test.ts similarity index 65% rename from test/e2e/filesystem-cache/filesystem-cache.ts rename to test/e2e/filesystem-cache/filesystem-cache.test.ts index e27687c4f74f62..9aef6d906ebe0e 100644 --- a/test/e2e/filesystem-cache/filesystem-cache.ts +++ b/test/e2e/filesystem-cache/filesystem-cache.test.ts @@ -1,8 +1,12 @@ import { nextTestSetup, isNextDev } from 'e2e-utils' import { waitFor } from 'next-test-utils' -describe('persistent-caching', () => { +describe('filesystem-caching', () => { process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' + // Make it easier to run in development, test directories are cleared between runs already so this is safe. + process.env.TURBO_ENGINE_DISABLE_VERSIONING = '1' + // decrease the idle timeout to make the test more reliable + process.env.TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS = '1000' const { skipped, next, isTurbopack } = nextTestSetup({ files: __dirname, skipDeployment: true, @@ -30,7 +34,7 @@ describe('persistent-caching', () => { async function stop() { if (isNextDev) { // Give FileSystem Cache time to write to disk - // Turbopack has an idle timeout of 2s + // Turbopack is configured to wait 1s above. // Webpack has an idle timeout (after large changes) of 1s // and we give time a bit more to allow writing to disk await waitFor(3000) @@ -43,6 +47,7 @@ describe('persistent-caching', () => { } it('should filesystem cache loaders', async () => { + process.env.ENABLE_CACHING = '1' let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp { const browser = await next.browser('/') @@ -114,7 +119,13 @@ describe('persistent-caching', () => { } } - const POTENTIAL_CHANGES = { + interface Change { + checkInitial(): Promise + withChange(previous: () => Promise): Promise + checkChanged(): Promise + fullInvalidation?: boolean + } + const POTENTIAL_CHANGES: Record = { 'RSC change': { checkInitial: makeTextCheck('/', 'hello world'), withChange: makeFileEdit('app/page.tsx'), @@ -189,80 +200,86 @@ describe('persistent-caching', () => { await textCheck('/env/client', 'hello filesystem cache') }, }, - } + } as const - const KEYS = Object.keys(POTENTIAL_CHANGES) - for (let bitset = 1; bitset < 1 << KEYS.length; bitset++) { - let combination = [] - for (let i = 0; i < KEYS.length; i++) { - if (bitset & (1 << i)) { - combination.push(KEYS[i]) - } - } - // Checking only single change and all combined for performance reasons. - if (combination.length !== 1 && combination.length !== KEYS.length) continue + // Checking only single change and all combined for performance reasons. + const combinations = Object.entries(POTENTIAL_CHANGES).map(([k, v]) => [ + k, + [v], + ]) as Array<[string, Array]> + combinations.push([ + Object.keys(POTENTIAL_CHANGES).join(', '), + Object.values(POTENTIAL_CHANGES), + ]) + for (const cacheEnabled of [true, false]) { + describe(`with cache ${cacheEnabled ? 'enabled' : 'disabled'}`, () => { + beforeAll(() => { + process.env.ENABLE_CACHING = cacheEnabled ? '1' : '0' + }) + for (const [name, changes] of combinations) { + it(`should allow to change files while stopped (${name})`, async () => { + let fullInvalidation = false + for (const change of changes) { + await change.checkInitial() + if (change.fullInvalidation) { + fullInvalidation = true + } + } - it(`should allow to change files while stopped (${combination.join(', ')})`, async () => { - let fullInvalidation = false - for (const key of combination) { - await POTENTIAL_CHANGES[key].checkInitial() - if (POTENTIAL_CHANGES[key].fullInvalidation) { - fullInvalidation = true - } - } + let unchangedTimestamp: string + if (!fullInvalidation) { + const browser = await next.browser('/unchanged') + unchangedTimestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } - let unchangedTimestamp - if (!fullInvalidation) { - const browser = await next.browser('/unchanged') - unchangedTimestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } + async function checkChanged() { + for (const change of changes) { + await change.checkChanged() + } - async function checkChanged() { - for (const key of combination) { - await POTENTIAL_CHANGES[key].checkChanged() - } + if (!fullInvalidation) { + const browser = await next.browser('/unchanged') + const timestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toEqual(timestamp) + await browser.close() + } + } - if (!fullInvalidation) { - const browser = await next.browser('/unchanged') - const timestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toEqual(timestamp) - await browser.close() - } - } + await stop() - await stop() + async function inner() { + await start() + await checkChanged() + // Some no-op change builds + for (let i = 0; i < 2; i++) { + await restartCycle() + await checkChanged() + } + await stop() + } - async function inner() { - await start() - await checkChanged() - // Some no-op change builds - for (let i = 0; i < 2; i++) { - await restartCycle() - await checkChanged() - } - await stop() - } + let current = inner + for (const change of changes) { + const prev = current + current = () => change.withChange(prev) + } + await current() - let current = inner - for (const key of combination) { - const prev = current - current = () => POTENTIAL_CHANGES[key].withChange(prev) - } - await current() - - await start() - for (const key of combination) { - await POTENTIAL_CHANGES[key].checkInitial() - } + await start() + for (const change of changes) { + await change.checkInitial() + } - if (!fullInvalidation) { - const browser = await next.browser('/unchanged') - const timestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toEqual(timestamp) - await browser.close() + if (!fullInvalidation) { + const browser = await next.browser('/unchanged') + const timestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toEqual(timestamp) + await browser.close() + } + }, 200000) } - }, 200000) + }) } }) diff --git a/test/e2e/filesystem-cache/next.config.js b/test/e2e/filesystem-cache/next.config.js index f72954fe4205b1..0a1cf6802ef2cc 100644 --- a/test/e2e/filesystem-cache/next.config.js +++ b/test/e2e/filesystem-cache/next.config.js @@ -1,3 +1,5 @@ +const enableCaching = !!process.env.ENABLE_CACHING + /** * @type {import('next').NextConfig} */ @@ -16,8 +18,8 @@ const nextConfig = { }, }, experimental: { - turbopackFileSystemCacheForDev: true, - turbopackFileSystemCacheForBuild: true, + turbopackFileSystemCacheForDev: enableCaching, + turbopackFileSystemCacheForBuild: enableCaching, }, env: { NEXT_PUBLIC_CONFIG_ENV: 'hello world', @@ -31,6 +33,9 @@ const nextConfig = { test: /app\/loader(?:\/client)?\/page\.tsx/, use: ['./my-loader.js'], }) + config.cache = Object.freeze({ + type: enableCaching ? 'filesystem' : 'memory', + }) if (dev) { // Make webpack consider the build as large change which makes it filesystem cache it sooner config.plugins.push((compiler) => { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index d9dc5cea2894a9..79830cd8c51594 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -12,7 +12,7 @@ use std::{ ops::Range, pin::Pin, sync::{ - Arc, + Arc, LazyLock, atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, }, }; @@ -72,6 +72,16 @@ use crate::{ const SNAPSHOT_REQUESTED_BIT: usize = 1 << (usize::BITS - 1); +/// Configurable idle timeout for snapshot persistence. +/// Defaults to 2 seconds if not set or if the value is invalid. +static IDLE_TIMEOUT: LazyLock = LazyLock::new(|| { + std::env::var("TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS") + .ok() + .and_then(|v| v.parse::().ok()) + .map(Duration::from_millis) + .unwrap_or(Duration::from_secs(2)) +}); + struct SnapshotRequest { snapshot_requested: bool, suspended_operations: FxHashSet>, @@ -2467,8 +2477,7 @@ impl TurboTasksBackendInner { loop { const FIRST_SNAPSHOT_WAIT: Duration = Duration::from_secs(300); const SNAPSHOT_INTERVAL: Duration = Duration::from_secs(120); - const IDLE_TIMEOUT: Duration = Duration::from_secs(2); - + let idle_timeout = *IDLE_TIMEOUT; let (time, mut reason) = if matches!(job, TurboTasksBackendJob::InitialSnapshot) { (FIRST_SNAPSHOT_WAIT, "initial snapshot timeout") @@ -2485,7 +2494,7 @@ impl TurboTasksBackendInner { let mut idle_start_listener = self.idle_start_event.listen(); let mut idle_end_listener = self.idle_end_event.listen(); let mut idle_time = if turbo_tasks.is_idle() { - Instant::now() + IDLE_TIMEOUT + Instant::now() + idle_timeout } else { far_future() }; @@ -2495,11 +2504,11 @@ impl TurboTasksBackendInner { return; }, _ = &mut idle_start_listener => { - idle_time = Instant::now() + IDLE_TIMEOUT; + idle_time = Instant::now() + idle_timeout; idle_start_listener = self.idle_start_event.listen() }, _ = &mut idle_end_listener => { - idle_time = until + IDLE_TIMEOUT; + idle_time = until + idle_timeout; idle_end_listener = self.idle_end_event.listen() }, _ = tokio::time::sleep_until(until) => { From fb64e3852a440f141228a35119951ea3e1f93ed8 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Thu, 13 Nov 2025 12:01:02 -0800 Subject: [PATCH 2/4] fix init --- test/e2e/filesystem-cache/filesystem-cache.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/filesystem-cache/filesystem-cache.test.ts b/test/e2e/filesystem-cache/filesystem-cache.test.ts index 9aef6d906ebe0e..35f554c4763121 100644 --- a/test/e2e/filesystem-cache/filesystem-cache.test.ts +++ b/test/e2e/filesystem-cache/filesystem-cache.test.ts @@ -4,9 +4,10 @@ import { waitFor } from 'next-test-utils' describe('filesystem-caching', () => { process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' // Make it easier to run in development, test directories are cleared between runs already so this is safe. - process.env.TURBO_ENGINE_DISABLE_VERSIONING = '1' + process.env.TURBO_ENGINE_IGNORE_DIRTY = '1' // decrease the idle timeout to make the test more reliable process.env.TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS = '1000' + process.env.ENABLE_CACHING = '1' const { skipped, next, isTurbopack } = nextTestSetup({ files: __dirname, skipDeployment: true, @@ -47,7 +48,6 @@ describe('filesystem-caching', () => { } it('should filesystem cache loaders', async () => { - process.env.ENABLE_CACHING = '1' let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp { const browser = await next.browser('/') @@ -214,6 +214,7 @@ describe('filesystem-caching', () => { for (const cacheEnabled of [true, false]) { describe(`with cache ${cacheEnabled ? 'enabled' : 'disabled'}`, () => { beforeAll(() => { + // Next is always started with caching, but this can disable it for the followup restarts process.env.ENABLE_CACHING = cacheEnabled ? '1' : '0' }) for (const [name, changes] of combinations) { From debe23fbcaa83fa1da680e53ea8f38ec3356ed37 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 14 Nov 2025 13:37:51 +0100 Subject: [PATCH 3/4] improve test case --- .../filesystem-cache/filesystem-cache.test.ts | 339 ++++++++++-------- test/e2e/filesystem-cache/next.config.js | 20 +- 2 files changed, 193 insertions(+), 166 deletions(-) diff --git a/test/e2e/filesystem-cache/filesystem-cache.test.ts b/test/e2e/filesystem-cache/filesystem-cache.test.ts index 35f554c4763121..0afee5f4d139c9 100644 --- a/test/e2e/filesystem-cache/filesystem-cache.test.ts +++ b/test/e2e/filesystem-cache/filesystem-cache.test.ts @@ -47,172 +47,193 @@ describe('filesystem-caching', () => { await next.start() } - it('should filesystem cache loaders', async () => { - let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp - { - const browser = await next.browser('/') - appTimestamp = await browser.elementByCss('main').text() - expect(appTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - { - const browser = await next.browser('/unchanged') - unchangedTimestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - { - const browser = await next.browser('/client') - appClientTimestamp = await browser.elementByCss('main').text() - expect(appClientTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - { - const browser = await next.browser('/pages') - pagesTimestamp = await browser.elementByCss('main').text() - expect(pagesTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - await restartCycle() - - { - const browser = await next.browser('/') - expect(await browser.elementByCss('main').text()).toBe(appTimestamp) - await browser.close() - } - { - const browser = await next.browser('/unchanged') - expect(await browser.elementByCss('main').text()).toBe(unchangedTimestamp) - await browser.close() - } - { - const browser = await next.browser('/client') - expect(await browser.elementByCss('main').text()).toBe(appClientTimestamp) - await browser.close() - } - { - const browser = await next.browser('/pages') - expect(await browser.elementByCss('main').text()).toBe(pagesTimestamp) - await browser.close() - } - }) + for (const cacheEnabled of [false, true]) { + describe(`with cache ${cacheEnabled ? 'enabled' : 'disabled'}`, () => { + it('should cache or not cache loaders', async () => { + let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp + { + const browser = await next.browser('/') + appTimestamp = await browser.elementByCss('main').text() + expect(appTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + { + const browser = await next.browser('/unchanged') + unchangedTimestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + { + const browser = await next.browser('/client') + appClientTimestamp = await browser.elementByCss('main').text() + expect(appClientTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + { + const browser = await next.browser('/pages') + pagesTimestamp = await browser.elementByCss('main').text() + expect(pagesTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + await restartCycle() - function makeTextCheck(url: string, text: string) { - return textCheck.bind(null, url, text) - } + { + const browser = await next.browser('/') + const newTimestamp = await browser.elementByCss('main').text() + if (cacheEnabled) { + expect(newTimestamp).toBe(appTimestamp) + } else { + expect(newTimestamp).not.toBe(appTimestamp) + } + await browser.close() + } + { + const browser = await next.browser('/unchanged') + const newTimestamp = await browser.elementByCss('main').text() + if (cacheEnabled) { + expect(newTimestamp).toBe(unchangedTimestamp) + } else { + expect(newTimestamp).not.toBe(unchangedTimestamp) + } + await browser.close() + } + { + const browser = await next.browser('/client') + const newTimestamp = await browser.elementByCss('main').text() + if (cacheEnabled) { + expect(newTimestamp).toBe(appClientTimestamp) + } else { + expect(newTimestamp).not.toBe(appClientTimestamp) + } + await browser.close() + } + { + const browser = await next.browser('/pages') + const newTimestamp = await browser.elementByCss('main').text() + if (cacheEnabled) { + expect(newTimestamp).toBe(pagesTimestamp) + } else { + expect(newTimestamp).not.toBe(pagesTimestamp) + } + await browser.close() + } + }) - async function textCheck(url: string, text: string) { - const browser = await next.browser(url) - expect(await browser.elementByCss('p').text()).toBe(text) - await browser.close() - } + function makeTextCheck(url: string, text: string) { + return textCheck.bind(null, url, text) + } - function makeFileEdit(file: string) { - return async (inner: () => Promise) => { - await next.patchFile( - file, - (content) => { - return content.replace('hello world', 'hello filesystem cache') - }, - inner - ) - } - } + async function textCheck(url: string, text: string) { + const browser = await next.browser(url) + expect(await browser.elementByCss('p').text()).toBe(text) + await browser.close() + } - interface Change { - checkInitial(): Promise - withChange(previous: () => Promise): Promise - checkChanged(): Promise - fullInvalidation?: boolean - } - const POTENTIAL_CHANGES: Record = { - 'RSC change': { - checkInitial: makeTextCheck('/', 'hello world'), - withChange: makeFileEdit('app/page.tsx'), - checkChanged: makeTextCheck('/', 'hello filesystem cache'), - }, - 'RCC change': { - checkInitial: makeTextCheck('/client', 'hello world'), - withChange: makeFileEdit('app/client/page.tsx'), - checkChanged: makeTextCheck('/client', 'hello filesystem cache'), - }, - 'Pages change': { - checkInitial: makeTextCheck('/pages', 'hello world'), - withChange: makeFileEdit('pages/pages.tsx'), - checkChanged: makeTextCheck('/pages', 'hello filesystem cache'), - }, - 'rename app page': { - checkInitial: makeTextCheck('/remove-me', 'hello world'), - async withChange(inner) { - await next.renameFolder('app/remove-me', 'app/add-me') - try { - await inner() - } finally { - await next.renameFolder('app/add-me', 'app/remove-me') - } - }, - checkChanged: makeTextCheck('/add-me', 'hello world'), - }, - // TODO fix this case with Turbopack - ...(isTurbopack - ? {} - : { - 'loader change': { - async checkInitial() { - await textCheck('/loader', 'hello world') - await textCheck('/loader/client', 'hello world') - }, - withChange: makeFileEdit('my-loader.js'), - async checkChanged() { - await textCheck('/loader', 'hello filesystem cache') - await textCheck('/loader/client', 'hello filesystem cache') + function makeFileEdit(file: string) { + return async (inner: () => Promise) => { + await next.patchFile( + file, + (content) => { + return content.replace('hello world', 'hello filesystem cache') }, - fullInvalidation: !isTurbopack, - }, - }), - 'next config change': { - async checkInitial() { - await textCheck('/next-config', 'hello world') - await textCheck('/next-config/client', 'hello world') - }, - withChange: makeFileEdit('next.config.js'), - async checkChanged() { - await textCheck('/next-config', 'hello filesystem cache') - await textCheck('/next-config/client', 'hello filesystem cache') - }, - fullInvalidation: !isTurbopack, - }, - 'env var change': { - async checkInitial() { - await textCheck('/env', 'hello world') - await textCheck('/env/client', 'hello world') - }, - async withChange(inner) { - process.env.NEXT_PUBLIC_ENV_VAR = 'hello filesystem cache' - try { - await inner() - } finally { - process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' + inner + ) } - }, - async checkChanged() { - await textCheck('/env', 'hello filesystem cache') - await textCheck('/env/client', 'hello filesystem cache') - }, - }, - } as const + } + + interface Change { + checkInitial(): Promise + withChange(previous: () => Promise): Promise + checkChanged(): Promise + fullInvalidation?: boolean + } + const POTENTIAL_CHANGES: Record = { + 'RSC change': { + checkInitial: makeTextCheck('/', 'hello world'), + withChange: makeFileEdit('app/page.tsx'), + checkChanged: makeTextCheck('/', 'hello filesystem cache'), + }, + 'RCC change': { + checkInitial: makeTextCheck('/client', 'hello world'), + withChange: makeFileEdit('app/client/page.tsx'), + checkChanged: makeTextCheck('/client', 'hello filesystem cache'), + }, + 'Pages change': { + checkInitial: makeTextCheck('/pages', 'hello world'), + withChange: makeFileEdit('pages/pages.tsx'), + checkChanged: makeTextCheck('/pages', 'hello filesystem cache'), + }, + 'rename app page': { + checkInitial: makeTextCheck('/remove-me', 'hello world'), + async withChange(inner) { + await next.renameFolder('app/remove-me', 'app/add-me') + try { + await inner() + } finally { + await next.renameFolder('app/add-me', 'app/remove-me') + } + }, + checkChanged: makeTextCheck('/add-me', 'hello world'), + }, + // TODO fix this case with Turbopack + ...(isTurbopack + ? {} + : { + 'loader change': { + async checkInitial() { + await textCheck('/loader', 'hello world') + await textCheck('/loader/client', 'hello world') + }, + withChange: makeFileEdit('my-loader.js'), + async checkChanged() { + await textCheck('/loader', 'hello filesystem cache') + await textCheck('/loader/client', 'hello filesystem cache') + }, + fullInvalidation: !isTurbopack, + }, + }), + 'next config change': { + async checkInitial() { + await textCheck('/next-config', 'hello world') + await textCheck('/next-config/client', 'hello world') + }, + withChange: makeFileEdit('next.config.js'), + async checkChanged() { + await textCheck('/next-config', 'hello filesystem cache') + await textCheck('/next-config/client', 'hello filesystem cache') + }, + fullInvalidation: !isTurbopack, + }, + 'env var change': { + async checkInitial() { + await textCheck('/env', 'hello world') + await textCheck('/env/client', 'hello world') + }, + async withChange(inner) { + process.env.NEXT_PUBLIC_ENV_VAR = 'hello filesystem cache' + try { + await inner() + } finally { + process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' + } + }, + async checkChanged() { + await textCheck('/env', 'hello filesystem cache') + await textCheck('/env/client', 'hello filesystem cache') + }, + }, + } as const + + // Checking only single change and all combined for performance reasons. + const combinations = Object.entries(POTENTIAL_CHANGES).map(([k, v]) => [ + k, + [v], + ]) as Array<[string, Array]> + combinations.push([ + Object.keys(POTENTIAL_CHANGES).join(', '), + Object.values(POTENTIAL_CHANGES), + ]) - // Checking only single change and all combined for performance reasons. - const combinations = Object.entries(POTENTIAL_CHANGES).map(([k, v]) => [ - k, - [v], - ]) as Array<[string, Array]> - combinations.push([ - Object.keys(POTENTIAL_CHANGES).join(', '), - Object.values(POTENTIAL_CHANGES), - ]) - for (const cacheEnabled of [true, false]) { - describe(`with cache ${cacheEnabled ? 'enabled' : 'disabled'}`, () => { beforeAll(() => { // Next is always started with caching, but this can disable it for the followup restarts process.env.ENABLE_CACHING = cacheEnabled ? '1' : '0' diff --git a/test/e2e/filesystem-cache/next.config.js b/test/e2e/filesystem-cache/next.config.js index 0a1cf6802ef2cc..89f644ec2b2cc3 100644 --- a/test/e2e/filesystem-cache/next.config.js +++ b/test/e2e/filesystem-cache/next.config.js @@ -17,10 +17,14 @@ const nextConfig = { }, }, }, - experimental: { - turbopackFileSystemCacheForDev: enableCaching, - turbopackFileSystemCacheForBuild: enableCaching, - }, + experimental: enableCaching + ? { + turbopackFileSystemCacheForBuild: true, + } + : { + turbopackFileSystemCacheForDev: false, + turbopackFileSystemCacheForBuild: false, + }, env: { NEXT_PUBLIC_CONFIG_ENV: 'hello world', }, @@ -33,9 +37,11 @@ const nextConfig = { test: /app\/loader(?:\/client)?\/page\.tsx/, use: ['./my-loader.js'], }) - config.cache = Object.freeze({ - type: enableCaching ? 'filesystem' : 'memory', - }) + if (enableCaching) { + config.cache = Object.freeze({ + type: 'memory', + }) + } if (dev) { // Make webpack consider the build as large change which makes it filesystem cache it sooner config.plugins.push((compiler) => { From b6bd681333b4956fe701ec085dd6ea2b282a17f1 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 17 Nov 2025 15:08:10 +0100 Subject: [PATCH 4/4] fix and improve test case --- .../filesystem-cache/filesystem-cache.test.ts | 536 +++++++++--------- 1 file changed, 271 insertions(+), 265 deletions(-) diff --git a/test/e2e/filesystem-cache/filesystem-cache.test.ts b/test/e2e/filesystem-cache/filesystem-cache.test.ts index 0afee5f4d139c9..38e11cbd7974d0 100644 --- a/test/e2e/filesystem-cache/filesystem-cache.test.ts +++ b/test/e2e/filesystem-cache/filesystem-cache.test.ts @@ -1,307 +1,313 @@ import { nextTestSetup, isNextDev } from 'e2e-utils' import { waitFor } from 'next-test-utils' -describe('filesystem-caching', () => { - process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' - // Make it easier to run in development, test directories are cleared between runs already so this is safe. - process.env.TURBO_ENGINE_IGNORE_DIRTY = '1' - // decrease the idle timeout to make the test more reliable - process.env.TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS = '1000' - process.env.ENABLE_CACHING = '1' - const { skipped, next, isTurbopack } = nextTestSetup({ - files: __dirname, - skipDeployment: true, - // We need to use npm here as pnpms symlinks trigger a weird bug (kernel bug?) - installCommand: 'npm i', - buildCommand: 'npm exec next build', - startCommand: isNextDev ? 'npm exec next dev' : 'npm exec next start', - }) +process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' +// Make it easier to run in development, test directories are cleared between runs already so this is safe. +process.env.TURBO_ENGINE_IGNORE_DIRTY = '1' +// decrease the idle timeout to make the test more reliable +process.env.TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS = '1000' - if (skipped) { - return - } +for (const cacheEnabled of [false, true]) { + describe(`filesystem-caching with cache ${cacheEnabled ? 'enabled' : 'disabled'}`, () => { + const { skipped, next, isTurbopack } = nextTestSetup({ + files: __dirname, + skipDeployment: true, + packageJson: { + scripts: { + build: `ENABLE_CACHING=${cacheEnabled ? '1' : ''} next build`, + dev: `ENABLE_CACHING=${cacheEnabled ? '1' : ''} next dev`, + start: 'next start', + }, + }, + // We need to use npm here as pnpms symlinks trigger a weird bug (kernel bug?) + installCommand: 'npm i', + // Next is always started with caching, but this can disable it for the followup restarts + buildCommand: `npm run build`, + startCommand: isNextDev ? 'npm run dev' : 'npm run start', + }) - beforeAll(() => { - // We can skip the dev watch delay since this is not an HMR test - ;(next as any).handleDevWatchDelayBeforeChange = () => {} - ;(next as any).handleDevWatchDelayAfterChange = () => {} - }) + if (skipped) { + return + } - async function restartCycle() { - await stop() - await start() - } + beforeAll(() => { + // We can skip the dev watch delay since this is not an HMR test + ;(next as any).handleDevWatchDelayBeforeChange = () => {} + ;(next as any).handleDevWatchDelayAfterChange = () => {} + }) - async function stop() { - if (isNextDev) { - // Give FileSystem Cache time to write to disk - // Turbopack is configured to wait 1s above. - // Webpack has an idle timeout (after large changes) of 1s - // and we give time a bit more to allow writing to disk - await waitFor(3000) + async function restartCycle() { + await stop() + await start() } - await next.stop() - } - async function start() { - await next.start() - } + async function stop() { + if (isNextDev) { + // Give FileSystem Cache time to write to disk + // Turbopack is configured to wait 1s above. + // Webpack has an idle timeout (after large changes) of 1s + // and we give time a bit more to allow writing to disk + await waitFor(3000) + } + await next.stop() + } - for (const cacheEnabled of [false, true]) { - describe(`with cache ${cacheEnabled ? 'enabled' : 'disabled'}`, () => { - it('should cache or not cache loaders', async () => { - let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp - { - const browser = await next.browser('/') - appTimestamp = await browser.elementByCss('main').text() - expect(appTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - { - const browser = await next.browser('/unchanged') - unchangedTimestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - { - const browser = await next.browser('/client') - appClientTimestamp = await browser.elementByCss('main').text() - expect(appClientTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - { - const browser = await next.browser('/pages') - pagesTimestamp = await browser.elementByCss('main').text() - expect(pagesTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - await restartCycle() + async function start() { + await next.start() + } - { - const browser = await next.browser('/') - const newTimestamp = await browser.elementByCss('main').text() - if (cacheEnabled) { - expect(newTimestamp).toBe(appTimestamp) - } else { - expect(newTimestamp).not.toBe(appTimestamp) - } - await browser.close() - } - { - const browser = await next.browser('/unchanged') - const newTimestamp = await browser.elementByCss('main').text() - if (cacheEnabled) { - expect(newTimestamp).toBe(unchangedTimestamp) - } else { - expect(newTimestamp).not.toBe(unchangedTimestamp) - } - await browser.close() - } - { - const browser = await next.browser('/client') - const newTimestamp = await browser.elementByCss('main').text() - if (cacheEnabled) { - expect(newTimestamp).toBe(appClientTimestamp) - } else { - expect(newTimestamp).not.toBe(appClientTimestamp) - } - await browser.close() + it('should cache or not cache loaders', async () => { + let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp + { + const browser = await next.browser('/') + appTimestamp = await browser.elementByCss('main').text() + expect(appTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + { + const browser = await next.browser('/unchanged') + unchangedTimestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + { + const browser = await next.browser('/client') + appClientTimestamp = await browser.elementByCss('main').text() + expect(appClientTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + { + const browser = await next.browser('/pages') + pagesTimestamp = await browser.elementByCss('main').text() + expect(pagesTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + await restartCycle() + + { + const browser = await next.browser('/') + const newTimestamp = await browser.elementByCss('main').text() + expect(newTimestamp).toMatch(/Timestamp = \d+$/) + if (cacheEnabled) { + expect(newTimestamp).toBe(appTimestamp) + } else { + expect(newTimestamp).not.toBe(appTimestamp) } - { - const browser = await next.browser('/pages') - const newTimestamp = await browser.elementByCss('main').text() - if (cacheEnabled) { - expect(newTimestamp).toBe(pagesTimestamp) - } else { - expect(newTimestamp).not.toBe(pagesTimestamp) - } - await browser.close() + await browser.close() + } + { + const browser = await next.browser('/unchanged') + const newTimestamp = await browser.elementByCss('main').text() + expect(newTimestamp).toMatch(/Timestamp = \d+$/) + if (cacheEnabled) { + expect(newTimestamp).toBe(unchangedTimestamp) + } else { + expect(newTimestamp).not.toBe(unchangedTimestamp) } - }) - - function makeTextCheck(url: string, text: string) { - return textCheck.bind(null, url, text) + await browser.close() } - - async function textCheck(url: string, text: string) { - const browser = await next.browser(url) - expect(await browser.elementByCss('p').text()).toBe(text) + { + const browser = await next.browser('/client') + const newTimestamp = await browser.elementByCss('main').text() + expect(newTimestamp).toMatch(/Timestamp = \d+$/) + if (cacheEnabled) { + expect(newTimestamp).toBe(appClientTimestamp) + } else { + expect(newTimestamp).not.toBe(appClientTimestamp) + } await browser.close() } - - function makeFileEdit(file: string) { - return async (inner: () => Promise) => { - await next.patchFile( - file, - (content) => { - return content.replace('hello world', 'hello filesystem cache') - }, - inner - ) + { + const browser = await next.browser('/pages') + const newTimestamp = await browser.elementByCss('main').text() + expect(newTimestamp).toMatch(/Timestamp = \d+$/) + if (cacheEnabled) { + expect(newTimestamp).toBe(pagesTimestamp) + } else { + expect(newTimestamp).not.toBe(pagesTimestamp) } + await browser.close() } + }) + + function makeTextCheck(url: string, text: string) { + return textCheck.bind(null, url, text) + } - interface Change { - checkInitial(): Promise - withChange(previous: () => Promise): Promise - checkChanged(): Promise - fullInvalidation?: boolean + async function textCheck(url: string, text: string) { + const browser = await next.browser(url) + expect(await browser.elementByCss('p').text()).toBe(text) + await browser.close() + } + + function makeFileEdit(file: string) { + return async (inner: () => Promise) => { + await next.patchFile( + file, + (content) => { + return content.replace('hello world', 'hello filesystem cache') + }, + inner + ) } - const POTENTIAL_CHANGES: Record = { - 'RSC change': { - checkInitial: makeTextCheck('/', 'hello world'), - withChange: makeFileEdit('app/page.tsx'), - checkChanged: makeTextCheck('/', 'hello filesystem cache'), + } + + interface Change { + checkInitial(): Promise + withChange(previous: () => Promise): Promise + checkChanged(): Promise + fullInvalidation?: boolean + } + const POTENTIAL_CHANGES: Record = { + 'RSC change': { + checkInitial: makeTextCheck('/', 'hello world'), + withChange: makeFileEdit('app/page.tsx'), + checkChanged: makeTextCheck('/', 'hello filesystem cache'), + }, + 'RCC change': { + checkInitial: makeTextCheck('/client', 'hello world'), + withChange: makeFileEdit('app/client/page.tsx'), + checkChanged: makeTextCheck('/client', 'hello filesystem cache'), + }, + 'Pages change': { + checkInitial: makeTextCheck('/pages', 'hello world'), + withChange: makeFileEdit('pages/pages.tsx'), + checkChanged: makeTextCheck('/pages', 'hello filesystem cache'), + }, + 'rename app page': { + checkInitial: makeTextCheck('/remove-me', 'hello world'), + async withChange(inner) { + await next.renameFolder('app/remove-me', 'app/add-me') + try { + await inner() + } finally { + await next.renameFolder('app/add-me', 'app/remove-me') + } }, - 'RCC change': { - checkInitial: makeTextCheck('/client', 'hello world'), - withChange: makeFileEdit('app/client/page.tsx'), - checkChanged: makeTextCheck('/client', 'hello filesystem cache'), + checkChanged: makeTextCheck('/add-me', 'hello world'), + }, + // TODO fix this case with Turbopack + ...(isTurbopack + ? {} + : { + 'loader change': { + async checkInitial() { + await textCheck('/loader', 'hello world') + await textCheck('/loader/client', 'hello world') + }, + withChange: makeFileEdit('my-loader.js'), + async checkChanged() { + await textCheck('/loader', 'hello filesystem cache') + await textCheck('/loader/client', 'hello filesystem cache') + }, + fullInvalidation: !isTurbopack, + }, + }), + 'next config change': { + async checkInitial() { + await textCheck('/next-config', 'hello world') + await textCheck('/next-config/client', 'hello world') }, - 'Pages change': { - checkInitial: makeTextCheck('/pages', 'hello world'), - withChange: makeFileEdit('pages/pages.tsx'), - checkChanged: makeTextCheck('/pages', 'hello filesystem cache'), + withChange: makeFileEdit('next.config.js'), + async checkChanged() { + await textCheck('/next-config', 'hello filesystem cache') + await textCheck('/next-config/client', 'hello filesystem cache') }, - 'rename app page': { - checkInitial: makeTextCheck('/remove-me', 'hello world'), - async withChange(inner) { - await next.renameFolder('app/remove-me', 'app/add-me') - try { - await inner() - } finally { - await next.renameFolder('app/add-me', 'app/remove-me') - } - }, - checkChanged: makeTextCheck('/add-me', 'hello world'), + fullInvalidation: !isTurbopack, + }, + 'env var change': { + async checkInitial() { + await textCheck('/env', 'hello world') + await textCheck('/env/client', 'hello world') }, - // TODO fix this case with Turbopack - ...(isTurbopack - ? {} - : { - 'loader change': { - async checkInitial() { - await textCheck('/loader', 'hello world') - await textCheck('/loader/client', 'hello world') - }, - withChange: makeFileEdit('my-loader.js'), - async checkChanged() { - await textCheck('/loader', 'hello filesystem cache') - await textCheck('/loader/client', 'hello filesystem cache') - }, - fullInvalidation: !isTurbopack, - }, - }), - 'next config change': { - async checkInitial() { - await textCheck('/next-config', 'hello world') - await textCheck('/next-config/client', 'hello world') - }, - withChange: makeFileEdit('next.config.js'), - async checkChanged() { - await textCheck('/next-config', 'hello filesystem cache') - await textCheck('/next-config/client', 'hello filesystem cache') - }, - fullInvalidation: !isTurbopack, + async withChange(inner) { + process.env.NEXT_PUBLIC_ENV_VAR = 'hello filesystem cache' + try { + await inner() + } finally { + process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' + } }, - 'env var change': { - async checkInitial() { - await textCheck('/env', 'hello world') - await textCheck('/env/client', 'hello world') - }, - async withChange(inner) { - process.env.NEXT_PUBLIC_ENV_VAR = 'hello filesystem cache' - try { - await inner() - } finally { - process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' - } - }, - async checkChanged() { - await textCheck('/env', 'hello filesystem cache') - await textCheck('/env/client', 'hello filesystem cache') - }, + async checkChanged() { + await textCheck('/env', 'hello filesystem cache') + await textCheck('/env/client', 'hello filesystem cache') }, - } as const + }, + } as const + + // Checking only single change and all combined for performance reasons. + const combinations = Object.entries(POTENTIAL_CHANGES).map(([k, v]) => [ + k, + [v], + ]) as Array<[string, Array]> + combinations.push([ + Object.keys(POTENTIAL_CHANGES).join(', '), + Object.values(POTENTIAL_CHANGES), + ]) - // Checking only single change and all combined for performance reasons. - const combinations = Object.entries(POTENTIAL_CHANGES).map(([k, v]) => [ - k, - [v], - ]) as Array<[string, Array]> - combinations.push([ - Object.keys(POTENTIAL_CHANGES).join(', '), - Object.values(POTENTIAL_CHANGES), - ]) + for (const [name, changes] of combinations) { + it(`should allow to change files while stopped (${name})`, async () => { + let fullInvalidation = !cacheEnabled + for (const change of changes) { + await change.checkInitial() + if (change.fullInvalidation) { + fullInvalidation = true + } + } - beforeAll(() => { - // Next is always started with caching, but this can disable it for the followup restarts - process.env.ENABLE_CACHING = cacheEnabled ? '1' : '0' - }) - for (const [name, changes] of combinations) { - it(`should allow to change files while stopped (${name})`, async () => { - let fullInvalidation = false + let unchangedTimestamp: string + if (!fullInvalidation) { + const browser = await next.browser('/unchanged') + unchangedTimestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + + async function checkChanged() { for (const change of changes) { - await change.checkInitial() - if (change.fullInvalidation) { - fullInvalidation = true - } + await change.checkChanged() } - let unchangedTimestamp: string if (!fullInvalidation) { const browser = await next.browser('/unchanged') - unchangedTimestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) + const timestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toEqual(timestamp) await browser.close() } + } - async function checkChanged() { - for (const change of changes) { - await change.checkChanged() - } - - if (!fullInvalidation) { - const browser = await next.browser('/unchanged') - const timestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toEqual(timestamp) - await browser.close() - } - } - - await stop() + await stop() - async function inner() { - await start() + async function inner() { + await start() + await checkChanged() + // Some no-op change builds + for (let i = 0; i < 2; i++) { + await restartCycle() await checkChanged() - // Some no-op change builds - for (let i = 0; i < 2; i++) { - await restartCycle() - await checkChanged() - } - await stop() } + await stop() + } - let current = inner - for (const change of changes) { - const prev = current - current = () => change.withChange(prev) - } - await current() + let current = inner + for (const change of changes) { + const prev = current + current = () => change.withChange(prev) + } + await current() - await start() - for (const change of changes) { - await change.checkInitial() - } + await start() + for (const change of changes) { + await change.checkInitial() + } - if (!fullInvalidation) { - const browser = await next.browser('/unchanged') - const timestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toEqual(timestamp) - await browser.close() - } - }, 200000) - } - }) - } -}) + if (!fullInvalidation) { + const browser = await next.browser('/unchanged') + const timestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toEqual(timestamp) + await browser.close() + } + }, 200000) + } + }) +}