diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index fa23ce95f6416..fab9f81736cae 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -415,6 +415,8 @@ export interface ExperimentalConfig { /** * Enable filesystem cache for the turbopack dev server. + * + * Defaults to `true` in canary releases. */ turbopackFileSystemCacheForDev?: boolean @@ -1532,6 +1534,8 @@ export const defaultConfig = Object.freeze({ proxyClientMaxBodySize: 10_485_760, // 10MB hideLogsAfterAbort: false, mcpServer: true, + turbopackFileSystemCacheForDev: !isStableBuild(), + turbopackFileSystemCacheForBuild: false, }, htmlLimitedBots: undefined, bundlePagesRouterDependencies: false, diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index 8c756a7d32a48..741452c4d8920 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -400,7 +400,7 @@ function assignDefaultsAndValidate( if (isStableBuild()) { // Prevents usage of certain experimental features outside of canary - if (result.experimental?.turbopackFileSystemCacheForBuild) { + if (result.experimental.turbopackFileSystemCacheForBuild) { throw new CanaryOnlyConfigError({ feature: 'experimental.turbopackFileSystemCacheForBuild', }) diff --git a/test/e2e/filesystem-cache/filesystem-cache.test.ts b/test/e2e/filesystem-cache/filesystem-cache.test.ts new file mode 100644 index 0000000000000..38e11cbd7974d --- /dev/null +++ b/test/e2e/filesystem-cache/filesystem-cache.test.ts @@ -0,0 +1,313 @@ +import { nextTestSetup, isNextDev } from 'e2e-utils' +import { waitFor } from 'next-test-utils' + +process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' +// Make it easier to run in development, test directories are cleared between runs already so this is safe. +process.env.TURBO_ENGINE_IGNORE_DIRTY = '1' +// decrease the idle timeout to make the test more reliable +process.env.TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS = '1000' + +for (const cacheEnabled of [false, true]) { + describe(`filesystem-caching with cache ${cacheEnabled ? 'enabled' : 'disabled'}`, () => { + const { skipped, next, isTurbopack } = nextTestSetup({ + files: __dirname, + skipDeployment: true, + packageJson: { + scripts: { + build: `ENABLE_CACHING=${cacheEnabled ? '1' : ''} next build`, + dev: `ENABLE_CACHING=${cacheEnabled ? '1' : ''} next dev`, + start: 'next start', + }, + }, + // We need to use npm here as pnpms symlinks trigger a weird bug (kernel bug?) + installCommand: 'npm i', + // Next is always started with caching, but this can disable it for the followup restarts + buildCommand: `npm run build`, + startCommand: isNextDev ? 'npm run dev' : 'npm run start', + }) + + if (skipped) { + return + } + + beforeAll(() => { + // We can skip the dev watch delay since this is not an HMR test + ;(next as any).handleDevWatchDelayBeforeChange = () => {} + ;(next as any).handleDevWatchDelayAfterChange = () => {} + }) + + async function restartCycle() { + await stop() + await start() + } + + async function stop() { + if (isNextDev) { + // Give FileSystem Cache time to write to disk + // Turbopack is configured to wait 1s above. + // Webpack has an idle timeout (after large changes) of 1s + // and we give time a bit more to allow writing to disk + await waitFor(3000) + } + await next.stop() + } + + async function start() { + await next.start() + } + + it('should cache or not cache loaders', async () => { + let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp + { + const browser = await next.browser('/') + appTimestamp = await browser.elementByCss('main').text() + expect(appTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + { + const browser = await next.browser('/unchanged') + unchangedTimestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + { + const browser = await next.browser('/client') + appClientTimestamp = await browser.elementByCss('main').text() + expect(appClientTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + { + const browser = await next.browser('/pages') + pagesTimestamp = await browser.elementByCss('main').text() + expect(pagesTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + await restartCycle() + + { + const browser = await next.browser('/') + const newTimestamp = await browser.elementByCss('main').text() + expect(newTimestamp).toMatch(/Timestamp = \d+$/) + if (cacheEnabled) { + expect(newTimestamp).toBe(appTimestamp) + } else { + expect(newTimestamp).not.toBe(appTimestamp) + } + await browser.close() + } + { + const browser = await next.browser('/unchanged') + const newTimestamp = await browser.elementByCss('main').text() + expect(newTimestamp).toMatch(/Timestamp = \d+$/) + if (cacheEnabled) { + expect(newTimestamp).toBe(unchangedTimestamp) + } else { + expect(newTimestamp).not.toBe(unchangedTimestamp) + } + await browser.close() + } + { + const browser = await next.browser('/client') + const newTimestamp = await browser.elementByCss('main').text() + expect(newTimestamp).toMatch(/Timestamp = \d+$/) + if (cacheEnabled) { + expect(newTimestamp).toBe(appClientTimestamp) + } else { + expect(newTimestamp).not.toBe(appClientTimestamp) + } + await browser.close() + } + { + const browser = await next.browser('/pages') + const newTimestamp = await browser.elementByCss('main').text() + expect(newTimestamp).toMatch(/Timestamp = \d+$/) + if (cacheEnabled) { + expect(newTimestamp).toBe(pagesTimestamp) + } else { + expect(newTimestamp).not.toBe(pagesTimestamp) + } + await browser.close() + } + }) + + function makeTextCheck(url: string, text: string) { + return textCheck.bind(null, url, text) + } + + async function textCheck(url: string, text: string) { + const browser = await next.browser(url) + expect(await browser.elementByCss('p').text()).toBe(text) + await browser.close() + } + + function makeFileEdit(file: string) { + return async (inner: () => Promise) => { + await next.patchFile( + file, + (content) => { + return content.replace('hello world', 'hello filesystem cache') + }, + inner + ) + } + } + + interface Change { + checkInitial(): Promise + withChange(previous: () => Promise): Promise + checkChanged(): Promise + fullInvalidation?: boolean + } + const POTENTIAL_CHANGES: Record = { + 'RSC change': { + checkInitial: makeTextCheck('/', 'hello world'), + withChange: makeFileEdit('app/page.tsx'), + checkChanged: makeTextCheck('/', 'hello filesystem cache'), + }, + 'RCC change': { + checkInitial: makeTextCheck('/client', 'hello world'), + withChange: makeFileEdit('app/client/page.tsx'), + checkChanged: makeTextCheck('/client', 'hello filesystem cache'), + }, + 'Pages change': { + checkInitial: makeTextCheck('/pages', 'hello world'), + withChange: makeFileEdit('pages/pages.tsx'), + checkChanged: makeTextCheck('/pages', 'hello filesystem cache'), + }, + 'rename app page': { + checkInitial: makeTextCheck('/remove-me', 'hello world'), + async withChange(inner) { + await next.renameFolder('app/remove-me', 'app/add-me') + try { + await inner() + } finally { + await next.renameFolder('app/add-me', 'app/remove-me') + } + }, + checkChanged: makeTextCheck('/add-me', 'hello world'), + }, + // TODO fix this case with Turbopack + ...(isTurbopack + ? {} + : { + 'loader change': { + async checkInitial() { + await textCheck('/loader', 'hello world') + await textCheck('/loader/client', 'hello world') + }, + withChange: makeFileEdit('my-loader.js'), + async checkChanged() { + await textCheck('/loader', 'hello filesystem cache') + await textCheck('/loader/client', 'hello filesystem cache') + }, + fullInvalidation: !isTurbopack, + }, + }), + 'next config change': { + async checkInitial() { + await textCheck('/next-config', 'hello world') + await textCheck('/next-config/client', 'hello world') + }, + withChange: makeFileEdit('next.config.js'), + async checkChanged() { + await textCheck('/next-config', 'hello filesystem cache') + await textCheck('/next-config/client', 'hello filesystem cache') + }, + fullInvalidation: !isTurbopack, + }, + 'env var change': { + async checkInitial() { + await textCheck('/env', 'hello world') + await textCheck('/env/client', 'hello world') + }, + async withChange(inner) { + process.env.NEXT_PUBLIC_ENV_VAR = 'hello filesystem cache' + try { + await inner() + } finally { + process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' + } + }, + async checkChanged() { + await textCheck('/env', 'hello filesystem cache') + await textCheck('/env/client', 'hello filesystem cache') + }, + }, + } as const + + // Checking only single change and all combined for performance reasons. + const combinations = Object.entries(POTENTIAL_CHANGES).map(([k, v]) => [ + k, + [v], + ]) as Array<[string, Array]> + combinations.push([ + Object.keys(POTENTIAL_CHANGES).join(', '), + Object.values(POTENTIAL_CHANGES), + ]) + + for (const [name, changes] of combinations) { + it(`should allow to change files while stopped (${name})`, async () => { + let fullInvalidation = !cacheEnabled + for (const change of changes) { + await change.checkInitial() + if (change.fullInvalidation) { + fullInvalidation = true + } + } + + let unchangedTimestamp: string + if (!fullInvalidation) { + const browser = await next.browser('/unchanged') + unchangedTimestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) + await browser.close() + } + + async function checkChanged() { + for (const change of changes) { + await change.checkChanged() + } + + if (!fullInvalidation) { + const browser = await next.browser('/unchanged') + const timestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toEqual(timestamp) + await browser.close() + } + } + + await stop() + + async function inner() { + await start() + await checkChanged() + // Some no-op change builds + for (let i = 0; i < 2; i++) { + await restartCycle() + await checkChanged() + } + await stop() + } + + let current = inner + for (const change of changes) { + const prev = current + current = () => change.withChange(prev) + } + await current() + + await start() + for (const change of changes) { + await change.checkInitial() + } + + if (!fullInvalidation) { + const browser = await next.browser('/unchanged') + const timestamp = await browser.elementByCss('main').text() + expect(unchangedTimestamp).toEqual(timestamp) + await browser.close() + } + }, 200000) + } + }) +} diff --git a/test/e2e/filesystem-cache/filesystem-cache.ts b/test/e2e/filesystem-cache/filesystem-cache.ts deleted file mode 100644 index e27687c4f74f6..0000000000000 --- a/test/e2e/filesystem-cache/filesystem-cache.ts +++ /dev/null @@ -1,268 +0,0 @@ -import { nextTestSetup, isNextDev } from 'e2e-utils' -import { waitFor } from 'next-test-utils' - -describe('persistent-caching', () => { - process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' - const { skipped, next, isTurbopack } = nextTestSetup({ - files: __dirname, - skipDeployment: true, - // We need to use npm here as pnpms symlinks trigger a weird bug (kernel bug?) - installCommand: 'npm i', - buildCommand: 'npm exec next build', - startCommand: isNextDev ? 'npm exec next dev' : 'npm exec next start', - }) - - if (skipped) { - return - } - - beforeAll(() => { - // We can skip the dev watch delay since this is not an HMR test - ;(next as any).handleDevWatchDelayBeforeChange = () => {} - ;(next as any).handleDevWatchDelayAfterChange = () => {} - }) - - async function restartCycle() { - await stop() - await start() - } - - async function stop() { - if (isNextDev) { - // Give FileSystem Cache time to write to disk - // Turbopack has an idle timeout of 2s - // Webpack has an idle timeout (after large changes) of 1s - // and we give time a bit more to allow writing to disk - await waitFor(3000) - } - await next.stop() - } - - async function start() { - await next.start() - } - - it('should filesystem cache loaders', async () => { - let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp - { - const browser = await next.browser('/') - appTimestamp = await browser.elementByCss('main').text() - expect(appTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - { - const browser = await next.browser('/unchanged') - unchangedTimestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - { - const browser = await next.browser('/client') - appClientTimestamp = await browser.elementByCss('main').text() - expect(appClientTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - { - const browser = await next.browser('/pages') - pagesTimestamp = await browser.elementByCss('main').text() - expect(pagesTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - await restartCycle() - - { - const browser = await next.browser('/') - expect(await browser.elementByCss('main').text()).toBe(appTimestamp) - await browser.close() - } - { - const browser = await next.browser('/unchanged') - expect(await browser.elementByCss('main').text()).toBe(unchangedTimestamp) - await browser.close() - } - { - const browser = await next.browser('/client') - expect(await browser.elementByCss('main').text()).toBe(appClientTimestamp) - await browser.close() - } - { - const browser = await next.browser('/pages') - expect(await browser.elementByCss('main').text()).toBe(pagesTimestamp) - await browser.close() - } - }) - - function makeTextCheck(url: string, text: string) { - return textCheck.bind(null, url, text) - } - - async function textCheck(url: string, text: string) { - const browser = await next.browser(url) - expect(await browser.elementByCss('p').text()).toBe(text) - await browser.close() - } - - function makeFileEdit(file: string) { - return async (inner: () => Promise) => { - await next.patchFile( - file, - (content) => { - return content.replace('hello world', 'hello filesystem cache') - }, - inner - ) - } - } - - const POTENTIAL_CHANGES = { - 'RSC change': { - checkInitial: makeTextCheck('/', 'hello world'), - withChange: makeFileEdit('app/page.tsx'), - checkChanged: makeTextCheck('/', 'hello filesystem cache'), - }, - 'RCC change': { - checkInitial: makeTextCheck('/client', 'hello world'), - withChange: makeFileEdit('app/client/page.tsx'), - checkChanged: makeTextCheck('/client', 'hello filesystem cache'), - }, - 'Pages change': { - checkInitial: makeTextCheck('/pages', 'hello world'), - withChange: makeFileEdit('pages/pages.tsx'), - checkChanged: makeTextCheck('/pages', 'hello filesystem cache'), - }, - 'rename app page': { - checkInitial: makeTextCheck('/remove-me', 'hello world'), - async withChange(inner) { - await next.renameFolder('app/remove-me', 'app/add-me') - try { - await inner() - } finally { - await next.renameFolder('app/add-me', 'app/remove-me') - } - }, - checkChanged: makeTextCheck('/add-me', 'hello world'), - }, - // TODO fix this case with Turbopack - ...(isTurbopack - ? {} - : { - 'loader change': { - async checkInitial() { - await textCheck('/loader', 'hello world') - await textCheck('/loader/client', 'hello world') - }, - withChange: makeFileEdit('my-loader.js'), - async checkChanged() { - await textCheck('/loader', 'hello filesystem cache') - await textCheck('/loader/client', 'hello filesystem cache') - }, - fullInvalidation: !isTurbopack, - }, - }), - 'next config change': { - async checkInitial() { - await textCheck('/next-config', 'hello world') - await textCheck('/next-config/client', 'hello world') - }, - withChange: makeFileEdit('next.config.js'), - async checkChanged() { - await textCheck('/next-config', 'hello filesystem cache') - await textCheck('/next-config/client', 'hello filesystem cache') - }, - fullInvalidation: !isTurbopack, - }, - 'env var change': { - async checkInitial() { - await textCheck('/env', 'hello world') - await textCheck('/env/client', 'hello world') - }, - async withChange(inner) { - process.env.NEXT_PUBLIC_ENV_VAR = 'hello filesystem cache' - try { - await inner() - } finally { - process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' - } - }, - async checkChanged() { - await textCheck('/env', 'hello filesystem cache') - await textCheck('/env/client', 'hello filesystem cache') - }, - }, - } - - const KEYS = Object.keys(POTENTIAL_CHANGES) - for (let bitset = 1; bitset < 1 << KEYS.length; bitset++) { - let combination = [] - for (let i = 0; i < KEYS.length; i++) { - if (bitset & (1 << i)) { - combination.push(KEYS[i]) - } - } - // Checking only single change and all combined for performance reasons. - if (combination.length !== 1 && combination.length !== KEYS.length) continue - - it(`should allow to change files while stopped (${combination.join(', ')})`, async () => { - let fullInvalidation = false - for (const key of combination) { - await POTENTIAL_CHANGES[key].checkInitial() - if (POTENTIAL_CHANGES[key].fullInvalidation) { - fullInvalidation = true - } - } - - let unchangedTimestamp - if (!fullInvalidation) { - const browser = await next.browser('/unchanged') - unchangedTimestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/) - await browser.close() - } - - async function checkChanged() { - for (const key of combination) { - await POTENTIAL_CHANGES[key].checkChanged() - } - - if (!fullInvalidation) { - const browser = await next.browser('/unchanged') - const timestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toEqual(timestamp) - await browser.close() - } - } - - await stop() - - async function inner() { - await start() - await checkChanged() - // Some no-op change builds - for (let i = 0; i < 2; i++) { - await restartCycle() - await checkChanged() - } - await stop() - } - - let current = inner - for (const key of combination) { - const prev = current - current = () => POTENTIAL_CHANGES[key].withChange(prev) - } - await current() - - await start() - for (const key of combination) { - await POTENTIAL_CHANGES[key].checkInitial() - } - - if (!fullInvalidation) { - const browser = await next.browser('/unchanged') - const timestamp = await browser.elementByCss('main').text() - expect(unchangedTimestamp).toEqual(timestamp) - await browser.close() - } - }, 200000) - } -}) diff --git a/test/e2e/filesystem-cache/next.config.js b/test/e2e/filesystem-cache/next.config.js index f72954fe4205b..89f644ec2b2cc 100644 --- a/test/e2e/filesystem-cache/next.config.js +++ b/test/e2e/filesystem-cache/next.config.js @@ -1,3 +1,5 @@ +const enableCaching = !!process.env.ENABLE_CACHING + /** * @type {import('next').NextConfig} */ @@ -15,10 +17,14 @@ const nextConfig = { }, }, }, - experimental: { - turbopackFileSystemCacheForDev: true, - turbopackFileSystemCacheForBuild: true, - }, + experimental: enableCaching + ? { + turbopackFileSystemCacheForBuild: true, + } + : { + turbopackFileSystemCacheForDev: false, + turbopackFileSystemCacheForBuild: false, + }, env: { NEXT_PUBLIC_CONFIG_ENV: 'hello world', }, @@ -31,6 +37,11 @@ const nextConfig = { test: /app\/loader(?:\/client)?\/page\.tsx/, use: ['./my-loader.js'], }) + if (enableCaching) { + config.cache = Object.freeze({ + type: 'memory', + }) + } if (dev) { // Make webpack consider the build as large change which makes it filesystem cache it sooner config.plugins.push((compiler) => { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index d9dc5cea2894a..79830cd8c5159 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -12,7 +12,7 @@ use std::{ ops::Range, pin::Pin, sync::{ - Arc, + Arc, LazyLock, atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, }, }; @@ -72,6 +72,16 @@ use crate::{ const SNAPSHOT_REQUESTED_BIT: usize = 1 << (usize::BITS - 1); +/// Configurable idle timeout for snapshot persistence. +/// Defaults to 2 seconds if not set or if the value is invalid. +static IDLE_TIMEOUT: LazyLock = LazyLock::new(|| { + std::env::var("TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS") + .ok() + .and_then(|v| v.parse::().ok()) + .map(Duration::from_millis) + .unwrap_or(Duration::from_secs(2)) +}); + struct SnapshotRequest { snapshot_requested: bool, suspended_operations: FxHashSet>, @@ -2467,8 +2477,7 @@ impl TurboTasksBackendInner { loop { const FIRST_SNAPSHOT_WAIT: Duration = Duration::from_secs(300); const SNAPSHOT_INTERVAL: Duration = Duration::from_secs(120); - const IDLE_TIMEOUT: Duration = Duration::from_secs(2); - + let idle_timeout = *IDLE_TIMEOUT; let (time, mut reason) = if matches!(job, TurboTasksBackendJob::InitialSnapshot) { (FIRST_SNAPSHOT_WAIT, "initial snapshot timeout") @@ -2485,7 +2494,7 @@ impl TurboTasksBackendInner { let mut idle_start_listener = self.idle_start_event.listen(); let mut idle_end_listener = self.idle_end_event.listen(); let mut idle_time = if turbo_tasks.is_idle() { - Instant::now() + IDLE_TIMEOUT + Instant::now() + idle_timeout } else { far_future() }; @@ -2495,11 +2504,11 @@ impl TurboTasksBackendInner { return; }, _ = &mut idle_start_listener => { - idle_time = Instant::now() + IDLE_TIMEOUT; + idle_time = Instant::now() + idle_timeout; idle_start_listener = self.idle_start_event.listen() }, _ = &mut idle_end_listener => { - idle_time = until + IDLE_TIMEOUT; + idle_time = until + idle_timeout; idle_end_listener = self.idle_end_event.listen() }, _ = tokio::time::sleep_until(until) => {