From 8d47d8edda67b9aab6ffb85734cc4ab5c1e81259 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Thu, 10 Jul 2025 13:15:04 -0700 Subject: [PATCH] Enable growable shared array buffers Currently this is still behind a flag so we never turn it on by default. Fixes: #24287 --- .circleci/config.yml | 4 ++- .../tools_reference/settings_reference.rst | 13 ++++++++ src/lib/libcore.js | 6 +++- src/runtime_common.js | 6 +++- src/settings.js | 7 ++++ test/test_browser.py | 10 ++++-- test/test_other.py | 33 +++++++++++++++++-- tools/building.py | 1 + tools/link.py | 7 ++-- tools/system_libs.py | 2 +- 10 files changed, 79 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 980194db371ce..5e75db480aaf3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -320,7 +320,7 @@ commands: CHROME_FLAGS_BASE: "--no-first-run -start-maximized --no-sandbox --use-gl=swiftshader --user-data-dir=/tmp/chrome-emscripten-profile --enable-experimental-web-platform-features --enable-features=JavaScriptSourcePhaseImports" # Increase the window size to avoid flaky sdl tests see #24236. CHROME_FLAGS_HEADLESS: "--headless=new --window-size=1024,768 --remote-debugging-port=1234" - CHROME_FLAGS_WASM: "--enable-experimental-webassembly-features --js-flags=\"--experimental-wasm-stack-switching --experimental-wasm-type-reflection\"" + CHROME_FLAGS_WASM: "--enable-experimental-webassembly-features --js-flags=\"--experimental-wasm-stack-switching --experimental-wasm-type-reflection --experimental-wasm-rab-integration\"" CHROME_FLAGS_NOCACHE: "--disk-cache-dir=/dev/null --disk-cache-size=1 --media-cache-size=1 --disable-application-cache --incognito" # The runners lack sound hardware so fallback to a dummy device (and # bypass the user gesture so audio tests work without interaction) @@ -820,6 +820,7 @@ jobs: other.test_gen_struct_info other.test_native_call_before_init other.test_node_unhandled_rejection + other.test_*growable_arraybuffers core2.test_hello_world core2.test_esm_integration* core0.test_esm_integration* @@ -1023,6 +1024,7 @@ jobs: skip:browser.test_webgl_offscreen_canvas_in_pthread skip:browser.test_webgl_offscreen_canvas_in_mainthread_after_pthread skip:browser.test_glut_glutget + skip:browser.test_*_growable_arraybuffers " test-browser-firefox-wasm64: executor: focal diff --git a/site/source/docs/tools_reference/settings_reference.rst b/site/source/docs/tools_reference/settings_reference.rst index 1554af79e3cf0..c9e6e02f913c2 100644 --- a/site/source/docs/tools_reference/settings_reference.rst +++ b/site/source/docs/tools_reference/settings_reference.rst @@ -3343,6 +3343,19 @@ or chrome. Default value: false +.. _growable_arraybuffers: + +GROWABLE_ARRAYBUFFERS +===================== + +Enable support for GrowableSharedArrayBuffer. +This features is only available behind a flag in recent versions of +node/chrome. + +.. note:: This is an experimental setting + +Default value: false + .. _wasm_js_types: WASM_JS_TYPES diff --git a/src/lib/libcore.js b/src/lib/libcore.js index 73791efc656a8..f0cb1c87f5e47 100644 --- a/src/lib/libcore.js +++ b/src/lib/libcore.js @@ -182,7 +182,9 @@ addToLibrary({ try { // round size grow request up to wasm page size (fixed 64KB per spec) wasmMemory.grow({{{ toIndexType('pages') }}}); // .grow() takes a delta compared to the previous size +#if !GROWABLE_ARRAYBUFFERS updateMemoryViews(); +#endif #if MEMORYPROFILER if (typeof emscriptenMemoryProfiler != 'undefined') { emscriptenMemoryProfiler.onMemoryResize(oldHeapSize, wasmMemory.buffer.byteLength); @@ -320,6 +322,7 @@ addToLibrary({ #endif // ALLOW_MEMORY_GROWTH }, +#if !GROWABLE_ARRAYBUFFERS // Called after wasm grows memory. At that time we need to update the views. // Without this notification, we'd need to check the buffer in JS every time // we return from any wasm, which adds overhead. See @@ -330,6 +333,7 @@ addToLibrary({ #endif updateMemoryViews(); }, +#endif _emscripten_system: (command) => { #if ENVIRONMENT_MAY_BE_NODE @@ -2415,7 +2419,7 @@ function wrapSyscallFunction(x, library, isWasi) { library[x + '__deps'] ??= []; -#if PURE_WASI +#if PURE_WASI && !GROWABLE_ARRAYBUFFERS // In PURE_WASI mode we can't assume the wasm binary was built by emscripten // and politely notify us on memory growth. Instead we have to check for // possible memory growth on each syscall. diff --git a/src/runtime_common.js b/src/runtime_common.js index 724a7b5b88599..6ef8e7194fd46 100644 --- a/src/runtime_common.js +++ b/src/runtime_common.js @@ -12,7 +12,7 @@ #include "runtime_safe_heap.js" #endif -#if SHARED_MEMORY && ALLOW_MEMORY_GROWTH +#if SHARED_MEMORY && ALLOW_MEMORY_GROWTH && !GROWABLE_ARRAYBUFFERS #include "growableHeap.js" #endif @@ -132,7 +132,11 @@ var runtimeExited = false; }}} function updateMemoryViews() { +#if GROWABLE_ARRAYBUFFERS + var b = wasmMemory.toResizableBuffer(); +#else var b = wasmMemory.buffer; +#endif {{{ maybeExportHeap('HEAP8') }}}HEAP8 = new Int8Array(b); {{{ maybeExportHeap('HEAP16') }}}HEAP16 = new Int16Array(b); {{{ maybeExportHeap('HEAPU8') }}}HEAPU8 = new Uint8Array(b); diff --git a/src/settings.js b/src/settings.js index b21cc9ddc2454..feb8ab9816c7f 100644 --- a/src/settings.js +++ b/src/settings.js @@ -2184,6 +2184,13 @@ var WASM_ESM_INTEGRATION = false; // [link] var JS_BASE64_API = false; +// Enable support for GrowableSharedArrayBuffer. +// This features is only available behind a flag in recent versions of +// node/chrome. +// [experimental] +// [link] +var GROWABLE_ARRAYBUFFERS = false; + // Experimental support for WebAssembly js-types proposal. // It's currently only available under a flag in certain browsers, // so we disable it by default to save on code size. diff --git a/test/test_browser.py b/test/test_browser.py index 971a8d87000e1..0be24695d4222 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -4739,17 +4739,21 @@ def test_minimal_runtime_hello_thread(self, opts): # Tests memory growth in pthreads mode, but still on the main thread. @parameterized({ '': ([], 1), + 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'], 1), 'proxy': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME'], 2), }) @no_2gb('uses INITIAL_MEMORY') @no_4gb('uses INITIAL_MEMORY') def test_pthread_growth_mainthread(self, cflags, pthread_pool_size): self.set_setting('PTHREAD_POOL_SIZE', pthread_pool_size) - self.btest_exit('pthread/test_pthread_memory_growth_mainthread.c', cflags=['-Wno-pthreads-mem-growth', '-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) + if '-sGROWABLE_ARRAYBUFFERS' not in cflags: + self.cflags.append('-Wno-pthreads-mem-growth') + self.btest_exit('pthread/test_pthread_memory_growth_mainthread.c', cflags=['-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) # Tests memory growth in a pthread. @parameterized({ '': ([],), + 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'],), 'assert': (['-sASSERTIONS'],), 'proxy': (['-sPROXY_TO_PTHREAD'], 2), 'minimal': (['-sMINIMAL_RUNTIME', '-sMODULARIZE', '-sEXPORT_NAME=MyModule'],), @@ -4758,7 +4762,9 @@ def test_pthread_growth_mainthread(self, cflags, pthread_pool_size): @no_4gb('uses INITIAL_MEMORY') def test_pthread_growth(self, cflags, pthread_pool_size = 1): self.set_setting('PTHREAD_POOL_SIZE', pthread_pool_size) - self.btest_exit('pthread/test_pthread_memory_growth.c', cflags=['-Wno-pthreads-mem-growth', '-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) + if '-sGROWABLE_ARRAYBUFFERS' not in cflags: + self.cflags.append('-Wno-pthreads-mem-growth') + self.btest_exit('pthread/test_pthread_memory_growth.c', cflags=['-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) # Tests that time in a pthread is relative to the main thread, so measurements # on different threads are still monotonic, as if checking a single central diff --git a/test/test_other.py b/test/test_other.py index 13e902d332f8c..0f706d9ffe190 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -14285,23 +14285,52 @@ def test_pthread_kill(self): @node_pthreads @parameterized({ '': ([], 1), + 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'], 1), 'proxy': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME'], 2), }) def test_pthread_growth_mainthread(self, cflags, pthread_pool_size): + if '-sGROWABLE_ARRAYBUFFERS' in cflags: + self.node_args.append('--experimental-wasm-rab-integration') + self.v8_args.append('--experimental-wasm-rab-integration') + self.require_node_canary() + else: + self.cflags.append('-Wno-pthreads-mem-growth') self.set_setting('PTHREAD_POOL_SIZE', pthread_pool_size) - self.do_runf('pthread/test_pthread_memory_growth_mainthread.c', cflags=['-Wno-pthreads-mem-growth', '-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) + self.do_runf('pthread/test_pthread_memory_growth_mainthread.c', cflags=['-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) + + @requires_node_canary + def test_growable_arraybuffers(self): + self.node_args.append('--experimental-wasm-rab-integration') + self.v8_args.append('--experimental-wasm-rab-integration') + self.do_runf('hello_world.c', + cflags=['-O2', '-pthread', '-sALLOW_MEMORY_GROWTH', '-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'], + output_basename='growable') + self.do_runf('hello_world.c', + cflags=['-O2', '-pthread', '-sALLOW_MEMORY_GROWTH', '-Wno-pthreads-mem-growth'], + output_basename='no_growable') + growable_size = os.path.getsize('growable.js') + no_growable_size = os.path.getsize('no_growable.js') + print('growable:', growable_size, 'no_growable:', no_growable_size) + self.assertLess(growable_size, no_growable_size) # Tests memory growth in a pthread. @node_pthreads @parameterized({ '': ([],), + 'growable_arraybuffers': (['-sGROWABLE_ARRAYBUFFERS', '-Wno-experimental'],), 'assert': (['-sASSERTIONS'],), 'proxy': (['-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME'], 2), 'minimal': (['-sMINIMAL_RUNTIME', '-sMODULARIZE', '-sEXPORT_NAME=MyModule'],), }) def test_pthread_growth(self, cflags, pthread_pool_size = 1): self.set_setting('PTHREAD_POOL_SIZE', pthread_pool_size) - self.do_runf('pthread/test_pthread_memory_growth.c', cflags=['-Wno-pthreads-mem-growth', '-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) + if '-sGROWABLE_ARRAYBUFFERS' in cflags: + self.node_args.append('--experimental-wasm-rab-integration') + self.v8_args.append('--experimental-wasm-rab-integration') + self.require_node_canary() + else: + self.cflags.append('-Wno-pthreads-mem-growth') + self.do_runf('pthread/test_pthread_memory_growth.c', cflags=['-pthread', '-sALLOW_MEMORY_GROWTH', '-sINITIAL_MEMORY=32MB', '-sMAXIMUM_MEMORY=256MB'] + cflags) @node_pthreads def test_emscripten_set_interval(self): diff --git a/tools/building.py b/tools/building.py index 2939139ada5ef..42a24ce678bd4 100644 --- a/tools/building.py +++ b/tools/building.py @@ -1059,6 +1059,7 @@ def little_endian_heap(js_file): def apply_wasm_memory_growth(js_file): + assert not settings.GROWABLE_ARRAYBUFFERS logger.debug('supporting wasm memory growth with pthreads') return acorn_optimizer(js_file, ['growableHeap']) diff --git a/tools/link.py b/tools/link.py index 24a1a748dadcb..9521b86092863 100644 --- a/tools/link.py +++ b/tools/link.py @@ -509,7 +509,7 @@ def setup_pthreads(): diagnostics.warning('experimental', '-sMAIN_MODULE + pthreads is experimental') elif settings.LINKABLE: diagnostics.warning('experimental', '-sLINKABLE + pthreads is experimental') - if settings.ALLOW_MEMORY_GROWTH: + if settings.ALLOW_MEMORY_GROWTH and not settings.GROWABLE_ARRAYBUFFERS: diagnostics.warning('pthreads-mem-growth', '-pthread + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, see https://github.com/WebAssembly/design/issues/1271') default_setting('DEFAULT_PTHREAD_STACK_SIZE', settings.STACK_SIZE) @@ -795,6 +795,9 @@ def phase_linker_setup(options, linker_args): # noqa: C901, PLR0912, PLR0915 if settings.JS_BASE64_API: diagnostics.warning('experimental', '-sJS_BASE64_API is still experimental and not yet supported in browsers') + if settings.GROWABLE_ARRAYBUFFERS: + diagnostics.warning('experimental', '-sGROWABLE_ARRAYBUFFERS is still experimental and not yet supported in browsers') + if settings.SOURCE_PHASE_IMPORTS: if not settings.EXPORT_ES6: exit_with_error('SOURCE_PHASE_IMPORTS requires EXPORT_ES6') @@ -2330,7 +2333,7 @@ def phase_binaryen(target, options, wasm_target): # unsigning pass. # we also must do this after the asan or safe_heap instrumentation, as they # wouldn't be able to recognize patterns produced by the growth pass. - if settings.SHARED_MEMORY and settings.ALLOW_MEMORY_GROWTH: + if settings.SHARED_MEMORY and settings.ALLOW_MEMORY_GROWTH and not settings.GROWABLE_ARRAYBUFFERS: with ToolchainProfiler.profile_block('apply_wasm_memory_growth'): final_js = building.apply_wasm_memory_growth(final_js) diff --git a/tools/system_libs.py b/tools/system_libs.py index 711dc4d5844ea..5764302a2e83a 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -2231,7 +2231,7 @@ def vary_on(cls): def get_default_variation(cls, **kwargs): return super().get_default_variation( is_mem_grow=settings.ALLOW_MEMORY_GROWTH, - is_pure=settings.PURE_WASI, + is_pure=settings.PURE_WASI or settings.GROWABLE_ARRAYBUFFERS, nocatch=settings.DISABLE_EXCEPTION_CATCHING and not settings.WASM_EXCEPTIONS, **kwargs, )