From 0bdd47b0fb66bb2d59d653cedfec0e8bc013cfdc Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 13:54:42 -0800 Subject: [PATCH 01/30] very wip --- emcc.py | 5 ----- emscripten.py | 2 ++ src/modules.js | 1 - src/parseTools.js | 5 ++++- src/preamble.js | 8 +++----- src/settings.js | 42 +++++++++++++++++++++++------------------- src/support.js | 12 ++---------- 7 files changed, 34 insertions(+), 41 deletions(-) diff --git a/emcc.py b/emcc.py index e0e5d2586dc6..4e16cb9ab8bf 100755 --- a/emcc.py +++ b/emcc.py @@ -1284,12 +1284,10 @@ def check(input_file): # placeholder strings for JS glue, to be replaced with subresource locations in do_binaryen shared.Settings.WASM_TEXT_FILE = shared.FilenameReplacementStrings.WASM_TEXT_FILE shared.Settings.WASM_BINARY_FILE = shared.FilenameReplacementStrings.WASM_BINARY_FILE - shared.Settings.ASMJS_CODE_FILE = shared.FilenameReplacementStrings.ASMJS_CODE_FILE else: # set file locations, so that JS glue can find what it needs shared.Settings.WASM_TEXT_FILE = shared.JS.escape_for_js_string(os.path.basename(wasm_text_target)) shared.Settings.WASM_BINARY_FILE = shared.JS.escape_for_js_string(os.path.basename(wasm_binary_target)) - shared.Settings.ASMJS_CODE_FILE = shared.JS.escape_for_js_string(os.path.basename(asm_target)) shared.Settings.ASM_JS = 2 # when targeting wasm, we use a wasm Memory, but that is not compatible with asm.js opts shared.Settings.GLOBAL_BASE = 1024 # leave some room for mapping global vars @@ -2605,9 +2603,6 @@ def do_binaryen(target, asm_target, options, memfile, wasm_binary_target, (wasm_binary_target, shared.FilenameReplacementStrings.WASM_BINARY_FILE, 'native-wasm' in shared.Settings.BINARYEN_METHOD), - (asm_target, - shared.FilenameReplacementStrings.ASMJS_CODE_FILE, - 'asmjs' in shared.Settings.BINARYEN_METHOD), ): if should_embed and os.path.isfile(target): js = js.replace(replacement_string, shared.JS.get_subresource_location(target)) diff --git a/emscripten.py b/emscripten.py index 687be35d15bc..e6bfd6679af1 100644 --- a/emscripten.py +++ b/emscripten.py @@ -581,6 +581,8 @@ def read_proxied_function_signatures(asmConsts): if not shared.Settings.WASM_BACKEND: shared.Settings.PROXIED_FUNCTION_SIGNATURES = read_proxied_function_signatures(metadata['asmConsts']) + shared.Settings.STATIC_BUMP = metadata['staticBump'] + def compile_settings(compiler_engine, libraries, temp_files): # Save settings to a file to work around v8 issue 1579 diff --git a/src/modules.js b/src/modules.js index 13c37c6957fb..cb00f7747c79 100644 --- a/src/modules.js +++ b/src/modules.js @@ -394,7 +394,6 @@ function exportRuntime() { 'FS_createDevice', 'FS_unlink', 'GL', - 'staticAlloc', 'dynamicAlloc', 'warnOnce', 'loadDynamicLibrary', diff --git a/src/parseTools.js b/src/parseTools.js index 8700db430c3c..88eccb5bb058 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1442,7 +1442,10 @@ function makeEval(code) { function makeStaticAlloc(size) { size = (size + (STACK_ALIGN-1)) & -STACK_ALIGN; - return 'STATICTOP; STATICTOP += ' + size + ';'; + STATIC_BUMP += size; + var ret = GLOBAL_BASE + STATIC_BUMP; + assert(!(ret & (STACK_ALIGN - 1))); + return '/* waka */' + ret; } function makeRetainedCompilerSettings() { diff --git a/src/preamble.js b/src/preamble.js index 6b04cc0a09d1..687b2d14abd5 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -286,9 +286,8 @@ function getValue(ptr, type, noSafe) { var ALLOC_NORMAL = 0; // Tries to use _malloc() var ALLOC_STACK = 1; // Lives for the duration of the current function call -var ALLOC_STATIC = 2; // Cannot be freed -var ALLOC_DYNAMIC = 3; // Cannot be freed except through sbrk -var ALLOC_NONE = 4; // Do not allocate +var ALLOC_DYNAMIC = 2; // Cannot be freed except through sbrk +var ALLOC_NONE = 3; // Do not allocate // allocate(): This is for internal use. You can use it yourself as well, but the interface // is a little tricky (see docs right below). The reason is that it is optimized @@ -320,7 +319,7 @@ function allocate(slab, types, allocator, ptr) { if (allocator == ALLOC_NONE) { ret = ptr; } else { - ret = [typeof _malloc === 'function' ? _malloc : staticAlloc, stackAlloc, staticAlloc, dynamicAlloc][allocator === undefined ? ALLOC_STATIC : allocator](Math.max(size, singleType ? 1 : types.length)); + ret = [_malloc, stackAlloc, dynamicAlloc][allocator](Math.max(size, singleType ? 1 : types.length)); } if (zeroinit) { @@ -377,7 +376,6 @@ function allocate(slab, types, allocator, ptr) { // Allocate memory during any stage of startup - static memory early on, dynamic memory later, malloc when ready function getMemory(size) { - if (!staticSealed) return staticAlloc(size); if (!runtimeInitialized) return dynamicAlloc(size); return _malloc(size); } diff --git a/src/settings.js b/src/settings.js index ac21fad4b742..469d2a89bff1 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1127,12 +1127,6 @@ var SDL2_IMAGE_FORMATS = []; // legalizer var DEBUG_TAGS_SHOWING = []; -// Internal: tracks the list of EM_ASM signatures that are proxied between threads. -var PROXIED_FUNCTION_SIGNATURES = []; - -// For internal use only -var ORIGINAL_EXPORTED_FUNCTIONS = []; - // The list of defines (C_DEFINES) was moved into struct_info.json in the same // directory. That file is automatically parsed by tools/gen_struct_info.py. // If you modify the headers, just clear your cache and emscripten libc should @@ -1283,41 +1277,51 @@ var ASMFS = 0; // then you can safely ignore this warning. var SINGLE_FILE = 0; -// For internal use only (name of the file containing wasm text, if relevant). +// if set to 1, then generated WASM files will contain a custom +// "emscripten_metadata" section that contains information necessary +// to execute the file without the accompanying JS file. +var EMIT_EMSCRIPTEN_METADATA = 0; + + +// Internal use only, from here + +// tracks the list of EM_ASM signatures that are proxied between threads. +var PROXIED_FUNCTION_SIGNATURES = []; + +var ORIGINAL_EXPORTED_FUNCTIONS = []; + +// name of the file containing wasm text, if relevant var WASM_TEXT_FILE = ''; -// For internal use only (name of the file containing wasm binary, if relevant). +// name of the file containing wasm binary, if relevant var WASM_BINARY_FILE = ''; -// For internal use only (name of the file containing asm.js, if relevant). -var ASMJS_CODE_FILE = ''; - -// For internal use only (name of the file containing the pthread *.worker.js, if relevant). +// name of the file containing the pthread *.worker.js, if relevant var PTHREAD_WORKER_FILE = ''; // Base URL the source mapfile, if relevant var SOURCE_MAP_BASE = ''; -// for internal use only var MEM_INIT_IN_WASM = 0; // If set to 1, src/base64Utils.js will be included in the bundle. // This is set internally when needed (SINGLE_FILE) var SUPPORT_BASE64_EMBEDDING = 0; -// For internal use only, the possible environments the code may run in. +// the possible environments the code may run in. var ENVIRONMENT_MAY_BE_WEB = 1; var ENVIRONMENT_MAY_BE_WORKER = 1; var ENVIRONMENT_MAY_BE_NODE = 1; var ENVIRONMENT_MAY_BE_SHELL = 1; var ENVIRONMENT_MAY_BE_WEB_OR_WORKER = 1; -// Internal: passes information to emscripten.py about whether to minify +// passes information to emscripten.py about whether to minify // JS -> asm.js import names. Controlled by optimization level, enabled // at -O1 and higher, but disabled at -g2 and higher. var MINIFY_ASMJS_IMPORT_NAMES = 0; -// if set to 1, then generated WASM files will contain a custom -// "emscripten_metadata" section that contains information necessary -// to execute the file without the accompanying JS file. -var EMIT_EMSCRIPTEN_METADATA = 0; +// the total static allocation, that is, how much to bump the start of memory +// for static globals. received from the backend, and possibly increased due +// to JS static allocations +var STATIC_BUMP = -1; + diff --git a/src/support.js b/src/support.js index ed3363cb27aa..e10ba72a1c50 100644 --- a/src/support.js +++ b/src/support.js @@ -13,19 +13,11 @@ var STACK_ALIGN = {{{ STACK_ALIGN }}}; stackSave = stackRestore = stackAlloc = function() { abort('cannot use the stack before compiled code is ready to run, and has provided stack access'); }; -#endif function staticAlloc(size) { -#if ASSERTIONS - assert(!staticSealed); -#endif - var ret = STATICTOP; - STATICTOP = (STATICTOP + size + 15) & -16; -#if ASSERTIONS - assert(STATICTOP < TOTAL_MEMORY, 'not enough memory for static allocation - increase TOTAL_MEMORY'); -#endif - return ret; + abort('staticAlloc is no longer available at runtime; instead, perform static allocations at compile time (using makeStaticAlloc)'); } +#endif function dynamicAlloc(size) { #if ASSERTIONS From e5bbc5258c0e9138d1e3fc735e8e1cd596bc0625 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 14:03:53 -0800 Subject: [PATCH 02/30] fix --- emcc.py | 5 +++++ src/jsifier.js | 2 +- src/settings.js | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/emcc.py b/emcc.py index 4e16cb9ab8bf..e0e5d2586dc6 100755 --- a/emcc.py +++ b/emcc.py @@ -1284,10 +1284,12 @@ def check(input_file): # placeholder strings for JS glue, to be replaced with subresource locations in do_binaryen shared.Settings.WASM_TEXT_FILE = shared.FilenameReplacementStrings.WASM_TEXT_FILE shared.Settings.WASM_BINARY_FILE = shared.FilenameReplacementStrings.WASM_BINARY_FILE + shared.Settings.ASMJS_CODE_FILE = shared.FilenameReplacementStrings.ASMJS_CODE_FILE else: # set file locations, so that JS glue can find what it needs shared.Settings.WASM_TEXT_FILE = shared.JS.escape_for_js_string(os.path.basename(wasm_text_target)) shared.Settings.WASM_BINARY_FILE = shared.JS.escape_for_js_string(os.path.basename(wasm_binary_target)) + shared.Settings.ASMJS_CODE_FILE = shared.JS.escape_for_js_string(os.path.basename(asm_target)) shared.Settings.ASM_JS = 2 # when targeting wasm, we use a wasm Memory, but that is not compatible with asm.js opts shared.Settings.GLOBAL_BASE = 1024 # leave some room for mapping global vars @@ -2603,6 +2605,9 @@ def do_binaryen(target, asm_target, options, memfile, wasm_binary_target, (wasm_binary_target, shared.FilenameReplacementStrings.WASM_BINARY_FILE, 'native-wasm' in shared.Settings.BINARYEN_METHOD), + (asm_target, + shared.FilenameReplacementStrings.ASMJS_CODE_FILE, + 'asmjs' in shared.Settings.BINARYEN_METHOD), ): if should_embed and os.path.isfile(target): js = js.replace(replacement_string, shared.JS.get_subresource_location(target)) diff --git a/src/jsifier.js b/src/jsifier.js index c11c8f3aa380..81564dfa70ff 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -543,7 +543,7 @@ function JSify(data, functionsOnly) { for(i in proxiedFunctionInvokers) print(proxiedFunctionInvokers[i]+'\n'); print('if (!ENVIRONMENT_IS_PTHREAD) {\n // Only main thread initializes these, pthreads copy them over at thread worker init time (in worker.js)'); } - print('DYNAMICTOP_PTR = staticAlloc(4);\n'); + print('DYNAMICTOP_PTR = ' + makeStaticAlloc(4) + ''); print('STACK_BASE = STACKTOP = alignMemory(STATICTOP);\n'); if (STACK_START > 0) print('if (STACKTOP < ' + STACK_START + ') STACK_BASE = STACKTOP = alignMemory(' + STACK_START + ');\n'); print('STACK_MAX = STACK_BASE + TOTAL_STACK;\n'); diff --git a/src/settings.js b/src/settings.js index 469d2a89bff1..a26fdef9361b 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1296,6 +1296,9 @@ var WASM_TEXT_FILE = ''; // name of the file containing wasm binary, if relevant var WASM_BINARY_FILE = ''; +// name of the file containing asm.js code, if relevant +var ASMJS_CODE_FILE = ''; + // name of the file containing the pthread *.worker.js, if relevant var PTHREAD_WORKER_FILE = ''; From afc4c1659cb537de30d95fe036f54f8bca98f002 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 14:38:08 -0800 Subject: [PATCH 03/30] remove staticAlloc/ALLOC_STATIC --- src/jsifier.js | 2 +- src/library.js | 29 +++++++++++++++-------------- src/library_fetch.js | 4 ++-- src/library_pthread.js | 10 +++++----- src/parseTools.js | 7 ++++++- 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/jsifier.js b/src/jsifier.js index 81564dfa70ff..5db4b7c5b0b7 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -488,7 +488,7 @@ function JSify(data, functionsOnly) { if (!SIDE_MODULE) { if (USE_PTHREADS) { print('var tempDoublePtr;'); - print('if (!ENVIRONMENT_IS_PTHREAD) tempDoublePtr = alignMemory(allocate(12, "i8", ALLOC_STATIC), 8);'); + print('if (!ENVIRONMENT_IS_PTHREAD) tempDoublePtr = ' + makeStaticAlloc(12) + ';'); } else { print('var tempDoublePtr = ' + makeStaticAlloc(8) + ''); } diff --git a/src/library.js b/src/library.js index c90aae70e7e2..7a49801e0bc7 100644 --- a/src/library.js +++ b/src/library.js @@ -19,16 +19,17 @@ // object. For convenience, the short name appears here. Note that if you add a // new function with an '_', it will not be found. -// Memory allocated during startup, in postsets, should only be ALLOC_STATIC +// Memory allocated during startup, in postsets, should only be static +// (using makeStaticAlloc) LibraryManager.library = { // keep this low in memory, because we flatten arrays with them in them #if USE_PTHREADS - stdin: '; if (ENVIRONMENT_IS_PTHREAD) _stdin = PthreadWorkerInit._stdin; else PthreadWorkerInit._stdin = _stdin = allocate(1, "i32*", ALLOC_STATIC)', - stdout: '; if (ENVIRONMENT_IS_PTHREAD) _stdout = PthreadWorkerInit._stdout; else PthreadWorkerInit._stdout = _stdout = allocate(1, "i32*", ALLOC_STATIC)', - stderr: '; if (ENVIRONMENT_IS_PTHREAD) _stderr = PthreadWorkerInit._stderr; else PthreadWorkerInit._stderr = _stderr = allocate(1, "i32*", ALLOC_STATIC)', - _impure_ptr: '; if (ENVIRONMENT_IS_PTHREAD) __impure_ptr = PthreadWorkerInit.__impure_ptr; else PthreadWorkerInit.__impure_ptr __impure_ptr = allocate(1, "i32*", ALLOC_STATIC)', - __dso_handle: '; if (ENVIRONMENT_IS_PTHREAD) ___dso_handle = PthreadWorkerInit.___dso_handle; else PthreadWorkerInit.___dso_handle = ___dso_handle = allocate(1, "i32*", ALLOC_STATIC)', + stdin: '; if (ENVIRONMENT_IS_PTHREAD) _stdin = PthreadWorkerInit._stdin; else PthreadWorkerInit._stdin = _stdin = {{{ makeStaticAlloc(4) }}}', + stdout: '; if (ENVIRONMENT_IS_PTHREAD) _stdout = PthreadWorkerInit._stdout; else PthreadWorkerInit._stdout = _stdout = {{{ makeStaticAlloc(4) }}}', + stderr: '; if (ENVIRONMENT_IS_PTHREAD) _stderr = PthreadWorkerInit._stderr; else PthreadWorkerInit._stderr = _stderr = {{{ makeStaticAlloc(4) }}}', + _impure_ptr: '; if (ENVIRONMENT_IS_PTHREAD) __impure_ptr = PthreadWorkerInit.__impure_ptr; else PthreadWorkerInit.__impure_ptr __impure_ptr = {{{ makeStaticAlloc(4) }}}', + __dso_handle: '; if (ENVIRONMENT_IS_PTHREAD) ___dso_handle = PthreadWorkerInit.___dso_handle; else PthreadWorkerInit.___dso_handle = ___dso_handle = {{{ makeStaticAlloc(4) }}}', #else stdin: '{{{ makeStaticAlloc(1) }}}', stdout: '{{{ makeStaticAlloc(1) }}}', @@ -1928,13 +1929,13 @@ LibraryManager.library = { // Statically allocated time struct. #if USE_PTHREADS - __tm_current: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_current = PthreadWorkerInit.___tm_current; else PthreadWorkerInit.___tm_current = ___tm_current = allocate({{{ C_STRUCTS.tm.__size__ }}}, "i8", ALLOC_STATIC)', - __tm_timezone: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_timezone = PthreadWorkerInit.___tm_timezone; else PthreadWorkerInit.___tm_timezone = ___tm_timezone = allocate(intArrayFromString("GMT"), "i8", ALLOC_STATIC)', - __tm_formatted: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_formatted = PthreadWorkerInit.___tm_formatted; else PthreadWorkerInit.___tm_formatted = ___tm_formatted = allocate({{{ C_STRUCTS.tm.__size__ }}}, "i8", ALLOC_STATIC)', + __tm_current: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_current = PthreadWorkerInit.___tm_current; else PthreadWorkerInit.___tm_current = ___tm_current = {{{ makeStaticAlloc(C_STRUCTS.tm.__size__) }}}', + __tm_timezone: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_timezone = PthreadWorkerInit.___tm_timezone; else PthreadWorkerInit.___tm_timezone = ___tm_timezone = {{{ makeStaticString("GMT") }}}', + __tm_formatted: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_formatted = PthreadWorkerInit.___tm_formatted; else PthreadWorkerInit.___tm_formatted = ___tm_formatted = {{{ makeStaticAlloc(C_STRUCTS.tm.__size__) }}}', #else __tm_current: '{{{ makeStaticAlloc(C_STRUCTS.tm.__size__) }}}', // Statically allocated copy of the string "GMT" for gmtime() to point to - __tm_timezone: 'allocate(intArrayFromString("GMT"), "i8", ALLOC_STATIC)', + __tm_timezone: '{{{ makeStaticString("GMT") }}}', // Statically allocated time strings. __tm_formatted: '{{{ makeStaticAlloc(C_STRUCTS.tm.__size__) }}}', #endif @@ -3259,13 +3260,13 @@ LibraryManager.library = { // ========================================================================== #if USE_PTHREADS - in6addr_any: '; if (ENVIRONMENT_IS_PTHREAD) _in6addr_any = PthreadWorkerInit._in6addr_any; else PthreadWorkerInit._in6addr_any = _in6addr_any = allocate([0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], "i8", ALLOC_STATIC)', - in6addr_loopback: '; if (ENVIRONMENT_IS_PTHREAD) _in6addr_loopback = PthreadWorkerInit._in6addr_loopback; else PthreadWorkerInit._in6addr_loopback = _in6addr_loopback = allocate([0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1], "i8", ALLOC_STATIC)', + in6addr_any: '; if (ENVIRONMENT_IS_PTHREAD) _in6addr_any = PthreadWorkerInit._in6addr_any; else PthreadWorkerInit._in6addr_any = _in6addr_any = {{{ makeStaticAlloc(16) }}}', + in6addr_loopback: '; if (ENVIRONMENT_IS_PTHREAD) _in6addr_loopback = PthreadWorkerInit._in6addr_loopback; else PthreadWorkerInit._in6addr_loopback = _in6addr_loopback = {{{ makeStaticAlloc(16) }}}', #else in6addr_any: - 'allocate([0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], "i8", ALLOC_STATIC)', + '{{{ makeStaticAlloc(16) }}}', in6addr_loopback: - 'allocate([0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1], "i8", ALLOC_STATIC)', + '{{{ makeStaticAlloc(16) }}}', #endif // ========================================================================== diff --git a/src/library_fetch.js b/src/library_fetch.js index ce2d21acac0a..a30e76711772 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -8,10 +8,10 @@ var LibraryFetch = { #if USE_PTHREADS $Fetch__postset: 'if (!ENVIRONMENT_IS_PTHREAD) Fetch.staticInit();', - fetch_work_queue: '; if (ENVIRONMENT_IS_PTHREAD) _fetch_work_queue = PthreadWorkerInit._fetch_work_queue; else PthreadWorkerInit._fetch_work_queue = _fetch_work_queue = allocate(12, "i32*", ALLOC_STATIC)', + fetch_work_queue: '; if (ENVIRONMENT_IS_PTHREAD) _fetch_work_queue = PthreadWorkerInit._fetch_work_queue; else PthreadWorkerInit._fetch_work_queue = _fetch_work_queue = {{{ makeStaticAlloc(12) }}}', #else $Fetch__postset: 'Fetch.staticInit();', - fetch_work_queue: 'allocate(12, "i32*", ALLOC_STATIC)', + fetch_work_queue: '{{{ makeStaticAlloc(12) }}}', #endif $Fetch: Fetch, _emscripten_get_fetch_work_queue__deps: ['fetch_work_queue'], diff --git a/src/library_pthread.js b/src/library_pthread.js index 0f173fe53090..660f2d42e519 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -21,7 +21,7 @@ var LibraryPThread = { // mainThreadBlock: undefined, initMainThreadBlock: function() { if (ENVIRONMENT_IS_PTHREAD) return undefined; - PThread.mainThreadBlock = allocate({{{ C_STRUCTS.pthread.__size__ }}}, "i32*", ALLOC_STATIC); + PThread.mainThreadBlock = {{{ makeStaticAlloc(C_STRUCTS.pthread.__size__) }}}; for (var i = 0; i < {{{ C_STRUCTS.pthread.__size__ }}}/4; ++i) HEAPU32[PThread.mainThreadBlock/4+i] = 0; @@ -34,7 +34,7 @@ var LibraryPThread = { {{{ makeSetValue('headPtr', 0, 'headPtr', 'i32') }}}; // Allocate memory for thread-local storage. - var tlsMemory = allocate({{{ cDefine('PTHREAD_KEYS_MAX') }}} * 4, "i32*", ALLOC_STATIC); + var tlsMemory = {{{ makeStaticAlloc(cDefine('PTHREAD_KEYS_MAX') * 4) }}}; for (var i = 0; i < {{{ cDefine('PTHREAD_KEYS_MAX') }}}; ++i) HEAPU32[tlsMemory/4+i] = 0; Atomics.store(HEAPU32, (PThread.mainThreadBlock + {{{ C_STRUCTS.pthread.tsd }}} ) >> 2, tlsMemory); // Init thread-local-storage memory array. Atomics.store(HEAPU32, (PThread.mainThreadBlock + {{{ C_STRUCTS.pthread.tid }}} ) >> 2, PThread.mainThreadBlock); // Main thread ID. @@ -54,7 +54,7 @@ var LibraryPThread = { #if PTHREADS_PROFILING createProfilerBlock: function(pthreadPtr) { - var profilerBlock = (pthreadPtr == PThread.mainThreadBlock) ? allocate({{{ C_STRUCTS.thread_profiler_block.__size__ }}}, "i32*", ALLOC_STATIC) : _malloc({{{ C_STRUCTS.thread_profiler_block.__size__ }}}); + var profilerBlock = (pthreadPtr == PThread.mainThreadBlock) ? {{{ makeStaticAlloc(C_STRUCTS.thread_profiler_block.__size__) }}} : _malloc({{{ C_STRUCTS.thread_profiler_block.__size__ }}}); Atomics.store(HEAPU32, (pthreadPtr + {{{ C_STRUCTS.pthread.profilerBlock }}} ) >> 2, profilerBlock); // Zero fill contents at startup. @@ -529,7 +529,7 @@ var LibraryPThread = { }, _num_logical_cores__deps: ['emscripten_force_num_logical_cores'], - _num_logical_cores: '; if (ENVIRONMENT_IS_PTHREAD) __num_logical_cores = PthreadWorkerInit.__num_logical_cores; else { PthreadWorkerInit.__num_logical_cores = __num_logical_cores = allocate(1, "i32*", ALLOC_STATIC); HEAPU32[__num_logical_cores>>2] = navigator["hardwareConcurrency"] || ' + {{{ PTHREAD_HINT_NUM_CORES }}} + '; }', + _num_logical_cores: '; if (ENVIRONMENT_IS_PTHREAD) __num_logical_cores = PthreadWorkerInit.__num_logical_cores; else { PthreadWorkerInit.__num_logical_cores = __num_logical_cores = makeStaticAlloc(1); HEAPU32[__num_logical_cores>>2] = navigator["hardwareConcurrency"] || ' + {{{ PTHREAD_HINT_NUM_CORES }}} + '; }', emscripten_has_threading_support: function() { return typeof SharedArrayBuffer !== 'undefined'; @@ -1031,7 +1031,7 @@ var LibraryPThread = { }, // Stores the memory address that the main thread is waiting on, if any. - _main_thread_futex_wait_address: '; if (ENVIRONMENT_IS_PTHREAD) __main_thread_futex_wait_address = PthreadWorkerInit.__main_thread_futex_wait_address; else PthreadWorkerInit.__main_thread_futex_wait_address = __main_thread_futex_wait_address = allocate(1, "i32*", ALLOC_STATIC)', + _main_thread_futex_wait_address: '; if (ENVIRONMENT_IS_PTHREAD) __main_thread_futex_wait_address = PthreadWorkerInit.__main_thread_futex_wait_address; else PthreadWorkerInit.__main_thread_futex_wait_address = __main_thread_futex_wait_address = makeStaticAlloc(1)', // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error. emscripten_futex_wait__deps: ['_main_thread_futex_wait_address', 'emscripten_main_thread_process_queued_calls'], diff --git a/src/parseTools.js b/src/parseTools.js index 88eccb5bb058..2f8704264a67 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1445,7 +1445,12 @@ function makeStaticAlloc(size) { STATIC_BUMP += size; var ret = GLOBAL_BASE + STATIC_BUMP; assert(!(ret & (STACK_ALIGN - 1))); - return '/* waka */' + ret; + return ret; +} + +function makeStaticString(string) { + var len = string.length + 1; + return 'stringToUTF8("' + string + '", ' + makeStaticAlloc(len) + ', ' + len + ')'; } function makeRetainedCompilerSettings() { From ae0bbb5c9b8c3b0056583d98ec8cbf2eafce63eb Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 15:15:13 -0800 Subject: [PATCH 04/30] fixes [ci skip] --- emscripten.py | 16 +++++++++++----- src/jsifier.js | 2 +- src/library.js | 1 + src/modules.js | 1 + src/parseTools.js | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/emscripten.py b/emscripten.py index e6bfd6679af1..09f6f463b411 100644 --- a/emscripten.py +++ b/emscripten.py @@ -527,6 +527,14 @@ def is_int(x): return False +def align_static_bump(metadata): + staticbump = metadata['staticBump'] + while staticbump % 16 != 0: + staticbump += 1 + metadata['staticBump'] = staticbump + return staticbump + + def update_settings_glue(metadata): if shared.Settings.CYBERDWARF: shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.append("cyberdwarf_Debugger") @@ -581,7 +589,7 @@ def read_proxied_function_signatures(asmConsts): if not shared.Settings.WASM_BACKEND: shared.Settings.PROXIED_FUNCTION_SIGNATURES = read_proxied_function_signatures(metadata['asmConsts']) - shared.Settings.STATIC_BUMP = metadata['staticBump'] + shared.Settings.STATIC_BUMP = align_static_bump(metadata) def compile_settings(compiler_engine, libraries, temp_files): @@ -608,8 +616,7 @@ def memory_and_global_initializers(pre, metadata, mem_init): pre = open(path_from_root(os.path.join('src', 'ecmascript_simd.js'))).read() + '\n\n' + pre staticbump = metadata['staticBump'] - while staticbump % 16 != 0: - staticbump += 1 + pthread = '' if shared.Settings.USE_PTHREADS: pthread = 'if (!ENVIRONMENT_IS_PTHREAD)' @@ -1880,8 +1887,7 @@ def emscript_wasm_backend(infile, outfile, memfile, libraries, compiler_engine, global_initializers = ', '.join('{ func: function() { %s() } }' % i for i in metadata['initializers']) staticbump = metadata['staticBump'] - while staticbump % 16 != 0: - staticbump += 1 + pre = pre.replace('STATICTOP = STATIC_BASE + 0;', '''STATICTOP = STATIC_BASE + %d; /* global initializers */ %s __ATINIT__.push(%s); ''' % (staticbump, diff --git a/src/jsifier.js b/src/jsifier.js index 5db4b7c5b0b7..01d38b6b20d0 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -321,7 +321,7 @@ function JSify(data, functionsOnly) { } }); }); - if (VERBOSE) printErr('adding ' + finalName + ' and deps ' + deps + ' : ' + (snippet + '').substr(0, 40)); + if (VERBOSE) printErr('adding ' + finalName + ' and deps ' + JSON.stringify(deps) + ' : ' + (snippet + '').substr(0, 40)); var depsText = (deps ? '\n' + deps.map(addFromLibrary).filter(function(x) { return x != '' }).join('\n') : ''); var contentText; if (isFunction) { diff --git a/src/library.js b/src/library.js index 7a49801e0bc7..167bb68d3ea4 100644 --- a/src/library.js +++ b/src/library.js @@ -4781,3 +4781,4 @@ function autoAddDeps(object, name) { } } } + diff --git a/src/modules.js b/src/modules.js index cb00f7747c79..6b99b7880a82 100644 --- a/src/modules.js +++ b/src/modules.js @@ -211,6 +211,7 @@ var LibraryManager = { target = lib[target]; } if (lib[target + '__asm']) continue; // This is an alias of an asm library function. Also needs to be fully optimized. + if (!isNaN(target)) continue; // This is a number, and so cannot be an alias target. if (typeof lib[target] === 'undefined' || typeof lib[target] === 'function') { lib[x] = new Function('return _' + target + '.apply(null, arguments)'); if (!lib[x + '__deps']) lib[x + '__deps'] = []; diff --git a/src/parseTools.js b/src/parseTools.js index 2f8704264a67..f009ca2b5911 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1444,7 +1444,7 @@ function makeStaticAlloc(size) { size = (size + (STACK_ALIGN-1)) & -STACK_ALIGN; STATIC_BUMP += size; var ret = GLOBAL_BASE + STATIC_BUMP; - assert(!(ret & (STACK_ALIGN - 1))); + assert(!(ret & (STACK_ALIGN - 1)), 'bad alignment'); return ret; } From 2e75cd85a3cd8172af930fa9645776d67484df98 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 15:25:01 -0800 Subject: [PATCH 05/30] more [ci skip] --- emcc.py | 8 ++++++++ src/jsifier.js | 1 - src/parseTools.js | 15 +++++++++++++-- src/preamble.js | 24 +++++++----------------- src/worker.js | 1 - 5 files changed, 28 insertions(+), 21 deletions(-) diff --git a/emcc.py b/emcc.py index e0e5d2586dc6..f79d05989e43 100755 --- a/emcc.py +++ b/emcc.py @@ -3057,6 +3057,14 @@ def validate_arg_level(level_string, max_level, err_msg, clamp=False): return level +''' +TODO: search and remove as needed all of + staticSealed + staticAlloc + ALLOC_STATIC +''' + + if __name__ == '__main__': try: sys.exit(run()) diff --git a/src/jsifier.js b/src/jsifier.js index 01d38b6b20d0..e927da5b85e1 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -554,7 +554,6 @@ function JSify(data, functionsOnly) { print('STACK_MAX = STACK_BASE;'); } print('HEAP32[DYNAMICTOP_PTR>>2] = DYNAMIC_BASE;\n'); - print('staticSealed = true; // seal the static portion of memory\n'); if (ASSERTIONS) print('assert(DYNAMIC_BASE < TOTAL_MEMORY, "TOTAL_MEMORY not big enough for stack");\n'); if (USE_PTHREADS) print('}\n'); } diff --git a/src/parseTools.js b/src/parseTools.js index f009ca2b5911..fbb57d53eeba 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1440,11 +1440,22 @@ function makeEval(code) { return ret; } +var STATIC_ALIGN = 8; // matches GLOBAL_BASE, which is 8. TODO do we need SIMD? + function makeStaticAlloc(size) { - size = (size + (STACK_ALIGN-1)) & -STACK_ALIGN; + size = (size + (STATIC_ALIGN-1)) & -STATIC_ALIGN; + + + +assert(!(STATIC_BUMP & (STATIC_ALIGN - 1)), 'bad alignment-1: ' + STATIC_BUMP); +assert(!(GLOBAL_BASE & (STATIC_ALIGN - 1)), 'bad alignment-2: ' + GLOBAL_BASE); +assert(!(size & (STATIC_ALIGN - 1)), 'bad alignment-3: ' + size); + + + STATIC_BUMP += size; var ret = GLOBAL_BASE + STATIC_BUMP; - assert(!(ret & (STACK_ALIGN - 1)), 'bad alignment'); + assert(!(ret & (STATIC_ALIGN - 1)), 'bad alignment'); return ret; } diff --git a/src/preamble.js b/src/preamble.js index 687b2d14abd5..32540897b6a4 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -34,13 +34,9 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { #endif if (dest <= 0) abort('segmentation fault storing ' + bytes + ' bytes to address ' + dest); if (dest % bytes !== 0) abort('alignment error storing to address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (staticSealed) { - if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); - assert(DYNAMICTOP_PTR); - assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); - } else { - if (dest + bytes > STATICTOP) abort('segmentation fault, exceeded the top of the available static heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP); - } + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); + assert(DYNAMICTOP_PTR); + assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); setValue(dest, value, getSafeHeapType(bytes, isFloat), 1); } function SAFE_HEAP_STORE_D(dest, value, bytes) { @@ -50,13 +46,9 @@ function SAFE_HEAP_STORE_D(dest, value, bytes) { function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { if (dest <= 0) abort('segmentation fault loading ' + bytes + ' bytes from address ' + dest); if (dest % bytes !== 0) abort('alignment error loading from address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (staticSealed) { - if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); - assert(DYNAMICTOP_PTR); - assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); - } else { - if (dest + bytes > STATICTOP) abort('segmentation fault, exceeded the top of the available static heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP); - } + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); + assert(DYNAMICTOP_PTR); + assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); var type = getSafeHeapType(bytes, isFloat); var ret = getValue(dest, type, 1); if (unsigned) ret = unSign(ret, parseInt(type.substr(1)), 1); @@ -924,7 +916,7 @@ function updateGlobalBufferViews() { Module['HEAPF64'] = HEAPF64 = new Float64Array(buffer); } -var STATIC_BASE, STATICTOP, staticSealed; // static area +var STATIC_BASE, STATICTOP; // static area var STACK_BASE, STACKTOP, STACK_MAX; // stack area var DYNAMIC_BASE, DYNAMICTOP_PTR; // dynamic area handled by sbrk @@ -932,14 +924,12 @@ var DYNAMIC_BASE, DYNAMICTOP_PTR; // dynamic area handled by sbrk if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variables in src/worker.js, where they were passed to the thread worker at startup time #endif STATIC_BASE = STATICTOP = STACK_BASE = STACKTOP = STACK_MAX = DYNAMIC_BASE = DYNAMICTOP_PTR = 0; - staticSealed = false; #if USE_PTHREADS } #endif #if USE_PTHREADS if (ENVIRONMENT_IS_PTHREAD) { - staticSealed = true; // The static memory area has been initialized already in the main thread, pthreads skip this. #if SEPARATE_ASM != 0 importScripts('{{{ SEPARATE_ASM }}}'); // load the separated-out asm.js #endif diff --git a/src/worker.js b/src/worker.js index e4219d3b10d5..610a1ada04f0 100644 --- a/src/worker.js +++ b/src/worker.js @@ -23,7 +23,6 @@ var buffer; // All pthreads share the same Emscripten HEAP as SharedArrayBuffer var DYNAMICTOP_PTR = 0; var TOTAL_MEMORY = 0; var STATICTOP = 0; -var staticSealed = true; // When threads are being initialized, the static memory area has been already sealed a long time ago. var DYNAMIC_BASE = 0; var ENVIRONMENT_IS_PTHREAD = true; From 4a62b1f0296ae379a7f06997ccda56f56b9294fb Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 15:33:14 -0800 Subject: [PATCH 06/30] do dynamic top emitting earlier in the js, so dynamicAlloc works everywhere (fixes emterpreter, possibly others) [ci skip] --- src/jsifier.js | 1 - src/preamble.js | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jsifier.js b/src/jsifier.js index e927da5b85e1..2a7fc98f4ed6 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -543,7 +543,6 @@ function JSify(data, functionsOnly) { for(i in proxiedFunctionInvokers) print(proxiedFunctionInvokers[i]+'\n'); print('if (!ENVIRONMENT_IS_PTHREAD) {\n // Only main thread initializes these, pthreads copy them over at thread worker init time (in worker.js)'); } - print('DYNAMICTOP_PTR = ' + makeStaticAlloc(4) + ''); print('STACK_BASE = STACKTOP = alignMemory(STATICTOP);\n'); if (STACK_START > 0) print('if (STACKTOP < ' + STACK_START + ') STACK_BASE = STACKTOP = alignMemory(' + STACK_START + ');\n'); print('STACK_MAX = STACK_BASE + TOTAL_STACK;\n'); diff --git a/src/preamble.js b/src/preamble.js index 32540897b6a4..596b028dad5d 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -918,12 +918,13 @@ function updateGlobalBufferViews() { var STATIC_BASE, STATICTOP; // static area var STACK_BASE, STACKTOP, STACK_MAX; // stack area -var DYNAMIC_BASE, DYNAMICTOP_PTR; // dynamic area handled by sbrk +var DYNAMIC_BASE, // dynamic area handled by sbrk + DYNAMICTOP_PTR = {{{ makeStaticAlloc(4) }}}; #if USE_PTHREADS if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variables in src/worker.js, where they were passed to the thread worker at startup time #endif - STATIC_BASE = STATICTOP = STACK_BASE = STACKTOP = STACK_MAX = DYNAMIC_BASE = DYNAMICTOP_PTR = 0; + STATIC_BASE = STATICTOP = STACK_BASE = STACKTOP = STACK_MAX = DYNAMIC_BASE; #if USE_PTHREADS } #endif From 35282e27f59a6c4b91ad4ded2088c60d92896642 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 16:11:15 -0800 Subject: [PATCH 07/30] new model [ci skip] --- emscripten.py | 37 +++++++++++++++++++++++++++++-------- src/jsifier.js | 11 ----------- src/parseTools.js | 16 ++++++++++++++++ src/preamble.js | 28 +++++++++++++++++++++------- src/settings.js | 3 --- 5 files changed, 66 insertions(+), 29 deletions(-) diff --git a/emscripten.py b/emscripten.py index 09f6f463b411..e3495ab0bb48 100644 --- a/emscripten.py +++ b/emscripten.py @@ -527,12 +527,13 @@ def is_int(x): return False +def align_memory(addr): + return (addr + 15) & -16 + + def align_static_bump(metadata): - staticbump = metadata['staticBump'] - while staticbump % 16 != 0: - staticbump += 1 - metadata['staticBump'] = staticbump - return staticbump + metadata['staticBump'] = align_memory(metadata['staticBump']) + return metadata['staticBump'] def update_settings_glue(metadata): @@ -609,6 +610,26 @@ def compile_settings(compiler_engine, libraries, temp_files): return glue, forwarded_data +def apply_memory(pre, metadata): + # Apply the statically-at-compile-time computed memory locations. + + # Memory layout: + # * first the static globals + global_start = shared.Settings.GLOBAL_BASE + # * then the stack + stack_start = align_memory(global_start + metadata['staticBump']) + # * then dynamic memory begins + dynamic_start = align_memory(stack_start + shared.Settings.TOTAL_STACK) + + # Write it all out + pre = pre.replace('{{{ STATIC_BUMP }}}', str(metadata['staticBump'])) + pre = pre.replace('{{{ STACK_BASE }}}', str(stack_start)) + pre = pre.replace('{{{ STACK_MAX }}}', str(dynamic_start)) + pre = pre.replace('{{{ DYNAMIC_BASE }}}', str(dynamic_start)) + + return pre + + def memory_and_global_initializers(pre, metadata, mem_init): global_initializers = ', '.join('{ func: function() { %s() } }' % i for i in metadata['initializers']) @@ -630,8 +651,8 @@ def memory_and_global_initializers(pre, metadata, mem_init): if shared.Settings.SIDE_MODULE: pre = pre.replace('GLOBAL_BASE', 'gb') - if shared.Settings.SIDE_MODULE or shared.Settings.WASM: - pre = pre.replace('{{{ STATIC_BUMP }}}', str(staticbump)) + + pre = apply_memory(pre, metadata) return pre @@ -1894,7 +1915,7 @@ def emscript_wasm_backend(infile, outfile, memfile, libraries, compiler_engine, 'if (!ENVIRONMENT_IS_PTHREAD)' if shared.Settings.USE_PTHREADS else '', global_initializers)) - pre = pre.replace('{{{ STATIC_BUMP }}}', str(staticbump)) + pre = apply_memory(pre, metadata) # merge forwarded data shared.Settings.EXPORTED_FUNCTIONS = forwarded_json['EXPORTED_FUNCTIONS'] diff --git a/src/jsifier.js b/src/jsifier.js index 2a7fc98f4ed6..1c397b1c3c1d 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -543,17 +543,6 @@ function JSify(data, functionsOnly) { for(i in proxiedFunctionInvokers) print(proxiedFunctionInvokers[i]+'\n'); print('if (!ENVIRONMENT_IS_PTHREAD) {\n // Only main thread initializes these, pthreads copy them over at thread worker init time (in worker.js)'); } - print('STACK_BASE = STACKTOP = alignMemory(STATICTOP);\n'); - if (STACK_START > 0) print('if (STACKTOP < ' + STACK_START + ') STACK_BASE = STACKTOP = alignMemory(' + STACK_START + ');\n'); - print('STACK_MAX = STACK_BASE + TOTAL_STACK;\n'); - print('DYNAMIC_BASE = alignMemory(STACK_MAX);\n'); - if (WASM_BACKEND) { - // wasm backend stack goes down - print('STACKTOP = STACK_BASE + TOTAL_STACK;'); - print('STACK_MAX = STACK_BASE;'); - } - print('HEAP32[DYNAMICTOP_PTR>>2] = DYNAMIC_BASE;\n'); - if (ASSERTIONS) print('assert(DYNAMIC_BASE < TOTAL_MEMORY, "TOTAL_MEMORY not big enough for stack");\n'); if (USE_PTHREADS) print('}\n'); } diff --git a/src/parseTools.js b/src/parseTools.js index fbb57d53eeba..018367e1d110 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1464,6 +1464,22 @@ function makeStaticString(string) { return 'stringToUTF8("' + string + '", ' + makeStaticAlloc(len) + ', ' + len + ')'; } +// We emit the dynamic and stack bases as strings that need to be further +// preprocessed, since during JS compiler time here we are still computing +// static allocations as we go. + +function getStackBase() { + return '{{{ STACK_BASE }}}'; +} + +function getStackMax() { + return '{{{ STACK_MAX }}}'; +} + +function getDynamicBase() { + return '{{{ DYNAMIC_BASE }}}'; +} + function makeRetainedCompilerSettings() { var blacklist = set('STRUCT_INFO'); var ret = {}; diff --git a/src/preamble.js b/src/preamble.js index 596b028dad5d..7e282142cf46 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -916,15 +916,17 @@ function updateGlobalBufferViews() { Module['HEAPF64'] = HEAPF64 = new Float64Array(buffer); } -var STATIC_BASE, STATICTOP; // static area -var STACK_BASE, STACKTOP, STACK_MAX; // stack area -var DYNAMIC_BASE, // dynamic area handled by sbrk - DYNAMICTOP_PTR = {{{ makeStaticAlloc(4) }}}; - #if USE_PTHREADS if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variables in src/worker.js, where they were passed to the thread worker at startup time #endif - STATIC_BASE = STATICTOP = STACK_BASE = STACKTOP = STACK_MAX = DYNAMIC_BASE; + +var STATIC_BASE = {{{ GLOBAL_BASE }}}, + STACK_BASE = {{{ getStackBase() }}}, + STACKTOP = STACK_BASE; + STACK_MAX = {{{ getStackMax() }}}, + DYNAMIC_BASE = {{{ getDynamicBase() }}}, + DYNAMICTOP_PTR = {{{ makeStaticAlloc(4) }}}; + #if USE_PTHREADS } #endif @@ -1109,7 +1111,11 @@ try { } #endif -var TOTAL_STACK = Module['TOTAL_STACK'] || {{{ TOTAL_STACK }}}; +var TOTAL_STACK = {{{ TOTAL_STACK }}}; +#if ASSERTIONS +if (Module['TOTAL_STACK']) assert(TOTAL_STACK === Module['TOTAL_STACK'], 'the stack size can no longer be determined at runtime') +#endif + var TOTAL_MEMORY = Module['TOTAL_MEMORY'] || {{{ TOTAL_MEMORY }}}; if (TOTAL_MEMORY < TOTAL_STACK) err('TOTAL_MEMORY should be larger than TOTAL_STACK, was ' + TOTAL_MEMORY + '! (TOTAL_STACK=' + TOTAL_STACK + ')'); @@ -1250,6 +1256,14 @@ updateGlobalBufferViews(); #endif // USE_PTHREADS +#if USE_PTHREADS +if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variables in src/worker.js, where they were passed to the thread worker at startup time +#endif +HEAP32[DYNAMICTOP_PTR>>2] = DYNAMIC_BASE; +#if USE_PTHREADS +} +#endif + function getTotalMemory() { return TOTAL_MEMORY; } diff --git a/src/settings.js b/src/settings.js index a26fdef9361b..bbdcc9ed2129 100644 --- a/src/settings.js +++ b/src/settings.js @@ -141,9 +141,6 @@ var GLOBAL_BASE = -1; // allocations, by forcing the stack to start in the same place their // memory usage patterns would be the same. -// Code embetterments -var STACK_START = -1; - // How to load and store 64-bit doubles. A potential risk is that doubles may // be only 32-bit aligned. Forcing 64-bit alignment in Clang itself should be // able to solve that, or as a workaround in DOUBLE_MODE 1 we will carefully From 30f2ff0064b45986c3fa47eac3ffe38eeda72544 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 16:26:32 -0800 Subject: [PATCH 08/30] fix [ci skip] --- emscripten.py | 3 +++ src/parseTools.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/emscripten.py b/emscripten.py index e3495ab0bb48..731a08716c57 100644 --- a/emscripten.py +++ b/emscripten.py @@ -621,6 +621,9 @@ def apply_memory(pre, metadata): # * then dynamic memory begins dynamic_start = align_memory(stack_start + shared.Settings.TOTAL_STACK) + # FIXME + logging.error('global_start: %d stack_start: %d, dynamic_start: %d', global_start, stack_start, dynamic_start) + # Write it all out pre = pre.replace('{{{ STATIC_BUMP }}}', str(metadata['staticBump'])) pre = pre.replace('{{{ STACK_BASE }}}', str(stack_start)) diff --git a/src/parseTools.js b/src/parseTools.js index 018367e1d110..d5900e420206 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1453,8 +1453,8 @@ assert(!(size & (STATIC_ALIGN - 1)), 'bad alignment-3: ' + size); - STATIC_BUMP += size; var ret = GLOBAL_BASE + STATIC_BUMP; + STATIC_BUMP += size; assert(!(ret & (STATIC_ALIGN - 1)), 'bad alignment'); return ret; } From d6b4c00d487c571dd4d206ccf0639568ec15810f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 16:41:14 -0800 Subject: [PATCH 09/30] test fix --- tests/core/legacy_exported_runtime_numbers.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/legacy_exported_runtime_numbers.txt b/tests/core/legacy_exported_runtime_numbers.txt index a12955e26f70..fd524e7bf08c 100644 --- a/tests/core/legacy_exported_runtime_numbers.txt +++ b/tests/core/legacy_exported_runtime_numbers.txt @@ -1 +1 @@ -|3| +|2| From 387034fbea696d632303b692896ee03f1f26cb11 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:06:04 -0800 Subject: [PATCH 10/30] forward static bump from js to emscripten.py [ci skip] --- emscripten.py | 21 ++++++++++++++++----- src/modules.js | 3 ++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/emscripten.py b/emscripten.py index 731a08716c57..1d7c3616e30e 100644 --- a/emscripten.py +++ b/emscripten.py @@ -593,6 +593,12 @@ def read_proxied_function_signatures(asmConsts): shared.Settings.STATIC_BUMP = align_static_bump(metadata) +def apply_forwarded_data(forwarded_data): + forwarded_json = json.loads(forwarded_data) + # Be aware of JS static allocations + shared.Settings.STATIC_BUMP = forwarded_json['STATIC_BUMP'] + + def compile_settings(compiler_engine, libraries, temp_files): # Save settings to a file to work around v8 issue 1579 with temp_files.get_file('.txt') as settings_file: @@ -607,6 +613,10 @@ def compile_settings(compiler_engine, libraries, temp_files): cwd=path_from_root('src'), env=env) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' glue, forwarded_data = out.split('//FORWARDED_DATA:') + + apply_forwarded_data(forwarded_data) + + logging.error(forwarded_data) # FIXME return glue, forwarded_data @@ -616,16 +626,17 @@ def apply_memory(pre, metadata): # Memory layout: # * first the static globals global_start = shared.Settings.GLOBAL_BASE + static_bump = shared.Settings.STATIC_BUMP # * then the stack - stack_start = align_memory(global_start + metadata['staticBump']) + stack_start = align_memory(global_start + static_bump) # * then dynamic memory begins dynamic_start = align_memory(stack_start + shared.Settings.TOTAL_STACK) # FIXME - logging.error('global_start: %d stack_start: %d, dynamic_start: %d', global_start, stack_start, dynamic_start) + logging.error('global_start: %d stack_start: %d, dynamic_start: %d, static bump: %d', global_start, stack_start, dynamic_start, static_bump) # Write it all out - pre = pre.replace('{{{ STATIC_BUMP }}}', str(metadata['staticBump'])) + pre = pre.replace('{{{ STATIC_BUMP }}}', str(static_bump)) pre = pre.replace('{{{ STACK_BASE }}}', str(stack_start)) pre = pre.replace('{{{ STACK_MAX }}}', str(dynamic_start)) pre = pre.replace('{{{ DYNAMIC_BASE }}}', str(dynamic_start)) @@ -639,7 +650,7 @@ def memory_and_global_initializers(pre, metadata, mem_init): if shared.Settings.SIMD == 1: pre = open(path_from_root(os.path.join('src', 'ecmascript_simd.js'))).read() + '\n\n' + pre - staticbump = metadata['staticBump'] + staticbump = shared.Settings.STATIC_BUMP pthread = '' if shared.Settings.USE_PTHREADS: @@ -1910,7 +1921,7 @@ def emscript_wasm_backend(infile, outfile, memfile, libraries, compiler_engine, global_initializers = ', '.join('{ func: function() { %s() } }' % i for i in metadata['initializers']) - staticbump = metadata['staticBump'] + staticbump = shared.Settings.STATIC_BUMP pre = pre.replace('STATICTOP = STATIC_BASE + 0;', '''STATICTOP = STATIC_BASE + %d; /* global initializers */ %s __ATINIT__.push(%s); diff --git a/src/modules.js b/src/modules.js index 6b99b7880a82..211403efe4cf 100644 --- a/src/modules.js +++ b/src/modules.js @@ -459,7 +459,8 @@ var PassManager = { serialize: function() { print('\n//FORWARDED_DATA:' + JSON.stringify({ Functions: Functions, - EXPORTED_FUNCTIONS: EXPORTED_FUNCTIONS + EXPORTED_FUNCTIONS: EXPORTED_FUNCTIONS, + STATIC_BUMP: STATIC_BUMP // updated with info from JS })); }, load: function(json) { From b07a65494ce29a661f834a734c615239d345e5d0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:08:49 -0800 Subject: [PATCH 11/30] fix static strings [ci skip] --- src/parseTools.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/parseTools.js b/src/parseTools.js index d5900e420206..5aebd64a435c 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1461,7 +1461,8 @@ assert(!(size & (STATIC_ALIGN - 1)), 'bad alignment-3: ' + size); function makeStaticString(string) { var len = string.length + 1; - return 'stringToUTF8("' + string + '", ' + makeStaticAlloc(len) + ', ' + len + ')'; + var ptr = makeStaticAlloc(len); + return '(stringToUTF8("' + string + '", ' + ptr + ', ' + len + '), ' + ptr + ')'; } // We emit the dynamic and stack bases as strings that need to be further From 1c57b5d55b1a91ec8f28538c3a7809b7c81266d1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:12:27 -0800 Subject: [PATCH 12/30] fixes [ci skip] --- src/modules.js | 1 - tests/test_core.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/modules.js b/src/modules.js index 211403efe4cf..39be02213f7b 100644 --- a/src/modules.js +++ b/src/modules.js @@ -435,7 +435,6 @@ function exportRuntime() { var runtimeNumbers = [ 'ALLOC_NORMAL', 'ALLOC_STACK', - 'ALLOC_STATIC', 'ALLOC_DYNAMIC', 'ALLOC_NONE', ]; diff --git a/tests/test_core.py b/tests/test_core.py index bcfccd4e518a..207f3925cc5f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -3790,7 +3790,7 @@ def test_dylink_jslib(self): def test_dylink_global_var_jslib(self): create_test_file('lib.js', r''' mergeInto(LibraryManager.library, { - jslib_x: 'allocate(1, "i32*", ALLOC_STATIC)', + jslib_x: '{{{ makeStaticAlloc(1) }}}', jslib_x__postset: 'HEAP32[_jslib_x>>2] = 148;', }); ''') From fdda39a0bcbe61f58ec4cbfc070cc64fc69a6024 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:31:08 -0800 Subject: [PATCH 13/30] fixes [ci skip] --- src/deterministic.js | 2 +- src/jsifier.js | 2 -- src/library_pthread.js | 1 - src/library_trace.js | 1 - src/memoryprofiler.js | 6 +----- src/preamble.js | 6 +++--- src/worker.js | 2 -- 7 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/deterministic.js b/src/deterministic.js index d12d9bfce184..07dd7c7dc4ea 100644 --- a/src/deterministic.js +++ b/src/deterministic.js @@ -19,7 +19,7 @@ Module['thisProgram'] = 'thisProgram'; // for consistency between different buil function hashMemory(id) { var ret = 0; - var len = Math.max(HEAP32[DYNAMICTOP_PTR>>2], STATICTOP); + var len = HEAP32[DYNAMICTOP_PTR>>2]; for (var i = 0; i < len; i++) { ret = (ret*17 + HEAPU8[i])|0; } diff --git a/src/jsifier.js b/src/jsifier.js index 1c397b1c3c1d..da69e16114f2 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -432,7 +432,6 @@ function JSify(data, functionsOnly) { // Globals are done, here is the rest of static memory if (!SIDE_MODULE) { print('STATIC_BASE = GLOBAL_BASE;\n'); - print('STATICTOP = STATIC_BASE + ' + Runtime.alignMemory(Variables.nextIndexedOffset) + ';\n'); } else { print('gb = alignMemory(getMemory({{{ STATIC_BUMP }}} + ' + MAX_GLOBAL_ALIGN + '), ' + MAX_GLOBAL_ALIGN + ' || 1);\n'); // The static area consists of explicitly initialized data, followed by zero-initialized data. @@ -441,7 +440,6 @@ function JSify(data, functionsOnly) { // here, we just zero the whole thing, which is suboptimal, but should at least resolve bugs // from uninitialized memory. print('for (var i = gb; i < gb + {{{ STATIC_BUMP }}}; ++i) HEAP8[i] = 0;\n'); - print('// STATICTOP = STATIC_BASE + ' + Runtime.alignMemory(Variables.nextIndexedOffset) + ';\n'); // comment as metadata only } if (WASM) { // export static base and bump, needed for linking in wasm binary's memory, dynamic linking, etc. diff --git a/src/library_pthread.js b/src/library_pthread.js index 660f2d42e519..f3dfc9802e82 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -400,7 +400,6 @@ var LibraryPThread = { #endif tempDoublePtr: tempDoublePtr, TOTAL_MEMORY: TOTAL_MEMORY, - STATICTOP: STATICTOP, DYNAMIC_BASE: DYNAMIC_BASE, DYNAMICTOP_PTR: DYNAMICTOP_PTR, PthreadWorkerInit: PthreadWorkerInit diff --git a/src/library_trace.js b/src/library_trace.js index 49dead959c3f..0e11375fb008 100644 --- a/src/library_trace.js +++ b/src/library_trace.js @@ -262,7 +262,6 @@ var LibraryTracing = { if (EmscriptenTrace.postEnabled) { var memory_layout = { 'static_base': STATIC_BASE, - 'static_top': STATICTOP, 'stack_base': STACK_BASE, 'stack_top': STACKTOP, 'stack_max': STACK_MAX, diff --git a/src/memoryprofiler.js b/src/memoryprofiler.js index ef7d8e3815c9..1f27df64e29a 100644 --- a/src/memoryprofiler.js +++ b/src/memoryprofiler.js @@ -308,9 +308,8 @@ var emscriptenMemoryProfiler = { var width = (nBits(TOTAL_MEMORY) + 3) / 4; // Pointer 'word width' var html = 'Total HEAP size: ' + this.formatBytes(TOTAL_MEMORY) + '.'; - html += '
' + colorBar('#202020') + 'STATIC memory area size: ' + this.formatBytes(STATICTOP - STATIC_BASE); + html += '
' + colorBar('#202020') + 'STATIC memory area size: ' + this.formatBytes(STACK_BASE - STATIC_BASE); html += '. STATIC_BASE: ' + toHex(STATIC_BASE, width); - html += '. STATICTOP: ' + toHex(STATICTOP, width) + '.'; html += '
' + colorBar('#FF8080') + 'STACK memory area size: ' + this.formatBytes(STACK_MAX - STACK_BASE); html += '. STACK_BASE: ' + toHex(STACK_BASE, width); @@ -336,9 +335,6 @@ var emscriptenMemoryProfiler = { this.drawContext.fillStyle = "#FFFFFF"; this.drawContext.fillRect(0, 0, this.canvas.width, this.canvas.height); - this.drawContext.fillStyle = "#202020"; - this.fillLine(STATIC_BASE, STATICTOP); - this.drawContext.fillStyle = "#FF8080"; this.fillLine(STACK_BASE, STACK_MAX); diff --git a/src/preamble.js b/src/preamble.js index 7e282142cf46..44bf0f626312 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -34,7 +34,7 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { #endif if (dest <= 0) abort('segmentation fault storing ' + bytes + ' bytes to address ' + dest); if (dest % bytes !== 0) abort('alignment error storing to address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); assert(DYNAMICTOP_PTR); assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); setValue(dest, value, getSafeHeapType(bytes, isFloat), 1); @@ -46,7 +46,7 @@ function SAFE_HEAP_STORE_D(dest, value, bytes) { function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { if (dest <= 0) abort('segmentation fault loading ' + bytes + ' bytes from address ' + dest); if (dest % bytes !== 0) abort('alignment error loading from address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); assert(DYNAMICTOP_PTR); assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); var type = getSafeHeapType(bytes, isFloat); @@ -922,7 +922,7 @@ if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variab var STATIC_BASE = {{{ GLOBAL_BASE }}}, STACK_BASE = {{{ getStackBase() }}}, - STACKTOP = STACK_BASE; + STACKTOP = STACK_BASE, STACK_MAX = {{{ getStackMax() }}}, DYNAMIC_BASE = {{{ getDynamicBase() }}}, DYNAMICTOP_PTR = {{{ makeStaticAlloc(4) }}}; diff --git a/src/worker.js b/src/worker.js index 610a1ada04f0..91a344f07ec4 100644 --- a/src/worker.js +++ b/src/worker.js @@ -22,7 +22,6 @@ var STACK_MAX = 0; var buffer; // All pthreads share the same Emscripten HEAP as SharedArrayBuffer with the main execution thread. var DYNAMICTOP_PTR = 0; var TOTAL_MEMORY = 0; -var STATICTOP = 0; var DYNAMIC_BASE = 0; var ENVIRONMENT_IS_PTHREAD = true; @@ -84,7 +83,6 @@ this.onmessage = function(e) { // Initialize the global "process"-wide fields: Module['TOTAL_MEMORY'] = TOTAL_MEMORY = e.data.TOTAL_MEMORY; - STATICTOP = e.data.STATICTOP; DYNAMIC_BASE = e.data.DYNAMIC_BASE; DYNAMICTOP_PTR = e.data.DYNAMICTOP_PTR; From de1ad4241477e026f8ba4a14a0abe2b74fda7dc0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:38:41 -0800 Subject: [PATCH 14/30] Revert "fixes [ci skip]" This reverts commit fdda39a0bcbe61f58ec4cbfc070cc64fc69a6024. --- src/deterministic.js | 2 +- src/jsifier.js | 2 ++ src/library_pthread.js | 1 + src/library_trace.js | 1 + src/memoryprofiler.js | 6 +++++- src/preamble.js | 6 +++--- src/worker.js | 2 ++ 7 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/deterministic.js b/src/deterministic.js index 07dd7c7dc4ea..d12d9bfce184 100644 --- a/src/deterministic.js +++ b/src/deterministic.js @@ -19,7 +19,7 @@ Module['thisProgram'] = 'thisProgram'; // for consistency between different buil function hashMemory(id) { var ret = 0; - var len = HEAP32[DYNAMICTOP_PTR>>2]; + var len = Math.max(HEAP32[DYNAMICTOP_PTR>>2], STATICTOP); for (var i = 0; i < len; i++) { ret = (ret*17 + HEAPU8[i])|0; } diff --git a/src/jsifier.js b/src/jsifier.js index da69e16114f2..1c397b1c3c1d 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -432,6 +432,7 @@ function JSify(data, functionsOnly) { // Globals are done, here is the rest of static memory if (!SIDE_MODULE) { print('STATIC_BASE = GLOBAL_BASE;\n'); + print('STATICTOP = STATIC_BASE + ' + Runtime.alignMemory(Variables.nextIndexedOffset) + ';\n'); } else { print('gb = alignMemory(getMemory({{{ STATIC_BUMP }}} + ' + MAX_GLOBAL_ALIGN + '), ' + MAX_GLOBAL_ALIGN + ' || 1);\n'); // The static area consists of explicitly initialized data, followed by zero-initialized data. @@ -440,6 +441,7 @@ function JSify(data, functionsOnly) { // here, we just zero the whole thing, which is suboptimal, but should at least resolve bugs // from uninitialized memory. print('for (var i = gb; i < gb + {{{ STATIC_BUMP }}}; ++i) HEAP8[i] = 0;\n'); + print('// STATICTOP = STATIC_BASE + ' + Runtime.alignMemory(Variables.nextIndexedOffset) + ';\n'); // comment as metadata only } if (WASM) { // export static base and bump, needed for linking in wasm binary's memory, dynamic linking, etc. diff --git a/src/library_pthread.js b/src/library_pthread.js index f3dfc9802e82..660f2d42e519 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -400,6 +400,7 @@ var LibraryPThread = { #endif tempDoublePtr: tempDoublePtr, TOTAL_MEMORY: TOTAL_MEMORY, + STATICTOP: STATICTOP, DYNAMIC_BASE: DYNAMIC_BASE, DYNAMICTOP_PTR: DYNAMICTOP_PTR, PthreadWorkerInit: PthreadWorkerInit diff --git a/src/library_trace.js b/src/library_trace.js index 0e11375fb008..49dead959c3f 100644 --- a/src/library_trace.js +++ b/src/library_trace.js @@ -262,6 +262,7 @@ var LibraryTracing = { if (EmscriptenTrace.postEnabled) { var memory_layout = { 'static_base': STATIC_BASE, + 'static_top': STATICTOP, 'stack_base': STACK_BASE, 'stack_top': STACKTOP, 'stack_max': STACK_MAX, diff --git a/src/memoryprofiler.js b/src/memoryprofiler.js index 1f27df64e29a..ef7d8e3815c9 100644 --- a/src/memoryprofiler.js +++ b/src/memoryprofiler.js @@ -308,8 +308,9 @@ var emscriptenMemoryProfiler = { var width = (nBits(TOTAL_MEMORY) + 3) / 4; // Pointer 'word width' var html = 'Total HEAP size: ' + this.formatBytes(TOTAL_MEMORY) + '.'; - html += '
' + colorBar('#202020') + 'STATIC memory area size: ' + this.formatBytes(STACK_BASE - STATIC_BASE); + html += '
' + colorBar('#202020') + 'STATIC memory area size: ' + this.formatBytes(STATICTOP - STATIC_BASE); html += '. STATIC_BASE: ' + toHex(STATIC_BASE, width); + html += '. STATICTOP: ' + toHex(STATICTOP, width) + '.'; html += '
' + colorBar('#FF8080') + 'STACK memory area size: ' + this.formatBytes(STACK_MAX - STACK_BASE); html += '. STACK_BASE: ' + toHex(STACK_BASE, width); @@ -335,6 +336,9 @@ var emscriptenMemoryProfiler = { this.drawContext.fillStyle = "#FFFFFF"; this.drawContext.fillRect(0, 0, this.canvas.width, this.canvas.height); + this.drawContext.fillStyle = "#202020"; + this.fillLine(STATIC_BASE, STATICTOP); + this.drawContext.fillStyle = "#FF8080"; this.fillLine(STACK_BASE, STACK_MAX); diff --git a/src/preamble.js b/src/preamble.js index 44bf0f626312..7e282142cf46 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -34,7 +34,7 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { #endif if (dest <= 0) abort('segmentation fault storing ' + bytes + ' bytes to address ' + dest); if (dest % bytes !== 0) abort('alignment error storing to address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); assert(DYNAMICTOP_PTR); assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); setValue(dest, value, getSafeHeapType(bytes, isFloat), 1); @@ -46,7 +46,7 @@ function SAFE_HEAP_STORE_D(dest, value, bytes) { function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { if (dest <= 0) abort('segmentation fault loading ' + bytes + ' bytes from address ' + dest); if (dest % bytes !== 0) abort('alignment error loading from address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); assert(DYNAMICTOP_PTR); assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); var type = getSafeHeapType(bytes, isFloat); @@ -922,7 +922,7 @@ if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variab var STATIC_BASE = {{{ GLOBAL_BASE }}}, STACK_BASE = {{{ getStackBase() }}}, - STACKTOP = STACK_BASE, + STACKTOP = STACK_BASE; STACK_MAX = {{{ getStackMax() }}}, DYNAMIC_BASE = {{{ getDynamicBase() }}}, DYNAMICTOP_PTR = {{{ makeStaticAlloc(4) }}}; diff --git a/src/worker.js b/src/worker.js index 91a344f07ec4..610a1ada04f0 100644 --- a/src/worker.js +++ b/src/worker.js @@ -22,6 +22,7 @@ var STACK_MAX = 0; var buffer; // All pthreads share the same Emscripten HEAP as SharedArrayBuffer with the main execution thread. var DYNAMICTOP_PTR = 0; var TOTAL_MEMORY = 0; +var STATICTOP = 0; var DYNAMIC_BASE = 0; var ENVIRONMENT_IS_PTHREAD = true; @@ -83,6 +84,7 @@ this.onmessage = function(e) { // Initialize the global "process"-wide fields: Module['TOTAL_MEMORY'] = TOTAL_MEMORY = e.data.TOTAL_MEMORY; + STATICTOP = e.data.STATICTOP; DYNAMIC_BASE = e.data.DYNAMIC_BASE; DYNAMICTOP_PTR = e.data.DYNAMICTOP_PTR; From 8f7ce9ca3e343739b12d3114bf996e0b11d4a74c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:39:38 -0800 Subject: [PATCH 15/30] Revert "Revert "fixes [ci skip]"" This reverts commit de1ad4241477e026f8ba4a14a0abe2b74fda7dc0. --- src/deterministic.js | 2 +- src/jsifier.js | 2 -- src/library_pthread.js | 1 - src/library_trace.js | 1 - src/memoryprofiler.js | 6 +----- src/preamble.js | 6 +++--- src/worker.js | 2 -- 7 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/deterministic.js b/src/deterministic.js index d12d9bfce184..07dd7c7dc4ea 100644 --- a/src/deterministic.js +++ b/src/deterministic.js @@ -19,7 +19,7 @@ Module['thisProgram'] = 'thisProgram'; // for consistency between different buil function hashMemory(id) { var ret = 0; - var len = Math.max(HEAP32[DYNAMICTOP_PTR>>2], STATICTOP); + var len = HEAP32[DYNAMICTOP_PTR>>2]; for (var i = 0; i < len; i++) { ret = (ret*17 + HEAPU8[i])|0; } diff --git a/src/jsifier.js b/src/jsifier.js index 1c397b1c3c1d..da69e16114f2 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -432,7 +432,6 @@ function JSify(data, functionsOnly) { // Globals are done, here is the rest of static memory if (!SIDE_MODULE) { print('STATIC_BASE = GLOBAL_BASE;\n'); - print('STATICTOP = STATIC_BASE + ' + Runtime.alignMemory(Variables.nextIndexedOffset) + ';\n'); } else { print('gb = alignMemory(getMemory({{{ STATIC_BUMP }}} + ' + MAX_GLOBAL_ALIGN + '), ' + MAX_GLOBAL_ALIGN + ' || 1);\n'); // The static area consists of explicitly initialized data, followed by zero-initialized data. @@ -441,7 +440,6 @@ function JSify(data, functionsOnly) { // here, we just zero the whole thing, which is suboptimal, but should at least resolve bugs // from uninitialized memory. print('for (var i = gb; i < gb + {{{ STATIC_BUMP }}}; ++i) HEAP8[i] = 0;\n'); - print('// STATICTOP = STATIC_BASE + ' + Runtime.alignMemory(Variables.nextIndexedOffset) + ';\n'); // comment as metadata only } if (WASM) { // export static base and bump, needed for linking in wasm binary's memory, dynamic linking, etc. diff --git a/src/library_pthread.js b/src/library_pthread.js index 660f2d42e519..f3dfc9802e82 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -400,7 +400,6 @@ var LibraryPThread = { #endif tempDoublePtr: tempDoublePtr, TOTAL_MEMORY: TOTAL_MEMORY, - STATICTOP: STATICTOP, DYNAMIC_BASE: DYNAMIC_BASE, DYNAMICTOP_PTR: DYNAMICTOP_PTR, PthreadWorkerInit: PthreadWorkerInit diff --git a/src/library_trace.js b/src/library_trace.js index 49dead959c3f..0e11375fb008 100644 --- a/src/library_trace.js +++ b/src/library_trace.js @@ -262,7 +262,6 @@ var LibraryTracing = { if (EmscriptenTrace.postEnabled) { var memory_layout = { 'static_base': STATIC_BASE, - 'static_top': STATICTOP, 'stack_base': STACK_BASE, 'stack_top': STACKTOP, 'stack_max': STACK_MAX, diff --git a/src/memoryprofiler.js b/src/memoryprofiler.js index ef7d8e3815c9..1f27df64e29a 100644 --- a/src/memoryprofiler.js +++ b/src/memoryprofiler.js @@ -308,9 +308,8 @@ var emscriptenMemoryProfiler = { var width = (nBits(TOTAL_MEMORY) + 3) / 4; // Pointer 'word width' var html = 'Total HEAP size: ' + this.formatBytes(TOTAL_MEMORY) + '.'; - html += '
' + colorBar('#202020') + 'STATIC memory area size: ' + this.formatBytes(STATICTOP - STATIC_BASE); + html += '
' + colorBar('#202020') + 'STATIC memory area size: ' + this.formatBytes(STACK_BASE - STATIC_BASE); html += '. STATIC_BASE: ' + toHex(STATIC_BASE, width); - html += '. STATICTOP: ' + toHex(STATICTOP, width) + '.'; html += '
' + colorBar('#FF8080') + 'STACK memory area size: ' + this.formatBytes(STACK_MAX - STACK_BASE); html += '. STACK_BASE: ' + toHex(STACK_BASE, width); @@ -336,9 +335,6 @@ var emscriptenMemoryProfiler = { this.drawContext.fillStyle = "#FFFFFF"; this.drawContext.fillRect(0, 0, this.canvas.width, this.canvas.height); - this.drawContext.fillStyle = "#202020"; - this.fillLine(STATIC_BASE, STATICTOP); - this.drawContext.fillStyle = "#FF8080"; this.fillLine(STACK_BASE, STACK_MAX); diff --git a/src/preamble.js b/src/preamble.js index 7e282142cf46..44bf0f626312 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -34,7 +34,7 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { #endif if (dest <= 0) abort('segmentation fault storing ' + bytes + ' bytes to address ' + dest); if (dest % bytes !== 0) abort('alignment error storing to address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when storing ' + bytes + ' bytes to address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); assert(DYNAMICTOP_PTR); assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); setValue(dest, value, getSafeHeapType(bytes, isFloat), 1); @@ -46,7 +46,7 @@ function SAFE_HEAP_STORE_D(dest, value, bytes) { function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { if (dest <= 0) abort('segmentation fault loading ' + bytes + ' bytes from address ' + dest); if (dest % bytes !== 0) abort('alignment error loading from address ' + dest + ', which was expected to be aligned to a multiple of ' + bytes); - if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. STATICTOP=' + STATICTOP + ', DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); + if (dest + bytes > HEAP32[DYNAMICTOP_PTR>>2]) abort('segmentation fault, exceeded the top of the available dynamic heap when loading ' + bytes + ' bytes from address ' + dest + '. DYNAMICTOP=' + HEAP32[DYNAMICTOP_PTR>>2]); assert(DYNAMICTOP_PTR); assert(HEAP32[DYNAMICTOP_PTR>>2] <= TOTAL_MEMORY); var type = getSafeHeapType(bytes, isFloat); @@ -922,7 +922,7 @@ if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variab var STATIC_BASE = {{{ GLOBAL_BASE }}}, STACK_BASE = {{{ getStackBase() }}}, - STACKTOP = STACK_BASE; + STACKTOP = STACK_BASE, STACK_MAX = {{{ getStackMax() }}}, DYNAMIC_BASE = {{{ getDynamicBase() }}}, DYNAMICTOP_PTR = {{{ makeStaticAlloc(4) }}}; diff --git a/src/worker.js b/src/worker.js index 610a1ada04f0..91a344f07ec4 100644 --- a/src/worker.js +++ b/src/worker.js @@ -22,7 +22,6 @@ var STACK_MAX = 0; var buffer; // All pthreads share the same Emscripten HEAP as SharedArrayBuffer with the main execution thread. var DYNAMICTOP_PTR = 0; var TOTAL_MEMORY = 0; -var STATICTOP = 0; var DYNAMIC_BASE = 0; var ENVIRONMENT_IS_PTHREAD = true; @@ -84,7 +83,6 @@ this.onmessage = function(e) { // Initialize the global "process"-wide fields: Module['TOTAL_MEMORY'] = TOTAL_MEMORY = e.data.TOTAL_MEMORY; - STATICTOP = e.data.STATICTOP; DYNAMIC_BASE = e.data.DYNAMIC_BASE; DYNAMICTOP_PTR = e.data.DYNAMICTOP_PTR; From ecedef6d9bca8d9a0baae8bf48f731c24680395e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:42:11 -0800 Subject: [PATCH 16/30] fix [ci skip] --- src/jsifier.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/jsifier.js b/src/jsifier.js index da69e16114f2..ec00d5c83ac3 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -441,6 +441,8 @@ function JSify(data, functionsOnly) { // from uninitialized memory. print('for (var i = gb; i < gb + {{{ STATIC_BUMP }}}; ++i) HEAP8[i] = 0;\n'); } + // emit "metadata" in a comment. FIXME make this nicer + print('// STATICTOP = STATIC_BASE + ' + Runtime.alignMemory(Variables.nextIndexedOffset) + ';\n'); if (WASM) { // export static base and bump, needed for linking in wasm binary's memory, dynamic linking, etc. print('var STATIC_BUMP = {{{ STATIC_BUMP }}};'); From 0338565d2ea8ab2118b4c88d1ed0490110c8bb31 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:48:22 -0800 Subject: [PATCH 17/30] test fix [ci skip] --- tests/test_core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index 207f3925cc5f..32ae930c614b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7811,7 +7811,6 @@ def test_memprof_requirements(self): check_memprof_requirements: function() { if (typeof TOTAL_MEMORY === 'number' && typeof STATIC_BASE === 'number' && - typeof STATICTOP === 'number' && typeof STACK_BASE === 'number' && typeof STACK_MAX === 'number' && typeof STACKTOP === 'number' && From 6d62a03e8f970370f08c0b08f297d517b097cce0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:50:52 -0800 Subject: [PATCH 18/30] remove fixmes --- emscripten.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/emscripten.py b/emscripten.py index 1d7c3616e30e..afc825658f69 100644 --- a/emscripten.py +++ b/emscripten.py @@ -616,7 +616,6 @@ def compile_settings(compiler_engine, libraries, temp_files): apply_forwarded_data(forwarded_data) - logging.error(forwarded_data) # FIXME return glue, forwarded_data @@ -632,9 +631,6 @@ def apply_memory(pre, metadata): # * then dynamic memory begins dynamic_start = align_memory(stack_start + shared.Settings.TOTAL_STACK) - # FIXME - logging.error('global_start: %d stack_start: %d, dynamic_start: %d, static bump: %d', global_start, stack_start, dynamic_start, static_bump) - # Write it all out pre = pre.replace('{{{ STATIC_BUMP }}}', str(static_bump)) pre = pre.replace('{{{ STACK_BASE }}}', str(stack_start)) From a470a71e5d3646f8592585457c42df8229cb3435 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 17:56:21 -0800 Subject: [PATCH 19/30] proper allocation --- src/parseTools.js | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/parseTools.js b/src/parseTools.js index 5aebd64a435c..79f1f8fcea43 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1440,22 +1440,10 @@ function makeEval(code) { return ret; } -var STATIC_ALIGN = 8; // matches GLOBAL_BASE, which is 8. TODO do we need SIMD? - function makeStaticAlloc(size) { - size = (size + (STATIC_ALIGN-1)) & -STATIC_ALIGN; - - - -assert(!(STATIC_BUMP & (STATIC_ALIGN - 1)), 'bad alignment-1: ' + STATIC_BUMP); -assert(!(GLOBAL_BASE & (STATIC_ALIGN - 1)), 'bad alignment-2: ' + GLOBAL_BASE); -assert(!(size & (STATIC_ALIGN - 1)), 'bad alignment-3: ' + size); - - - - var ret = GLOBAL_BASE + STATIC_BUMP; - STATIC_BUMP += size; - assert(!(ret & (STATIC_ALIGN - 1)), 'bad alignment'); + size = alignMemory(size); + var ret = alignMemory(GLOBAL_BASE + STATIC_BUMP); + STATIC_BUMP = ret + size - GLOBAL_BASE; return ret; } From 9d5c349ddc8bfbcd50b07ec75932a6de81ff0bf4 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 18:21:55 -0800 Subject: [PATCH 20/30] work [ci skip] --- emcc.py | 5 ++++- emscripten.py | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/emcc.py b/emcc.py index f79d05989e43..5c22b72a53b4 100755 --- a/emcc.py +++ b/emcc.py @@ -1292,7 +1292,10 @@ def check(input_file): shared.Settings.ASMJS_CODE_FILE = shared.JS.escape_for_js_string(os.path.basename(asm_target)) shared.Settings.ASM_JS = 2 # when targeting wasm, we use a wasm Memory, but that is not compatible with asm.js opts - shared.Settings.GLOBAL_BASE = 1024 # leave some room for mapping global vars + # for asm2wasm, a higher global base is useful for optimizing load/store offsets + # (and historically for mapping asm.js globals) + # TODO: for the wasm backend, we don't need this? + shared.Settings.GLOBAL_BASE = 1024 if shared.Settings.ELIMINATE_DUPLICATE_FUNCTIONS: logger.warning('for wasm there is no need to set ELIMINATE_DUPLICATE_FUNCTIONS, the binaryen optimizer does it automatically') shared.Settings.ELIMINATE_DUPLICATE_FUNCTIONS = 0 diff --git a/emscripten.py b/emscripten.py index afc825658f69..5b15485750f9 100644 --- a/emscripten.py +++ b/emscripten.py @@ -626,10 +626,15 @@ def apply_memory(pre, metadata): # * first the static globals global_start = shared.Settings.GLOBAL_BASE static_bump = shared.Settings.STATIC_BUMP - # * then the stack - stack_start = align_memory(global_start + static_bump) + # * then the stack (up on fastcomp, down on upstream) + stack_low = align_memory(global_start + static_bump) + stack_high = stack_low + shared.Settings.TOTAL_STACK + if shared.Settings.WASM_BACKEND: + stack_start = stack_high + else: + stack_start = stack_low # * then dynamic memory begins - dynamic_start = align_memory(stack_start + shared.Settings.TOTAL_STACK) + dynamic_start = align_memory(stack_high) # Write it all out pre = pre.replace('{{{ STATIC_BUMP }}}', str(static_bump)) @@ -637,6 +642,8 @@ def apply_memory(pre, metadata): pre = pre.replace('{{{ STACK_MAX }}}', str(dynamic_start)) pre = pre.replace('{{{ DYNAMIC_BASE }}}', str(dynamic_start)) + logging.error('global_start: %d stack_start: %d, dynamic_start: %d, static bump: %d', global_start, stack_start, dynamic_start, static_bump) + return pre From a07858fb6b5afc070a642f21ca9eb5b48867686c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Jan 2019 18:25:09 -0800 Subject: [PATCH 21/30] fix --- emscripten.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/emscripten.py b/emscripten.py index 5b15485750f9..7b6953427c26 100644 --- a/emscripten.py +++ b/emscripten.py @@ -631,18 +631,20 @@ def apply_memory(pre, metadata): stack_high = stack_low + shared.Settings.TOTAL_STACK if shared.Settings.WASM_BACKEND: stack_start = stack_high + stack_max = stack_low else: stack_start = stack_low + stack_max = stack_high # * then dynamic memory begins dynamic_start = align_memory(stack_high) # Write it all out pre = pre.replace('{{{ STATIC_BUMP }}}', str(static_bump)) pre = pre.replace('{{{ STACK_BASE }}}', str(stack_start)) - pre = pre.replace('{{{ STACK_MAX }}}', str(dynamic_start)) + pre = pre.replace('{{{ STACK_MAX }}}', str(stack_max)) pre = pre.replace('{{{ DYNAMIC_BASE }}}', str(dynamic_start)) - logging.error('global_start: %d stack_start: %d, dynamic_start: %d, static bump: %d', global_start, stack_start, dynamic_start, static_bump) + logging.error('global_start: %d stack_start: %d, stack_max: %d, dynamic_start: %d, static bump: %d', global_start, stack_start, stack_max, dynamic_start, static_bump) return pre From 214f93620264aa0ecc6332518be32193edbb50e0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 10 Jan 2019 10:25:00 -0800 Subject: [PATCH 22/30] fix zlib compiler error --- tests/zlib/gzwrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/zlib/gzwrite.c b/tests/zlib/gzwrite.c index e8defc6887a1..ed2074972ef5 100644 --- a/tests/zlib/gzwrite.c +++ b/tests/zlib/gzwrite.c @@ -321,7 +321,7 @@ int ZEXPORTVA gzprintf (gzFile file, const char *format, ...) for (len = 0; len < size; len++) if (state->in[len] == 0) break; # else - len = vsprintf(state->in, format, va); + len = vsprintf((char*)state->in, format, va); va_end(va); # endif #else From afbac79451d01110abb3b21c24a2f2ec13c6ea73 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 10 Jan 2019 10:29:57 -0800 Subject: [PATCH 23/30] fix logger --- emscripten.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emscripten.py b/emscripten.py index 7b6953427c26..3baf6978fccd 100644 --- a/emscripten.py +++ b/emscripten.py @@ -644,7 +644,7 @@ def apply_memory(pre, metadata): pre = pre.replace('{{{ STACK_MAX }}}', str(stack_max)) pre = pre.replace('{{{ DYNAMIC_BASE }}}', str(dynamic_start)) - logging.error('global_start: %d stack_start: %d, stack_max: %d, dynamic_start: %d, static bump: %d', global_start, stack_start, stack_max, dynamic_start, static_bump) + logger.debug('global_start: %d stack_start: %d, stack_max: %d, dynamic_start: %d, static bump: %d', global_start, stack_start, stack_max, dynamic_start, static_bump) return pre From 332849f78992075a38b5598ce338c4e1bc85c833 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 10 Jan 2019 10:37:59 -0800 Subject: [PATCH 24/30] fix pthreads --- src/library_pthread.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index f3dfc9802e82..8550e5dfa80e 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -528,7 +528,7 @@ var LibraryPThread = { }, _num_logical_cores__deps: ['emscripten_force_num_logical_cores'], - _num_logical_cores: '; if (ENVIRONMENT_IS_PTHREAD) __num_logical_cores = PthreadWorkerInit.__num_logical_cores; else { PthreadWorkerInit.__num_logical_cores = __num_logical_cores = makeStaticAlloc(1); HEAPU32[__num_logical_cores>>2] = navigator["hardwareConcurrency"] || ' + {{{ PTHREAD_HINT_NUM_CORES }}} + '; }', + _num_logical_cores: '; if (ENVIRONMENT_IS_PTHREAD) __num_logical_cores = PthreadWorkerInit.__num_logical_cores; else { PthreadWorkerInit.__num_logical_cores = __num_logical_cores = {{{ makeStaticAlloc(1) }}}; HEAPU32[__num_logical_cores>>2] = navigator["hardwareConcurrency"] || ' + {{{ PTHREAD_HINT_NUM_CORES }}} + '; }', emscripten_has_threading_support: function() { return typeof SharedArrayBuffer !== 'undefined'; @@ -1030,7 +1030,7 @@ var LibraryPThread = { }, // Stores the memory address that the main thread is waiting on, if any. - _main_thread_futex_wait_address: '; if (ENVIRONMENT_IS_PTHREAD) __main_thread_futex_wait_address = PthreadWorkerInit.__main_thread_futex_wait_address; else PthreadWorkerInit.__main_thread_futex_wait_address = __main_thread_futex_wait_address = makeStaticAlloc(1)', + _main_thread_futex_wait_address: '; if (ENVIRONMENT_IS_PTHREAD) __main_thread_futex_wait_address = PthreadWorkerInit.__main_thread_futex_wait_address; else PthreadWorkerInit.__main_thread_futex_wait_address = __main_thread_futex_wait_address = {{{ makeStaticAlloc(1) }}}', // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error. emscripten_futex_wait__deps: ['_main_thread_futex_wait_address', 'emscripten_main_thread_process_queued_calls'], From 6d8a2c1ebd95af4a5d547defc79a10feb06fe4d9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 10 Jan 2019 15:27:01 -0800 Subject: [PATCH 25/30] remove irrelevant test - without full stack overflow tests, anything can happen --- tests/test_core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 32ae930c614b..eba01d9c7c8f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7843,8 +7843,6 @@ def test_fs_dict(self): @no_wasm_backend("wasm backend has no support for fastcomp's -emscripten-assertions flag") def test_stack_overflow_check(self): args = self.emcc_args + ['-s', 'TOTAL_STACK=1048576'] - self.emcc_args = args + ['-s', 'STACK_OVERFLOW_CHECK=1', '-s', 'ASSERTIONS=0'] - self.do_run(open(path_from_root('tests', 'stack_overflow.cpp')).read(), 'Stack overflow! Stack cookie has been overwritten' if not self.get_setting('SAFE_HEAP') else 'segmentation fault') self.emcc_args = args + ['-s', 'STACK_OVERFLOW_CHECK=2', '-s', 'ASSERTIONS=0'] self.do_run(open(path_from_root('tests', 'stack_overflow.cpp')).read(), 'Stack overflow! Attempted to allocate') From dccd4f16f02ab15765da211fcdca31cf90385162 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 10 Jan 2019 15:33:23 -0800 Subject: [PATCH 26/30] cleanup --- emcc.py | 8 -------- site/source/docs/api_reference/preamble.js.rst | 3 +-- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/emcc.py b/emcc.py index 371b94e8f6dc..37f6bbd5ec01 100755 --- a/emcc.py +++ b/emcc.py @@ -3051,14 +3051,6 @@ def validate_arg_level(level_string, max_level, err_msg, clamp=False): return level -''' -TODO: search and remove as needed all of - staticSealed - staticAlloc - ALLOC_STATIC -''' - - if __name__ == '__main__': try: sys.exit(run()) diff --git a/site/source/docs/api_reference/preamble.js.rst b/site/source/docs/api_reference/preamble.js.rst index bf2c3c67887f..f097913c0f98 100644 --- a/site/source/docs/api_reference/preamble.js.rst +++ b/site/source/docs/api_reference/preamble.js.rst @@ -405,7 +405,7 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t .. COMMENT (not rendered) : The following methods are explicitly not part of the public API and not documented. Note that in some case referred to by function name, other cases by Module assignment. function allocate(slab, types, allocator, ptr) — Internal and use is discouraged. Documentation can remain in source code but not here. - associated contants ALLOC_NORMAL, ALLOC_STACK, ALLOC_STATIC, ALLOC_DYNAMIC, ALLOC_NONE + associated contants ALLOC_NORMAL, ALLOC_STACK, ALLOC_DYNAMIC, ALLOC_NONE function addOnPreRun function addOnInit @@ -414,7 +414,6 @@ The :ref:`emscripten-memory-model` uses a typed array buffer (``ArrayBuffer``) t function addOnPostRun Module['ALLOC_NORMAL'] = ALLOC_NORMAL; Module['ALLOC_STACK'] = ALLOC_STACK; - Module['ALLOC_STATIC'] = ALLOC_STATIC; Module['ALLOC_DYNAMIC'] = ALLOC_DYNAMIC; Module['ALLOC_NONE'] = ALLOC_NONE; Module['HEAP'] = HEAP; From c6e3ea3dce5da0af12a4b8b8c5c03995937ba9cd Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 10 Jan 2019 15:37:58 -0800 Subject: [PATCH 27/30] cleanup --- src/jsifier.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jsifier.js b/src/jsifier.js index 74330a592428..b9cd0a7f2bfb 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -320,7 +320,7 @@ function JSify(data, functionsOnly) { } }); }); - if (VERBOSE) printErr('adding ' + finalName + ' and deps ' + JSON.stringify(deps) + ' : ' + (snippet + '').substr(0, 40)); + if (VERBOSE) printErr('adding ' + finalName + ' and deps ' + deps + ' : ' + (snippet + '').substr(0, 40)); var depsText = (deps ? '\n' + deps.map(addFromLibrary).filter(function(x) { return x != '' }).join('\n') : ''); var contentText; if (isFunction) { From c8aed5d35e9800394ca67038ed297964c736e58b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 11 Jan 2019 12:51:34 -0800 Subject: [PATCH 28/30] feedback --- emscripten.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/emscripten.py b/emscripten.py index 53f9e78ecdea..c4e3972da50e 100644 --- a/emscripten.py +++ b/emscripten.py @@ -621,6 +621,9 @@ def compile_settings(compiler_engine, libraries, temp_files): def apply_memory(pre, metadata): # Apply the statically-at-compile-time computed memory locations. + # Note: if RELOCATABLE, then only relative sizes can be computed, and we don't + # actually write out any absolute memory locations ({{{ STACK_BASE }}} + # does not exist, etc.) # Memory layout: # * first the static globals @@ -628,7 +631,7 @@ def apply_memory(pre, metadata): static_bump = shared.Settings.STATIC_BUMP # * then the stack (up on fastcomp, down on upstream) stack_low = align_memory(global_start + static_bump) - stack_high = stack_low + shared.Settings.TOTAL_STACK + stack_high = align_memory(stack_low + shared.Settings.TOTAL_STACK) if shared.Settings.WASM_BACKEND: stack_start = stack_high stack_max = stack_low From 4b48ca94204bcf35e36c0463baf1010b5cebb33b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 11 Jan 2019 13:38:47 -0800 Subject: [PATCH 29/30] assert that the stack and heap must be aligned --- src/preamble.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/preamble.js b/src/preamble.js index 011c2ee817e9..237906d6768f 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -930,6 +930,11 @@ var STATIC_BASE = {{{ GLOBAL_BASE }}}, DYNAMIC_BASE = {{{ getDynamicBase() }}}, DYNAMICTOP_PTR = {{{ makeStaticAlloc(4) }}}; +#if ASSERTIONS +assert(STACK_BASE % 16 === 0, 'stack must start aligned'); +assert(DYNAMIC_BASE % 16 === 0, 'heap must start aligned'); +#endif + #if USE_PTHREADS } #endif From 9e44fd6785f02964c74820064971984bc4c7103f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 14 Jan 2019 08:52:47 -0800 Subject: [PATCH 30/30] review feedback --- src/library_pthread.js | 4 ++-- src/parseTools.js | 27 ++++++++++++++++++++++++++- src/preamble.js | 25 +------------------------ tests/test_core.py | 2 +- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index 8550e5dfa80e..7d7189adcefb 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -528,7 +528,7 @@ var LibraryPThread = { }, _num_logical_cores__deps: ['emscripten_force_num_logical_cores'], - _num_logical_cores: '; if (ENVIRONMENT_IS_PTHREAD) __num_logical_cores = PthreadWorkerInit.__num_logical_cores; else { PthreadWorkerInit.__num_logical_cores = __num_logical_cores = {{{ makeStaticAlloc(1) }}}; HEAPU32[__num_logical_cores>>2] = navigator["hardwareConcurrency"] || ' + {{{ PTHREAD_HINT_NUM_CORES }}} + '; }', + _num_logical_cores: '; if (ENVIRONMENT_IS_PTHREAD) __num_logical_cores = PthreadWorkerInit.__num_logical_cores; else { PthreadWorkerInit.__num_logical_cores = __num_logical_cores = {{{ makeStaticAlloc(4) }}}; HEAPU32[__num_logical_cores>>2] = navigator["hardwareConcurrency"] || ' + {{{ PTHREAD_HINT_NUM_CORES }}} + '; }', emscripten_has_threading_support: function() { return typeof SharedArrayBuffer !== 'undefined'; @@ -1030,7 +1030,7 @@ var LibraryPThread = { }, // Stores the memory address that the main thread is waiting on, if any. - _main_thread_futex_wait_address: '; if (ENVIRONMENT_IS_PTHREAD) __main_thread_futex_wait_address = PthreadWorkerInit.__main_thread_futex_wait_address; else PthreadWorkerInit.__main_thread_futex_wait_address = __main_thread_futex_wait_address = {{{ makeStaticAlloc(1) }}}', + _main_thread_futex_wait_address: '; if (ENVIRONMENT_IS_PTHREAD) __main_thread_futex_wait_address = PthreadWorkerInit.__main_thread_futex_wait_address; else PthreadWorkerInit.__main_thread_futex_wait_address = __main_thread_futex_wait_address = {{{ makeStaticAlloc(4) }}}', // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error. emscripten_futex_wait__deps: ['_main_thread_futex_wait_address', 'emscripten_main_thread_process_queued_calls'], diff --git a/src/parseTools.js b/src/parseTools.js index 2d00f3efff5a..13af6086cc99 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1391,6 +1391,31 @@ function charCode(char) { return char.charCodeAt(0); } +// Returns the number of bytes the given Javascript string takes if encoded as a UTF8 byte array, EXCLUDING the null terminator byte. +function lengthBytesUTF8(str) { + var len = 0; + for (var i = 0; i < str.length; ++i) { + // Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code unit, not a Unicode code point of the character! So decode UTF16->UTF32->UTF8. + // See http://unicode.org/faq/utf_bom.html#utf16-3 + var u = str.charCodeAt(i); // possibly a lead surrogate + if (u >= 0xD800 && u <= 0xDFFF) u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt(++i) & 0x3FF); + if (u <= 0x7F) { + ++len; + } else if (u <= 0x7FF) { + len += 2; + } else if (u <= 0xFFFF) { + len += 3; + } else if (u <= 0x1FFFFF) { + len += 4; + } else if (u <= 0x3FFFFFF) { + len += 5; + } else { + len += 6; + } + } + return len; +} + function getTypeFromHeap(suffix) { switch (suffix) { case '8': return 'i8'; @@ -1448,7 +1473,7 @@ function makeStaticAlloc(size) { } function makeStaticString(string) { - var len = string.length + 1; + var len = lengthBytesUTF8(string) + 1; var ptr = makeStaticAlloc(len); return '(stringToUTF8("' + string + '", ' + ptr + ', ' + len + '), ' + ptr + ')'; } diff --git a/src/preamble.js b/src/preamble.js index 237906d6768f..fd9ab0a8d1a8 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -589,30 +589,7 @@ function stringToUTF8(str, outPtr, maxBytesToWrite) { } // Returns the number of bytes the given Javascript string takes if encoded as a UTF8 byte array, EXCLUDING the null terminator byte. - -function lengthBytesUTF8(str) { - var len = 0; - for (var i = 0; i < str.length; ++i) { - // Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code unit, not a Unicode code point of the character! So decode UTF16->UTF32->UTF8. - // See http://unicode.org/faq/utf_bom.html#utf16-3 - var u = str.charCodeAt(i); // possibly a lead surrogate - if (u >= 0xD800 && u <= 0xDFFF) u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt(++i) & 0x3FF); - if (u <= 0x7F) { - ++len; - } else if (u <= 0x7FF) { - len += 2; - } else if (u <= 0xFFFF) { - len += 3; - } else if (u <= 0x1FFFFF) { - len += 4; - } else if (u <= 0x3FFFFFF) { - len += 5; - } else { - len += 6; - } - } - return len; -} +{{{ lengthBytesUTF8 }}} // Given a pointer 'ptr' to a null-terminated UTF16LE-encoded string in the emscripten HEAP, returns // a copy of that string as a Javascript String object. diff --git a/tests/test_core.py b/tests/test_core.py index 7f05706a1d25..4c53b942e747 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -3790,7 +3790,7 @@ def test_dylink_jslib(self): def test_dylink_global_var_jslib(self): create_test_file('lib.js', r''' mergeInto(LibraryManager.library, { - jslib_x: '{{{ makeStaticAlloc(1) }}}', + jslib_x: '{{{ makeStaticAlloc(4) }}}', jslib_x__postset: 'HEAP32[_jslib_x>>2] = 148;', }); ''')