diff --git a/emcc.py b/emcc.py index db7767efe7fa..c3b16e32e2b4 100755 --- a/emcc.py +++ b/emcc.py @@ -277,6 +277,10 @@ def do_minify(self): self.cleanup_shell = True +def embed_memfile(options): + return shared.Settings.SINGLE_FILE or (shared.Settings.MEM_INIT_METHOD == 0 and (not shared.Settings.MAIN_MODULE and not shared.Settings.SIDE_MODULE and options.debug_level < 4)) + + # # Main run() function # @@ -801,7 +805,7 @@ def detect_fixed_language_mode(args): options.separate_asm = True logging.warning('forcing separate asm output (--separate-asm), because -s PRECISE_F32=2 or -s USE_PTHREADS=2 was passed.') if options.separate_asm: - shared.Settings.SEPARATE_ASM = os.path.basename(asm_target) + shared.Settings.SEPARATE_ASM = shared.JS.get_subresource_location(asm_target) if 'EMCC_STRICT' in os.environ: shared.Settings.STRICT = os.environ.get('EMCC_STRICT') != '0' @@ -918,6 +922,8 @@ def check(input_file): logging.warning('disabling closure because debug info was requested') options.use_closure_compiler = False + assert not (shared.Settings.EMTERPRETIFY_FILE and shared.Settings.SINGLE_FILE), 'cannot have both EMTERPRETIFY_FILE and SINGLE_FILE enabled at the same time' + assert not (shared.Settings.NO_DYNAMIC_EXECUTION and options.use_closure_compiler), 'cannot have both NO_DYNAMIC_EXECUTION and closure compiler enabled at the same time' if options.use_closure_compiler: @@ -1110,10 +1116,16 @@ def check(input_file): os.environ['EMCC_WASM_BACKEND_BINARYEN'] = '1' if shared.Settings.BINARYEN: - # set file locations, so that JS glue can find what it needs - shared.Settings.WASM_TEXT_FILE = os.path.basename(wasm_text_target) - shared.Settings.WASM_BINARY_FILE = os.path.basename(wasm_binary_target) - shared.Settings.ASMJS_CODE_FILE = os.path.basename(asm_target) + if shared.Settings.SINGLE_FILE: + # placeholder strings for JS glue, to be replaced with subresource locations in do_binaryen + shared.Settings.WASM_TEXT_FILE = shared.FilenameReplacementStrings.WASM_TEXT_FILE + shared.Settings.WASM_BINARY_FILE = shared.FilenameReplacementStrings.WASM_BINARY_FILE + shared.Settings.ASMJS_CODE_FILE = shared.FilenameReplacementStrings.ASMJS_CODE_FILE + else: + # set file locations, so that JS glue can find what it needs + shared.Settings.WASM_TEXT_FILE = shared.JS.get_subresource_location(wasm_text_target) + shared.Settings.WASM_BINARY_FILE = shared.JS.get_subresource_location(wasm_binary_target) + shared.Settings.ASMJS_CODE_FILE = shared.JS.get_subresource_location(asm_target) shared.Settings.ASM_JS = 2 # when targeting wasm, we use a wasm Memory, but that is not compatible with asm.js opts shared.Settings.GLOBAL_BASE = 1024 # leave some room for mapping global vars @@ -1538,6 +1550,10 @@ def get_final(): shared.Settings.MEM_INIT_METHOD = 1 else: assert shared.Settings.MEM_INIT_METHOD != 1 + + if embed_memfile(options): + shared.Settings.SUPPORT_BASE64_EMBEDDING = 1 + final = shared.Building.emscripten(final, append_ext=False, extra_args=extra_args) if DEBUG: save_intermediate('original') @@ -1625,7 +1641,8 @@ def get_final(): with ToolchainProfiler.profile_block('memory initializer'): memfile = None - if shared.Settings.MEM_INIT_METHOD > 0: + + if shared.Settings.MEM_INIT_METHOD > 0 or embed_memfile(options): memfile = target + '.mem' shared.try_delete(memfile) def repl(m): @@ -1636,20 +1653,17 @@ def repl(m): while membytes and membytes[-1] == 0: membytes.pop() if not membytes: return '' - if not options.memory_init_file: + if shared.Settings.MEM_INIT_METHOD == 2: # memory initializer in a string literal return "memoryInitializer = '%s';" % shared.JS.generate_string_initializer(list(membytes)) open(memfile, 'wb').write(''.join(map(chr, membytes))) if DEBUG: # Copy into temp dir as well, so can be run there too shared.safe_copy(memfile, os.path.join(shared.get_emscripten_temp_dir(), os.path.basename(memfile))) - if not shared.Settings.BINARYEN: - return 'memoryInitializer = "%s";' % os.path.basename(memfile) + if not shared.Settings.BINARYEN or 'asmjs' in shared.Settings.BINARYEN_METHOD or 'interpret-asm2wasm' in shared.Settings.BINARYEN_METHOD: + return 'memoryInitializer = "%s";' % shared.JS.get_subresource_location(memfile, embed_memfile(options)) else: - # with wasm, we may have the mem init file in the wasm binary already - return ('memoryInitializer = Module["wasmJSMethod"].indexOf("asmjs") >= 0 || ' - 'Module["wasmJSMethod"].indexOf("interpret-asm2wasm") >= 0 ? "%s" : null;' - % os.path.basename(memfile)) + return 'memoryInitializer = null;' src = re.sub(shared.JS.memory_initializer_pattern, repl, open(final).read(), count=1) open(final + '.mem.js', 'w').write(src) final += '.mem.js' @@ -1661,15 +1675,6 @@ def repl(m): logging.debug('wrote memory initialization to %s', memfile) else: logging.debug('did not see memory initialization') - elif not shared.Settings.MAIN_MODULE and not shared.Settings.SIDE_MODULE and options.debug_level < 4: - # not writing a binary init, but we can at least optimize them by splitting them up - src = open(final).read() - src = shared.JS.optimize_initializer(src) - if src is not None: - logging.debug('optimizing memory initialization') - open(final + '.mem.js', 'w').write(src) - final += '.mem.js' - src = None if shared.Settings.USE_PTHREADS: target_dir = os.path.dirname(os.path.abspath(target)) @@ -1829,6 +1834,9 @@ def get_eliminate(): if options.proxy_to_worker: generate_worker_js(target, js_target, target_basename) + if embed_memfile(options): + shared.try_delete(memfile) + for f in generated_text_files_with_native_eols: tools.line_endings.convert_line_endings_in_file(f, os.linesep, options.output_eol) log_time('final emitting') @@ -2352,6 +2360,24 @@ def do_binaryen(final, target, asm_target, options, memfile, wasm_binary_target, passes.append('minifyWhitespace') final = shared.Building.js_optimizer_no_asmjs(final, passes) if DEBUG: save_intermediate('postclean', 'js') + # replace placeholder strings with correct subresource locations + if shared.Settings.SINGLE_FILE: + f = open(final, 'rb') + js = f.read() + f.close() + f = open(final, 'wb') + for target, replacement_string, should_embed in [ + (wasm_text_target, shared.FilenameReplacementStrings.WASM_TEXT_FILE, True), + (wasm_binary_target, shared.FilenameReplacementStrings.WASM_BINARY_FILE, True), + (asm_target, shared.FilenameReplacementStrings.ASMJS_CODE_FILE, not shared.Building.is_wasm_only()) + ]: + if should_embed and os.path.isfile(target): + js = js.replace(replacement_string, shared.JS.get_subresource_location(target)) + else: + js = js.replace(replacement_string, '') + shared.try_delete(target) + f.write(js) + f.close() return final @@ -2399,11 +2425,17 @@ def generate_html(target, options, js_target, target_basename, } else { // note: no support for code mods (PRECISE_F32==2) console.log('running code on the main thread'); + var filename = '%s'; + var fileBytes = tryParseAsDataURI(filename); var script = document.createElement('script'); - script.src = "%s.js"; + if (fileBytes) { + script.innerHTML = intArrayToString(fileBytes); + } else { + script.src = filename; + } document.body.appendChild(script); } -''' % proxy_worker_filename +''' % shared.JS.get_subresource_location(proxy_worker_filename + '.js') else: # Normal code generation path script.src = base_js_target @@ -2417,33 +2449,40 @@ def generate_html(target, options, js_target, target_basename, # We need to load the emterpreter file before anything else, it has to be synchronously ready script.un_src() script.inline = ''' + var emterpretURL = '%s'; var emterpretXHR = new XMLHttpRequest(); - emterpretXHR.open('GET', '%s', true); + emterpretXHR.open('GET', emterpretURL, true); emterpretXHR.responseType = 'arraybuffer'; emterpretXHR.onload = function() { - Module.emterpreterFile = emterpretXHR.response; + if (emterpretXHR.status === 200 || emterpretXHR.status === 0) { + Module.emterpreterFile = emterpretXHR.response; + } else { + var emterpretURLBytes = tryParseAsDataURI(emterpretURL); + if (emterpretURLBytes) { + Module.emterpreterFile = emterpretURLBytes.buffer; + } + } %s }; emterpretXHR.send(null); -''' % (shared.Settings.EMTERPRETIFY_FILE, script.inline) +''' % (shared.JS.get_subresource_location(shared.Settings.EMTERPRETIFY_FILE), script.inline) if options.memory_init_file: # start to load the memory init file in the HTML, in parallel with the JS script.un_src() script.inline = (''' - (function() { - var memoryInitializer = '%s'; - if (typeof Module['locateFile'] === 'function') { - memoryInitializer = Module['locateFile'](memoryInitializer); - } else if (Module['memoryInitializerPrefixURL']) { - memoryInitializer = Module['memoryInitializerPrefixURL'] + memoryInitializer; - } - var meminitXHR = Module['memoryInitializerRequest'] = new XMLHttpRequest(); - meminitXHR.open('GET', memoryInitializer, true); - meminitXHR.responseType = 'arraybuffer'; - meminitXHR.send(null); - })(); -''' % os.path.basename(memfile)) + script.inline + var memoryInitializer = '%s'; + if (typeof Module['locateFile'] === 'function') { + memoryInitializer = Module['locateFile'](memoryInitializer); + } else if (Module['memoryInitializerPrefixURL']) { + memoryInitializer = Module['memoryInitializerPrefixURL'] + memoryInitializer; + } + Module['memoryInitializerRequestURL'] = memoryInitializer; + var meminitXHR = Module['memoryInitializerRequest'] = new XMLHttpRequest(); + meminitXHR.open('GET', memoryInitializer, true); + meminitXHR.responseType = 'arraybuffer'; + meminitXHR.send(null); +''' % shared.JS.get_subresource_location(memfile)) + script.inline # Download .asm.js if --separate-asm was passed in an asm.js build, or if 'asmjs' is one # of the wasm run methods. @@ -2454,22 +2493,37 @@ def generate_html(target, options, js_target, target_basename, if len(asm_mods) == 0: # just load the asm, then load the rest script.inline = ''' + var filename = '%s'; + var fileBytes = tryParseAsDataURI(filename); var script = document.createElement('script'); - script.src = "%s"; + if (fileBytes) { + script.innerHTML = intArrayToString(fileBytes); + } else { + script.src = filename; + } script.onload = function() { setTimeout(function() { %s }, 1); // delaying even 1ms is enough to allow compilation memory to be reclaimed }; document.body.appendChild(script); -''' % (os.path.basename(asm_target), script.inline) +''' % (shared.JS.get_subresource_location(asm_target), script.inline) else: # may need to modify the asm code, load it as text, modify, and load asynchronously script.inline = ''' + var codeURL = '%s'; var codeXHR = new XMLHttpRequest(); - codeXHR.open('GET', '%s', true); + codeXHR.open('GET', codeURL, true); codeXHR.onload = function() { - var code = codeXHR.responseText; + var code; + if (codeXHR.status === 200 || codeXHR.status === 0) { + code = codeXHR.responseText; + } else { + var codeURLBytes = tryParseAsDataURI(codeURL); + if (codeURLBytes) { + code = intArrayToString(codeURLBytes); + } + } %s var blob = new Blob([code], { type: 'text/javascript' }); codeXHR = null; @@ -2485,21 +2539,36 @@ def generate_html(target, options, js_target, target_basename, document.body.appendChild(script); }; codeXHR.send(null); -''' % (os.path.basename(asm_target), '\n'.join(asm_mods), script.inline) +''' % (shared.JS.get_subresource_location(asm_target), '\n'.join(asm_mods), script.inline) if shared.Settings.BINARYEN and not shared.Settings.BINARYEN_ASYNC_COMPILATION: # We need to load the wasm file before anything else, it has to be synchronously ready TODO: optimize script.un_src() script.inline = ''' + var wasmURL = '%s'; var wasmXHR = new XMLHttpRequest(); - wasmXHR.open('GET', '%s', true); + wasmXHR.open('GET', wasmURL, true); wasmXHR.responseType = 'arraybuffer'; wasmXHR.onload = function() { - Module.wasmBinary = wasmXHR.response; + if (wasmXHR.status === 200 || wasmXHR.status === 0) { + Module.wasmBinary = wasmXHR.response; + } else { + var wasmURLBytes = tryParseAsDataURI(wasmURL); + if (wasmURLBytes) { + Module.wasmBinary = wasmURLBytes.buffer; + } + } %s }; wasmXHR.send(null); -''' % (os.path.basename(wasm_binary_target), script.inline) +''' % (shared.JS.get_subresource_location(wasm_binary_target), script.inline) + + # when script.inline isn't empty, add required helper functions such as tryParseAsDataURI + if script.inline: + for file in ['src/arrayUtils.js', 'src/base64Utils.js']: + f = open(shared.path_from_root(file), 'r') + script.inline = f.read() + script.inline + f.close() html = open(target, 'wb') html_contents = shell.replace('{{{ SCRIPT }}}', script.replacement()) diff --git a/src/arrayUtils.js b/src/arrayUtils.js new file mode 100644 index 000000000000..3720fbb5b499 --- /dev/null +++ b/src/arrayUtils.js @@ -0,0 +1,38 @@ +// All functions here should be maybeExported from jsifier.js + +/** @type {function(string, boolean=, number=)} */ +function intArrayFromString(stringy, dontAddNull, length) { + var len = length > 0 ? length : lengthBytesUTF8(stringy)+1; + var u8array = new Array(len); + var numBytesWritten = stringToUTF8Array(stringy, u8array, 0, u8array.length); + if (dontAddNull) u8array.length = numBytesWritten; + return u8array; +} + +// Temporarily duplicating function pending Python preprocessor support +var ASSERTIONS; +var intArrayToString = ASSERTIONS ? + function (array) { + var ret = []; + for (var i = 0; i < array.length; i++) { + var chr = array[i]; + if (chr > 0xFF) { + assert(false, 'Character code ' + chr + ' (' + String.fromCharCode(chr) + ') at offset ' + i + ' not in 0x00-0xFF.'); + chr &= 0xFF; + } + ret.push(String.fromCharCode(chr)); + } + return ret.join(''); + } : + function (array) { + var ret = []; + for (var i = 0; i < array.length; i++) { + var chr = array[i]; + if (chr > 0xFF) { + chr &= 0xFF; + } + ret.push(String.fromCharCode(chr)); + } + return ret.join(''); + } +; diff --git a/src/base64Utils.js b/src/base64Utils.js new file mode 100644 index 000000000000..8d8b04e7b701 --- /dev/null +++ b/src/base64Utils.js @@ -0,0 +1,83 @@ +// All functions here should be maybeExported from jsifier.js + +// Copied from https://github.com/strophe/strophejs/blob/e06d027/src/polyfills.js#L149 + +// This code was written by Tyler Akins and has been placed in the +// public domain. It would be nice if you left this header intact. +// Base64 code from Tyler Akins -- http://rumkin.com + +var keyStr = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='; + +var decodeBase64 = typeof atob === 'function' ? atob : function (input) { + /** + * Decodes a base64 string. + * @param {String} input The string to decode. + */ + var output = ''; + var chr1, chr2, chr3; + var enc1, enc2, enc3, enc4; + var i = 0; + // remove all characters that are not A-Z, a-z, 0-9, +, /, or = + input = input.replace(/[^A-Za-z0-9\+\/\=]/g, ''); + do { + enc1 = keyStr.indexOf(input.charAt(i++)); + enc2 = keyStr.indexOf(input.charAt(i++)); + enc3 = keyStr.indexOf(input.charAt(i++)); + enc4 = keyStr.indexOf(input.charAt(i++)); + + chr1 = (enc1 << 2) | (enc2 >> 4); + chr2 = ((enc2 & 15) << 4) | (enc3 >> 2); + chr3 = ((enc3 & 3) << 6) | enc4; + + output = output + String.fromCharCode(chr1); + + if (enc3 !== 64) { + output = output + String.fromCharCode(chr2); + } + if (enc4 !== 64) { + output = output + String.fromCharCode(chr3); + } + } while (i < input.length); + return output; +}; + +// Converts a string of base64 into a byte array. +// Throws error on invalid input. +function intArrayFromBase64(s) { + if (typeof ENVIRONMENT_IS_NODE === 'boolean' && ENVIRONMENT_IS_NODE) { + var buf; + try { + buf = Buffer.from(s, 'base64'); + } catch (_) { + buf = new Buffer(s, 'base64'); + } + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); + } + + try { + var decoded = decodeBase64(s); + var bytes = new Uint8Array(decoded.length); + for (var i = 0 ; i < decoded.length ; ++i) { + bytes[i] = decoded.charCodeAt(i); + } + return bytes; + } catch (_) { + throw new Error('Converting base64 string to bytes failed.'); + } +} + +// If filename is a base64 data URI, parses and returns data (Buffer on node, +// Uint8Array otherwise). If filename is not a base64 data URI, returns undefined. +function tryParseAsDataURI(filename) { + var dataURIPrefix = 'data:application/octet-stream;base64,'; + + if (!( + String.prototype.startsWith ? + filename.startsWith(dataURIPrefix) : + filename.indexOf(dataURIPrefix) === 0 + )) { + return; + } + + return intArrayFromBase64(filename.slice(dataURIPrefix.length)); +} diff --git a/src/jsifier.js b/src/jsifier.js index 350028bf8678..4166791b7d34 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -522,6 +522,18 @@ function JSify(data, functionsOnly) { print('assert(STACK_MAX < SPLIT_MEMORY, "SPLIT_MEMORY size must be big enough so the entire static memory + stack can fit in one chunk, need " + STACK_MAX);\n'); } + print(preprocess(read('arrayUtils.js'))); + // Export all arrayUtils.js functions + print(maybeExport('intArrayFromString')); + print(maybeExport('intArrayToString')); + + if (SUPPORT_BASE64_EMBEDDING) { + print(preprocess(read('base64Utils.js'))); + // Export all base64Utils.js functions + print(maybeExport('intArrayFromBase64')); + print(maybeExport('tryParseAsDataURI')); + } + if (asmLibraryFunctions.length > 0) { print('// ASM_LIBRARY FUNCTIONS'); function fix(f) { // fix indenting to not confuse js optimizer @@ -539,6 +551,8 @@ function JSify(data, functionsOnly) { // "Final shape that will be created"). print('// EMSCRIPTEN_END_FUNCS\n'); + if (ASSERTIONS) print('var ASSERTIONS = true;\n'); + if (HEADLESS) { print('if (!ENVIRONMENT_IS_WEB) {'); print(read('headlessCanvas.js')); diff --git a/src/postamble.js b/src/postamble.js index 6e52c58c2730..ce813ed5b9de 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -34,7 +34,6 @@ if (memoryInitializer) (function(s) { } })(memoryInitializer); #else -#if MEM_INIT_METHOD == 1 #if USE_PTHREADS if (memoryInitializer && !ENVIRONMENT_IS_PTHREAD) { #else @@ -69,19 +68,35 @@ if (memoryInitializer) { throw 'could not load memory initializer ' + memoryInitializer; }); } +#if SUPPORT_BASE64_EMBEDDING + var memoryInitializerBytes = tryParseAsDataURI(memoryInitializer); + if (memoryInitializerBytes) { + applyMemoryInitializer(memoryInitializerBytes.buffer); + } else +#endif if (Module['memoryInitializerRequest']) { // a network request has already been created, just use that function useRequest() { var request = Module['memoryInitializerRequest']; + var response = request.response; if (request.status !== 200 && request.status !== 0) { - // If you see this warning, the issue may be that you are using locateFile or memoryInitializerPrefixURL, and defining them in JS. That - // means that the HTML file doesn't know about them, and when it tries to create the mem init request early, does it to the wrong place. - // Look in your browser's devtools network console to see what's going on. - console.warn('a problem seems to have happened with Module.memoryInitializerRequest, status: ' + request.status + ', retrying ' + memoryInitializer); - doBrowserLoad(); - return; +#if SUPPORT_BASE64_EMBEDDING + var data = tryParseAsDataURI(Module['memoryInitializerRequestURL']); + if (data) { + response = data.buffer; + } else { +#endif + // If you see this warning, the issue may be that you are using locateFile or memoryInitializerPrefixURL, and defining them in JS. That + // means that the HTML file doesn't know about them, and when it tries to create the mem init request early, does it to the wrong place. + // Look in your browser's devtools network console to see what's going on. + console.warn('a problem seems to have happened with Module.memoryInitializerRequest, status: ' + request.status + ', retrying ' + memoryInitializer); + doBrowserLoad(); + return; +#if SUPPORT_BASE64_EMBEDDING + } +#endif } - applyMemoryInitializer(request.response); + applyMemoryInitializer(response); } if (Module['memoryInitializerRequest'].response) { setTimeout(useRequest, 0); // it's already here; but, apply it asynchronously @@ -95,7 +110,6 @@ if (memoryInitializer) { } } #endif -#endif #if CYBERDWARF Module['cyberdwarf'] = _cyberdwarf_Debugger(cyberDWARFFile); diff --git a/src/preamble.js b/src/preamble.js index ec0a9975a3e9..daf3cb1bbd30 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -1705,34 +1705,6 @@ function addOnPostRun(cb) { } {{{ maybeExport('addOnPostRun') }}} -// Tools - -/** @type {function(string, boolean=, number=)} */ -function intArrayFromString(stringy, dontAddNull, length) { - var len = length > 0 ? length : lengthBytesUTF8(stringy)+1; - var u8array = new Array(len); - var numBytesWritten = stringToUTF8Array(stringy, u8array, 0, u8array.length); - if (dontAddNull) u8array.length = numBytesWritten; - return u8array; -} -{{{ maybeExport('intArrayFromString') }}} - -function intArrayToString(array) { - var ret = []; - for (var i = 0; i < array.length; i++) { - var chr = array[i]; - if (chr > 0xFF) { -#if ASSERTIONS - assert(false, 'Character code ' + chr + ' (' + String.fromCharCode(chr) + ') at offset ' + i + ' not in 0x00-0xFF.'); -#endif - chr &= 0xFF; - } - ret.push(String.fromCharCode(chr)); - } - return ret.join(''); -} -{{{ maybeExport('intArrayToString') }}} - // Deprecated: This function should not be called because it is unsafe and does not provide // a maximum length limit of how many bytes it is allowed to write. Prefer calling the // function stringToUTF8Array() instead, which takes in a maximum length that can be used @@ -2213,16 +2185,20 @@ function integrateWasmJS() { function getBinary() { try { - var binary; if (Module['wasmBinary']) { - binary = Module['wasmBinary']; - binary = new Uint8Array(binary); - } else if (Module['readBinary']) { - binary = Module['readBinary'](wasmBinaryFile); + return new Uint8Array(Module['wasmBinary']); + } +#if SUPPORT_BASE64_EMBEDDING + var binary = tryParseAsDataURI(wasmBinaryFile); + if (binary) { + return binary; + } +#endif + if (Module['readBinary']) { + return Module['readBinary'](wasmBinaryFile); } else { throw "on the web, we need the wasm binary to be preloaded and set on Module['wasmBinary']. emcc.py will do that for you when generating HTML (but not JS)"; } - return binary; } catch (err) { abort(err); @@ -2238,6 +2214,8 @@ function integrateWasmJS() { throw "failed to load wasm binary file at '" + wasmBinaryFile + "'"; } return response['arrayBuffer'](); + }).catch(function () { + return getBinary(); }); } // Otherwise, getBinary should be able to get it synchronously @@ -2336,7 +2314,7 @@ function integrateWasmJS() { }); } // Prefer streaming instantiation if available. - if (!Module['wasmBinary'] && typeof WebAssembly.instantiateStreaming === 'function') { + if (!Module['wasmBinary'] && typeof WebAssembly.instantiateStreaming === 'function' && wasmBinaryFile.indexOf('data:') !== 0) { WebAssembly.instantiateStreaming(fetch(wasmBinaryFile, { credentials: 'same-origin' }), info) .then(receiveInstantiatedSource) .catch(function(reason) { diff --git a/src/proxyClient.js b/src/proxyClient.js index 392843b299a8..93ae743e1253 100644 --- a/src/proxyClient.js +++ b/src/proxyClient.js @@ -95,9 +95,22 @@ var IDBStore = {{{ IDBStore.js }}}; var frameId = 0; +// Temporarily handling this at run-time pending Python preprocessor support + +var SUPPORT_BASE64_EMBEDDING; + // Worker -var worker = new Worker('{{{ filename }}}.js'); +var filename = '{{{ filename }}}.js'; + +var workerURL = filename; +if (SUPPORT_BASE64_EMBEDDING) { + var fileBytes = tryParseAsDataURI(filename); + if (fileBytes) { + workerURL = URL.createObjectURL(new Blob([fileBytes], {type: 'application/javascript'})); + } +} +var worker = new Worker(filename); WebGLClient.prefetch(); @@ -108,7 +121,7 @@ setTimeout(function() { height: Module.canvas.height, boundingClientRect: cloneObject(Module.canvas.getBoundingClientRect()), URL: document.URL, - currentScriptUrl: '{{{ filename }}}.js', + currentScriptUrl: filename, preMain: true }); }, 0); // delay til next frame, to make sure html is ready @@ -119,6 +132,7 @@ worker.onmessage = function worker_onmessage(event) { if (!workerResponded) { workerResponded = true; if (Module.setStatus) Module.setStatus(''); + if (SUPPORT_BASE64_EMBEDDING && workerURL !== filename) URL.revokeObjectURL(workerURL); } var data = event.data; diff --git a/src/settings.js b/src/settings.js index 76c016dee9f2..e9809a770758 100644 --- a/src/settings.js +++ b/src/settings.js @@ -47,7 +47,7 @@ var INVOKE_RUN = 1; // Whether we will run the main() function. Disable if you e var NO_EXIT_RUNTIME = 0; // If set, the runtime is not quit when main() completes (allowing code to // run afterwards, for example from the browser main event loop). var MEM_INIT_METHOD = 0; // How to represent the initial memory content. - // 0: keep array literal representing the initial memory data + // 0: embed a base64 string literal representing the initial memory data // 1: create a *.mem file containing the binary data of the initial memory; // use the --memory-init-file command line switch to select this method // 2: embed a string literal representing that initial memory data @@ -862,7 +862,21 @@ var FETCH = 0; // If nonzero, enables emscripten_fetch API. var ASMFS = 0; // If set to 1, uses the multithreaded filesystem that is implemented within the asm.js module, using emscripten_fetch. Implies -s FETCH=1. +var SINGLE_FILE = 0; // If set to 1, embeds all subresources in the emitted file as base64 string + // literals. Embedded subresources may include (but aren't limited to) + // wasm, asm.js, and static memory initialization code. + // + // When using code that depends on this option, your Content Security Policy + // may need to be updated. Specifically, embedding asm.js requires the + // script-src directive to whitelist 'unsafe-inline', and using a Worker + // requires the child-src directive to whitelist blob:. If you aren't using + // Content Security Policy, or your CSP header doesn't include either + // script-src or child-src, then you can safely ignore this warning. + var WASM_TEXT_FILE = ''; // name of the file containing wasm text, if relevant var WASM_BINARY_FILE = ''; // name of the file containing wasm binary, if relevant var ASMJS_CODE_FILE = ''; // name of the file containing asm.js, if relevant var SOURCE_MAP_BASE = ''; // Base URL the source mapfile, if relevant + +var SUPPORT_BASE64_EMBEDDING = 0; // If set to 1, src/base64Utils.js will be included in the bundle. + // This is set internally when needed (SINGLE_FILE) diff --git a/src/shell.js b/src/shell.js index 4404f5310542..a83d7041f2d1 100644 --- a/src/shell.js +++ b/src/shell.js @@ -81,10 +81,18 @@ if (ENVIRONMENT_IS_NODE) { var nodePath; Module['read'] = function shell_read(filename, binary) { - if (!nodeFS) nodeFS = require('fs'); - if (!nodePath) nodePath = require('path'); - filename = nodePath['normalize'](filename); - var ret = nodeFS['readFileSync'](filename); + var ret; +#if SUPPORT_BASE64_EMBEDDING + ret = tryParseAsDataURI(filename); + if (!ret) { +#endif + if (!nodeFS) nodeFS = require('fs'); + if (!nodePath) nodePath = require('path'); + filename = nodePath['normalize'](filename); + ret = nodeFS['readFileSync'](filename); +#if SUPPORT_BASE64_EMBEDDING + } +#endif return binary ? ret : ret.toString(); }; @@ -135,16 +143,31 @@ else if (ENVIRONMENT_IS_SHELL) { if (typeof printErr != 'undefined') Module['printErr'] = printErr; // not present in v8 or older sm if (typeof read != 'undefined') { - Module['read'] = read; + Module['read'] = function shell_read(f) { +#if SUPPORT_BASE64_EMBEDDING + var data = tryParseAsDataURI(f); + if (data) { + return intArrayToString(data); + } +#endif + return read(f); + }; } else { Module['read'] = function shell_read() { throw 'no read() available' }; } Module['readBinary'] = function readBinary(f) { + var data; +#if SUPPORT_BASE64_EMBEDDING + data = tryParseAsDataURI(f); + if (data) { + return data; + } +#endif if (typeof readbuffer === 'function') { return new Uint8Array(readbuffer(f)); } - var data = read(f, 'binary'); + data = read(f, 'binary'); assert(typeof data === 'object'); return data; }; @@ -167,19 +190,43 @@ else if (ENVIRONMENT_IS_SHELL) { } else if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { Module['read'] = function shell_read(url) { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, false); - xhr.send(null); - return xhr.responseText; +#if SUPPORT_BASE64_EMBEDDING + try { +#endif + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, false); + xhr.send(null); + return xhr.responseText; +#if SUPPORT_BASE64_EMBEDDING + } catch (err) { + var data = tryParseAsDataURI(url); + if (data) { + return intArrayToString(data); + } + throw err; + } +#endif }; if (ENVIRONMENT_IS_WORKER) { Module['readBinary'] = function readBinary(url) { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, false); - xhr.responseType = 'arraybuffer'; - xhr.send(null); - return new Uint8Array(xhr.response); +#if SUPPORT_BASE64_EMBEDDING + try { +#endif + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, false); + xhr.responseType = 'arraybuffer'; + xhr.send(null); + return new Uint8Array(xhr.response); +#if SUPPORT_BASE64_EMBEDDING + } catch (err) { + var data = tryParseAsDataURI(f); + if (data) { + return data; + } + throw err; + } +#endif }; } @@ -190,9 +237,16 @@ else if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { xhr.onload = function xhr_onload() { if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 onload(xhr.response); - } else { - onerror(); + return; + } +#if SUPPORT_BASE64_EMBEDDING + var data = tryParseAsDataURI(url); + if (data) { + onload(data.buffer); + return; } +#endif + onerror(); }; xhr.onerror = onerror; xhr.send(null); @@ -228,8 +282,8 @@ else if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { } } else { - // Unreachable because SHELL is dependant on the others - throw 'Unknown runtime environment. Where are we?'; + // Unreachable because SHELL is dependent on the others + throw new Error('Unknown runtime environment. Where are we?'); } function globalEval(x) { diff --git a/tests/test_browser.py b/tests/test_browser.py index 46909d6302e3..387c3493d35c 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3655,3 +3655,31 @@ def test_load_js_from_blob_with_pthreads(self): Popen([PYTHON, EMCC, 'src.c', '-s', 'USE_PTHREADS=1', '-o', 'hello_thread_with_blob_url.js']).communicate() shutil.copyfile(path_from_root('tests', 'pthread', 'main_js_as_blob_loader.html'), os.path.join(self.get_dir(), 'hello_thread_with_blob_url.html')) self.run_browser('hello_thread_with_blob_url.html', 'hello from thread!', '/report_result?1') + + # Tests that base64 utils work in browser with no native atob function + def test_base64_atob_fallback(self): + opts = ['-s', 'SINGLE_FILE=1', '-s', 'WASM=1', '-s', "BINARYEN_METHOD='interpret-binary'"] + src = r''' + #include + #include + int main() { + REPORT_RESULT(0); + return 0; + } + ''' + open('test.c', 'w').write(self.with_report_result(src)) + # generate a dummy file + open('dummy_file', 'w').write('dummy') + # compile the code with the modularize feature and the preload-file option enabled + Popen([PYTHON, EMCC, 'test.c', '-s', 'MODULARIZE=1', '-s', 'EXPORT_NAME="Foo"', '--preload-file', 'dummy_file'] + opts).communicate() + open('a.html', 'w').write(''' + + + + ''') + self.run_browser('a.html', '...', '/report_result?0') diff --git a/tests/test_other.py b/tests/test_other.py index efd9dcb829a6..f82d21eb18a5 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -222,9 +222,9 @@ def test_emcc(self): # emcc -s INLINING_LIMIT=0 src.cpp ==> should pass -s to emscripten.py. for params, test, text in [ - (['-O2'], lambda generated: 'function intArrayToString' in generated, 'shell has unminified utilities'), - (['-O2', '--closure', '1'], lambda generated: 'function intArrayToString' not in generated and ';function' in generated, 'closure minifies the shell, removes whitespace'), - (['-O2', '--closure', '1', '-g1'], lambda generated: 'function intArrayToString' not in generated and ';function' not in generated, 'closure minifies the shell, -g1 makes it keep whitespace'), + (['-O2'], lambda generated: 'function addRunDependency' in generated, 'shell has unminified utilities'), + (['-O2', '--closure', '1'], lambda generated: 'function addRunDependency' not in generated and ';function' in generated, 'closure minifies the shell, removes whitespace'), + (['-O2', '--closure', '1', '-g1'], lambda generated: 'function addRunDependency' not in generated and ';function' not in generated, 'closure minifies the shell, -g1 makes it keep whitespace'), (['-O2'], lambda generated: 'var b=0' in generated and not 'function _main' in generated, 'registerize/minify is run by default in -O2'), (['-O2', '--minify', '0'], lambda generated: 'var b = 0' in generated and not 'function _main' in generated, 'minify is cancelled, but not registerize'), (['-O2', '--js-opts', '0'], lambda generated: 'var b=0' not in generated and 'var b = 0' not in generated and 'function _main' in generated, 'js opts are cancelled'), @@ -7927,3 +7927,68 @@ def test_include_system_header_in_c(self): print inc open('a.c', 'w').write(inc) subprocess.check_call([PYTHON, EMCC, '-std=c89', 'a.c']) + + def test_single_file(self): + for single_file_enabled in [True, False]: + for meminit1_enabled in [True, False]: + for debug_enabled in [True, False]: + for emterpreter_enabled in [True, False]: + for emterpreter_file_enabled in [True, False]: + for closure_enabled in [True, False]: + for wasm_enabled in [True, False]: + for asmjs_fallback_enabled in [True, False]: + # skip unhelpful option combinations + if ( + (asmjs_fallback_enabled and not wasm_enabled) or + (emterpreter_file_enabled and not emterpreter_enabled) + ): + continue + + expect_asmjs_code = asmjs_fallback_enabled and wasm_enabled + expect_emterpretify_file = emterpreter_file_enabled + expect_meminit = (meminit1_enabled and not wasm_enabled) or (wasm_enabled and asmjs_fallback_enabled) + expect_success = not (emterpreter_file_enabled and single_file_enabled) + expect_wasm = wasm_enabled + expect_wast = debug_enabled and wasm_enabled + + # currently, the emterpreter always fails with JS output since we do not preload the emterpreter file, which in non-HTML we would need to do manually + should_run_js = expect_success and not emterpreter_enabled + + cmd = [PYTHON, EMCC, path_from_root('tests', 'hello_world.c')] + + if single_file_enabled: + expect_asmjs_code = False + expect_emterpretify_file = False + expect_meminit = False + expect_wasm = False + expect_wast = False + cmd += ['-s', 'SINGLE_FILE=1'] + if meminit1_enabled: + cmd += ['--memory-init-file', '1'] + if debug_enabled: + cmd += ['-g'] + if emterpreter_enabled: + cmd += ['-s', 'EMTERPRETIFY=1'] + if emterpreter_file_enabled: + cmd += ['-s', "EMTERPRETIFY_FILE='a.out.dat'"] + if closure_enabled: + cmd += ['--closure', '1'] + if wasm_enabled: + method = 'interpret-binary' + if asmjs_fallback_enabled: + method += ',asmjs' + cmd += ['-s', 'WASM=1', '-s', "BINARYEN_METHOD='" + method + "'"] + + print ' '.join(cmd) + self.clear() + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + output, err = proc.communicate() + print(os.listdir('.')) + assert expect_success == (proc.returncode == 0) + assert expect_asmjs_code == os.path.exists('a.out.asm.js') + assert expect_emterpretify_file == os.path.exists('a.out.dat') + assert expect_meminit == (os.path.exists('a.out.mem') or os.path.exists('a.out.js.mem')) + assert expect_wasm == os.path.exists('a.out.wasm') + assert expect_wast == os.path.exists('a.out.wast') + if should_run_js: + self.assertContained('hello, world!', run_js('a.out.js')) diff --git a/tools/shared.py b/tools/shared.py index c9de1c6ba98a..4e53c3fe8fde 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -1,6 +1,6 @@ from __future__ import print_function from toolchain_profiler import ToolchainProfiler -import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, hashlib, cPickle, re, errno +import shutil, time, os, sys, base64, json, tempfile, copy, shlex, atexit, subprocess, hashlib, cPickle, re, errno from subprocess import Popen, PIPE, STDOUT from tempfile import mkstemp from distutils.spawn import find_executable @@ -2326,6 +2326,12 @@ def reconfigure_cache(): global Cache Cache = cache.Cache(debug=DEBUG_CACHE) +# Placeholder strings used for SINGLE_FILE +class FilenameReplacementStrings: + WASM_TEXT_FILE = '{{{ FILENAME_REPLACEMENT_STRINGS_WASM_TEXT_FILE }}}' + WASM_BINARY_FILE = '{{{ FILENAME_REPLACEMENT_STRINGS_WASM_BINARY_FILE }}}' + ASMJS_CODE_FILE = '{{{ FILENAME_REPLACEMENT_STRINGS_ASMJS_CODE_FILE }}}' + class JS(object): memory_initializer_pattern = '/\* memory initializer \*/ allocate\(\[([\d, ]*)\], "i8", ALLOC_NONE, ([\d+Runtime\.GLOBAL_BASEHgb]+)\);' no_memory_initializer_pattern = '/\* no memory initializer \*/' @@ -2338,6 +2344,19 @@ class JS(object): def to_nice_ident(ident): # limited version of the JS function toNiceIdent return ident.replace('%', '$').replace('@', '_').replace('.', '_') + # Returns the subresource location for run-time access + @staticmethod + def get_subresource_location(path, data_uri=None): + if data_uri is None: + data_uri = Settings.SINGLE_FILE + if data_uri: + f = open(path, 'rb') + data = base64.b64encode(f.read()) + f.close() + return 'data:application/octet-stream;base64,' + data + else: + return os.path.basename(path) + @staticmethod def make_initializer(sig, settings=None): settings = settings or Settings @@ -2450,75 +2469,6 @@ def align(x, by): while x % by != 0: x += 1 return x - INITIALIZER_CHUNK_SIZE = 10240 - - @staticmethod - def collect_initializers(src): - ret = [] - max_offset = -1 - for init in re.finditer(JS.memory_initializer_pattern, src): - contents = init.group(1).split(',') - offset = sum([int(x) if x[0] != 'R' else 0 for x in init.group(2).split('+')]) - ret.append((offset, contents)) - assert offset > max_offset - max_offset = offset - return ret - - @staticmethod - def split_initializer(contents): - # given a memory initializer (see memory_initializer_pattern), split it up into multiple initializers to avoid long runs of zeros or a single overly-large allocator - ret = [] - l = len(contents) - maxx = JS.INITIALIZER_CHUNK_SIZE - i = 0 - start = 0 - while 1: - if i - start >= maxx or (i > start and i == l): - #print >> sys.stderr, 'new', start, i-start - ret.append((start, contents[start:i])) - start = i - if i == l: break - if contents[i] != '0': - i += 1 - else: - # look for a sequence of zeros - j = i + 1 - while j < l and contents[j] == '0': j += 1 - if j-i > maxx/10 or j-start >= maxx: - #print >> sys.stderr, 'skip', start, i-start, j-start - ret.append((start, contents[start:i])) # skip over the zeros starting at i and ending at j - start = j - i = j - return ret - - @staticmethod - def replace_initializers(src, inits): - class State(object): - first = True - def rep(m): - if not State.first: return '' - # write out all the new initializers in place of the first old one - State.first = False - def gen_init(init): - offset, contents = init - return '/* memory initializer */ allocate([%s], "i8", ALLOC_NONE, Runtime.GLOBAL_BASE%s);' % ( - ','.join(contents), - '' if offset == 0 else ('+%d' % offset) - ) - return '\n'.join(map(gen_init, inits)) - return re.sub(JS.memory_initializer_pattern, rep, src) - - @staticmethod - def optimize_initializer(src): - inits = JS.collect_initializers(src) - if len(inits) == 0: return None - assert len(inits) == 1 - init = inits[0] - offset, contents = init - assert offset == 0 # offset 0, singleton - if len(contents) <= JS.INITIALIZER_CHUNK_SIZE: return None - return JS.replace_initializers(src, JS.split_initializer(contents)) - @staticmethod def generate_string_initializer(s): if Settings.ASSERTIONS: