diff --git a/ChangeLog.md b/ChangeLog.md index 3dc0fe52e465d..9fefcb399c3f2 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -17,6 +17,11 @@ See docs/process.md for how version tagging works. Current Trunk ------------- +- Add new `COMPILER_WRAPPER` settings (with corresponding `EM_COMPILER_WRAPPER` + environment variable. This replaces the existing `EMMAKEN_COMPILER` + environment variable which is deprecated, but still works for the time being. + The main differences is that `EM_COMPILER_WRAPPER` only wrapps the configured + version of clang rather than replacing it. - ASAN_SHADOW_SIZE is deprecated. When using AddressSanitizer, the correct amount of shadow memory will now be calculated automatically. diff --git a/emcc.py b/emcc.py index c7e8a58344b00..49d02de18b167 100755 --- a/emcc.py +++ b/emcc.py @@ -725,6 +725,13 @@ def run(args): ''' % (shared.EMSCRIPTEN_VERSION, revision)) return 0 + CXX = [shared.CLANG_CXX] + CC = [shared.CLANG_CC] + if shared.COMPILER_WRAPPER: + logger.debug('using compiler wrapper: %s', shared.COMPILER_WRAPPER) + CXX.insert(0, shared.COMPILER_WRAPPER) + CC.insert(0, shared.COMPILER_WRAPPER) + if run_via_emxx: clang = shared.CLANG_CXX else: @@ -733,7 +740,7 @@ def run(args): if len(args) == 1 and args[0] == '-v': # -v with no inputs # autoconf likes to see 'GNU' in the output to enable shared object support print('emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) %s' % shared.EMSCRIPTEN_VERSION, file=sys.stderr) - code = run_process([clang, '-v'], check=False).returncode + code = shared.check_call([clang, '-v'], check=False).returncode shared.check_sanity(force=True) return code @@ -850,15 +857,15 @@ def need_llvm_debug_info(): options, settings_changes, user_js_defines, newargs = parse_args(newargs) - CXX = shared.CLANG_CXX - CC = shared.CLANG_CC if 'EMMAKEN_COMPILER' in os.environ: - diagnostics.warning('deprecated', 'EMMAKEN_COMPILER is deprecated. Please set LLVM_ROOT in config file or EM_LLVM_ROOT in the environment') - CXX = os.environ['EMMAKEN_COMPILER'] - CC = cxx_to_c_compiler(CXX) + diagnostics.warning('deprecated', '`EMMAKEN_COMPILER` is deprecated.\n' + 'To use an alteranative LLVM build set `LLVM_ROOT` in the config file (or `EM_LLVM_ROOT` env var).\n' + 'To wrap invocations of clang use the `COMPILER_WRAPPER` setting (or `EM_COMPILER_WRAPPER` env var.\n') + CXX = [os.environ['EMMAKEN_COMPILER']] + CC = [cxx_to_c_compiler(os.environ['EMMAKEN_COMPILER'])] if '-print-search-dirs' in newargs: - return run_process([CC, '-print-search-dirs'], check=False).returncode + return run_process(CC + ['-print-search-dirs'], check=False).returncode if options.emrun: options.pre_js += open(shared.path_from_root('src', 'emrun_prejs.js')).read() + '\n' @@ -1919,12 +1926,12 @@ def get_compiler(cxx): def get_clang_command(src_file): cxx = use_cxx(src_file) base_cflags = shared.get_cflags(args, cxx) - cmd = [get_compiler(cxx)] + base_cflags + cflags + compile_args + [src_file] + cmd = get_compiler(cxx) + base_cflags + cflags + compile_args + [src_file] return system_libs.process_args(cmd, shared.Settings) def get_clang_command_asm(src_file): asflags = shared.get_asmflags() - return [get_compiler(use_cxx(src_file))] + asflags + compile_args + [src_file] + return get_compiler(use_cxx(src_file)) + asflags + compile_args + [src_file] # preprocessor-only (-E) support if has_dash_E or '-M' in newargs or '-MM' in newargs or '-fsyntax-only' in newargs: @@ -1949,9 +1956,8 @@ def get_clang_command_asm(src_file): if not header.endswith(HEADER_ENDINGS): exit_with_error('cannot mix precompile headers with non-header inputs: ' + str(headers) + ' : ' + header) cxx = use_cxx(header) - compiler = get_compiler(cxx) base_cflags = shared.get_cflags(args, cxx) - cmd = [compiler] + base_cflags + cflags + compile_args + [header] + cmd = get_compiler(cxx) + base_cflags + cflags + compile_args + [header] if specified_target: cmd += ['-o', specified_target] cmd = system_libs.process_args(cmd, shared.Settings) diff --git a/site/source/docs/compiling/Building-Projects.rst b/site/source/docs/compiling/Building-Projects.rst index 995e4c77d9512..95450f0ddb52e 100644 --- a/site/source/docs/compiling/Building-Projects.rst +++ b/site/source/docs/compiling/Building-Projects.rst @@ -327,6 +327,21 @@ Emscripten provides the following preprocessor macros that can be used to identi * If targeting the pthreads multithreading support with the compiler & linker flag ``-s USE_PTHREADS=1``, the preprocessor define ``__EMSCRIPTEN_PTHREADS__`` will be present. +Using a compiler wrapper +======================== + +Sometimes it can be useful to use a compiler wrapper in order to do things like +``ccache``, ``distcc`` or ``gomacc``. For ``ccache`` the normal method of +simply wrapping the entire compiler should work, e.g. ``ccache emcc``. For +distributed builds it can be beneficial to run the emscripten driver locally and +distribute only the underlying clang commands. If this is desirable, the +``COMPILER_WRAPPER`` setting in the config file can be used to add a wrapper +around the internal calls to clang. Like other config settings this can also be +set via an environment variable. e.g:: + + EM_COMPILER_WRAPPER=gomacc emcc -c hello.c + + Examples / test code ==================== @@ -335,8 +350,6 @@ The Emscripten test suite (`tests/runner.py `_ project. - - Troubleshooting =============== diff --git a/tests/runner.py b/tests/runner.py index e36ec3a2d9c54..4059c73d7af1c 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -44,7 +44,7 @@ import shutil import string import subprocess -import sys +import stat import tempfile import time import unittest @@ -273,6 +273,10 @@ def create_test_file(name, contents, binary=False): f.write(contents) +def make_executable(name): + os.chmod(name, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + + # The core test modes core_test_modes = [ 'wasm0', diff --git a/tests/test_other.py b/tests/test_other.py index f8ae391864101..02e5a327c58a2 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -32,7 +32,7 @@ from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, LLVM_ROOT, EM_BUILD_VERBOSE from tools.shared import CLANG_CC, CLANG_CXX, LLVM_AR, LLVM_DWARFDUMP from tools.shared import NODE_JS, JS_ENGINES, WASM_ENGINES, V8_ENGINE -from runner import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled +from runner import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled, make_executable from runner import env_modify, no_windows, requires_native_clang, chdir, with_env_modify, create_test_file, parameterized from runner import js_engines_modify, NON_ZERO from tools import shared, building @@ -9409,7 +9409,20 @@ def test_getrusage(self): @with_env_modify({'EMMAKEN_COMPILER': shared.CLANG_CC}) def test_emmaken_compiler(self): stderr = self.run_process([EMCC, '-c', path_from_root('tests', 'core', 'test_hello_world.c')], stderr=PIPE).stderr - self.assertContained('warning: EMMAKEN_COMPILER is deprecated', stderr) + self.assertContained('warning: `EMMAKEN_COMPILER` is deprecated', stderr) + + @no_windows('relies on a shell script') + def test_compiler_wrapper(self): + create_test_file('wrapper.sh', '''\ +#!/bin/sh +echo "wrapping compiler call: $@" +exec "$@" + ''') + make_executable('wrapper.sh') + with env_modify({'EM_COMPILER_WRAPPER': './wrapper.sh'}): + stdout = self.run_process([EMCC, '-c', path_from_root('tests', 'core', 'test_hello_world.c')], stdout=PIPE).stdout + self.assertContained('wrapping compiler call: ', stdout) + self.assertExists('test_hello_world.o') def test_llvm_option_dash_o(self): # emcc used to interpret -mllvm's option value as the output file if it diff --git a/tests/test_sanity.py b/tests/test_sanity.py index 5f8acea62ab85..a5e3be9c43800 100644 --- a/tests/test_sanity.py +++ b/tests/test_sanity.py @@ -6,7 +6,6 @@ import os import platform import shutil -import stat import time import re import tempfile @@ -14,7 +13,7 @@ from subprocess import PIPE, STDOUT from runner import RunnerCore, path_from_root, env_modify, chdir -from runner import create_test_file, ensure_dir +from runner import create_test_file, ensure_dir, make_executable from tools.shared import NODE_JS, PYTHON, EMCC, SPIDERMONKEY_ENGINE, V8_ENGINE from tools.shared import CONFIG_FILE, EM_CONFIG, LLVM_ROOT, CANONICAL_TEMP_DIR from tools.shared import try_delete @@ -65,9 +64,7 @@ def make_fake_wasm_opt(filename, version): f.write('#!/bin/sh\n') f.write('echo "wasm-opt version %s"\n' % version) f.write('echo "..."\n') - shutil.copyfile(filename, filename + '++') - os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) - os.chmod(filename + '++', stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(filename) def make_fake_clang(filename, version): @@ -81,8 +78,8 @@ def make_fake_clang(filename, version): f.write('echo "clang version %s"\n' % version) f.write('echo "..."\n') shutil.copyfile(filename, filename + '++') - os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) - os.chmod(filename + '++', stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(filename) + make_executable(filename + '++') def make_fake_llc(filename, targets): @@ -94,7 +91,7 @@ def make_fake_llc(filename, targets): with open(filename, 'w') as f: f.write('#!/bin/sh\n') f.write('echo "llc fake output\nRegistered Targets:\n%s"' % targets) - os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(filename) def make_fake_lld(filename): @@ -102,7 +99,7 @@ def make_fake_lld(filename): with open(filename, 'w') as f: f.write('#!/bin/sh\n') f.write('exit 0\n') - os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(filename) SANITY_MESSAGE = 'Emscripten: Running sanity checks' @@ -184,9 +181,9 @@ def test_firstrun(self): for command in commands: wipe() - def make_executable(name): - with open(os.path.join(temp_bin, name), 'w') as f: - os.fchmod(f.fileno(), stat.S_IRWXU) + def make_new_executable(name): + open(os.path.join(temp_bin, name), 'w').close() + make_executable(os.path.join(temp_bin, name)) env = os.environ.copy() if 'EM_CONFIG' in env: @@ -194,8 +191,8 @@ def make_executable(name): try: temp_bin = tempfile.mkdtemp() - make_executable('llvm-dis') - make_executable('node') + make_new_executable('llvm-dis') + make_new_executable('node') env['PATH'] = temp_bin + os.pathsep + os.environ['PATH'] output = self.do(command, env=env) finally: @@ -326,7 +323,7 @@ def test_node(self): fi ''' % (version, NODE_JS)) f.close() - os.chmod(self.in_dir('fake', 'nodejs'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(self.in_dir('fake', 'nodejs')) if not succeed: if version[0] == 'v': self.check_working(EMCC, NODE_WARNING) @@ -626,7 +623,7 @@ def test_js_engine_path(self): with open(test_engine_path, 'w') as f: f.write('#!/bin/sh\n') f.write('exec %s $@\n' % (engine)) - os.chmod(test_engine_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(test_engine_path) out = self.run_js(sample_script, engine=test_engine_path, args=['--foo']) diff --git a/tools/scons/site_scons/site_tools/emscripten/emscripten.py b/tools/scons/site_scons/site_tools/emscripten/emscripten.py index f963348c01695..7ff743798c218 100644 --- a/tools/scons/site_scons/site_tools/emscripten/emscripten.py +++ b/tools/scons/site_scons/site_tools/emscripten/emscripten.py @@ -22,7 +22,7 @@ def generate(env, emscripten_path=None, **kw): # by Emscripten to the child. for var in ['EM_CACHE', 'EMCC_DEBUG', 'EMTEST_BROWSER', 'EMMAKEN_JUST_CONFIGURE', 'EMCC_CFLAGS', 'EMCC_TEMP_DIR', - 'EMCC_AUTODEBUG', + 'EMCC_AUTODEBUG', 'EM_COMPILER_WRAPPER', 'EMMAKEN_COMPILER', 'EMMAKEN_CFLAGS', 'EMCC_JSOPT_BLACKLIST', 'MOZ_DISABLE_AUTO_SAFE_MODE', 'EMCC_STDERR_FILE', 'EMSCRIPTEN_SUPPRESS_USAGE_WARNING', 'NODE_PATH', 'EMCC_JSOPT_MIN_CHUNK_SIZE', diff --git a/tools/shared.py b/tools/shared.py index c0124d817652c..9e28f4a738d9b 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -362,6 +362,7 @@ def parse_config_file(): 'FROZEN_CACHE', 'CACHE', 'PORTS', + 'COMPILER_WRAPPER', ) # Only propagate certain settings from the config file. @@ -1473,6 +1474,7 @@ def read_and_preprocess(filename, expand_macros=False): CACHE = None PORTS = None FROZEN_CACHE = False +COMPILER_WRAPPER = None # Emscripten compiler spawns other processes, which can reimport shared.py, so # make sure that those child processes get the same configuration file by