From a888f12cae3b3390ec697475c581b9df4057ab78 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Wed, 30 Sep 2020 08:31:39 -0700 Subject: [PATCH 1/3] Add EM_COMPILER_WRAPPER as replacement for EMMAKEN_COMPILER This allows a compiler wrapper to be installed around the underlying clang, but without changing the location of the llvm installation or having to know where it is. This is in contrast to other option LLVM_ROOT which points emscripten at a different llvm build (and also replaces not just clang but all the other llvm tools). This mimikes the corresponding CMAKE option CMAKE__COMPILER_WRAPPER. The reason we allow the inner compiler to be wrapped like this is that distributing the whole of emscripten to a distcc farm or gomacc farm would otherwise be required. Fixes: #12340 --- emcc.py | 33 +++++++++++-------- tests/runner.py | 6 +++- tests/test_other.py | 15 ++++++++- tests/test_sanity.py | 29 ++++++++-------- .../site_tools/emscripten/emscripten.py | 2 +- tools/shared.py | 2 ++ 6 files changed, 54 insertions(+), 33 deletions(-) diff --git a/emcc.py b/emcc.py index c7e8a58344b00..eeaa46017b5f5 100755 --- a/emcc.py +++ b/emcc.py @@ -725,15 +725,21 @@ def run(args): ''' % (shared.EMSCRIPTEN_VERSION, revision)) return 0 - if run_via_emxx: - clang = shared.CLANG_CXX - else: - clang = shared.CLANG_CC + CXX = [shared.CLANG_CXX] + CC = [shared.CLANG_CC] + if shared.COMPILER_WRAPPER: + logger.debug('using compiler wrapper: %s', shared.COMPILER_WRAPPER) + CXX.insert(0, shared.COMPILER_WRAPPER) + CC.insert(0, shared.COMPILER_WRAPPER) if len(args) == 1 and args[0] == '-v': # -v with no inputs # autoconf likes to see 'GNU' in the output to enable shared object support print('emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) %s' % shared.EMSCRIPTEN_VERSION, file=sys.stderr) - code = run_process([clang, '-v'], check=False).returncode + if run_via_emxx: + clang = CXX + else: + clang = CC + code = shared.check_call(clang + ['-v'], check=False).returncode shared.check_sanity(force=True) return code @@ -850,12 +856,12 @@ def need_llvm_debug_info(): options, settings_changes, user_js_defines, newargs = parse_args(newargs) - CXX = shared.CLANG_CXX - CC = shared.CLANG_CC if 'EMMAKEN_COMPILER' in os.environ: - diagnostics.warning('deprecated', 'EMMAKEN_COMPILER is deprecated. Please set LLVM_ROOT in config file or EM_LLVM_ROOT in the environment') - CXX = os.environ['EMMAKEN_COMPILER'] - CC = cxx_to_c_compiler(CXX) + diagnostics.warning('deprecated', '`EMMAKEN_COMPILER` is deprecated.\n' + 'To use an alteranative LLVM build set `LLVM_ROOT` in the config file (or `EM_LLVM_ROOT` env var).\n' + 'To wrap invocations of clang use the `COMPILER_WRAPPER` setting (or `EM_COMPILER_WRAPPER` env var.\n') + CXX = [os.environ['EMMAKEN_COMPILER']] + CC = [cxx_to_c_compiler(CXX)] if '-print-search-dirs' in newargs: return run_process([CC, '-print-search-dirs'], check=False).returncode @@ -1919,12 +1925,12 @@ def get_compiler(cxx): def get_clang_command(src_file): cxx = use_cxx(src_file) base_cflags = shared.get_cflags(args, cxx) - cmd = [get_compiler(cxx)] + base_cflags + cflags + compile_args + [src_file] + cmd = get_compiler(cxx) + base_cflags + cflags + compile_args + [src_file] return system_libs.process_args(cmd, shared.Settings) def get_clang_command_asm(src_file): asflags = shared.get_asmflags() - return [get_compiler(use_cxx(src_file))] + asflags + compile_args + [src_file] + return get_compiler(use_cxx(src_file)) + asflags + compile_args + [src_file] # preprocessor-only (-E) support if has_dash_E or '-M' in newargs or '-MM' in newargs or '-fsyntax-only' in newargs: @@ -1949,9 +1955,8 @@ def get_clang_command_asm(src_file): if not header.endswith(HEADER_ENDINGS): exit_with_error('cannot mix precompile headers with non-header inputs: ' + str(headers) + ' : ' + header) cxx = use_cxx(header) - compiler = get_compiler(cxx) base_cflags = shared.get_cflags(args, cxx) - cmd = [compiler] + base_cflags + cflags + compile_args + [header] + cmd = get_compiler(cxx) + base_cflags + cflags + compile_args + [header] if specified_target: cmd += ['-o', specified_target] cmd = system_libs.process_args(cmd, shared.Settings) diff --git a/tests/runner.py b/tests/runner.py index e36ec3a2d9c54..4059c73d7af1c 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -44,7 +44,7 @@ import shutil import string import subprocess -import sys +import stat import tempfile import time import unittest @@ -273,6 +273,10 @@ def create_test_file(name, contents, binary=False): f.write(contents) +def make_executable(name): + os.chmod(name, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + + # The core test modes core_test_modes = [ 'wasm0', diff --git a/tests/test_other.py b/tests/test_other.py index f8ae391864101..85f0917b19ea1 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -32,7 +32,7 @@ from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, LLVM_ROOT, EM_BUILD_VERBOSE from tools.shared import CLANG_CC, CLANG_CXX, LLVM_AR, LLVM_DWARFDUMP from tools.shared import NODE_JS, JS_ENGINES, WASM_ENGINES, V8_ENGINE -from runner import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled +from runner import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled, make_executable from runner import env_modify, no_windows, requires_native_clang, chdir, with_env_modify, create_test_file, parameterized from runner import js_engines_modify, NON_ZERO from tools import shared, building @@ -9411,6 +9411,19 @@ def test_emmaken_compiler(self): stderr = self.run_process([EMCC, '-c', path_from_root('tests', 'core', 'test_hello_world.c')], stderr=PIPE).stderr self.assertContained('warning: EMMAKEN_COMPILER is deprecated', stderr) + @no_windows('relies of shell script') + def test_compiler_wrapper(self): + create_test_file('wrapper.sh', '''\ +#!/bin/sh +echo "wrapping compiler call: $@" +exec "$@" + ''') + make_executable('wrapper.sh') + with env_modify({'EM_COMPILER_WRAPPER': './wrapper.sh'}): + stdout = self.run_process([EMCC, '-c', path_from_root('tests', 'core', 'test_hello_world.c')], stdout=PIPE).stdout + self.assertContained('wrapping compiler call: ', stdout) + self.assertExists('test_hello_world.o') + def test_llvm_option_dash_o(self): # emcc used to interpret -mllvm's option value as the output file if it # began with -o diff --git a/tests/test_sanity.py b/tests/test_sanity.py index 5f8acea62ab85..a5e3be9c43800 100644 --- a/tests/test_sanity.py +++ b/tests/test_sanity.py @@ -6,7 +6,6 @@ import os import platform import shutil -import stat import time import re import tempfile @@ -14,7 +13,7 @@ from subprocess import PIPE, STDOUT from runner import RunnerCore, path_from_root, env_modify, chdir -from runner import create_test_file, ensure_dir +from runner import create_test_file, ensure_dir, make_executable from tools.shared import NODE_JS, PYTHON, EMCC, SPIDERMONKEY_ENGINE, V8_ENGINE from tools.shared import CONFIG_FILE, EM_CONFIG, LLVM_ROOT, CANONICAL_TEMP_DIR from tools.shared import try_delete @@ -65,9 +64,7 @@ def make_fake_wasm_opt(filename, version): f.write('#!/bin/sh\n') f.write('echo "wasm-opt version %s"\n' % version) f.write('echo "..."\n') - shutil.copyfile(filename, filename + '++') - os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) - os.chmod(filename + '++', stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(filename) def make_fake_clang(filename, version): @@ -81,8 +78,8 @@ def make_fake_clang(filename, version): f.write('echo "clang version %s"\n' % version) f.write('echo "..."\n') shutil.copyfile(filename, filename + '++') - os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) - os.chmod(filename + '++', stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(filename) + make_executable(filename + '++') def make_fake_llc(filename, targets): @@ -94,7 +91,7 @@ def make_fake_llc(filename, targets): with open(filename, 'w') as f: f.write('#!/bin/sh\n') f.write('echo "llc fake output\nRegistered Targets:\n%s"' % targets) - os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(filename) def make_fake_lld(filename): @@ -102,7 +99,7 @@ def make_fake_lld(filename): with open(filename, 'w') as f: f.write('#!/bin/sh\n') f.write('exit 0\n') - os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(filename) SANITY_MESSAGE = 'Emscripten: Running sanity checks' @@ -184,9 +181,9 @@ def test_firstrun(self): for command in commands: wipe() - def make_executable(name): - with open(os.path.join(temp_bin, name), 'w') as f: - os.fchmod(f.fileno(), stat.S_IRWXU) + def make_new_executable(name): + open(os.path.join(temp_bin, name), 'w').close() + make_executable(os.path.join(temp_bin, name)) env = os.environ.copy() if 'EM_CONFIG' in env: @@ -194,8 +191,8 @@ def make_executable(name): try: temp_bin = tempfile.mkdtemp() - make_executable('llvm-dis') - make_executable('node') + make_new_executable('llvm-dis') + make_new_executable('node') env['PATH'] = temp_bin + os.pathsep + os.environ['PATH'] output = self.do(command, env=env) finally: @@ -326,7 +323,7 @@ def test_node(self): fi ''' % (version, NODE_JS)) f.close() - os.chmod(self.in_dir('fake', 'nodejs'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(self.in_dir('fake', 'nodejs')) if not succeed: if version[0] == 'v': self.check_working(EMCC, NODE_WARNING) @@ -626,7 +623,7 @@ def test_js_engine_path(self): with open(test_engine_path, 'w') as f: f.write('#!/bin/sh\n') f.write('exec %s $@\n' % (engine)) - os.chmod(test_engine_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + make_executable(test_engine_path) out = self.run_js(sample_script, engine=test_engine_path, args=['--foo']) diff --git a/tools/scons/site_scons/site_tools/emscripten/emscripten.py b/tools/scons/site_scons/site_tools/emscripten/emscripten.py index f963348c01695..7ff743798c218 100644 --- a/tools/scons/site_scons/site_tools/emscripten/emscripten.py +++ b/tools/scons/site_scons/site_tools/emscripten/emscripten.py @@ -22,7 +22,7 @@ def generate(env, emscripten_path=None, **kw): # by Emscripten to the child. for var in ['EM_CACHE', 'EMCC_DEBUG', 'EMTEST_BROWSER', 'EMMAKEN_JUST_CONFIGURE', 'EMCC_CFLAGS', 'EMCC_TEMP_DIR', - 'EMCC_AUTODEBUG', + 'EMCC_AUTODEBUG', 'EM_COMPILER_WRAPPER', 'EMMAKEN_COMPILER', 'EMMAKEN_CFLAGS', 'EMCC_JSOPT_BLACKLIST', 'MOZ_DISABLE_AUTO_SAFE_MODE', 'EMCC_STDERR_FILE', 'EMSCRIPTEN_SUPPRESS_USAGE_WARNING', 'NODE_PATH', 'EMCC_JSOPT_MIN_CHUNK_SIZE', diff --git a/tools/shared.py b/tools/shared.py index c0124d817652c..9e28f4a738d9b 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -362,6 +362,7 @@ def parse_config_file(): 'FROZEN_CACHE', 'CACHE', 'PORTS', + 'COMPILER_WRAPPER', ) # Only propagate certain settings from the config file. @@ -1473,6 +1474,7 @@ def read_and_preprocess(filename, expand_macros=False): CACHE = None PORTS = None FROZEN_CACHE = False +COMPILER_WRAPPER = None # Emscripten compiler spawns other processes, which can reimport shared.py, so # make sure that those child processes get the same configuration file by From a067e8f91d5d2d94c623c04ddec97ccd0fa7f7bf Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Wed, 30 Sep 2020 09:17:27 -0700 Subject: [PATCH 2/3] feedback --- .../source/docs/compiling/Building-Projects.rst | 17 +++++++++++++++-- tests/test_other.py | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/site/source/docs/compiling/Building-Projects.rst b/site/source/docs/compiling/Building-Projects.rst index 995e4c77d9512..95450f0ddb52e 100644 --- a/site/source/docs/compiling/Building-Projects.rst +++ b/site/source/docs/compiling/Building-Projects.rst @@ -327,6 +327,21 @@ Emscripten provides the following preprocessor macros that can be used to identi * If targeting the pthreads multithreading support with the compiler & linker flag ``-s USE_PTHREADS=1``, the preprocessor define ``__EMSCRIPTEN_PTHREADS__`` will be present. +Using a compiler wrapper +======================== + +Sometimes it can be useful to use a compiler wrapper in order to do things like +``ccache``, ``distcc`` or ``gomacc``. For ``ccache`` the normal method of +simply wrapping the entire compiler should work, e.g. ``ccache emcc``. For +distributed builds it can be beneficial to run the emscripten driver locally and +distribute only the underlying clang commands. If this is desirable, the +``COMPILER_WRAPPER`` setting in the config file can be used to add a wrapper +around the internal calls to clang. Like other config settings this can also be +set via an environment variable. e.g:: + + EM_COMPILER_WRAPPER=gomacc emcc -c hello.c + + Examples / test code ==================== @@ -335,8 +350,6 @@ The Emscripten test suite (`tests/runner.py `_ project. - - Troubleshooting =============== diff --git a/tests/test_other.py b/tests/test_other.py index 85f0917b19ea1..b5ba00f1ec5b9 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -9411,7 +9411,7 @@ def test_emmaken_compiler(self): stderr = self.run_process([EMCC, '-c', path_from_root('tests', 'core', 'test_hello_world.c')], stderr=PIPE).stderr self.assertContained('warning: EMMAKEN_COMPILER is deprecated', stderr) - @no_windows('relies of shell script') + @no_windows('relies on a shell script') def test_compiler_wrapper(self): create_test_file('wrapper.sh', '''\ #!/bin/sh From cb4dc68d8c211c49b778760d9632fe2b32cb7e93 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Wed, 30 Sep 2020 09:21:13 -0700 Subject: [PATCH 3/3] . --- ChangeLog.md | 5 +++++ emcc.py | 15 ++++++++------- tests/test_other.py | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 3dc0fe52e465d..9fefcb399c3f2 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -17,6 +17,11 @@ See docs/process.md for how version tagging works. Current Trunk ------------- +- Add new `COMPILER_WRAPPER` settings (with corresponding `EM_COMPILER_WRAPPER` + environment variable. This replaces the existing `EMMAKEN_COMPILER` + environment variable which is deprecated, but still works for the time being. + The main differences is that `EM_COMPILER_WRAPPER` only wrapps the configured + version of clang rather than replacing it. - ASAN_SHADOW_SIZE is deprecated. When using AddressSanitizer, the correct amount of shadow memory will now be calculated automatically. diff --git a/emcc.py b/emcc.py index eeaa46017b5f5..49d02de18b167 100755 --- a/emcc.py +++ b/emcc.py @@ -732,14 +732,15 @@ def run(args): CXX.insert(0, shared.COMPILER_WRAPPER) CC.insert(0, shared.COMPILER_WRAPPER) + if run_via_emxx: + clang = shared.CLANG_CXX + else: + clang = shared.CLANG_CC + if len(args) == 1 and args[0] == '-v': # -v with no inputs # autoconf likes to see 'GNU' in the output to enable shared object support print('emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) %s' % shared.EMSCRIPTEN_VERSION, file=sys.stderr) - if run_via_emxx: - clang = CXX - else: - clang = CC - code = shared.check_call(clang + ['-v'], check=False).returncode + code = shared.check_call([clang, '-v'], check=False).returncode shared.check_sanity(force=True) return code @@ -861,10 +862,10 @@ def need_llvm_debug_info(): 'To use an alteranative LLVM build set `LLVM_ROOT` in the config file (or `EM_LLVM_ROOT` env var).\n' 'To wrap invocations of clang use the `COMPILER_WRAPPER` setting (or `EM_COMPILER_WRAPPER` env var.\n') CXX = [os.environ['EMMAKEN_COMPILER']] - CC = [cxx_to_c_compiler(CXX)] + CC = [cxx_to_c_compiler(os.environ['EMMAKEN_COMPILER'])] if '-print-search-dirs' in newargs: - return run_process([CC, '-print-search-dirs'], check=False).returncode + return run_process(CC + ['-print-search-dirs'], check=False).returncode if options.emrun: options.pre_js += open(shared.path_from_root('src', 'emrun_prejs.js')).read() + '\n' diff --git a/tests/test_other.py b/tests/test_other.py index b5ba00f1ec5b9..02e5a327c58a2 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -9409,7 +9409,7 @@ def test_getrusage(self): @with_env_modify({'EMMAKEN_COMPILER': shared.CLANG_CC}) def test_emmaken_compiler(self): stderr = self.run_process([EMCC, '-c', path_from_root('tests', 'core', 'test_hello_world.c')], stderr=PIPE).stderr - self.assertContained('warning: EMMAKEN_COMPILER is deprecated', stderr) + self.assertContained('warning: `EMMAKEN_COMPILER` is deprecated', stderr) @no_windows('relies on a shell script') def test_compiler_wrapper(self):