Add EM_COMPILER_WRAPPER as replacement for EMMAKEN_COMPILER

ChangeLog.md

@@ -17,6 +17,11 @@ See docs/process.md for how version tagging works.
 
 Current Trunk
 -------------
+- Add new `COMPILER_WRAPPER` settings (with corresponding `EM_COMPILER_WRAPPER`
+  environment variable.  This replaces the existing `EMMAKEN_COMPILER`
+  environment variable which is deprecated, but still works for the time being.
+  The main differences is that `EM_COMPILER_WRAPPER` only wrapps the configured
+  version of clang rather than replacing it.
 - ASAN_SHADOW_SIZE is deprecated. When using AddressSanitizer, the correct
   amount of shadow memory will now be calculated automatically.
 

emcc.py

@@ -725,6 +725,13 @@ def run(args):
   ''' % (shared.EMSCRIPTEN_VERSION, revision))
     return 0
 
+  CXX = [shared.CLANG_CXX]
+  CC = [shared.CLANG_CC]
+  if shared.COMPILER_WRAPPER:
+    logger.debug('using compiler wrapper: %s', shared.COMPILER_WRAPPER)
+    CXX.insert(0, shared.COMPILER_WRAPPER)
+    CC.insert(0, shared.COMPILER_WRAPPER)
+
   if run_via_emxx:
     clang = shared.CLANG_CXX
   else:
@@ -733,7 +740,7 @@ def run(args):
   if len(args) == 1 and args[0] == '-v': # -v with no inputs
     # autoconf likes to see 'GNU' in the output to enable shared object support
     print('emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) %s' % shared.EMSCRIPTEN_VERSION, file=sys.stderr)
-    code = run_process([clang, '-v'], check=False).returncode
+    code = shared.check_call([clang, '-v'], check=False).returncode
     shared.check_sanity(force=True)
     return code
 
@@ -850,15 +857,15 @@ def need_llvm_debug_info():
 
     options, settings_changes, user_js_defines, newargs = parse_args(newargs)
 
-    CXX = shared.CLANG_CXX
-    CC = shared.CLANG_CC
     if 'EMMAKEN_COMPILER' in os.environ:
-      diagnostics.warning('deprecated', 'EMMAKEN_COMPILER is deprecated.  Please set LLVM_ROOT in config file or EM_LLVM_ROOT in the environment')
-      CXX = os.environ['EMMAKEN_COMPILER']
-      CC = cxx_to_c_compiler(CXX)
+      diagnostics.warning('deprecated', '`EMMAKEN_COMPILER` is deprecated.\n'
+                          'To use an alteranative LLVM build set `LLVM_ROOT` in the config file (or `EM_LLVM_ROOT` env var).\n'
+                          'To wrap invocations of clang use the `COMPILER_WRAPPER` setting (or `EM_COMPILER_WRAPPER` env var.\n')
+      CXX = [os.environ['EMMAKEN_COMPILER']]
+      CC = [cxx_to_c_compiler(os.environ['EMMAKEN_COMPILER'])]
 
     if '-print-search-dirs' in newargs:
-      return run_process([CC, '-print-search-dirs'], check=False).returncode
+      return run_process(CC + ['-print-search-dirs'], check=False).returncode
 
     if options.emrun:
       options.pre_js += open(shared.path_from_root('src', 'emrun_prejs.js')).read() + '\n'
@@ -1919,12 +1926,12 @@ def get_compiler(cxx):
       def get_clang_command(src_file):
         cxx = use_cxx(src_file)
         base_cflags = shared.get_cflags(args, cxx)
-        cmd = [get_compiler(cxx)] + base_cflags + cflags + compile_args + [src_file]
+        cmd = get_compiler(cxx) + base_cflags + cflags + compile_args + [src_file]
         return system_libs.process_args(cmd, shared.Settings)
 
       def get_clang_command_asm(src_file):
         asflags = shared.get_asmflags()
-        return [get_compiler(use_cxx(src_file))] + asflags + compile_args + [src_file]
+        return get_compiler(use_cxx(src_file)) + asflags + compile_args + [src_file]
 
       # preprocessor-only (-E) support
       if has_dash_E or '-M' in newargs or '-MM' in newargs or '-fsyntax-only' in newargs:
@@ -1949,9 +1956,8 @@ def get_clang_command_asm(src_file):
           if not header.endswith(HEADER_ENDINGS):
             exit_with_error('cannot mix precompile headers with non-header inputs: ' + str(headers) + ' : ' + header)
           cxx = use_cxx(header)
-          compiler = get_compiler(cxx)
           base_cflags = shared.get_cflags(args, cxx)
-          cmd = [compiler] + base_cflags + cflags + compile_args + [header]
+          cmd = get_compiler(cxx) + base_cflags + cflags + compile_args + [header]
           if specified_target:
             cmd += ['-o', specified_target]
           cmd = system_libs.process_args(cmd, shared.Settings)

site/source/docs/compiling/Building-Projects.rst

@@ -327,6 +327,21 @@ Emscripten provides the following preprocessor macros that can be used to identi
  * If targeting the pthreads multithreading support with the compiler & linker flag ``-s USE_PTHREADS=1``, the preprocessor define ``__EMSCRIPTEN_PTHREADS__`` will be present.
 
 
+Using a compiler wrapper
+========================
+
+Sometimes it can be useful to use a compiler wrapper in order to do things like
+``ccache``, ``distcc`` or ``gomacc``.  For ``ccache`` the normal method of
+simply wrapping the entire compiler should work, e.g. ``ccache emcc``.  For
+distributed builds it can be beneficial to run the emscripten driver locally and
+distribute only the underlying clang commands.  If this is desirable, the
+``COMPILER_WRAPPER`` setting in the config file can be used to add a wrapper
+around the internal calls to clang.  Like other config settings this can also be
+set via an environment variable.  e.g::
+
+  EM_COMPILER_WRAPPER=gomacc emcc -c hello.c
+
+
 Examples / test code
 ====================
 
@@ -335,8 +350,6 @@ The Emscripten test suite (`tests/runner.py <https://github.com/emscripten-core/
 It is also worth looking at the build scripts in the `ammo.js <https://github.com/kripken/ammo.js/blob/master/make.py>`_ project.
 
 
-
-
 Troubleshooting
 ===============
 

tests/runner.py

@@ -44,7 +44,7 @@
 import shutil
 import string
 import subprocess
-import sys
+import stat
 import tempfile
 import time
 import unittest
@@ -273,6 +273,10 @@ def create_test_file(name, contents, binary=False):
     f.write(contents)
 
 
+def make_executable(name):
+  os.chmod(name, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+
+
 # The core test modes
 core_test_modes = [
   'wasm0',

tests/test_other.py

@@ -32,7 +32,7 @@
 from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, LLVM_ROOT, EM_BUILD_VERBOSE
 from tools.shared import CLANG_CC, CLANG_CXX, LLVM_AR, LLVM_DWARFDUMP
 from tools.shared import NODE_JS, JS_ENGINES, WASM_ENGINES, V8_ENGINE
-from runner import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled
+from runner import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled, make_executable
 from runner import env_modify, no_windows, requires_native_clang, chdir, with_env_modify, create_test_file, parameterized
 from runner import js_engines_modify, NON_ZERO
 from tools import shared, building
@@ -9409,7 +9409,20 @@ def test_getrusage(self):
   @with_env_modify({'EMMAKEN_COMPILER': shared.CLANG_CC})
   def test_emmaken_compiler(self):
     stderr = self.run_process([EMCC, '-c', path_from_root('tests', 'core', 'test_hello_world.c')], stderr=PIPE).stderr
-    self.assertContained('warning: EMMAKEN_COMPILER is deprecated', stderr)
+    self.assertContained('warning: `EMMAKEN_COMPILER` is deprecated', stderr)
+
+  @no_windows('relies on a shell script')
+  def test_compiler_wrapper(self):
+    create_test_file('wrapper.sh', '''\
+#!/bin/sh
+echo "wrapping compiler call: $@"
+exec "$@"
+    ''')
+    make_executable('wrapper.sh')
+    with env_modify({'EM_COMPILER_WRAPPER': './wrapper.sh'}):
+      stdout = self.run_process([EMCC, '-c', path_from_root('tests', 'core', 'test_hello_world.c')], stdout=PIPE).stdout
+    self.assertContained('wrapping compiler call: ', stdout)
+    self.assertExists('test_hello_world.o')
 
   def test_llvm_option_dash_o(self):
     # emcc used to interpret -mllvm's option value as the output file if it

tests/test_sanity.py

@@ -6,15 +6,14 @@
 import os
 import platform
 import shutil
-import stat
 import time
 import re
 import tempfile
 import zipfile
 from subprocess import PIPE, STDOUT
 
 from runner import RunnerCore, path_from_root, env_modify, chdir
-from runner import create_test_file, ensure_dir
+from runner import create_test_file, ensure_dir, make_executable
 from tools.shared import NODE_JS, PYTHON, EMCC, SPIDERMONKEY_ENGINE, V8_ENGINE
 from tools.shared import CONFIG_FILE, EM_CONFIG, LLVM_ROOT, CANONICAL_TEMP_DIR
 from tools.shared import try_delete
@@ -65,9 +64,7 @@ def make_fake_wasm_opt(filename, version):
     f.write('#!/bin/sh\n')
     f.write('echo "wasm-opt version %s"\n' % version)
     f.write('echo "..."\n')
-  shutil.copyfile(filename, filename + '++')
-  os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
-  os.chmod(filename + '++', stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+  make_executable(filename)
 
 
 def make_fake_clang(filename, version):
@@ -81,8 +78,8 @@ def make_fake_clang(filename, version):
     f.write('echo "clang version %s"\n' % version)
     f.write('echo "..."\n')
   shutil.copyfile(filename, filename + '++')
-  os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
-  os.chmod(filename + '++', stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+  make_executable(filename)
+  make_executable(filename + '++')
 
 
 def make_fake_llc(filename, targets):
@@ -94,15 +91,15 @@ def make_fake_llc(filename, targets):
   with open(filename, 'w') as f:
     f.write('#!/bin/sh\n')
     f.write('echo "llc fake output\nRegistered Targets:\n%s"' % targets)
-  os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+  make_executable(filename)
 
 
 def make_fake_lld(filename):
   print('make_fake_lld: %s' % filename)
   with open(filename, 'w') as f:
     f.write('#!/bin/sh\n')
     f.write('exit 0\n')
-  os.chmod(filename, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+  make_executable(filename)
 
 
 SANITY_MESSAGE = 'Emscripten: Running sanity checks'
@@ -184,18 +181,18 @@ def test_firstrun(self):
     for command in commands:
       wipe()
 
-      def make_executable(name):
-        with open(os.path.join(temp_bin, name), 'w') as f:
-          os.fchmod(f.fileno(), stat.S_IRWXU)
+      def make_new_executable(name):
+        open(os.path.join(temp_bin, name), 'w').close()
+        make_executable(os.path.join(temp_bin, name))
 
       env = os.environ.copy()
       if 'EM_CONFIG' in env:
         del env['EM_CONFIG']
 
       try:
         temp_bin = tempfile.mkdtemp()
-        make_executable('llvm-dis')
-        make_executable('node')
+        make_new_executable('llvm-dis')
+        make_new_executable('node')
         env['PATH'] = temp_bin + os.pathsep + os.environ['PATH']
         output = self.do(command, env=env)
       finally:
@@ -326,7 +323,7 @@ def test_node(self):
 fi
 ''' % (version, NODE_JS))
         f.close()
-        os.chmod(self.in_dir('fake', 'nodejs'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+        make_executable(self.in_dir('fake', 'nodejs'))
         if not succeed:
           if version[0] == 'v':
             self.check_working(EMCC, NODE_WARNING)
@@ -626,7 +623,7 @@ def test_js_engine_path(self):
         with open(test_engine_path, 'w') as f:
           f.write('#!/bin/sh\n')
           f.write('exec %s $@\n' % (engine))
-        os.chmod(test_engine_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
+        make_executable(test_engine_path)
 
         out = self.run_js(sample_script, engine=test_engine_path, args=['--foo'])
 

tools/scons/site_scons/site_tools/emscripten/emscripten.py

@@ -22,7 +22,7 @@ def generate(env, emscripten_path=None, **kw):
   # by Emscripten to the child.
   for var in ['EM_CACHE', 'EMCC_DEBUG', 'EMTEST_BROWSER',
               'EMMAKEN_JUST_CONFIGURE', 'EMCC_CFLAGS', 'EMCC_TEMP_DIR',
-              'EMCC_AUTODEBUG',
+              'EMCC_AUTODEBUG', 'EM_COMPILER_WRAPPER',
               'EMMAKEN_COMPILER', 'EMMAKEN_CFLAGS', 'EMCC_JSOPT_BLACKLIST',
               'MOZ_DISABLE_AUTO_SAFE_MODE', 'EMCC_STDERR_FILE',
               'EMSCRIPTEN_SUPPRESS_USAGE_WARNING', 'NODE_PATH', 'EMCC_JSOPT_MIN_CHUNK_SIZE',

tools/shared.py

@@ -362,6 +362,7 @@ def parse_config_file():
     'FROZEN_CACHE',
     'CACHE',
     'PORTS',
+    'COMPILER_WRAPPER',
   )
 
   # Only propagate certain settings from the config file.
@@ -1473,6 +1474,7 @@ def read_and_preprocess(filename, expand_macros=False):
 CACHE = None
 PORTS = None
 FROZEN_CACHE = False
+COMPILER_WRAPPER = None
 
 # Emscripten compiler spawns other processes, which can reimport shared.py, so
 # make sure that those child processes get the same configuration file by