From 9a8dec8caedcf153df49b56d04b101d3caf9dad7 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Wed, 6 May 2020 14:33:34 -0700 Subject: [PATCH 1/2] [lldb/Test] Run dotest.py with the Python LLDB was built with. The Python used to run lit can be different from the Python LLDB was build with. One scenario where this happens is when LLVM can find the Python 3 interpreter, but not the Python 3 libraries, in which case LLDB build and links against Python 3. Without this change, you end up with an ModuleNotFoundError because of the mismatch. Instead of looking at the Python interpreter that's used to run lit, lldbtest should use the interpreter that matches the Python version LLDB was build against. Differential revision: https://reviews.llvm.org/D79519 (cherry picked from commit 7c6420e43178b0c03618f445fd008606a8b2c95e) --- lldb/test/API/lldbtest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lldb/test/API/lldbtest.py b/lldb/test/API/lldbtest.py index 60ac42a2fe0f8b..774a9281a9e107 100644 --- a/lldb/test/API/lldbtest.py +++ b/lldb/test/API/lldbtest.py @@ -63,10 +63,15 @@ def execute(self, test, litConfig): return (lit.Test.UNSUPPORTED, 'Test is unsupported') testPath, testFile = os.path.split(test.getSourcePath()) + + # The Python used to run lit can be different from the Python LLDB was + # build with. + executable = test.config.python_executable + # On Windows, the system does not always correctly interpret # shebang lines. To make sure we can execute the tests, add # python exe as the first parameter of the command. - cmd = [sys.executable] + self.dotest_cmd + [testPath, '-p', testFile] + cmd = [executable] + self.dotest_cmd + [testPath, '-p', testFile] # On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim # python binary as the ASan interceptors get loaded too late. Also, @@ -82,7 +87,7 @@ def execute(self, test, litConfig): if not os.path.isfile(copied_python): import shutil, subprocess python = subprocess.check_output([ - sys.executable, + executable, os.path.join(os.path.dirname(os.path.realpath(__file__)), 'get_darwin_real_python.py') ]).decode('utf-8').strip() From 711673d537e92addb46ed3ee6e96c265fec8df7e Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 11 Jun 2020 19:29:26 -0700 Subject: [PATCH 2/2] [lldb/Test] Fix ASan/TSan workaround for Xcode Python 3 The Python 3 interpreter in Xcode has a relative RPATH and dyld fails to load it when we copy it into the build directory. This patch adds an additional check that the copied binary can be executed. If it doesn't, we assume we're dealing with the Xcode python interpreter and return the path to the real executable. That is sufficient for the sanitizers because only system binaries need to be copied to work around SIP. This patch also moves all that logic out of LLDBTest and into the lit configuration so that it's executed only once per test run, instead of once for every test. Although I didn't benchmark the difference this should result in a mild speedup. Differential revision: https://reviews.llvm.org/D81696 (cherry picked from commit 526e0c8d15216e6c49b1769c63f5433df6841c64) --- lldb/test/API/lit.cfg.py | 85 ++++++++++++++++++++++++++++++++------- lldb/test/API/lldbtest.py | 39 ------------------ 2 files changed, 71 insertions(+), 53 deletions(-) diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py index 49cd7d3a305161..49cf72af7ffa88 100644 --- a/lldb/test/API/lit.cfg.py +++ b/lldb/test/API/lit.cfg.py @@ -20,20 +20,25 @@ config.test_source_root = os.path.dirname(__file__) config.test_exec_root = config.test_source_root -if 'Address' in config.llvm_use_sanitizer: - config.environment['ASAN_OPTIONS'] = 'detect_stack_use_after_return=1' - # Swift's libReflection builds without ASAN, which causes a known - # false positive in std::vector. - config.environment['ASAN_OPTIONS'] += ':detect_container_overflow=0' - # macOS flags needed for LLDB built with address sanitizer. - if 'Darwin' in config.host_os and 'x86' in config.host_triple: - import subprocess - resource_dir = subprocess.check_output( - [config.cmake_cxx_compiler, - '-print-resource-dir']).decode('utf-8').strip() - runtime = os.path.join(resource_dir, 'lib', 'darwin', - 'libclang_rt.asan_osx_dynamic.dylib') - config.environment['DYLD_INSERT_LIBRARIES'] = runtime + +def mkdir_p(path): + import errno + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + if not os.path.isdir(path): + raise OSError(errno.ENOTDIR, "%s is not a directory"%path) + + +def find_sanitizer_runtime(name): + import subprocess + resource_dir = subprocess.check_output( + [config.cmake_cxx_compiler, + '-print-resource-dir']).decode('utf-8').strip() + return os.path.join(resource_dir, 'lib', 'darwin', name) + def find_shlibpath_var(): if platform.system() in ['Linux', 'FreeBSD', 'NetBSD', 'SunOS']: @@ -43,6 +48,58 @@ def find_shlibpath_var(): elif platform.system() == 'Windows': yield 'PATH' + +# On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim python +# binary as the ASan interceptors get loaded too late. Also, when SIP is +# enabled, we can't inject libraries into system binaries at all, so we need a +# copy of the "real" python to work with. +def find_python_interpreter(): + # Avoid doing any work if we already copied the binary. + copied_python = os.path.join(config.lldb_build_directory, 'copied-python') + if os.path.isfile(copied_python): + return copied_python + + # Find the "real" python binary. + import shutil, subprocess + real_python = subprocess.check_output([ + config.python_executable, + os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'get_darwin_real_python.py') + ]).decode('utf-8').strip() + + shutil.copy(real_python, copied_python) + + # Now make sure the copied Python works. The Python in Xcode has a relative + # RPATH and cannot be copied. + try: + # We don't care about the output, just make sure it runs. + subprocess.check_output([copied_python, '-V'], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + # The copied Python didn't work. Assume we're dealing with the Python + # interpreter in Xcode. Given that this is not a system binary SIP + # won't prevent us form injecting the interceptors so we get away with + # not copying the executable. + os.remove(copied_python) + return real_python + + # The copied Python works. + return copied_python + + +if 'Address' in config.llvm_use_sanitizer: + config.environment['ASAN_OPTIONS'] = 'detect_stack_use_after_return=1' + if 'Darwin' in config.host_os and 'x86' in config.host_triple: + config.environment['DYLD_INSERT_LIBRARIES'] = find_sanitizer_runtime( + 'libclang_rt.asan_osx_dynamic.dylib') + +if 'Thread' in config.llvm_use_sanitizer: + if 'Darwin' in config.host_os and 'x86' in config.host_triple: + config.environment['DYLD_INSERT_LIBRARIES'] = find_sanitizer_runtime( + 'libclang_rt.tsan_osx_dynamic.dylib') + +if 'DYLD_INSERT_LIBRARIES' in config.environment and platform.system() == 'Darwin': + config.python_executable = find_python_interpreter() + # Shared library build of LLVM may require LD_LIBRARY_PATH or equivalent. if config.shared_libs: for shlibpath_var in find_shlibpath_var(): diff --git a/lldb/test/API/lldbtest.py b/lldb/test/API/lldbtest.py index 774a9281a9e107..55d2c8428396cc 100644 --- a/lldb/test/API/lldbtest.py +++ b/lldb/test/API/lldbtest.py @@ -10,24 +10,6 @@ import lit.util from lit.formats.base import TestFormat -def getBuildDir(cmd): - found = False - for arg in cmd: - if found: - return arg - if arg == '--build-dir': - found = True - return None - -def mkdir_p(path): - import errno - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - if not os.path.isdir(path): - raise OSError(errno.ENOTDIR, "%s is not a directory"%path) class LLDBTest(TestFormat): def __init__(self, dotest_cmd): @@ -73,27 +55,6 @@ def execute(self, test, litConfig): # python exe as the first parameter of the command. cmd = [executable] + self.dotest_cmd + [testPath, '-p', testFile] - # On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim - # python binary as the ASan interceptors get loaded too late. Also, - # when SIP is enabled, we can't inject libraries into system binaries - # at all, so we need a copy of the "real" python to work with. - # - # Find the "real" python binary, copy it, and invoke it. - if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \ - platform.system() == 'Darwin': - builddir = getBuildDir(cmd) - mkdir_p(builddir) - copied_python = os.path.join(builddir, 'copied-system-python') - if not os.path.isfile(copied_python): - import shutil, subprocess - python = subprocess.check_output([ - executable, - os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'get_darwin_real_python.py') - ]).decode('utf-8').strip() - shutil.copy(python, copied_python) - cmd[0] = copied_python - timeoutInfo = None try: out, err, exitCode = lit.util.executeCommand(