diff --git a/emcc.py b/emcc.py index 7339957d17261..dd85c1492dfc5 100644 --- a/emcc.py +++ b/emcc.py @@ -35,7 +35,6 @@ from tools import shared, system_libs, utils, cmdline from tools import diagnostics, building, compile -from tools.shared import unsuffixed_basename, get_file_suffix from tools.shared import exit_with_error, DEBUG from tools.shared import in_temp from tools.shared import DYLIB_EXTENSIONS @@ -44,7 +43,7 @@ from tools import config from tools import cache from tools.settings import default_setting, user_settings, settings, COMPILE_TIME_SETTINGS -from tools.utils import read_file +from tools.utils import read_file, unsuffixed_basename, get_file_suffix logger = logging.getLogger('emcc') @@ -566,7 +565,7 @@ def compile_source_file(input_file): if not shared.SKIP_SUBPROCS: assert os.path.exists(output_file) if options.save_temps: - shutil.copyfile(output_file, shared.unsuffixed_basename(input_file) + '.o') + shutil.copyfile(output_file, utils.unsuffixed_basename(input_file) + '.o') return output_file # Compile input files individually to temporary locations. diff --git a/test/common.py b/test/common.py index 3cd0a8fae6772..45c84ab700327 100644 --- a/test/common.py +++ b/test/common.py @@ -38,7 +38,7 @@ import clang_native import jsrun import line_endings -from tools.shared import EMCC, EMXX, DEBUG, exe_suffix +from tools.shared import EMCC, EMXX, DEBUG from tools.shared import get_canonical_temp_dir, path_from_root from tools.utils import MACOS, WINDOWS, read_file, read_binary, write_binary, exit_with_error from tools.settings import COMPILE_TIME_SETTINGS @@ -126,7 +126,7 @@ class FirefoxConfig: data_dir_flag = '-profile ' default_flags = ('-new-instance',) headless_flags = '-headless' - executable_name = exe_suffix('firefox') + executable_name = utils.exe_suffix('firefox') @staticmethod def configure(data_dir): @@ -158,13 +158,13 @@ def configure(data_dir): DEFAULT_BROWSER_DATA_DIR = path_from_root('out/browser-profile') -WEBIDL_BINDER = shared.bat_suffix(path_from_root('tools/webidl_binder')) +WEBIDL_BINDER = utils.bat_suffix(path_from_root('tools/webidl_binder')) -EMBUILDER = shared.bat_suffix(path_from_root('embuilder')) -EMMAKE = shared.bat_suffix(path_from_root('emmake')) -EMCMAKE = shared.bat_suffix(path_from_root('emcmake')) -EMCONFIGURE = shared.bat_suffix(path_from_root('emconfigure')) -EMRUN = shared.bat_suffix(shared.path_from_root('emrun')) +EMBUILDER = utils.bat_suffix(path_from_root('embuilder')) +EMMAKE = utils.bat_suffix(path_from_root('emmake')) +EMCMAKE = utils.bat_suffix(path_from_root('emcmake')) +EMCONFIGURE = utils.bat_suffix(path_from_root('emconfigure')) +EMRUN = utils.bat_suffix(shared.path_from_root('emrun')) WASM_DIS = os.path.join(building.get_binaryen_bin(), 'wasm-dis') LLVM_OBJDUMP = shared.llvm_tool_path('llvm-objdump') PYTHON = sys.executable @@ -241,7 +241,7 @@ def get_browser_config(): def compiler_for(filename, force_c=False): - if shared.suffix(filename) in ('.cc', '.cxx', '.cpp') and not force_c: + if utils.suffix(filename) in ('.cc', '.cxx', '.cpp') and not force_c: return EMXX else: return EMCC @@ -1532,7 +1532,7 @@ def build(self, filename, libraries=None, includes=None, force_c=False, cflags=N compiler = [compiler_for(filename, force_c)] if force_c: - assert shared.suffix(filename) != '.c', 'force_c is not needed for source files ending in .c' + assert utils.suffix(filename) != '.c', 'force_c is not needed for source files ending in .c' compiler.append('-xc') all_cflags = self.get_cflags(main_file=True) @@ -1544,7 +1544,7 @@ def build(self, filename, libraries=None, includes=None, force_c=False, cflags=N if output_basename: output = output_basename + output_suffix else: - output = shared.unsuffixed_basename(filename) + output_suffix + output = utils.unsuffixed_basename(filename) + output_suffix cmd = compiler + [str(filename), '-o', output] + all_cflags if libraries: cmd += libraries @@ -1984,7 +1984,7 @@ def _test_dylink_dso_needed(self, do_run): so = '.wasm' if self.is_wasm() else '.js' def ccshared(src, linkto=None): - cmdv = [EMCC, src, '-o', shared.unsuffixed(src) + so, '-sSIDE_MODULE'] + self.get_cflags() + cmdv = [EMCC, src, '-o', utils.unsuffixed(src) + so, '-sSIDE_MODULE'] + self.get_cflags() if linkto: cmdv += linkto self.run_process(cmdv) @@ -2060,7 +2060,7 @@ def do_runf(self, filename, expected_output=None, **kwargs): def do_run_in_out_file_test(self, srcfile, **kwargs): srcfile = maybe_test_file(srcfile) out_suffix = kwargs.pop('out_suffix', '') - outfile = shared.unsuffixed(srcfile) + out_suffix + '.out' + outfile = utils.unsuffixed(srcfile) + out_suffix + '.out' if EMTEST_REBASELINE: expected = None else: diff --git a/test/jsrun.py b/test/jsrun.py index 3b490b3ee8127..6c7d21d4b46af 100644 --- a/test/jsrun.py +++ b/test/jsrun.py @@ -11,7 +11,7 @@ import sys from subprocess import PIPE, CalledProcessError -from tools import shared, utils +from tools import utils WORKING_ENGINES = {} # Holds all configured engines and whether they work: maps path -> True/False DEFAULT_TIMEOUT = 5 * 60 @@ -22,7 +22,7 @@ def make_command(filename, engine, args=None): # if no engine is needed, indicated by None, then there is a native executable # provided which we can just run if engine[0] is None: - executable = shared.replace_suffix(os.path.abspath(filename), '.exe') + executable = utils.replace_suffix(os.path.abspath(filename), '.exe') return [executable] + args # Emscripten supports multiple javascript runtimes. The default is nodejs but # it can also use d8 (the v8 engine shell) or jsc (JavaScript Core aka @@ -43,7 +43,7 @@ def make_command(filename, engine, args=None): command_flags += ['run'] if is_wasmer or is_wasmtime: # in a wasm runtime, run the wasm, not the js - filename = shared.replace_suffix(filename, '.wasm') + filename = utils.replace_suffix(filename, '.wasm') # Separates engine flags from script flags flag_separator = ['--'] if is_d8 or is_jsc else [] return engine + command_flags + [filename] + flag_separator + args diff --git a/test/test_benchmark.py b/test/test_benchmark.py index 070bc01982757..e8adcda47e3e0 100644 --- a/test/test_benchmark.py +++ b/test/test_benchmark.py @@ -24,7 +24,7 @@ from tools.shared import CLANG_CC, CLANG_CXX from common import test_file, read_file, read_binary, needs_make from tools.shared import run_process, PIPE, EMCC, config -from tools import building, utils, shared +from tools import building, utils # standard arguments for timing: # 0: no runtime, just startup @@ -250,7 +250,7 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat self.cmd = cmd run_process(cmd, env=self.env) if self.binaryen_opts: - run_binaryen_opts(shared.replace_suffix(final, '.wasm'), self.binaryen_opts) + run_binaryen_opts(utils.replace_suffix(final, '.wasm'), self.binaryen_opts) self.filename = final def run(self, args): @@ -260,12 +260,12 @@ def get_output_files(self): ret = [self.filename] if 'WASM=0' in self.cmd: if 'MINIMAL_RUNTIME=0' not in self.cmd: - ret.append(shared.replace_suffix(self.filename, '.asm.js')) - ret.append(shared.replace_suffix(self.filename, '.mem')) + ret.append(utils.replace_suffix(self.filename, '.asm.js')) + ret.append(utils.replace_suffix(self.filename, '.mem')) else: ret.append(self.filename + '.mem') else: - ret.append(shared.replace_suffix(self.filename, '.wasm')) + ret.append(utils.replace_suffix(self.filename, '.wasm')) return ret @@ -346,7 +346,7 @@ def run(self, args): return jsrun.run_js(self.filename, engine=self.engine, args=args, stderr=PIPE) def get_output_files(self): - return [self.filename, shared.replace_suffix(self.filename, '.wasm')] + return [self.filename, utils.replace_suffix(self.filename, '.wasm')] # Benchmarkers diff --git a/test/test_core.py b/test/test_core.py index 838448a1476f0..201d55b4efb5b 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -4067,7 +4067,7 @@ def dylink_testf(self, main, side=None, expected=None, force_c=False, main_cflag # Same as dylink_test but takes source code as filenames on disc. old_args = self.cflags.copy() if not expected: - outfile = shared.replace_suffix(main, '.out') + outfile = utils.replace_suffix(main, '.out') expected = read_file(outfile) if not side: side, ext = os.path.splitext(main) diff --git a/test/test_other.py b/test/test_other.py index 8184bb6ca3b02..dbd85194a1217 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -51,13 +51,13 @@ from tools.settings import settings from tools.system_libs import DETERMINISTIC_PREFIX -emmake = shared.bat_suffix(path_from_root('emmake')) -emconfig = shared.bat_suffix(path_from_root('em-config')) -emsize = shared.bat_suffix(path_from_root('emsize')) -empath_split = shared.bat_suffix(path_from_root('empath-split')) -emprofile = shared.bat_suffix(path_from_root('emprofile')) -emstrip = shared.bat_suffix(path_from_root('emstrip')) -emsymbolizer = shared.bat_suffix(path_from_root('emsymbolizer')) +emmake = utils.bat_suffix(path_from_root('emmake')) +emconfig = utils.bat_suffix(path_from_root('em-config')) +emsize = utils.bat_suffix(path_from_root('emsize')) +empath_split = utils.bat_suffix(path_from_root('empath-split')) +emprofile = utils.bat_suffix(path_from_root('emprofile')) +emstrip = utils.bat_suffix(path_from_root('emstrip')) +emsymbolizer = utils.bat_suffix(path_from_root('emsymbolizer')) def is_bitcode(filename): @@ -756,7 +756,7 @@ def test_print_prog_name(self): output = self.run_process([EMCC, '--print-prog-name=clang'], stdout=PIPE).stdout expected = CLANG_CC if WINDOWS: - expected = os.path.normpath(shared.unsuffixed(CLANG_CC)) + expected = os.path.normpath(utils.unsuffixed(CLANG_CC)) self.assertContained(expected, output) @crossplatform @@ -2947,7 +2947,7 @@ def test_js_optimizer(self, passes, filename=None): testname = self.id().split('.')[-1] filename = utils.removeprefix(testname, 'test_js_optimizer_') + '.js' filename = test_file('js_optimizer', filename) - expected_file = shared.unsuffixed(filename) + '-output.js' + expected_file = utils.unsuffixed(filename) + '-output.js' # test calling optimizer js = self.run_process(config.NODE_JS + [path_from_root('tools/acorn-optimizer.mjs'), filename] + passes, stdin=PIPE, stdout=PIPE).stdout if common.EMTEST_REBASELINE: @@ -3982,7 +3982,7 @@ def test_file_packager_depfile(self, embed): else: self.run_process([FILE_PACKAGER, 'test.data', '--js-output=test.js', '--depfile=test.data.d', '--from-emcc', '--preload', '.']) output = read_file('test.data.d') - file_packager = utils.normalize_path(shared.replace_suffix(FILE_PACKAGER, '.py')) + file_packager = utils.normalize_path(utils.replace_suffix(FILE_PACKAGER, '.py')) file_packager = file_packager.replace(' ', '\\ ') lines = output.splitlines() split = lines.index(': \\') @@ -6903,7 +6903,7 @@ def test_sdl2_config(self): [['--cflags', '--libs'], '-sUSE_SDL=2'], ]: print(args, expected) - out = self.run_process([shared.bat_suffix(cache.get_sysroot_dir('bin/sdl2-config'))] + args, + out = self.run_process([utils.bat_suffix(cache.get_sysroot_dir('bin/sdl2-config'))] + args, stdout=PIPE, stderr=PIPE).stdout self.assertContained(expected, out) print('via emmake') diff --git a/test/test_sanity.py b/test/test_sanity.py index 1fcfae216597e..67f70cc73105a 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -488,7 +488,7 @@ def make_readonly(filename): with env_modify({'EM_CACHE': self.in_dir('test_cache')}): self.run_process([EMCC, test_file('hello_world.c'), '-c']) finally: - for_all_files(path_from_root('system/include'), shared.make_writable) + for_all_files(path_from_root('system/include'), utils.make_writable) @parameterized({ '': [False, False], @@ -831,7 +831,7 @@ def test_bootstrap_without_em_config(self): # Remove from PATH every directory that contains clang.exe so that bootstrap.py cannot # accidentally succeed by virtue of locating tools in PATH. - new_path = [d for d in env['PATH'].split(os.pathsep) if not os.path.isfile(os.path.join(d, shared.exe_suffix('clang')))] + new_path = [d for d in env['PATH'].split(os.pathsep) if not os.path.isfile(os.path.join(d, utils.exe_suffix('clang')))] env['PATH'] = os.pathsep.join(new_path) # Running bootstrap.py should not fail diff --git a/tools/building.py b/tools/building.py index 8db0e2371738e..646d4eac3f6ed 100644 --- a/tools/building.py +++ b/tools/building.py @@ -31,7 +31,7 @@ from .shared import path_from_root from .shared import asmjs_mangle, DEBUG from .shared import LLVM_DWARFDUMP, demangle_c_symbol_name -from .shared import get_emscripten_temp_dir, exe_suffix, is_c_symbol +from .shared import get_emscripten_temp_dir, is_c_symbol from .utils import WINDOWS from .settings import settings from .feature_matrix import UNSUPPORTED @@ -384,9 +384,9 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False, work return check_call(cmd, stdout=PIPE).stdout acorn_optimizer.counter += 1 - basename = shared.unsuffixed(original_filename) + basename = utils.unsuffixed(original_filename) if '.jso' in basename: - basename = shared.unsuffixed(basename) + basename = utils.unsuffixed(basename) output_file = basename + '.jso%d.js' % acorn_optimizer.counter shared.get_temp_files().note(output_file) cmd += ['-o', output_file] @@ -1181,7 +1181,7 @@ def get_binaryen_feature_flags(): def check_binaryen(bindir): - opt = os.path.join(bindir, exe_suffix('wasm-opt')) + opt = os.path.join(bindir, utils.exe_suffix('wasm-opt')) if not os.path.exists(opt): exit_with_error('binaryen executable not found (%s). Please check your binaryen installation' % opt) try: diff --git a/tools/empath-split.py b/tools/empath-split.py index 3fabebb0ce883..7a056a9ae9832 100755 --- a/tools/empath-split.py +++ b/tools/empath-split.py @@ -89,7 +89,7 @@ def parse_args(): if args.preserve_manifest: args.verbose = True if not args.wasm_split: - args.wasm_split = os.path.join(building.get_binaryen_bin(), shared.exe_suffix('wasm-split')) + args.wasm_split = os.path.join(building.get_binaryen_bin(), utils.exe_suffix('wasm-split')) if '--manifest' in forwarded_args: parser.error('manifest file will be generated by this script and should not be given') diff --git a/tools/file_packager.py b/tools/file_packager.py index 5c7b7c2f26ccb..3a4eb7d340ba8 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -255,7 +255,7 @@ def generate_object_file(data_files): embed_files = [f for f in data_files if f.mode == 'embed'] assert embed_files - asm_file = shared.replace_suffix(options.obj_output, '.s') + asm_file = utils.replace_suffix(options.obj_output, '.s') used = set() for f in embed_files: diff --git a/tools/gen_struct_info.py b/tools/gen_struct_info.py index b7adf76d08568..4c8a30dceee69 100755 --- a/tools/gen_struct_info.py +++ b/tools/gen_struct_info.py @@ -266,7 +266,7 @@ def inspect_headers(headers, cflags): if os.path.exists(js_file_path): os.unlink(js_file_path) - wasm_file_path = shared.replace_suffix(js_file_path, '.wasm') + wasm_file_path = utils.replace_suffix(js_file_path, '.wasm') os.unlink(wasm_file_path) # Parse the output of the program into a dict. diff --git a/tools/link.py b/tools/link.py index f06b05d650f83..22123b5d4ed82 100644 --- a/tools/link.py +++ b/tools/link.py @@ -34,11 +34,11 @@ from . import webassembly from . import extract_metadata from .cmdline import OFormat -from .utils import read_file, write_file, delete_file +from .utils import read_file, write_file, delete_file, safe_copy from .utils import removeprefix, exit_with_error -from .shared import in_temp, safe_copy, do_replace +from .utils import unsuffixed, unsuffixed_basename, get_file_suffix +from .shared import in_temp, do_replace from .shared import DEBUG, WINDOWS, DYLIB_EXTENSIONS -from .shared import unsuffixed, unsuffixed_basename, get_file_suffix from .settings import settings, default_setting, user_settings, JS_ONLY_SETTINGS, DEPRECATED_SETTINGS from .minimal_runtime_shell import generate_minimal_runtime_html @@ -910,7 +910,7 @@ def limit_incoming_module_api(): elif settings.SINGLE_FILE or settings.WASM == 0: # In SINGLE_FILE or WASM2JS mode the wasm file is not part of the output at # all so we generate it the temp directory. - wasm_target = in_temp(shared.replace_suffix(target, '.wasm')) + wasm_target = in_temp(utils.replace_suffix(target, '.wasm')) else: # Otherwise the wasm file is produced alongside the final target. wasm_target = get_secondary_target(target, '.wasm') @@ -1567,7 +1567,7 @@ def limit_incoming_module_api(): settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['_load_secondary_module'] # wasm side modules have suffix .wasm - if settings.SIDE_MODULE and shared.suffix(target) in ('.js', '.mjs'): + if settings.SIDE_MODULE and utils.suffix(target) in ('.js', '.mjs'): diagnostics.warning('emcc', 'JavaScript output suffix requested, but wasm side modules are just wasm files; emitting only a .wasm, no .js') if options.sanitize: @@ -2633,7 +2633,7 @@ def generate_worker_js(target, options, js_target, target_basename): proxy_worker_filename = get_subresource_location_js(js_target) else: # compiler output goes in .worker.js file - move_file(js_target, shared.replace_suffix(js_target, get_worker_js_suffix())) + move_file(js_target, utils.replace_suffix(js_target, get_worker_js_suffix())) worker_target_basename = target_basename + '.worker' proxy_worker_filename = (settings.PROXY_TO_WORKER_FILENAME or worker_target_basename) + '.js' @@ -2998,7 +2998,7 @@ def package_files(options, target): rtn.append(object_file) cmd = building.get_command_with_possible_response_file( - [shared.FILE_PACKAGER, shared.replace_suffix(target, '.data')] + file_args) + [shared.FILE_PACKAGER, utils.replace_suffix(target, '.data')] + file_args) if options.preload_files: # Preloading files uses --pre-js code that runs before the module is loaded. file_code = shared.check_call(cmd, stdout=PIPE).stdout diff --git a/tools/maint/rebaseline_tests.py b/tools/maint/rebaseline_tests.py index 1b9af57c30b34..2dec1429ff757 100755 --- a/tools/maint/rebaseline_tests.py +++ b/tools/maint/rebaseline_tests.py @@ -20,7 +20,7 @@ root_dir = os.path.dirname(os.path.dirname(script_dir)) sys.path.insert(0, root_dir) -from tools import utils, shared +from tools import utils def run(cmd, **args): @@ -79,7 +79,7 @@ def main(): print('tree is not clean') return 1 - subprocess.check_call([shared.bat_suffix(os.path.join('test', 'runner')), '--rebaseline', 'codesize'], cwd=root_dir) + subprocess.check_call([utils.bat_suffix(os.path.join('test', 'runner')), '--rebaseline', 'codesize'], cwd=root_dir) output = run(['git', 'status', '-uno', '--porcelain']) filenames = [] diff --git a/tools/ports/__init__.py b/tools/ports/__init__.py index cc1445806db0e..65e55ae4b7327 100644 --- a/tools/ports/__init__.py +++ b/tools/ports/__init__.py @@ -119,7 +119,7 @@ def init_external_port(name, port): def load_port(path, name=None): if not name: - name = shared.unsuffixed_basename(path) + name = utils.unsuffixed_basename(path) if name in ports_by_name: utils.exit_with_error(f'port path [`{path}`] is invalid: duplicate port name `{name}`') port = load_port_module(f'tools.ports.{name}', path) @@ -148,7 +148,7 @@ def read_ports(): for filename in os.listdir(contrib_dir): if not filename.endswith('.py') or filename == '__init__.py': continue - name = 'contrib.' + shared.unsuffixed(filename) + name = 'contrib.' + utils.unsuffixed(filename) load_port(os.path.join(contrib_dir, filename), name) @@ -241,7 +241,7 @@ def build_port(src_dir, output_path, port_name, includes=[], flags=[], cxxflags= if ex in dirs: dirs.remove(ex) for f in files: - ext = shared.suffix(f) + ext = utils.suffix(f) if ext in ('.c', '.cpp') and not any((excluded in f) for excluded in exclude_files): srcs.append(os.path.join(root, f)) @@ -265,7 +265,7 @@ def build_port(src_dir, output_path, port_name, includes=[], flags=[], cxxflags= dirname = os.path.dirname(obj) os.makedirs(dirname, exist_ok=True) cmd = [shared.EMCC, '-c', src, '-o', obj] + cflags - if shared.suffix(src) in ('.cc', '.cxx', '.cpp'): + if utils.suffix(src) in ('.cc', '.cxx', '.cpp'): cmd[0] = shared.EMXX cmd += cxxflags commands.append(cmd) diff --git a/tools/ports/bullet.py b/tools/ports/bullet.py index aa45d93b4dc5b..0deda7a038156 100644 --- a/tools/ports/bullet.py +++ b/tools/ports/bullet.py @@ -24,7 +24,7 @@ def create(final): dest_include_path = ports.get_include_dir('bullet') for base, _, files in os.walk(src_path): for f in files: - if shared.suffix(f) != '.h': + if os.path.splitext(f)[1] != '.h': continue fullpath = os.path.join(base, f) relpath = os.path.relpath(fullpath, src_path) diff --git a/tools/shared.py b/tools/shared.py index 4ab4b361f2a6d..495cd325f2a97 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -3,6 +3,9 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. +"""Shared code specific to emscripten. General purpose and low-level helpers belong instead in +utils.py.""" + from .toolchain_profiler import ToolchainProfiler from subprocess import PIPE @@ -10,11 +13,9 @@ import logging import os import re -import shutil import shlex import subprocess import signal -import stat import sys import tempfile @@ -38,7 +39,7 @@ logging.basicConfig(format='%(name)s:%(levelname)s: %(message)s', level=log_level) colored_logger.enable() -from .utils import path_from_root, exit_with_error, safe_ensure_dirs, WINDOWS, set_version_globals, memoize +from .utils import path_from_root, exit_with_error, safe_ensure_dirs, WINDOWS, set_version_globals, memoize, bat_suffix from . import cache, tempfiles from . import diagnostics from . import config @@ -480,7 +481,7 @@ def llvm_tool_path_with_suffix(tool, suffix): if suffix: tool += '-' + suffix llvm_root = os.path.expanduser(config.LLVM_ROOT) - return os.path.join(llvm_root, exe_suffix(tool)) + return os.path.join(llvm_root, utils.exe_suffix(tool)) # Some distributions ship with multiple llvm versions so they add @@ -495,25 +496,12 @@ def clang_tool_path(tool): return llvm_tool_path_with_suffix(tool, config.CLANG_ADD_VERSION) -def exe_suffix(cmd): - return cmd + '.exe' if WINDOWS else cmd - - -def bat_suffix(cmd): - return cmd + '.bat' if WINDOWS else cmd - - -def replace_suffix(filename, new_suffix): - assert new_suffix[0] == '.' - return os.path.splitext(filename)[0] + new_suffix - - # In MINIMAL_RUNTIME mode, keep suffixes of generated files simple # ('.mem' instead of '.js.mem'; .'symbols' instead of '.js.symbols' etc) # Retain the original naming scheme in traditional runtime. def replace_or_append_suffix(filename, new_suffix): assert new_suffix[0] == '.' - return replace_suffix(filename, new_suffix) if settings.MINIMAL_RUNTIME else filename + new_suffix + return utils.replace_suffix(filename, new_suffix) if settings.MINIMAL_RUNTIME else filename + new_suffix # Temp dir. Create a random one, unless EMCC_DEBUG is set, in which case use the canonical @@ -646,56 +634,6 @@ def asmjs_mangle(name): return name -def suffix(name): - """Return the file extension""" - return os.path.splitext(name)[1] - - -def unsuffixed(name): - """Return the filename without the extension. - - If there are multiple extensions this strips only the final one. - """ - return os.path.splitext(name)[0] - - -def unsuffixed_basename(name): - return os.path.basename(unsuffixed(name)) - - -def get_file_suffix(filename): - """Parses the essential suffix of a filename, discarding Unix-style version - numbers in the name. For example for 'libz.so.1.2.8' returns '.so'""" - while filename: - filename, suffix = os.path.splitext(filename) - if not suffix[1:].isdigit(): - return suffix - return '' - - -def make_writable(filename): - assert os.path.exists(filename) - old_mode = stat.S_IMODE(os.stat(filename).st_mode) - os.chmod(filename, old_mode | stat.S_IWUSR) - - -def safe_copy(src, dst): - logging.debug('copy: %s -> %s', src, dst) - src = os.path.abspath(src) - dst = os.path.abspath(dst) - if os.path.isdir(dst): - dst = os.path.join(dst, os.path.basename(src)) - if src == dst: - return - if dst == os.devnull: - return - # Copies data and permission bits, but not other metadata such as timestamp - shutil.copy(src, dst) - # We always want the target file to be writable even when copying from - # read-only source. (e.g. a read-only install of emscripten). - make_writable(dst) - - def do_replace(input_, pattern, replacement): if pattern not in input_: exit_with_error('expected to find pattern in input JS: %s' % pattern) diff --git a/tools/system_libs.py b/tools/system_libs.py index 54296e0923bf4..b5e0710040fc7 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -140,7 +140,7 @@ def objectfile_sort_key(filename): def create_lib(libname, inputs): """Create a library from a set of input objects.""" - suffix = shared.suffix(libname) + suffix = utils.suffix(libname) inputs = sorted(inputs, key=objectfile_sort_key) if suffix in ('.bc', '.o'): @@ -229,13 +229,13 @@ def create_ninja_file(input_files, filename, libname, cflags, asflags=None, cust description = AR $out ''' - suffix = shared.suffix(libname) + suffix = utils.suffix(libname) build_dir = os.path.dirname(filename) if suffix == '.o': assert len(input_files) == 1 input_file = escape_ninja_path(input_files[0]) - depfile = shared.unsuffixed_basename(input_file) + '.d' + depfile = utils.unsuffixed_basename(input_file) + '.d' out += f'build {escape_ninja_path(libname)}: direct_cc {input_file}\n' out += f' with_depfile = {depfile}\n' else: @@ -245,7 +245,7 @@ def create_ninja_file(input_files, filename, libname, cflags, asflags=None, cust # insensitive filesystem to handle, for example, _exit.o and _Exit.o. # This is done even on case sensitive filesystem so that builds are # reproducible across platforms. - object_basename = shared.unsuffixed_basename(src).lower() + object_basename = utils.unsuffixed_basename(src).lower() o = os.path.join(build_dir, object_basename + '.o') object_uuid = 0 # Find a unique basename @@ -253,7 +253,7 @@ def create_ninja_file(input_files, filename, libname, cflags, asflags=None, cust object_uuid += 1 o = os.path.join(build_dir, f'{object_basename}__{object_uuid}.o') objects.append(o) - ext = shared.suffix(src) + ext = utils.suffix(src) if ext == '.s': cmd = 'asm' flags = asflags @@ -447,7 +447,7 @@ def get_link_flag(self): if self.get_ext() != '.a': return fullpath # For libraries (.a) files, we pass the abbreviated `-l` form. - base = shared.unsuffixed_basename(fullpath) + base = utils.unsuffixed_basename(fullpath) return '-l' + utils.removeprefix(base, 'lib') def get_files(self): @@ -493,7 +493,7 @@ def build_objects(self, build_dir): objects = set() cflags = self.get_cflags() for src in self.get_files(): - ext = shared.suffix(src) + ext = utils.suffix(src) if ext in {'.s', '.S', '.c'}: cmd = shared.EMCC else: @@ -508,7 +508,7 @@ def build_objects(self, build_dir): cmd += cflags cmd = self.customize_build_cmd(cmd, src) - object_basename = shared.unsuffixed_basename(src).lower() + object_basename = utils.unsuffixed_basename(src).lower() o = os.path.join(build_dir, object_basename + '.o') if o in objects: # If we have seen a file with the same name before, we need a separate @@ -529,7 +529,7 @@ def build_objects(self, build_dir): src = utils.normalize_path(src) batches.setdefault(tuple(cmd), []).append(src) # No -o in command, use original file name. - o = os.path.join(build_dir, shared.unsuffixed_basename(src) + '.o') + o = os.path.join(build_dir, utils.unsuffixed_basename(src) + '.o') else: commands.append(cmd + [src, '-o', o]) objects.add(o) @@ -2496,7 +2496,7 @@ def safe_copytree(src, dst, excludes=None): if entry.is_dir(): safe_copytree(srcname, dstname, excludes) else: - shared.safe_copy(srcname, dstname) + utils.safe_copy(srcname, dstname) def install_system_headers(stamp): diff --git a/tools/utils.py b/tools/utils.py index 36ffce1078daa..14e0c1ec1b8f8 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -3,10 +3,16 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. +"""General purpose utility functions. The code in this file should mostly be +not emscripten-specific, but general purpose enough to be useful in any command +line utility.""" + +import functools +import logging import os import shutil +import stat import sys -import functools from pathlib import Path from . import diagnostics @@ -16,6 +22,8 @@ MACOS = sys.platform == 'darwin' LINUX = sys.platform.startswith('linux') +logger = logging.getLogger('utils') + def exit_with_error(msg, *args): diagnostics.error(msg, *args) @@ -25,6 +33,46 @@ def path_from_root(*pathelems): return str(Path(__rootpath__, *pathelems)) +def suffix(name): + """Return the file extension""" + return os.path.splitext(name)[1] + + +def exe_suffix(cmd): + return cmd + '.exe' if WINDOWS else cmd + + +def bat_suffix(cmd): + return cmd + '.bat' if WINDOWS else cmd + + +def replace_suffix(filename, new_suffix): + assert new_suffix[0] == '.' + return os.path.splitext(filename)[0] + new_suffix + + +def unsuffixed(name): + """Return the filename without the extension. + + If there are multiple extensions this strips only the final one. + """ + return os.path.splitext(name)[0] + + +def unsuffixed_basename(name): + return os.path.basename(unsuffixed(name)) + + +def get_file_suffix(filename): + """Parses the essential suffix of a filename, discarding Unix-style version + numbers in the name. For example for 'libz.so.1.2.8' returns '.so'""" + while filename: + filename, suffix = os.path.splitext(filename) + if not suffix[1:].isdigit(): + return suffix + return '' + + def normalize_path(path): """Normalize path separators to UNIX-style forward slashes. @@ -40,6 +88,29 @@ def safe_ensure_dirs(dirname): os.makedirs(dirname, exist_ok=True) +def make_writable(filename): + assert os.path.exists(filename) + old_mode = stat.S_IMODE(os.stat(filename).st_mode) + os.chmod(filename, old_mode | stat.S_IWUSR) + + +def safe_copy(src, dst): + logger.debug('copy: %s -> %s', src, dst) + src = os.path.abspath(src) + dst = os.path.abspath(dst) + if os.path.isdir(dst): + dst = os.path.join(dst, os.path.basename(src)) + if src == dst: + return + if dst == os.devnull: + return + # Copies data and permission bits, but not other metadata such as timestamp + shutil.copy(src, dst) + # We always want the target file to be writable even when copying from + # read-only source. (e.g. a read-only install of emscripten). + make_writable(dst) + + # TODO(sbc): Replace with str.removeprefix once we update to python3.9 def removeprefix(string, prefix): if string.startswith(prefix):