diff --git a/utils/build-script b/utils/build-script index 8a35436dc117a..23c1f04de1089 100755 --- a/utils/build-script +++ b/utils/build-script @@ -14,6 +14,7 @@ from __future__ import print_function import argparse import multiprocessing import os +import platform import shutil import sys import textwrap @@ -32,11 +33,15 @@ from SwiftBuildSupport import ( get_preset_options, print_with_argv0, quote_shell_command, + WorkingDirectory, ) sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support')) import swift_build_support.clang import swift_build_support.cmake +import swift_build_support.debug +import swift_build_support.ninja +import swift_build_support.tar import swift_build_support.targets from swift_build_support.migration import migrate_impl_args @@ -333,6 +338,10 @@ details of the setups of other systems or automated environments.""") extra_actions_group.add_argument("--export-compile-commands", help="generate compilation databases in addition to building", action="store_true") + extra_actions_group.add_argument("--symbols-package", + metavar="PATH", + help="if provided, an archive of the symbols directory will be " + "generated at this path") build_variant_group = parser.add_mutually_exclusive_group(required=False) build_variant_group.add_argument("-d", "--debug", @@ -530,6 +539,14 @@ also build for Apple watchos, but disallow tests that require an watchOS device" name of the directory under $SWIFT_BUILD_ROOT where the build products will be placed""", metavar="PATH") + parser.add_argument("--install-prefix", + help="The installation prefix. This is where built Swift products " + "(like bin, lib, and include) will be installed.", + metavar="PATH", + default=swift_build_support.targets.install_prefix()) + parser.add_argument("--install-symroot", + help="the path to install debug symbols into", + metavar="PATH") parser.add_argument("-j", "--jobs", help=""" @@ -544,6 +561,9 @@ the number of parallel build jobs to use""", parser.add_argument("--cmake", help="the path to a CMake executable that will be " "used to build Swift") + parser.add_argument("--show-sdks", + help="print installed Xcode and SDK versions", + action="store_true") parser.add_argument("--extra-swift-args", help=textwrap.dedent(""" Pass through extra flags to swift in the form of a cmake list 'module_regexp;flag'. Can @@ -559,12 +579,27 @@ the number of parallel build jobs to use""", '--darwin-xcrun-toolchain', '--cmake', '--host-target', + '--skip-build', + '--show-sdks', + '--install-prefix', + '--install-symroot', + '--symbols-package', ])) if args.host_target is None: print_with_argv0("Unknown operating system.") return 1 + if args.symbols_package: + if not os.path.isabs(args.symbols_package): + print('--symbols-package must be an absolute path ' + '(was \'{}\')'.format(args.symbols_package)) + return 1 + if not args.install_symroot: + print_with_argv0("--install-symroot is required when specifying " + "--symbols-package.") + return 1 + # Build cmark if any cmark-related options were specified. if (args.cmark_build_variant is not None): args.build_cmark = True @@ -726,7 +761,32 @@ the number of parallel build jobs to use""", if args.skip_build: build_script_impl_inferred_args += [ - "--skip-build" + "--skip-build-cmark", + "--skip-build-llvm", + "--skip-build-swift", + "--skip-build-osx", + "--skip-build-ios", + "--skip-build-ios-device", + "--skip-build-ios-simulator", + "--skip-build-tvos", + "--skip-build-tvos_device", + "--skip-build-tvos-simulator", + "--skip-build-watchos", + "--skip-build-watchos-device", + "--skip-build-watchos-simulator", + "--skip-build-lldb", + "--skip-build-llbuild", + "--skip-build-swiftpm", + "--skip-build-xctest", + "--skip-build-foundation", + "--skip-build-libdispatch", + "--skip-build-benchmarks", + ] + + if platform.system() == 'Darwin': + build_script_impl_inferred_args += [ + "--toolchain-prefix", + swift_build_support.targets.darwin_toolchain_prefix(args.install_prefix), ] if args.build_subdir is None: @@ -804,6 +864,7 @@ the number of parallel build jobs to use""", build_script_impl_args = [ os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "build-script-impl"), "--build-dir", build_dir, + "--install-prefix", os.path.abspath(args.install_prefix), "--host-target", args.host_target, "--host-cc", host_clang.cc, "--host-cxx", host_clang.cxx, @@ -826,6 +887,9 @@ the number of parallel build jobs to use""", build_script_impl_args += [ "--foundation-build-type", args.foundation_build_variant ] + if args.cmake_generator == 'Ninja' and \ + not swift_build_support.ninja.is_ninja_installed(): + build_script_impl_args += ["--build-ninja"] build_script_impl_args += build_script_impl_inferred_args # If we have extra_swift_args, combine all of them together and then add @@ -839,8 +903,33 @@ the number of parallel build jobs to use""", for v in ['MAKEFLAGS', 'SDKROOT', 'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET']: os.environ.pop(v, None) + if args.show_sdks: + swift_build_support.debug.print_xcodebuild_versions([ + 'iphonesimulator', + 'appletvsimulator', + 'watchsimulator', + ]) + check_call(build_script_impl_args) + if args.symbols_package: + print('--- Creating symbols package ---') + print('-- Package file: {} --'.format(args.symbols_package)) + + if platform.system() == 'Darwin': + prefix = swift_build_support.targets.darwin_toolchain_prefix( + args.install_prefix) + else: + prefix = args.install_prefix + + # As a security measure, `tar` normally strips leading '/' from paths + # it is archiving. To stay safe, we change working directories, then + # run `tar` without the leading '/' (we remove it ourselves to keep + # `tar` from emitting a warning). + with WorkingDirectory(args.install_symroot): + swift_build_support.tar.tar(source=prefix.lstrip('/'), + destination=args.symbols_package) + return 0 diff --git a/utils/build-script-impl b/utils/build-script-impl index 9c223f64e673e..1fcb503b7d63d 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -86,14 +86,13 @@ KNOWN_SETTINGS=( cmake-generator "Unix Makefiles" "kind of build system to generate; see output of 'cmake --help' for choices" verbose-build "" "print the commands executed during the build" install-prefix "" "installation prefix" + toolchain-prefix "" "the path to the .xctoolchain directory that houses the install prefix path" install-destdir "" "the path to use as the filesystem root for the installation" install-symroot "" "the path to install debug symbols into" swift-install-components "" "a semicolon-separated list of Swift components to install" llvm-install-components "" "a semicolon-separated list of LLVM components to install" installable-package "" "the path to the archive of the installation directory" test-installable-package "" "whether to run post-packaging tests on the produced package" - symbols-package "" "the path to the archive of the symbols directory" - show-sdks "" "print installed Xcode and SDK versions" reconfigure "" "force a CMake configuration run even if CMakeCache.txt already exists" swift-sdks "" "build target binaries only for specified SDKs (semicolon-separated list)" swift-primary-variant-sdk "" "default SDK for target binaries" @@ -101,7 +100,6 @@ KNOWN_SETTINGS=( skip-ios "" "set to skip everything iOS-related" skip-tvos "" "set to skip everything tvOS-related" skip-watchos "" "set to skip everything watchOS-related" - skip-build "" "set to skip building anything" skip-build-cmark "" "set to skip building CommonMark" skip-build-llvm "" "set to skip building LLVM/Clang" skip-build-swift "" "set to skip building Swift" @@ -539,29 +537,6 @@ while [[ "$1" ]] ; do shift done -if [[ "${SKIP_BUILD}" ]]; then - SKIP_BUILD_CMARK=1 - SKIP_BUILD_LLVM=1 - SKIP_BUILD_SWIFT=1 - SKIP_BUILD_OSX=1 - SKIP_BUILD_IOS=1 - SKIP_BUILD_IOS_DEVICE=1 - SKIP_BUILD_IOS_SIMULATOR=1 - SKIP_BUILD_TVOS=1 - SKIP_BUILD_TVOS_DEVICE=1 - SKIP_BUILD_TVOS_SIMULATOR=1 - SKIP_BUILD_WATCHOS=1 - SKIP_BUILD_WATCHOS_DEVICE=1 - SKIP_BUILD_WATCHOS_SIMULATOR=1 - SKIP_BUILD_LLDB=1 - SKIP_BUILD_LLBUILD=1 - SKIP_BUILD_SWIFTPM=1 - SKIP_BUILD_XCTEST=1 - SKIP_BUILD_FOUNDATION=1 - SKIP_BUILD_LIBDISPATCH=1 - SKIP_BUILD_BENCHMARKS=1 -fi - if [[ "${SKIP_IOS}" ]] ; then SKIP_BUILD_IOS=1 SKIP_BUILD_IOS_DEVICE=1 @@ -658,12 +633,6 @@ if [[ "${SKIP_TEST_WATCHOS}" ]] ; then SKIP_TEST_WATCHOS_SIMULATOR=1 fi -if [[ "${CMAKE_GENERATOR}" == "Ninja" && \ - -z "$(which ninja 2>/dev/null)" && - -z "$(which ninja-build 2>/dev/null)" ]] ; then - BUILD_NINJA=1 -fi - # WORKSPACE, BUILD_DIR and INSTALLABLE_PACKAGE must be absolute paths case "${WORKSPACE}" in /*) ;; @@ -692,14 +661,6 @@ case "${INSTALLABLE_PACKAGE}" in exit 1 ;; esac -case "${SYMBOLS_PACKAGE}" in - /*) ;; - "") ;; - *) - echo "symbols-package must be an absolute path (was '${SYMBOLS_PACKAGE}')" - exit 1 - ;; -esac # WORKSPACE must exist if [ ! -e "${WORKSPACE}" ] ; then @@ -745,23 +706,6 @@ function true_false() { esac } -# -# Set default values for command-line parameters. -# - -if [[ "${INSTALL_PREFIX}" == "" ]] ; then - if [[ "$(uname -s)" == "Darwin" ]] ; then - INSTALL_PREFIX="/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr" - else - INSTALL_PREFIX="/usr" - fi -fi - -if [[ "$(uname -s)" == "Darwin" ]] ; then - TOOLCHAIN_PREFIX=$(echo ${INSTALL_PREFIX} | sed -E 's/\/usr$//') -fi - - # A list of deployment targets to cross-compile the Swift host tools for. # We can't run the resulting binaries on the build machine. CROSS_TOOLS_DEPLOYMENT_TARGETS=() @@ -1147,24 +1091,6 @@ if [[ "${CLANG_COMPILER_VERSION}" ]] ; then ) fi -# -# Record SDK and tools versions for posterity -# -if [[ "${SHOW_SDKS}" ]] ; then - echo "--- SDK versions ---" - xcodebuild -version || : - echo - if [[ ! "${SKIP_IOS}" ]] ; then - xcodebuild -version -sdk iphonesimulator || : - fi - if [[ ! "${SKIP_TVOS}" ]] ; then - xcodebuild -version -sdk appletvsimulator || : - fi - if [[ ! "${SKIP_WATCHOS}" ]] ; then - xcodebuild -version -sdk watchsimulator || : - fi -fi - function build_directory() { deployment_target=$1 product=$2 @@ -2389,15 +2315,3 @@ if [[ "${INSTALLABLE_PACKAGE}" ]] ; then { set +x; } 2>/dev/null fi fi - -if [[ "${SYMBOLS_PACKAGE}" ]] ; then - echo "--- Creating symbols package ---" - echo "-- Package file: ${SYMBOLS_PACKAGE} --" - if [[ "$(uname -s)" == "Darwin" ]] ; then - (cd "${INSTALL_SYMROOT}" && - tar -c -z -f "${SYMBOLS_PACKAGE}" "${TOOLCHAIN_PREFIX/#\/}") - else - (cd "${INSTALL_SYMROOT}" && - tar -c -z -f "${SYMBOLS_PACKAGE}" --owner=0 --group=0 "${INSTALL_PREFIX/#\/}") - fi -fi diff --git a/utils/swift_build_support/swift_build_support/debug.py b/utils/swift_build_support/swift_build_support/debug.py new file mode 100644 index 0000000000000..c008d48bd6790 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/debug.py @@ -0,0 +1,41 @@ +# swift_build_support/debug.py - Print information on the build -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- +# +# Convenient functions for printing out information on the build process. +# +# ---------------------------------------------------------------------------- + +from __future__ import print_function + +import subprocess +import sys + + +def _output(args): + try: + out = subprocess.check_output(args, stderr=subprocess.PIPE) + return out.rstrip() + except subprocess.CalledProcessError: + return None + + +def print_xcodebuild_versions(sdks, file=sys.stdout): + """ + Print the host machine's `xcodebuild` version, as well as version + information for each of the given SDKs (for a full list of available + SDKs, invoke `xcodebuild -showsdks` on the command line). + """ + print(u'--- SDK versions ---', file=file) + print(u'{}\n'.format(_output(['xcodebuild', '-version'])), file=file) + for sdk in sdks: + print(u'{}\n'.format(_output(['xcodebuild', '-version', '-sdk', sdk])), + file=file) diff --git a/utils/swift_build_support/swift_build_support/ninja.py b/utils/swift_build_support/swift_build_support/ninja.py new file mode 100644 index 0000000000000..772db5fc774ea --- /dev/null +++ b/utils/swift_build_support/swift_build_support/ninja.py @@ -0,0 +1,21 @@ +# swift_build_support/ninja.py - Detect host machine's Ninja -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +from .which import which + + +def is_ninja_installed(): + """ + Return whether `ninja` or `ninja-build` are available on the host machine. + """ + if which('ninja') or which('ninja-build'): + return True + else: + return False diff --git a/utils/swift_build_support/swift_build_support/tar.py b/utils/swift_build_support/swift_build_support/tar.py new file mode 100644 index 0000000000000..52a96c570dd1a --- /dev/null +++ b/utils/swift_build_support/swift_build_support/tar.py @@ -0,0 +1,31 @@ +# swift_build_support/tar.py - Call tar from Python -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +import platform +import subprocess + + +def tar(source, destination): + """ + Create a gzip archive of the file at 'source' at the given + 'destination' path. + """ + # We do not use `tarfile` here because: + # - We wish to support LZMA2 compression while also supporting Python 2.7. + # - We wish to explicitly set the owner and group of the archive. + args = ['tar', '-c', '-z', '-f', destination] + + if platform.system() != 'Darwin': + args += ['--owner=0', '--group=0'] + + # Capture stderr output such as 'tar: Failed to open ...'. We'll detect + # these cases using the exit code, which should cause 'check_call' to + # raise. + subprocess.check_call(args + [source], stderr=subprocess.PIPE) diff --git a/utils/swift_build_support/swift_build_support/targets.py b/utils/swift_build_support/swift_build_support/targets.py index a4bdf99a03af3..4643531376b36 100644 --- a/utils/swift_build_support/swift_build_support/targets.py +++ b/utils/swift_build_support/swift_build_support/targets.py @@ -8,6 +8,7 @@ # See http://swift.org/LICENSE.txt for license information # See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +import os import platform @@ -44,3 +45,24 @@ def host_target(): return 'freebsd-x86_64' return None + + +def install_prefix(): + """ + Returns the default path at which built Swift products (like bin, lib, + and include) will be installed, based on the host machine's operating + system. + """ + if platform.system() == 'Darwin': + return '/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr' + else: + return '/usr' + + +def darwin_toolchain_prefix(darwin_install_prefix): + """ + Given the install prefix for a Darwin system, and assuming that that path + is to a .xctoolchain directory, return the path to the .xctoolchain + directory. + """ + return os.path.split(darwin_install_prefix)[0] diff --git a/utils/swift_build_support/tests/test_debug.py b/utils/swift_build_support/tests/test_debug.py new file mode 100644 index 0000000000000..45c0365ef7485 --- /dev/null +++ b/utils/swift_build_support/tests/test_debug.py @@ -0,0 +1,46 @@ +# test_debug.py - Unit tests for swift_build_support.debug -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +import platform +import unittest + +# StringIO import path differs across Python 2 and 3. +try: + from io import StringIO +except ImportError: + from cStringIO import StringIO + +from swift_build_support import debug + + +class PrintXcodebuildVersionsTestCase(unittest.TestCase): + def setUp(self): + if platform.system() != 'Darwin': + self.skipTest('print_xcodebuild_version() tests should only be ' + 'run on OS X') + self._out = StringIO() + + def test_outputs_xcode_version(self): + debug.print_xcodebuild_versions([], file=self._out) + actual = self._out.getvalue().splitlines() + self.assertEqual(actual[0], '--- SDK versions ---') + self.assertTrue(actual[1].startswith('Xcode ')) + self.assertTrue(actual[2].startswith('Build version ')) + + def test_outputs_all_sdks(self): + debug.print_xcodebuild_versions( + ['iphonesimulator', 'watchsimulator'], file=self._out) + actual = self._out.getvalue() + self.assertNotEqual(actual.find('iPhoneSimulator'), -1) + self.assertNotEqual(actual.find('WatchSimulator'), -1) + + +if __name__ == '__main__': + unittest.main() diff --git a/utils/swift_build_support/tests/test_tar.py b/utils/swift_build_support/tests/test_tar.py new file mode 100644 index 0000000000000..4b04a0e91761c --- /dev/null +++ b/utils/swift_build_support/tests/test_tar.py @@ -0,0 +1,33 @@ +# test_tar.py - Unit tests for swift_build_support.tar -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +import os +import subprocess +import tempfile +import unittest + +from swift_build_support.tar import tar + + +class TarTestCase(unittest.TestCase): + def test_tar_this_file_succeeds(self): + # `tar` complains about absolute paths, so use a relative path here. + source = os.path.relpath(__file__) + _, destination = tempfile.mkstemp() + tar(source=source, destination=destination) + + def test_tar_nonexistent_file_raises(self): + with self.assertRaises(subprocess.CalledProcessError): + tar(source='/path/to/something/that/surely/doesnt/exist', + destination='/another/path/that/shouldnt/exist') + + +if __name__ == '__main__': + unittest.main()