From f662d506840fdd50f71d9fea3cf358f1a358fab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ro=CC=88nnqvist?= Date: Tue, 12 Oct 2021 14:41:30 -0700 Subject: [PATCH 1/3] Add build script products for Swift-DocC and Swift-DocC-Render (rdar://79703353) --- utils/build-presets.ini | 54 ++++++ utils/build-toolchain | 6 +- .../build_swift/driver_arguments.py | 14 ++ .../contents.xcworkspacedata | 3 + utils/build_swift/tests/expected_options.py | 12 ++ .../build_script_invocation.py | 4 + .../swift_build_support/products/__init__.py | 4 + .../swift_build_support/products/swiftdocc.py | 163 ++++++++++++++++++ .../products/swiftdoccrender.py | 78 +++++++++ .../update-checkout-config.json | 46 ++++- 10 files changed, 380 insertions(+), 4 deletions(-) create mode 100644 utils/swift_build_support/swift_build_support/products/swiftdocc.py create mode 100644 utils/swift_build_support/swift_build_support/products/swiftdoccrender.py diff --git a/utils/build-presets.ini b/utils/build-presets.ini index 84ebc533d74ad..f6df4d9c39fa7 100644 --- a/utils/build-presets.ini +++ b/utils/build-presets.ini @@ -815,6 +815,7 @@ install-libicu install-prefix=/usr install-libcxx install-sourcekit-lsp +install-swiftdocc build-swift-static-stdlib build-swift-static-sdk-overlay build-swift-stdlib-unittest-extra @@ -854,6 +855,7 @@ foundation libdispatch indexstore-db sourcekit-lsp +swiftdocc lit-args=-v --time-tests # rdar://problem/31454823 @@ -877,6 +879,7 @@ skip-test-playgroundsupport skip-test-libicu skip-test-indexstore-db skip-test-sourcekit-lsp +skip-test-swiftdocc # Linux package with out test [preset: buildbot_linux,no_test] @@ -884,6 +887,21 @@ mixin-preset= buildbot_linux mixin_buildbot_linux,no_test +# Linux buidbot that include Swift-DocC-Render. +[preset: buildbot_linux,swiftdoccrender] +mixin-preset=buildbot_linux + +swiftdoccrender +install-swiftdoccrender + +# Linux buidbot without tests that include Swift-DocC-Render. +[preset: buildbot_linux,swiftdoccrender,no_test] +mixin-preset=buildbot_linux,no_test + +swiftdoccrender +install-swiftdoccrender +skip-test-swiftdoccrender + [preset: buildbot_linux_crosscompile_android,tools=RA,stdlib=RD,build] mixin-preset= mixin_lightweight_assertions,no-stdlib-asserts @@ -1221,6 +1239,7 @@ playgroundsupport libcxx indexstore-db sourcekit-lsp +swiftdocc # Build concurrency back-deployment binaries back-deploy-concurrency @@ -1270,6 +1289,7 @@ install-playgroundsupport install-libcxx install-sourcekit-lsp install-back-deploy-concurrency +install-swiftdocc install-destdir=%(install_destdir)s @@ -1377,6 +1397,7 @@ skip-test-playgroundsupport skip-test-swiftsyntax skip-test-skstresstester skip-test-swiftevolve +skip-test-swiftdocc [preset: buildbot_osx_package,no_assertions,no_test,use_os_runtime] mixin-preset= @@ -1411,6 +1432,7 @@ skip-test-playgroundsupport skip-test-swiftsyntax skip-test-skstresstester skip-test-swiftevolve +skip-test-swiftdocc # macOS package with out test [preset: buildbot_osx_package,no_test] @@ -1426,6 +1448,36 @@ mixin-preset= mixin_buildbot_osx_package,no_test mixin_osx_package,use_os_runtime +# macOS package that include Swift-DocC-Render. +[preset: buildbot_osx_package,swiftdoccrender] +mixin-preset=buildbot_osx_package + +swiftdoccrender +install-swiftdoccrender + +# macOS package without tests that include Swift-DocC-Render. +[preset: buildbot_osx_package,swiftdoccrender,no_test] +mixin-preset=buildbot_osx_package,no_test + +swiftdoccrender +install-swiftdoccrender +skip-test-swiftdoccrender + +# macOS package that use the OS runtime and include Swift-DocC-Render. +[preset: buildbot_osx_package,swiftdoccrender,use_os_runtime] +mixin-preset=buildbot_osx_package,use_os_runtime + +swiftdoccrender +install-swiftdoccrender + +# macOS package without tests that use the OS runtime and include Swift-DocC-Render. +[preset: buildbot_osx_package,swiftdoccrender,no_test,use_os_runtime] +mixin-preset=buildbot_osx_package,no_test,use_os_runtime + +swiftdoccrender +install-swiftdoccrender +skip-test-swiftdoccrender + #===------------------------------------------------------------------------===# # LLDB build configurations # @@ -1555,6 +1607,7 @@ swiftpm # swift-inspect swift-driver swiftsyntax +swiftdocc # Don't generate the SwiftSyntax gyb files. Instead verify that up-to-date ones # are checked in. @@ -1569,6 +1622,7 @@ install-llbuild install-swiftpm install-swift-driver install-libcxx +install-swiftdocc # Build the stress tester and SwiftEvolve skstresstester diff --git a/utils/build-toolchain b/utils/build-toolchain index 34eef3844ae7a..c65fe5b3c794c 100755 --- a/utils/build-toolchain +++ b/utils/build-toolchain @@ -50,6 +50,7 @@ DRY_RUN= BUNDLE_PREFIX= PRESET_FILE_FLAGS= PRESET_PREFIX= +SWIFT_DOCC_RENDER= NO_TEST=",no_test" USE_OS_RUNTIME= @@ -94,6 +95,9 @@ while [ $# -ne 0 ]; do ;; --use-os-runtime) USE_OS_RUNTIME=",use_os_runtime" + ;; + --swift-docc-render) + SWIFT_DOCC_RENDER=",swiftdoccrender" ;; -h|--help) usage @@ -145,7 +149,7 @@ SCCACHE_FLAG="${SCCACHE_FLAG}" ./utils/build-script ${DRY_RUN} ${DISTCC_FLAG} ${PRESET_FILE_FLAGS} \ ${SCCACHE_FLAG} \ - --preset="${PRESET_PREFIX}${SWIFT_PACKAGE}${NO_TEST}${USE_OS_RUNTIME}" \ + --preset="${PRESET_PREFIX}${SWIFT_PACKAGE}${SWIFT_DOCC_RENDER}${NO_TEST}${USE_OS_RUNTIME}" \ install_destdir="${SWIFT_INSTALL_DIR}" \ installable_package="${SWIFT_INSTALLABLE_PACKAGE}" \ install_toolchain_dir="${SWIFT_TOOLCHAIN_DIR}" \ diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index 569fd40182172..dc0c070075c40 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -193,6 +193,8 @@ def _apply_default_arguments(args): args.test_swiftformat = False args.test_swiftevolve = False args.test_toolchainbenchmarks = False + args.test_swiftdocc = False + args.test_swiftdoccrender = False # --test implies --test-early-swift-driver # (unless explicitly skipped with `--skip-test-early-swift-driver`) @@ -632,6 +634,10 @@ def create_argument_parser(): option(['--swift-driver'], toggle_true('build_swift_driver'), help='build swift-driver') + option(['--swiftdocc'], toggle_true('build_swiftdocc'), + help='build Swift DocC') + option(['--swiftdoccrender'], toggle_true('build_swiftdoccrender'), + help='build Swift DocC Render') option(['--skip-early-swift-driver'], toggle_false('build_early_swift_driver'), help='skip building the early swift-driver') @@ -660,6 +666,10 @@ def create_argument_parser(): help='install new Swift driver') option(['--install-swiftevolve'], toggle_true('install_swiftevolve'), help='install SwiftEvolve') + option(['--install-swiftdocc'], toggle_true('install_swiftdocc'), + help='install Swift DocC') + option(['--install-swiftdoccrender'], toggle_true('install_swiftdoccrender'), + help='install Swift DocC Render') option(['--toolchain-benchmarks'], toggle_true('build_toolchainbenchmarks'), help='build Swift Benchmarks using swiftpm against the just built ' @@ -1119,6 +1129,10 @@ def create_argument_parser(): option('--skip-test-swift-inspect', toggle_false('test_swift_inspect'), help='skip testing swift_inspect') + option('--skip-test-swiftdocc', toggle_false('test_swiftdocc'), + help='skip testing swift-docc') + option('--skip-test-swiftdoccrender', toggle_false('test_swiftdoccrender'), + help='skip testing swift-docc-render') # ------------------------------------------------------------------------- in_group('Build settings specific for LLVM') diff --git a/utils/build_swift/resources/SwiftPM-Unified-Build.xcworkspace/contents.xcworkspacedata b/utils/build_swift/resources/SwiftPM-Unified-Build.xcworkspace/contents.xcworkspacedata index 92ad6dd28df5b..c3e3faefe53b7 100644 --- a/utils/build_swift/resources/SwiftPM-Unified-Build.xcworkspace/contents.xcworkspacedata +++ b/utils/build_swift/resources/SwiftPM-Unified-Build.xcworkspace/contents.xcworkspacedata @@ -13,4 +13,7 @@ + + diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index 4004159d2966d..2a4b0f7a9e465 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -96,6 +96,8 @@ 'build_skstresstester': False, 'build_swiftformat': False, 'build_swiftevolve': False, + 'build_swiftdocc': False, + 'build_swiftdoccrender': False, 'build_indexstoredb': False, 'test_indexstoredb_sanitize_all': False, 'test_sourcekitlsp_sanitize_all': False, @@ -103,6 +105,8 @@ 'install_swiftpm': False, 'install_swiftsyntax': False, 'install_swift_driver': False, + 'install_swiftdocc': False, + 'install_swiftdoccrender': False, 'swiftsyntax_verify_generated_files': False, 'install_playgroundsupport': False, 'install_sourcekitlsp': False, @@ -262,6 +266,8 @@ 'test_skstresstester': False, 'test_swiftformat': False, 'test_swiftevolve': False, + 'test_swiftdocc': False, + 'test_swiftdoccrender': False, 'test_toolchainbenchmarks': False, 'tvos': False, 'tvos_all': False, @@ -509,6 +515,8 @@ class BuildScriptImplOption(_BaseOption): SetTrueOption('--skstresstester', dest='build_skstresstester'), SetTrueOption('--swiftformat', dest='build_swiftformat'), SetTrueOption('--swiftevolve', dest='build_swiftevolve'), + SetTrueOption('--swiftdocc', dest='build_swiftdocc'), + SetTrueOption('--swiftdoccrender', dest='build_swiftdoccrender'), SetTrueOption('-B', dest='benchmark'), SetTrueOption('-S', dest='skip_build'), SetTrueOption('-b', dest='build_llbuild'), @@ -563,6 +571,8 @@ class BuildScriptImplOption(_BaseOption): EnableOption('--install-sourcekit-lsp', dest='install_sourcekitlsp'), EnableOption('--install-skstresstester', dest='install_skstresstester'), EnableOption('--install-swiftevolve', dest='install_swiftevolve'), + EnableOption('--install-swiftdocc', dest='install_swiftdocc'), + EnableOption('--install-swiftdoccrender', dest='install_swiftdoccrender'), EnableOption('--toolchain-benchmarks', dest='build_toolchainbenchmarks'), EnableOption('--swift-inspect', dest='build_swift_inspect'), EnableOption('--tsan-libdispatch-test'), @@ -644,6 +654,8 @@ class BuildScriptImplOption(_BaseOption): DisableOption('--skip-test-skstresstester', dest='test_skstresstester'), DisableOption('--skip-test-swiftformat', dest='test_swiftformat'), DisableOption('--skip-test-swiftevolve', dest='test_swiftevolve'), + DisableOption('--skip-test-swiftdocc', dest='test_swiftdocc'), + DisableOption('--skip-test-swiftdoccrender', dest='test_swiftdoccrender'), DisableOption('--skip-test-toolchain-benchmarks', dest='test_toolchainbenchmarks'), DisableOption('--skip-test-swift-inspect', diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index fb1237dd6d633..6de96b4865f22 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -605,6 +605,10 @@ def compute_product_pipelines(self): is_enabled=self.args.build_swift_inspect) builder.add_product(products.TSanLibDispatch, is_enabled=self.args.tsan_libdispatch_test) + builder.add_product(products.SwiftDocC, + is_enabled=self.args.build_swiftdocc) + builder.add_product(products.SwiftDocCRender, + is_enabled=self.args.build_swiftdoccrender) # Keep SwiftDriver at last. # swift-driver's integration with the build scripts is not fully diff --git a/utils/swift_build_support/swift_build_support/products/__init__.py b/utils/swift_build_support/swift_build_support/products/__init__.py index 5fe35e9035c68..7cdd61fb504fb 100644 --- a/utils/swift_build_support/swift_build_support/products/__init__.py +++ b/utils/swift_build_support/swift_build_support/products/__init__.py @@ -27,6 +27,8 @@ from .skstresstester import SKStressTester from .sourcekitlsp import SourceKitLSP from .swift import Swift +from .swiftdocc import SwiftDocC +from .swiftdoccrender import SwiftDocCRender from .swiftdriver import SwiftDriver from .swiftevolve import SwiftEvolve from .swiftformat import SwiftFormat @@ -63,4 +65,6 @@ 'SourceKitLSP', 'Benchmarks', 'TSanLibDispatch', + 'SwiftDocC', + 'SwiftDocCRender' ] diff --git a/utils/swift_build_support/swift_build_support/products/swiftdocc.py b/utils/swift_build_support/swift_build_support/products/swiftdocc.py new file mode 100644 index 0000000000000..7790eb4a9735b --- /dev/null +++ b/utils/swift_build_support/swift_build_support/products/swiftdocc.py @@ -0,0 +1,163 @@ +# swift_build_support/products/swiftdocc.py ---------------------*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- + +import os + +from build_swift.build_swift.constants import MULTIROOT_DATA_FILE_PATH + +from . import cmark +from . import foundation +from . import libcxx +from . import libdispatch +from . import libicu +from . import llbuild +from . import llvm +from . import product +from . import swift +from . import swiftpm +from . import swiftsyntax +from . import xctest +from .. import shell + + +class SwiftDocC(product.Product): + @classmethod + def product_source_name(cls): + """product_source_name() -> str + + The name of the source code directory of this product. + """ + return "swift-docc" + + @classmethod + def is_build_script_impl_product(cls): + return False + + @classmethod + def is_before_build_script_impl_product(cls): + return False + + @classmethod + def is_swiftpm_unified_build_product(cls): + return True + + def run_build_script_helper(self, action, host_target, additional_params=[]): + script_path = os.path.join( + self.source_dir, 'build-script-helper.py') + + configuration = 'release' if self.is_release() else 'debug' + + helper_cmd = [ + script_path, + action, + '--toolchain', self.install_toolchain_path(host_target), + '--configuration', configuration, + '--build-dir', self.build_dir, + '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH, + ] + if action != 'install': + helper_cmd.extend([ + # There might have been a Package.resolved created by other builds + # or by the package being opened using Xcode. Discard that and + # reset the dependencies to be local. + '--update' + ]) + if self.args.verbose_build: + helper_cmd.append('--verbose') + helper_cmd.extend(additional_params) + + shell.call(helper_cmd) + + def should_build(self, host_target): + return True + + def build(self, host_target): + self.run_build_script_helper('build', host_target) + + def should_test(self, host_target): + return self.args.test_swiftdocc + + def test(self, host_target): + self.run_build_script_helper('test', host_target) + + def should_install(self, host_target): + return self.args.install_swiftdocc + + def install(self, host_target): + # swift-docc is installed at '/usr/bin/docc' in the built toolchain. + install_toolchain_path = self.install_toolchain_path(host_target) + install_dir = os.path.join(install_toolchain_path, 'bin') + + additional_params = ['--install-dir', install_dir] + + # PLEASE READ: + # The code below does something different than other build-script products do. + # + # Installing swift-docc without also building swift-docc-render will find the + # already built swift-docc-render template in the host toolchain and copy it + # over to the install toolchain. + # This allows developers with local swift-docc changes but no swift-docc-render + # changes to build and run their swift-docc changes without requiring 'node' + # which is required to build and install swift-docc-render from source. + # (Displaying the output from swift-docc with built swift-docc-render template + # doesn't require 'node' or other tools.) + if not self.args.build_swiftdoccrender: + # swift-docc and swift-docc-render are expected to come in pairs. + # When building swift-docc without building swift-docc-render, + # attempt to copy the prebuilt swift-docc-render from the host toolchain. + docc_path = self.toolchain.find_tool("docc") + if docc_path is None: + warn_msg = 'Host toolchain could not locate an prebuilt'\ + 'swift-docc-render to copy.'\ + '(Use `--swiftdoccrender` to build swift-docc-render)' + print('-- Warning: {}', warn_msg) + else: + additional_params.append('--copy-doccrender-from') + # Drop two path components from the docc path to get to the `/usr/` dir. + host_toolchain_path = os.path.dirname(os.path.dirname(docc_path)) + # Get the location of the swift-docc-render in the host toolchain. + built_render_dir = self.get_render_install_destdir( + host_toolchain_path) + additional_params.append(built_render_dir) + + additional_params.append('--copy-doccrender-to') + # Get the install location of the swift-docc-render in the install + # toolchain. + render_install_dir = self.get_render_install_destdir( + install_toolchain_path) + additional_params.append(render_install_dir) + note_msg = 'Copying already built swift-docc-render output '\ + 'from %s to %s.' % (built_render_dir, render_install_dir) + print('-- Note: {}', note_msg) + + self.run_build_script_helper('install', host_target, additional_params) + + # This is defined on swift-docc to avoid a circular import. + @classmethod + def get_render_install_destdir(cls, toolchain_install_dir): + # swift-docc-render is installed at '/usr/share/docc/render' in the built + # toolchain. + return os.path.join(toolchain_install_dir, 'share', 'docc', 'render') + + @classmethod + def get_dependencies(cls): + return [cmark.CMark, + llvm.LLVM, + libcxx.LibCXX, + libicu.LibICU, + swift.Swift, + libdispatch.LibDispatch, + foundation.Foundation, + xctest.XCTest, + llbuild.LLBuild, + swiftpm.SwiftPM, + swiftsyntax.SwiftSyntax] diff --git a/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py b/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py new file mode 100644 index 0000000000000..e91115a54b137 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py @@ -0,0 +1,78 @@ +# swift_build_support/products/swiftdoccrender.py ---------------*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- + +import os + +from . import product +from . import swiftdocc +from .. import shell + + +class SwiftDocCRender(product.Product): + @classmethod + def product_source_name(cls): + """product_source_name() -> str + + The name of the source code directory of this product. + """ + return "swift-docc-render" + + @classmethod + def is_build_script_impl_product(cls): + return False + + @classmethod + def is_before_build_script_impl_product(cls): + return False + + @classmethod + def is_swiftpm_unified_build_product(cls): + return False + + def run_build_script_helper(self, action, host_target, additional_params=[]): + script_path = os.path.join( + self.source_dir, 'build-script-helper.py') + + helper_cmd = [ + script_path, + action + ] + if self.args.verbose_build: + helper_cmd.append('--verbose') + helper_cmd.extend(additional_params) + + shell.call(helper_cmd) + + def should_build(self, host_target): + return True + + def build(self, host_target): + self.run_build_script_helper('build', host_target) + + def should_test(self, host_target): + return self.args.test_swiftdoccrender + + def test(self, host_target): + self.run_build_script_helper('test', host_target) + + def should_install(self, host_target): + return self.args.install_swiftdoccrender + + def install(self, host_target): + install_dir = swiftdocc.SwiftDocC.get_render_install_destdir( + self.install_toolchain_path(host_target)) + additional_params = ['--install-dir', install_dir] + self.run_build_script_helper('install', host_target, additional_params) + + @classmethod + def get_dependencies(cls): + return [swiftdocc.SwiftDocC] diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 6c81249d67871..c45abc7a94055 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -61,6 +61,22 @@ "remote": { "id": "apple/swift-format" } }, "swift-installer-scripts": { "remote": { "id": "apple/swift-installer-scripts" } }, + "swift-docc": { + "remote": { "id": "apple/swift-docc" } }, + "swift-lmdb": { + "remote": { "id": "apple/swift-lmdb" } }, + "swift-docc-render": { + "remote": { "id": "apple/swift-docc-render" } }, + "swift-docc-symbolkit": { + "remote": { "id": "apple/swift-docc-symbolkit" } }, + "swift-markdown": { + "remote": { "id": "apple/swift-markdown" } }, + "swift-cmark-gfm": { + "remote": { "id": "apple/swift-cmark" } }, + "swift-nio": { + "remote": { "id": "apple/swift-nio" } }, + "swift-nio-ssl": { + "remote": { "id": "apple/swift-nio-ssl" } }, "llvm-project": { "remote": { "id": "apple/llvm-project" } } }, @@ -96,7 +112,15 @@ "indexstore-db": "main", "sourcekit-lsp": "main", "swift-format": "main", - "swift-installer-scripts": "main" + "swift-installer-scripts": "main", + "swift-docc": "main", + "swift-lmdb": "main", + "swift-docc-render": "main", + "swift-docc-symbolkit": "main", + "swift-markdown": "main", + "swift-cmark-gfm": "gfm", + "swift-nio": "2.31.2", + "swift-nio-ssl": "2.15.0" } }, "release/5.5": { @@ -125,7 +149,15 @@ "indexstore-db": "release/5.5", "sourcekit-lsp": "release/5.5", "swift-format": "main", - "swift-installer-scripts": "main" + "swift-installer-scripts": "main", + "swift-docc": "main", + "swift-lmdb": "main", + "swift-docc-render": "main", + "swift-docc-symbolkit": "main", + "swift-markdown": "main", + "swift-cmark-gfm": "gfm", + "swift-nio": "2.31.2", + "swift-nio-ssl": "2.15.0" } }, "release/5.5-08092021": { @@ -154,7 +186,15 @@ "indexstore-db": "release/5.5-08092021", "sourcekit-lsp": "release/5.5-08092021", "swift-format": "main", - "swift-installer-scripts": "main" + "swift-installer-scripts": "main", + "swift-docc": "main", + "swift-lmdb": "main", + "swift-docc-render": "main", + "swift-docc-symbolkit": "main", + "swift-markdown": "main", + "swift-cmark-gfm": "gfm", + "swift-nio": "2.31.2", + "swift-nio-ssl": "2.15.0" } }, "next" : { From 6fff54d2150524d8ffe1bae46d3b3feea4252271 Mon Sep 17 00:00:00 2001 From: Ethan Kusters Date: Mon, 18 Oct 2021 11:54:12 -0700 Subject: [PATCH 2/3] Use swift-docc-render-artifact instead of building swift-docc-render --- utils/build-presets.ini | 45 --------------- utils/build-toolchain | 6 +- .../build_swift/driver_arguments.py | 7 --- utils/build_swift/tests/expected_options.py | 6 -- .../build_script_invocation.py | 2 +- .../swift_build_support/products/swiftdocc.py | 47 ---------------- .../products/swiftdoccrender.py | 56 ++++++++----------- .../update-checkout-config.json | 10 ++-- 8 files changed, 30 insertions(+), 149 deletions(-) diff --git a/utils/build-presets.ini b/utils/build-presets.ini index f6df4d9c39fa7..cceddab9d9bf5 100644 --- a/utils/build-presets.ini +++ b/utils/build-presets.ini @@ -887,21 +887,6 @@ mixin-preset= buildbot_linux mixin_buildbot_linux,no_test -# Linux buidbot that include Swift-DocC-Render. -[preset: buildbot_linux,swiftdoccrender] -mixin-preset=buildbot_linux - -swiftdoccrender -install-swiftdoccrender - -# Linux buidbot without tests that include Swift-DocC-Render. -[preset: buildbot_linux,swiftdoccrender,no_test] -mixin-preset=buildbot_linux,no_test - -swiftdoccrender -install-swiftdoccrender -skip-test-swiftdoccrender - [preset: buildbot_linux_crosscompile_android,tools=RA,stdlib=RD,build] mixin-preset= mixin_lightweight_assertions,no-stdlib-asserts @@ -1448,36 +1433,6 @@ mixin-preset= mixin_buildbot_osx_package,no_test mixin_osx_package,use_os_runtime -# macOS package that include Swift-DocC-Render. -[preset: buildbot_osx_package,swiftdoccrender] -mixin-preset=buildbot_osx_package - -swiftdoccrender -install-swiftdoccrender - -# macOS package without tests that include Swift-DocC-Render. -[preset: buildbot_osx_package,swiftdoccrender,no_test] -mixin-preset=buildbot_osx_package,no_test - -swiftdoccrender -install-swiftdoccrender -skip-test-swiftdoccrender - -# macOS package that use the OS runtime and include Swift-DocC-Render. -[preset: buildbot_osx_package,swiftdoccrender,use_os_runtime] -mixin-preset=buildbot_osx_package,use_os_runtime - -swiftdoccrender -install-swiftdoccrender - -# macOS package without tests that use the OS runtime and include Swift-DocC-Render. -[preset: buildbot_osx_package,swiftdoccrender,no_test,use_os_runtime] -mixin-preset=buildbot_osx_package,no_test,use_os_runtime - -swiftdoccrender -install-swiftdoccrender -skip-test-swiftdoccrender - #===------------------------------------------------------------------------===# # LLDB build configurations # diff --git a/utils/build-toolchain b/utils/build-toolchain index c65fe5b3c794c..34eef3844ae7a 100755 --- a/utils/build-toolchain +++ b/utils/build-toolchain @@ -50,7 +50,6 @@ DRY_RUN= BUNDLE_PREFIX= PRESET_FILE_FLAGS= PRESET_PREFIX= -SWIFT_DOCC_RENDER= NO_TEST=",no_test" USE_OS_RUNTIME= @@ -95,9 +94,6 @@ while [ $# -ne 0 ]; do ;; --use-os-runtime) USE_OS_RUNTIME=",use_os_runtime" - ;; - --swift-docc-render) - SWIFT_DOCC_RENDER=",swiftdoccrender" ;; -h|--help) usage @@ -149,7 +145,7 @@ SCCACHE_FLAG="${SCCACHE_FLAG}" ./utils/build-script ${DRY_RUN} ${DISTCC_FLAG} ${PRESET_FILE_FLAGS} \ ${SCCACHE_FLAG} \ - --preset="${PRESET_PREFIX}${SWIFT_PACKAGE}${SWIFT_DOCC_RENDER}${NO_TEST}${USE_OS_RUNTIME}" \ + --preset="${PRESET_PREFIX}${SWIFT_PACKAGE}${NO_TEST}${USE_OS_RUNTIME}" \ install_destdir="${SWIFT_INSTALL_DIR}" \ installable_package="${SWIFT_INSTALLABLE_PACKAGE}" \ install_toolchain_dir="${SWIFT_TOOLCHAIN_DIR}" \ diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index dc0c070075c40..a833b8763bf8f 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -194,7 +194,6 @@ def _apply_default_arguments(args): args.test_swiftevolve = False args.test_toolchainbenchmarks = False args.test_swiftdocc = False - args.test_swiftdoccrender = False # --test implies --test-early-swift-driver # (unless explicitly skipped with `--skip-test-early-swift-driver`) @@ -636,8 +635,6 @@ def create_argument_parser(): help='build swift-driver') option(['--swiftdocc'], toggle_true('build_swiftdocc'), help='build Swift DocC') - option(['--swiftdoccrender'], toggle_true('build_swiftdoccrender'), - help='build Swift DocC Render') option(['--skip-early-swift-driver'], toggle_false('build_early_swift_driver'), help='skip building the early swift-driver') @@ -668,8 +665,6 @@ def create_argument_parser(): help='install SwiftEvolve') option(['--install-swiftdocc'], toggle_true('install_swiftdocc'), help='install Swift DocC') - option(['--install-swiftdoccrender'], toggle_true('install_swiftdoccrender'), - help='install Swift DocC Render') option(['--toolchain-benchmarks'], toggle_true('build_toolchainbenchmarks'), help='build Swift Benchmarks using swiftpm against the just built ' @@ -1131,8 +1126,6 @@ def create_argument_parser(): help='skip testing swift_inspect') option('--skip-test-swiftdocc', toggle_false('test_swiftdocc'), help='skip testing swift-docc') - option('--skip-test-swiftdoccrender', toggle_false('test_swiftdoccrender'), - help='skip testing swift-docc-render') # ------------------------------------------------------------------------- in_group('Build settings specific for LLVM') diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index 2a4b0f7a9e465..4f3082a931100 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -97,7 +97,6 @@ 'build_swiftformat': False, 'build_swiftevolve': False, 'build_swiftdocc': False, - 'build_swiftdoccrender': False, 'build_indexstoredb': False, 'test_indexstoredb_sanitize_all': False, 'test_sourcekitlsp_sanitize_all': False, @@ -106,7 +105,6 @@ 'install_swiftsyntax': False, 'install_swift_driver': False, 'install_swiftdocc': False, - 'install_swiftdoccrender': False, 'swiftsyntax_verify_generated_files': False, 'install_playgroundsupport': False, 'install_sourcekitlsp': False, @@ -267,7 +265,6 @@ 'test_swiftformat': False, 'test_swiftevolve': False, 'test_swiftdocc': False, - 'test_swiftdoccrender': False, 'test_toolchainbenchmarks': False, 'tvos': False, 'tvos_all': False, @@ -516,7 +513,6 @@ class BuildScriptImplOption(_BaseOption): SetTrueOption('--swiftformat', dest='build_swiftformat'), SetTrueOption('--swiftevolve', dest='build_swiftevolve'), SetTrueOption('--swiftdocc', dest='build_swiftdocc'), - SetTrueOption('--swiftdoccrender', dest='build_swiftdoccrender'), SetTrueOption('-B', dest='benchmark'), SetTrueOption('-S', dest='skip_build'), SetTrueOption('-b', dest='build_llbuild'), @@ -572,7 +568,6 @@ class BuildScriptImplOption(_BaseOption): EnableOption('--install-skstresstester', dest='install_skstresstester'), EnableOption('--install-swiftevolve', dest='install_swiftevolve'), EnableOption('--install-swiftdocc', dest='install_swiftdocc'), - EnableOption('--install-swiftdoccrender', dest='install_swiftdoccrender'), EnableOption('--toolchain-benchmarks', dest='build_toolchainbenchmarks'), EnableOption('--swift-inspect', dest='build_swift_inspect'), EnableOption('--tsan-libdispatch-test'), @@ -655,7 +650,6 @@ class BuildScriptImplOption(_BaseOption): DisableOption('--skip-test-swiftformat', dest='test_swiftformat'), DisableOption('--skip-test-swiftevolve', dest='test_swiftevolve'), DisableOption('--skip-test-swiftdocc', dest='test_swiftdocc'), - DisableOption('--skip-test-swiftdoccrender', dest='test_swiftdoccrender'), DisableOption('--skip-test-toolchain-benchmarks', dest='test_toolchainbenchmarks'), DisableOption('--skip-test-swift-inspect', diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index 6de96b4865f22..0a269317a2a77 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -608,7 +608,7 @@ def compute_product_pipelines(self): builder.add_product(products.SwiftDocC, is_enabled=self.args.build_swiftdocc) builder.add_product(products.SwiftDocCRender, - is_enabled=self.args.build_swiftdoccrender) + is_enabled=self.args.install_swiftdocc) # Keep SwiftDriver at last. # swift-driver's integration with the build scripts is not fully diff --git a/utils/swift_build_support/swift_build_support/products/swiftdocc.py b/utils/swift_build_support/swift_build_support/products/swiftdocc.py index 7790eb4a9735b..5e438591baa2c 100644 --- a/utils/swift_build_support/swift_build_support/products/swiftdocc.py +++ b/utils/swift_build_support/swift_build_support/products/swiftdocc.py @@ -99,55 +99,8 @@ def install(self, host_target): additional_params = ['--install-dir', install_dir] - # PLEASE READ: - # The code below does something different than other build-script products do. - # - # Installing swift-docc without also building swift-docc-render will find the - # already built swift-docc-render template in the host toolchain and copy it - # over to the install toolchain. - # This allows developers with local swift-docc changes but no swift-docc-render - # changes to build and run their swift-docc changes without requiring 'node' - # which is required to build and install swift-docc-render from source. - # (Displaying the output from swift-docc with built swift-docc-render template - # doesn't require 'node' or other tools.) - if not self.args.build_swiftdoccrender: - # swift-docc and swift-docc-render are expected to come in pairs. - # When building swift-docc without building swift-docc-render, - # attempt to copy the prebuilt swift-docc-render from the host toolchain. - docc_path = self.toolchain.find_tool("docc") - if docc_path is None: - warn_msg = 'Host toolchain could not locate an prebuilt'\ - 'swift-docc-render to copy.'\ - '(Use `--swiftdoccrender` to build swift-docc-render)' - print('-- Warning: {}', warn_msg) - else: - additional_params.append('--copy-doccrender-from') - # Drop two path components from the docc path to get to the `/usr/` dir. - host_toolchain_path = os.path.dirname(os.path.dirname(docc_path)) - # Get the location of the swift-docc-render in the host toolchain. - built_render_dir = self.get_render_install_destdir( - host_toolchain_path) - additional_params.append(built_render_dir) - - additional_params.append('--copy-doccrender-to') - # Get the install location of the swift-docc-render in the install - # toolchain. - render_install_dir = self.get_render_install_destdir( - install_toolchain_path) - additional_params.append(render_install_dir) - note_msg = 'Copying already built swift-docc-render output '\ - 'from %s to %s.' % (built_render_dir, render_install_dir) - print('-- Note: {}', note_msg) - self.run_build_script_helper('install', host_target, additional_params) - # This is defined on swift-docc to avoid a circular import. - @classmethod - def get_render_install_destdir(cls, toolchain_install_dir): - # swift-docc-render is installed at '/usr/share/docc/render' in the built - # toolchain. - return os.path.join(toolchain_install_dir, 'share', 'docc', 'render') - @classmethod def get_dependencies(cls): return [cmark.CMark, diff --git a/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py b/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py index e91115a54b137..fdca4e4ff955a 100644 --- a/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py +++ b/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py @@ -13,7 +13,6 @@ import os from . import product -from . import swiftdocc from .. import shell @@ -24,7 +23,7 @@ def product_source_name(cls): The name of the source code directory of this product. """ - return "swift-docc-render" + return "swift-docc-render-artifact" @classmethod def is_build_script_impl_product(cls): @@ -38,41 +37,32 @@ def is_before_build_script_impl_product(cls): def is_swiftpm_unified_build_product(cls): return False - def run_build_script_helper(self, action, host_target, additional_params=[]): - script_path = os.path.join( - self.source_dir, 'build-script-helper.py') - - helper_cmd = [ - script_path, - action - ] - if self.args.verbose_build: - helper_cmd.append('--verbose') - helper_cmd.extend(additional_params) - - shell.call(helper_cmd) - def should_build(self, host_target): - return True - - def build(self, host_target): - self.run_build_script_helper('build', host_target) + # Swift-DocC-Render is a pre-built, installable artifact. + return False def should_test(self, host_target): - return self.args.test_swiftdoccrender - - def test(self, host_target): - self.run_build_script_helper('test', host_target) + # Swift-DocC-Render is a pre-built, installable artifact. + return False def should_install(self, host_target): - return self.args.install_swiftdoccrender + # Swift-DocC-Render should always be installed if Swift-DocC is being installed + return self.args.install_swiftdocc def install(self, host_target): - install_dir = swiftdocc.SwiftDocC.get_render_install_destdir( - self.install_toolchain_path(host_target)) - additional_params = ['--install-dir', install_dir] - self.run_build_script_helper('install', host_target, additional_params) - - @classmethod - def get_dependencies(cls): - return [swiftdocc.SwiftDocC] + # Swift-DocC-Render is installed at '/usr/share/docc/render' in the built + # toolchain. + install_toolchain_path = self.install_toolchain_path(host_target) + install_path = os.path.join(install_toolchain_path, 'share', 'docc', 'render') + + # The pre-built version of Swift-DocC-Render is distributed in the 'dist' + # folder at the root of the swift-docc-render-artifact repository + artifact_dist_path = os.path.join(self.source_dir, 'dist') + + # Add a trailing slash so that we copy the contents of the 'dist' directory + # instead of the 'dist' directory itself. + artifact_dist_path_with_trailing_slash = os.path.join(artifact_dist_path, '') + + shell.call(["mkdir", "-p", install_path]) + shell.call( + ["rsync", "-a", artifact_dist_path_with_trailing_slash, install_path]) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index c45abc7a94055..9bc0750e78ecc 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -65,8 +65,8 @@ "remote": { "id": "apple/swift-docc" } }, "swift-lmdb": { "remote": { "id": "apple/swift-lmdb" } }, - "swift-docc-render": { - "remote": { "id": "apple/swift-docc-render" } }, + "swift-docc-render-artifact": { + "remote": { "id": "apple/swift-docc-render-artifact" } }, "swift-docc-symbolkit": { "remote": { "id": "apple/swift-docc-symbolkit" } }, "swift-markdown": { @@ -115,7 +115,7 @@ "swift-installer-scripts": "main", "swift-docc": "main", "swift-lmdb": "main", - "swift-docc-render": "main", + "swift-docc-render-artifact": "main", "swift-docc-symbolkit": "main", "swift-markdown": "main", "swift-cmark-gfm": "gfm", @@ -152,7 +152,7 @@ "swift-installer-scripts": "main", "swift-docc": "main", "swift-lmdb": "main", - "swift-docc-render": "main", + "swift-docc-render-artifact": "main", "swift-docc-symbolkit": "main", "swift-markdown": "main", "swift-cmark-gfm": "gfm", @@ -189,7 +189,7 @@ "swift-installer-scripts": "main", "swift-docc": "main", "swift-lmdb": "main", - "swift-docc-render": "main", + "swift-docc-render-artifact": "main", "swift-docc-symbolkit": "main", "swift-markdown": "main", "swift-cmark-gfm": "gfm", From 4d7383366b853d9ff6eb623d38d541535e6fd898 Mon Sep 17 00:00:00 2001 From: Ethan Kusters Date: Fri, 29 Oct 2021 13:58:19 -0700 Subject: [PATCH 3/3] Add `get_dependencies` method to Swift-DocC-Render script Adding the required `get_dependencies` class method should resolve the remaining test failures we're seeing for macOS toolchain builds. --- .../swift_build_support/products/swiftdoccrender.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py b/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py index fdca4e4ff955a..44e512388124f 100644 --- a/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py +++ b/utils/swift_build_support/swift_build_support/products/swiftdoccrender.py @@ -66,3 +66,7 @@ def install(self, host_target): shell.call(["mkdir", "-p", install_path]) shell.call( ["rsync", "-a", artifact_dist_path_with_trailing_slash, install_path]) + + @classmethod + def get_dependencies(cls): + return []