diff --git a/.gitignore b/.gitignore index fcd681e7bd9c5..59a4590b1a66f 100644 --- a/.gitignore +++ b/.gitignore @@ -130,4 +130,9 @@ app.*.symbols # Prebuilt binaries. /prebuilts/ + +# GN build support for protobufs vended by Fuchsia. /build/secondary/third_party/protobuf + +# RBE support configurations and scripts vended from CIPD +/build/rbe diff --git a/DEPS b/DEPS index 1c539b40d6abe..84ef232c3e621 100644 --- a/DEPS +++ b/DEPS @@ -54,7 +54,7 @@ vars = { # https://chrome-infra-packages.appspot.com/p/fuchsia/third_party/goma/client 'goma_version': ' git_revision:41b3bcb64014144a844153fd5588c36411fffb56', - 'reclient_version': 'git_revision:81e819b39d4743462857cc55430d898b9fcca1af', + 'reclient_version': 'git_revision:f3883c2237b0eb9cc9524cb571b5ab8378f257e4', 'gcloud_version': 'version:2@444.0.0.chromium.3', @@ -127,6 +127,10 @@ vars = { # specified by GOMA_DIR, or installed in the default goma install location. 'use_cipd_goma': False, + # When this is true, the Flutter Engine's configuration files and scripts for + # RBE will be downloaded from CIPD. This option is only usable by Googlers. + 'use_rbe': False, + # This is not downloaded be default because it increases the # `gclient sync` time by between 1 and 3 minutes. This option is enabled # in flutter/ci/builders/mac_impeller_cmake_example.json, and is likely to @@ -267,7 +271,7 @@ allowed_hosts = [ ] deps = { - 'src': 'https://github.com/flutter/buildroot.git' + '@' + '8c274b21f1ad4f2aec0a5e0ae8f4264393045b4b', + 'src': 'https://github.com/flutter/buildroot.git' + '@' + 'a43582b52d361bc3da156a8e6eab6dd947ca339d', 'src/flutter/third_party/rapidjson': Var('flutter_git') + '/third_party/rapidjson' + '@' + 'ef3564c5c8824989393b87df25355baf35ff544b', @@ -845,7 +849,7 @@ deps = { 'version': Var('clang_version'), } ], - 'condition': 'host_os == "linux" and host_cpu == "x64"', + 'condition': 'host_os == "linux" or host_os == "mac"', 'dep_type': 'cipd', }, @@ -905,7 +909,7 @@ deps = { 'dep_type': 'cipd', }, - # reclient. + # RBE binaries and configs. 'src/buildtools/linux-x64/reclient': { 'packages': [ { @@ -913,7 +917,40 @@ deps = { 'version': Var('reclient_version'), } ], - 'condition': 'host_os == "linux" and host_cpu == "x64"', + 'condition': 'use_rbe and host_os == "linux" and host_cpu == "x64"', + 'dep_type': 'cipd', + }, + + 'src/buildtools/mac-arm64/reclient': { + 'packages': [ + { + 'package': 'infra/rbe/client/${{platform}}', + 'version': Var('reclient_version'), + } + ], + 'condition': 'use_rbe and host_os == "mac" and host_cpu == "arm64"', + 'dep_type': 'cipd', + }, + + 'src/buildtools/mac-x64/reclient': { + 'packages': [ + { + 'package': 'infra/rbe/client/${{platform}}', + 'version': Var('reclient_version'), + } + ], + 'condition': 'use_rbe and host_os == "mac" and host_cpu == "x64"', + 'dep_type': 'cipd', + }, + + 'src/flutter/build/rbe': { + 'packages': [ + { + 'package': 'flutter_internal/rbe/reclient_cfgs', + 'version': 'U42C0v8jI-_YREjd8rbDEt0evvqvLWJ_NTkaiJ_Clt8C', + } + ], + 'condition': 'use_rbe', 'dep_type': 'cipd', }, @@ -925,7 +962,18 @@ deps = { 'version': Var('gcloud_version'), } ], - 'condition': 'host_os == "linux" and host_cpu == "x64"', + 'condition': 'use_rbe and host_os == "linux" and host_cpu == "x64"', + 'dep_type': 'cipd', + }, + + 'src/buildtools/mac-arm64/gcloud': { + 'packages': [ + { + 'package': 'infra/3pp/tools/gcloud/${{platform}}', + 'version': Var('gcloud_version'), + } + ], + 'condition': 'use_rbe and host_os == "mac" and host_cpu == "arm64"', 'dep_type': 'cipd', }, diff --git a/ci/builders/linux_android_emulator.json b/ci/builders/linux_android_emulator.json index 847d161edbe47..bc5217a92c7ac 100644 --- a/ci/builders/linux_android_emulator.json +++ b/ci/builders/linux_android_emulator.json @@ -7,6 +7,9 @@ "kvm=1", "cores=8" ], + "gclient_variables": { + "use_rbe": true + }, "gn": [ "--android", "--android-cpu=x64", @@ -69,6 +72,9 @@ "kvm=1", "cores=8" ], + "gclient_variables": { + "use_rbe": true + }, "gn": [ "--android", "--android-cpu=x86", diff --git a/ci/builders/linux_unopt.json b/ci/builders/linux_unopt.json index 32b9e0201e599..89bd7170dc02d 100644 --- a/ci/builders/linux_unopt.json +++ b/ci/builders/linux_unopt.json @@ -6,6 +6,9 @@ "os=Linux", "cores=32" ], + "gclient_variables": { + "use_rbe": true + }, "gn": [ "--runtime-mode", "debug", @@ -84,6 +87,9 @@ "os=Linux" ], "dependencies": [], + "gclient_variables": { + "use_rbe": true + }, "gn": [ "--android", "--embedder-for-target", @@ -112,6 +118,9 @@ "version": "last_updated:2023-02-03T15:32:01-0800" } ], + "gclient_variables": { + "use_rbe": true + }, "gn": [ "--android", "--unoptimized", diff --git a/ci/builders/mac_unopt.json b/ci/builders/mac_unopt.json index ae72400e34ef2..2e5f1a23139d4 100644 --- a/ci/builders/mac_unopt.json +++ b/ci/builders/mac_unopt.json @@ -16,7 +16,8 @@ "mac_model=Macmini8,1" ], "gclient_variables": { - "download_android_deps": false + "download_android_deps": false, + "use_rbe": true }, "gn": [ "--runtime-mode", @@ -24,7 +25,10 @@ "--unoptimized", "--no-lto", "--prebuilt-dart-sdk", - "--enable-impeller-3d" + "--enable-impeller-3d", + "--rbe", + "--no-goma", + "--xcode-symlinks" ], "name": "host_debug_unopt", "ninja": { @@ -75,14 +79,18 @@ "cpu=x86" ], "gclient_variables": { - "download_android_deps": false + "download_android_deps": false, + "use_rbe": true }, "gn": [ "--ios", "--runtime-mode", "debug", "--simulator", - "--no-lto" + "--no-lto", + "--rbe", + "--no-goma", + "--xcode-symlinks" ], "name": "ios_debug_sim", "ninja": { @@ -132,7 +140,8 @@ "cpu=arm64" ], "gclient_variables": { - "download_android_deps": false + "download_android_deps": false, + "use_rbe": true }, "gn": [ "--runtime-mode", @@ -142,7 +151,10 @@ "--prebuilt-dart-sdk", "--force-mac-arm64", "--mac-cpu", - "arm64" + "arm64", + "--rbe", + "--no-goma", + "--xcode-symlinks" ], "name": "host_debug_unopt_arm64", "ninja": { @@ -181,7 +193,8 @@ "cpu=arm64" ], "gclient_variables": { - "download_android_deps": false + "download_android_deps": false, + "use_rbe": true }, "gn": [ "--ios", @@ -191,7 +204,10 @@ "--no-lto", "--force-mac-arm64", "--simulator-cpu", - "arm64" + "arm64", + "--rbe", + "--no-goma", + "--xcode-symlinks" ], "name": "ios_debug_sim_arm64", "ninja": { @@ -251,7 +267,8 @@ "cpu=arm64" ], "gclient_variables": { - "download_android_deps": false + "download_android_deps": false, + "use_rbe": true }, "gn": [ "--ios", @@ -262,7 +279,10 @@ "--force-mac-arm64", "--simulator-cpu", "arm64", - "--darwin-extension-safe" + "--darwin-extension-safe", + "--rbe", + "--no-goma", + "--xcode-symlinks" ], "name": "ios_debug_sim_arm64_extension_safe", "ninja": { diff --git a/ci/builders/standalone/linux_benchmarks.json b/ci/builders/standalone/linux_benchmarks.json index a408d9cf77866..b118daf944f52 100644 --- a/ci/builders/standalone/linux_benchmarks.json +++ b/ci/builders/standalone/linux_benchmarks.json @@ -4,7 +4,8 @@ "os=Linux" ], "gclient_variables": { - "download_android_deps": false + "download_android_deps": false, + "use_rbe": true }, "gn": [ "--runtime-mode", diff --git a/tools/gn b/tools/gn index 6ceda846269fe..7b9b2fe76e9d4 100755 --- a/tools/gn +++ b/tools/gn @@ -205,8 +205,6 @@ def buildtools_dir(): host_cpu = get_host_cpu() if host_os == 'win': host_os = 'windows' - if host_os == 'mac' and host_cpu == 'arm64': - host_cpu = 'x64' return '%s-%s' % (host_os, host_cpu) @@ -217,9 +215,10 @@ def setup_rbe(args): if not args.rbe: return rbe_gn_args - if get_host_os() not in ['linux']: + if get_host_os() not in ['linux', 'mac']: print( - 'The --rbe flag has no effect. RBE is currently only supported on Linux.' + 'The --rbe flag has no effect. RBE is currently only supported on ' + 'macOS and Linux.' ) return rbe_gn_args @@ -239,7 +238,9 @@ def setup_rbe(args): ) bootstrap_path = os.path.join(cipd_reclient_dir, 'bootstrap') reproxy_path = os.path.join(cipd_reclient_dir, 'reproxy') - rbe_cfg_path = os.path.join(SRC_ROOT, 'build', 'rbe.cfg') + rbe_cfg_path = os.path.join( + SRC_ROOT, 'flutter', 'build', 'rbe', 'reclient.cfg' + ) bootstrap_cmd = [ bootstrap_path, '--re_proxy=' + reproxy_path, @@ -260,8 +261,20 @@ def setup_rbe(args): rbe_gn_args['rbe_dial_timeout'] = args.rbe_dial_timeout if args.rbe_platform: rbe_gn_args['rbe_platform'] = args.rbe_platform - if args.rbe_dir: - rbe_gn_args['rbe_dir'] = args.rbe_dir + + rbe_gn_args['rbe_dir'] = os.path.join( + SRC_ROOT, 'buildtools', buildtools_dir(), 'reclient' + ) + + rbe_gn_args['rbe_cfg'] = os.path.join( + SRC_ROOT, 'flutter', 'build', 'rbe', + 'rewrapper-' + buildtools_dir() + '.cfg' + ) + + if sys.platform == 'darwin': + if (not running_on_luci or args.xcode_symlinks or + os.getenv('FLUTTER_GOMA_CREATE_XCODE_SYMLINKS', '0') == '1'): + rbe_gn_args['create_xcode_symlinks'] = True return rbe_gn_args @@ -285,8 +298,16 @@ def setup_goma(args): # care of starting and stopping the compiler proxy. running_on_luci = os.environ.get('LUCI_CONTEXT') is not None + # The GOMA client has no arm64 binary, so run the x64 binary through + # Rosetta. + buildtools_platform = buildtools_dir() + if buildtools_platform == 'mac-arm64': + buildtools_platform = 'mac-x64' + # Prefer the goma fetched by gclient if it exists. - cipd_goma_dir = os.path.join(SRC_ROOT, 'buildtools', buildtools_dir(), 'goma') + cipd_goma_dir = os.path.join( + SRC_ROOT, 'buildtools', buildtools_platform, 'goma' + ) # Next, if GOMA_DIR is set, use that install. goma_dir = os.environ.get('GOMA_DIR')