From d26b899fa9172a83ef68c18ef038ed77abe6d10b Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Thu, 13 Jun 2024 09:27:38 +0100 Subject: [PATCH] [RN][CI]Update the testing-script to use github actions --- .github/workflows/test-all.yml | 5 + scripts/release-testing/test-e2e-local.js | 103 +++++---- .../utils/github-actions-utils.js | 195 ++++++++++++++++++ .../release-testing/utils/testing-utils.js | 124 ++++++++--- 4 files changed, 365 insertions(+), 62 deletions(-) create mode 100644 scripts/release-testing/utils/github-actions-utils.js diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index e27ce5c233ce77..203b4d7ac7df19 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -747,6 +747,11 @@ jobs: - name: Publish NPM shell: bash run: | + # The checkout command puts react-native in a folder that is not "safe" for git + # Every git command run in an unsafe folder fails. We need to pick the current commit to use it as part of the version + # The following line marks the folder where react-native lives as "safe" + git config --global --add safe.directory /__w/react-native/react-native + echo "GRADLE_OPTS = $GRADLE_OPTS" # We can't have a separate step because each command is executed in a separate shell # so variables exported in a command are not visible in another. diff --git a/scripts/release-testing/test-e2e-local.js b/scripts/release-testing/test-e2e-local.js index e7476030917dd8..0242df6dfa3a69 100644 --- a/scripts/release-testing/test-e2e-local.js +++ b/scripts/release-testing/test-e2e-local.js @@ -26,6 +26,7 @@ const { maybeLaunchAndroidEmulator, prepareArtifacts, setupCircleCIArtifacts, + setupGHAArtifacts, } = require('./utils/testing-utils'); const chalk = require('chalk'); const debug = require('debug')('test-e2e-local'); @@ -56,7 +57,7 @@ const argv = yargs default: true, }) .option('c', { - alias: 'circleciToken', + alias: 'ciToken', type: 'string', }) .option('useLastSuccessfulPipeline', { @@ -78,7 +79,7 @@ const argv = yargs * - @onReleaseBranch whether we are on a release branch or not */ async function testRNTesterIOS( - circleCIArtifacts /*: Unwrap> */, + ciArtifacts /*: Unwrap> */, onReleaseBranch /*: boolean */, ) { console.info( @@ -90,14 +91,17 @@ async function testRNTesterIOS( // remember that for this to be successful // you should have run bundle install once // in your local setup - if (argv.hermes === true && circleCIArtifacts != null) { - const hermesURL = await circleCIArtifacts.artifactURLHermesDebug(); - const hermesPath = path.join( - circleCIArtifacts.baseTmpPath(), - 'hermes-ios-debug.tar.gz', - ); + if (argv.hermes === true && ciArtifacts != null) { + const hermesURL = await ciArtifacts.artifactURLHermesDebug(); + const hermesZipPath = path.join(ciArtifacts.baseTmpPath(), 'hermes.zip'); // download hermes source code from manifold - circleCIArtifacts.downloadArtifact(hermesURL, hermesPath); + ciArtifacts.downloadArtifact(hermesURL, hermesZipPath); + // GHA zips by default the artifacts. + const outputFolder = path.join(ciArtifacts.baseTmpPath(), 'hermes'); + exec(`rm -rf ${outputFolder}`); + exec(`unzip ${hermesZipPath} -d ${outputFolder}`); + const hermesPath = path.join(outputFolder, 'hermes-ios-Debug.tar.gz'); + console.info(`Downloaded Hermes in ${hermesPath}`); exec( `HERMES_ENGINE_TARBALL_PATH=${hermesPath} RCT_NEW_ARCH_ENABLED=1 bundle exec pod install --ansi`, @@ -115,7 +119,9 @@ async function testRNTesterIOS( launchPackagerInSeparateWindow(pwd().toString()); // launch the app on iOS simulator - exec('npx react-native run-ios --scheme RNTester --simulator "iPhone 14"'); + exec( + 'npx react-native run-ios --scheme RNTester --simulator "iPhone 15 Pro"', + ); } /** @@ -125,7 +131,7 @@ async function testRNTesterIOS( * - @circleCIArtifacts manager object to manage all the download of CircleCIArtifacts. If null, it will fallback not to use them. */ async function testRNTesterAndroid( - circleCIArtifacts /*: Unwrap> */, + ciArtifacts /*: Unwrap> */, ) { maybeLaunchAndroidEmulator(); @@ -143,22 +149,37 @@ async function testRNTesterAndroid( "adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done; input keyevent 82'", ); - if (circleCIArtifacts != null) { - const downloadPath = path.join( - circleCIArtifacts.baseTmpPath(), - 'rntester.apk', - ); + if (ciArtifacts != null) { + const downloadPath = path.join(ciArtifacts.baseTmpPath(), 'rntester.zip'); const emulatorArch = exec('adb shell getprop ro.product.cpu.abi').trim(); - const rntesterAPKURL = - argv.hermes === true - ? await circleCIArtifacts.artifactURLForHermesRNTesterAPK(emulatorArch) - : await circleCIArtifacts.artifactURLForJSCRNTesterAPK(emulatorArch); + // Github Actions zips all the APKs in a single archive console.info('Start Downloading APK'); - circleCIArtifacts.downloadArtifact(rntesterAPKURL, downloadPath); + const rntesterAPKURL = + await ciArtifacts.artifactURLForHermesRNTesterAPK(emulatorArch); + ciArtifacts.downloadArtifact(rntesterAPKURL, downloadPath); + const unzipFolder = path.join(ciArtifacts.baseTmpPath(), 'rntester-apks'); + exec(`rm -rf ${unzipFolder}`); + exec(`unzip ${downloadPath} -d ${unzipFolder}`); + let apkPath; + if (argv.hermes === true) { + apkPath = path.join( + unzipFolder, + 'hermes', + 'release', + `app-hermes-${emulatorArch}-release.apk`, + ); + } else { + apkPath = path.join( + unzipFolder, + 'jsc', + 'release', + `app-jsc-${emulatorArch}-release.apk`, + ); + } - exec(`adb install ${downloadPath}`); + exec(`adb install ${apkPath}`); } else { exec( `../../gradlew :packages:rn-tester:android:app:${ @@ -205,7 +226,7 @@ async function testRNTester( // === RNTestProject === // async function testRNTestProject( - circleCIArtifacts /*: Unwrap> */, + ciArtifacts /*: Unwrap> */, ) { console.info("We're going to test a fresh new RN project"); @@ -227,12 +248,12 @@ async function testRNTestProject( const localNodeTGZPath = `${reactNativePackagePath}/react-native-${releaseVersion}.tgz`; const mavenLocalPath = - circleCIArtifacts != null - ? path.join(circleCIArtifacts.baseTmpPath(), 'maven-local') + ciArtifacts != null + ? path.join(ciArtifacts.baseTmpPath(), 'maven-local') : '/private/tmp/maven-local'; - const hermesPath = await prepareArtifacts( - circleCIArtifacts, + const {hermesPath, newLocalNodeTGZ} = await prepareArtifacts( + ciArtifacts, mavenLocalPath, localNodeTGZPath, releaseVersion, @@ -241,12 +262,12 @@ async function testRNTestProject( ); // If artifacts were built locally, we need to pack the react-native package - if (circleCIArtifacts == null) { + if (ciArtifacts == null) { exec('npm pack --pack-destination ', {cwd: reactNativePackagePath}); // node pack does not creates a version of React Native with the right name on main. // Let's add some defensive programming checks: - if (!fs.existsSync(localNodeTGZPath)) { + if (!fs.existsSync(newLocalNodeTGZ)) { const tarfile = fs .readdirSync(reactNativePackagePath) .find( @@ -256,13 +277,13 @@ async function testRNTestProject( throw new Error("Couldn't find a zipped version of react-native"); } exec( - `cp ${path.join(reactNativePackagePath, tarfile)} ${localNodeTGZPath}`, + `cp ${path.join(reactNativePackagePath, tarfile)} ${newLocalNodeTGZ}`, ); } } updateTemplatePackage({ - 'react-native': `file://${localNodeTGZPath}`, + 'react-native': `file://${newLocalNodeTGZ}`, }); pushd('/tmp/'); @@ -281,14 +302,20 @@ async function testRNTestProject( // /tmp/maven-local subfolder struct. // When we generate the project manually, there is no such structure. const expandedMavenLocal = - circleCIArtifacts == null - ? mavenLocalPath - : `${mavenLocalPath}/tmp/maven-local`; + ciArtifacts == null ? mavenLocalPath : `${mavenLocalPath}/maven-local`; // need to do this here so that Android will be properly setup either way exec( `echo "react.internal.mavenLocalRepo=${expandedMavenLocal}" >> android/gradle.properties`, ); + // Only build the simulator architecture. CI is however generating only that one. + sed( + '-i', + 'reactNativeArchitectures=armeabi-v7a,arm64-v8a,x86,x86_64', + 'reactNativeArchitectures=arm64-v8a', + 'android/gradle.properties', + ); + // Update gradle properties to set Hermes as false if (argv.hermes == null) { sed( @@ -336,18 +363,18 @@ async function main() { }).stdout.trim(); const onReleaseBranch = branchName.endsWith('-stable'); - let circleCIArtifacts = await setupCircleCIArtifacts( + let ghaArtifacts = await setupGHAArtifacts( // $FlowIgnoreError[prop-missing] - argv.circleciToken, + argv.ciToken, branchName, // $FlowIgnoreError[prop-missing] argv.useLastSuccessfulPipeline, ); if (argv.target === 'RNTester') { - await testRNTester(circleCIArtifacts, onReleaseBranch); + await testRNTester(ghaArtifacts, onReleaseBranch); } else { - await testRNTestProject(circleCIArtifacts); + await testRNTestProject(ghaArtifacts); console.warn( chalk.yellow(` diff --git a/scripts/release-testing/utils/github-actions-utils.js b/scripts/release-testing/utils/github-actions-utils.js new file mode 100644 index 00000000000000..1efd6353852bb8 --- /dev/null +++ b/scripts/release-testing/utils/github-actions-utils.js @@ -0,0 +1,195 @@ +#!/usr/bin/env node +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + */ + +'use strict'; + +const {execSync: exec} = require('child_process'); +const fetch = require('node-fetch'); + +/*:: +type CIHeaders = { + Authorization: string, + Accept: string, + 'X-GitHub-Api-Version': string +} + +type WorkflowRun = { + id: number, + name: string, + run_number: number, + status: string, + workflow_id: number, + url: string, + created_at: string, +}; + + +type Artifact = { + id: number, + name: string, + url: string, + archive_download_url: string, +} + +type WorkflowRuns = { + total_count: number, + workflow_runs: Array, +} + +type Artifacts = { + total_count: number, + artifacts: Array, +} +*/ + +let token; +let ciHeaders; +let artifacts; +let branch; +let baseTemporaryPath; + +const reactNativeRepo = 'https://api.github.com/repos/facebook/react-native/'; +const reactNativeActionsURL = `${reactNativeRepo}actions/runs`; + +async function _getActionRunsOnBranch() /*: Promise */ { + const url = `${reactNativeActionsURL}?branch=${branch}`; + const options = { + method: 'GET', + headers: ciHeaders, + }; + + // $FlowIgnore[prop-missing] Conflicting .flowconfig in Meta's monorepo + // $FlowIgnore[incompatible-call] + const response = await fetch(url, options); + if (!response.ok) { + throw new Error(JSON.stringify(await response.json())); + } + + const body = await response + // eslint-disable-next-line func-call-spacing + .json /*::*/ + (); + return body; +} + +async function _getArtifacts(run_id /*: number */) /*: Promise */ { + const url = `${reactNativeActionsURL}/${run_id}/artifacts`; + const options = { + method: 'GET', + headers: ciHeaders, + }; + + // $FlowIgnore[prop-missing] Conflicting .flowconfig in Meta's monorepo + // $FlowIgnore[incompatible-call] + const response = await fetch(url, options); + if (!response.ok) { + throw new Error(JSON.stringify(await response.json())); + } + + const body = await response + // eslint-disable-next-line func-call-spacing + .json /*::*/ + (); + return body; +} + +// === Public Interface === // +async function initialize( + ciToken /*: string */, + baseTempPath /*: string */, + branchName /*: string */, + useLastSuccessfulPipeline /*: boolean */ = false, +) { + console.info('Getting GHA information'); + baseTemporaryPath = baseTempPath; + exec(`mkdir -p ${baseTemporaryPath}`); + + branch = branchName; + + token = ciToken; + ciHeaders = { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + }; + + const testAllWorkflow = (await _getActionRunsOnBranch()).workflow_runs + .filter(w => w.name === 'Test All') + .sort((a, b) => (a.created_at > b.created_at ? -1 : 1))[0]; + + artifacts = await _getArtifacts(testAllWorkflow.id); +} + +function downloadArtifact( + artifactURL /*: string */, + destination /*: string */, +) { + exec(`rm -rf ${destination}`); + + const command = `curl ${artifactURL} \ + -Lo ${destination} \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${token}" \ + -H "X-GitHub-Api-Version: 2022-11-28"`; + + exec(command, {stdio: 'inherit'}); +} + +async function artifactURLForJSCRNTesterAPK( + emulatorArch /*: string */, +) /*: Promise */ { + const url = artifacts.artifacts.filter(a => a.name === 'rntester-apk')[0] + .archive_download_url; + return Promise.resolve(url); +} + +async function artifactURLForHermesRNTesterAPK( + emulatorArch /*: string */, +) /*: Promise */ { + const url = artifacts.artifacts.filter(a => a.name === 'rntester-apk')[0] + .archive_download_url; + return Promise.resolve(url); +} + +async function artifactURLForMavenLocal() /*: Promise */ { + const url = artifacts.artifacts.filter(a => a.name === 'maven-local')[0] + .archive_download_url; + return Promise.resolve(url); +} + +async function artifactURLHermesDebug() /*: Promise */ { + const url = artifacts.artifacts.filter( + a => a.name === 'hermes-darwin-bin-Debug', + )[0].archive_download_url; + return Promise.resolve(url); +} + +async function artifactURLForReactNative() /*: Promise */ { + const url = artifacts.artifacts.filter( + a => a.name === 'react-native-package', + )[0].archive_download_url; + return Promise.resolve(url); +} + +function baseTmpPath() /*: string */ { + return baseTemporaryPath; +} + +module.exports = { + initialize, + downloadArtifact, + artifactURLForJSCRNTesterAPK, + artifactURLForHermesRNTesterAPK, + artifactURLForMavenLocal, + artifactURLHermesDebug, + artifactURLForReactNative, + baseTmpPath, +}; diff --git a/scripts/release-testing/utils/testing-utils.js b/scripts/release-testing/utils/testing-utils.js index 0496f21a87f354..735476b47c2b0a 100644 --- a/scripts/release-testing/utils/testing-utils.js +++ b/scripts/release-testing/utils/testing-utils.js @@ -20,6 +20,7 @@ const { generateiOSArtifacts, } = require('../../releases/utils/release-utils'); const circleCIArtifactsUtils = require('./circle-ci-artifacts-utils.js'); +const ghaArtifactsUtils = require('./github-actions-utils.js'); const fs = require('fs'); // $FlowIgnore[cannot-resolve-module] const {spawn} = require('node:child_process'); @@ -177,30 +178,101 @@ async function setupCircleCIArtifacts( return circleCIArtifactsUtils; } -async function downloadArtifactsFromCircleCI( - circleCIArtifacts /*: typeof circleCIArtifactsUtils */, +/** + * Setups the CircleCIArtifacts if a token has been passed + * + * Parameters: + * - @circleciToken a valid CircleCI Token. + * - @branchName the branch of the name we want to use to fetch the artifacts. + */ +async function setupGHAArtifacts( + ciToken /*: ?string */, + branchName /*: string */, + useLastSuccessfulPipeline /*: boolean */, +) /*: Promise */ { + if (ciToken == null) { + return null; + } + + const baseTmpPath = '/tmp/react-native-tmp'; + await ghaArtifactsUtils.initialize( + ciToken, + baseTmpPath, + branchName, + useLastSuccessfulPipeline, + ); + return ghaArtifactsUtils; +} + +async function downloadArtifacts( + ciArtifacts /*: typeof circleCIArtifactsUtils */, mavenLocalPath /*: string */, localNodeTGZPath /*: string */, ) { - const mavenLocalURL = await circleCIArtifacts.artifactURLForMavenLocal(); - const hermesURL = await circleCIArtifacts.artifactURLHermesDebug(); - const reactNativeURL = await circleCIArtifacts.artifactURLForReactNative(); + const mavenLocalURL = await ciArtifacts.artifactURLForMavenLocal(); + const hermesURLZip = await ciArtifacts.artifactURLHermesDebug(); + const reactNativeURLZip = await ciArtifacts.artifactURLForReactNative(); - const hermesPath = path.join( - circleCIArtifacts.baseTmpPath(), - 'hermes-ios-debug.tar.gz', + // Cleanup destination folder + exec(`rm -rf ${ciArtifacts.baseTmpPath()}`); + exec(`mkdir ${ciArtifacts.baseTmpPath()}`); + + const hermesPathZip = path.join( + ciArtifacts.baseTmpPath(), + 'hermes-ios-debug.zip', ); - console.info(`[Download] Maven Local Artifacts from ${mavenLocalURL}`); const mavenLocalZipPath = `${mavenLocalPath}.zip`; - circleCIArtifacts.downloadArtifact(mavenLocalURL, mavenLocalZipPath); + console.info( + `\n[Download] Maven Local Artifacts from ${mavenLocalURL} into ${mavenLocalZipPath}`, + ); + ciArtifacts.downloadArtifact(mavenLocalURL, mavenLocalZipPath); + console.info(`Unzipping into ${mavenLocalPath}`); exec(`unzip -oq ${mavenLocalZipPath} -d ${mavenLocalPath}`); - console.info('[Download] Hermes'); - circleCIArtifacts.downloadArtifact(hermesURL, hermesPath); - console.info(`[Download] React Native from ${reactNativeURL}`); - circleCIArtifacts.downloadArtifact(reactNativeURL, localNodeTGZPath); - return hermesPath; + // Github Actions are zipping a zip. Needs to move it to the right place and unzip it again + exec(`rm -rf ${mavenLocalZipPath}`); + exec(`mv ${mavenLocalPath}/maven-local.zip ${mavenLocalZipPath}`); + exec(`unzip -oq ${mavenLocalZipPath} -d ${mavenLocalPath}`); + + console.info('\n[Download] Hermes'); + ciArtifacts.downloadArtifact(hermesURLZip, hermesPathZip); + exec(`unzip ${hermesPathZip} -d ${ciArtifacts.baseTmpPath()}/hermes`); + const hermesPath = path.join( + ciArtifacts.baseTmpPath(), + 'hermes', + 'hermes-ios-debug.tar.gz', + ); + + console.info(`\n[Download] React Native from ${reactNativeURLZip}`); + const reactNativeDestPath = path.join( + ciArtifacts.baseTmpPath(), + 'react-native', + ); + const reactNativeZipDestPath = `${reactNativeDestPath}.zip`; + ciArtifacts.downloadArtifact(reactNativeURLZip, reactNativeZipDestPath); + exec(`unzip ${reactNativeZipDestPath} -d ${reactNativeDestPath}`); + // For some reason, the commit on which the Github Action is running is not the same as the one + // that is running locally. This make so that the react-native package is created with a different + // commit sha in CI wrt what is used locally. + // As a result the react-native tgz is different. The next section of code use package that is created + // in CI as source of truth and sends back the new localNodeTGZ path so that the new apps can + // use it. + const tgzName = fs.readdirSync(reactNativeDestPath).filter(file => { + console.log(file); + return file.endsWith('.tgz'); + })[0]; + + if (tgzName == null) { + throw new Error('Could not find the tgz file in the react-native folder'); + } + + const basePath = path.dirname(localNodeTGZPath); + const newLocalNodeTGZ = path.join(basePath, tgzName); + const reactNativeTGZ = path.join(reactNativeDestPath, tgzName); + exec(`mv ${reactNativeTGZ} ${newLocalNodeTGZ}`); + + return {hermesPath, newLocalNodeTGZ}; } function buildArtifactsLocally( @@ -281,20 +353,23 @@ function buildArtifactsLocally( * - @hermesPath the path to hermes for iOS */ async function prepareArtifacts( - circleCIArtifacts /*: ?typeof circleCIArtifactsUtils */, + ciArtifacts /*: ?typeof circleCIArtifactsUtils */, mavenLocalPath /*: string */, localNodeTGZPath /*: string */, releaseVersion /*: string */, buildType /*: BuildType */, reactNativePackagePath /*: string */, -) /*: Promise */ { - return circleCIArtifacts != null - ? await downloadArtifactsFromCircleCI( - circleCIArtifacts, - mavenLocalPath, - localNodeTGZPath, - ) - : buildArtifactsLocally(releaseVersion, buildType, reactNativePackagePath); +) /*: Promise<{hermesPath: string, newLocalNodeTGZ: string }> */ { + return ciArtifacts != null + ? await downloadArtifacts(ciArtifacts, mavenLocalPath, localNodeTGZPath) + : { + hermesPath: buildArtifactsLocally( + releaseVersion, + buildType, + reactNativePackagePath, + ), + newLocalNodeTGZ: localNodeTGZPath, + }; } module.exports = { @@ -303,5 +378,6 @@ module.exports = { isPackagerRunning, launchPackagerInSeparateWindow, setupCircleCIArtifacts, + setupGHAArtifacts, prepareArtifacts, };