improve treedump capabilities

.ado/templates/e2e-test-job.yml

@@ -91,6 +91,13 @@ jobs:
           script: yarn run e2etest
           workingDirectory: packages/E2ETest
 
+      - task: CopyFiles@2
+        displayName: Copy test report
+        inputs:
+          sourceFolder: $(Build.SourcesDirectory)\packages\E2ETest\reports
+          targetFolder: $(Build.StagingDirectory)/ReactUWPTestAppTreeDump/reports
+        condition: succeededOrFailed()
+
       - task: CopyFiles@2
         displayName: Copy tree dump output files
         inputs:

.ado/windows-vs-pr.yml

@@ -65,6 +65,13 @@ jobs:
         clean: false
         submodules: false
 
+      - task: PowerShell@2
+        displayName: "Check if this environment meets the development dependencies"
+        inputs:
+          targetType: filePath
+          filePath: $(Build.SourcesDirectory)\vnext\Scripts\rnw-dependencies.ps1
+          arguments: -NoPrompt
+
       - template: templates/build-rnw.yml
         parameters:
           yarnBuildCmd: build

change/react-native-windows-2020-05-11-17-22-16-rnw-dependencies.json

@@ -0,0 +1,8 @@
+{
+  "type": "prerelease",
+  "comment": "add rnw-dependencies.ps1",
+  "packageName": "react-native-windows",
+  "email": "[email protected]",
+  "dependentChangeType": "patch",
+  "date": "2020-05-12T00:22:16.000Z"
+}

change/react-native-windows-2020-05-12-11-18-16-MS_UIQueue.json

@@ -0,0 +1,8 @@
+{
+  "type": "prerelease",
+  "comment": "Use DispatcherQueue instead of CoreDispatcher for Mso::DispatchQueue",
+  "packageName": "react-native-windows",
+  "email": "[email protected]",
+  "dependentChangeType": "patch",
+  "date": "2020-05-12T18:18:15.956Z"
+}

change/react-native-windows-2020-05-12-18-57-59-UnsafePropertyId.json

@@ -0,0 +1,8 @@
+{
+  "type": "prerelease",
+  "comment": "Allow storing non-WinRT types in ReactPropertyBag",
+  "packageName": "react-native-windows",
+  "email": "[email protected]",
+  "dependentChangeType": "patch",
+  "date": "2020-05-13T01:57:59.370Z"
+}

change/react-native-windows-2020-05-13-12-47-43-tabfocus-fix.json

@@ -0,0 +1,8 @@
+{
+  "type": "prerelease",
+  "comment": "revert dfc57fcf2504f57baab20f550b36a618eaa99e56",
+  "packageName": "react-native-windows",
+  "email": "[email protected]",
+  "dependentChangeType": "patch",
+  "date": "2020-05-13T19:47:43.325Z"
+}

change/react-native-windows-2020-05-13-14-16-07-rnw-dep-dontexit.json

@@ -0,0 +1,8 @@
+{
+  "type": "prerelease",
+  "comment": "don't exit the powershell session and pause for keyboard input if interactive",
+  "packageName": "react-native-windows",
+  "email": "[email protected]",
+  "dependentChangeType": "patch",
+  "date": "2020-05-13T21:16:07.200Z"
+}

docs/building-rnw.md

@@ -22,7 +22,7 @@ Note that react-native-windows is a monorepo and relies on monorepo tools like y
   There are two ways to run the app.  In a fully managed easy way, or by manually running all the required steps:
 
 ## Automatic
-The playground app can be run in a completely automatic way by using `react-native run-windows`.
+The playground app can be run in a completely automatic way by using `react-native run-windows --sln windows\playground.sln`.
 
 If you haven't already, install the react-native-cli  (One time only!)
 ```cmd
@@ -33,7 +33,7 @@ Then
 
 ```cmd
 cd packages\playground
-react-native run-windows
+react-native run-windows --sln windows\playground.sln
 ```
 
 ## Manual

docs/e2e-testing.md

@@ -436,3 +436,72 @@ this.submitButton.click();
 You can easily to use By(string) to locate a element which associated with testID in the app.
 
 It's recommended to define a `get` for each locator like above.
+
+## E2E Tests and masters
+
+E2E tests can be summarized as follows:
+- they are tests that run the ReactUWPTestApp
+- use UI Automation to navigate between pages, query the state of elements, click on them, etc.
+- the ReactUWPTestApp has code to produce a dump of the its own visual tree ("tree dump output") and compares it with a checked in copy ("tree dump masters") to make sure nothing has regressed. The tree dumps are produced in Json format (there is also an option to produce them in a custom text format of key=value, but that is deprecated now).
+
+So you've added or updated some tests: great! you get a cookie*. But now you probably need to update the masters, or the tests will fail and break the CI.
+
+\* void where prohibited, prizes and participation may vary.
+
+![testFail](img/e2e-testfail.png)
+
+The best way to do this is by letting the CI run and fail, then downloading the generated tree dump output files, and comparing to the masters. Make sure the differences are expected, copy over them and check them in. The reason is that the masters will include things like the size elements rendered at, which can be dependent on DPI, scale factor, resolution, and in some cases (due to bugs) even differ based on bitness (see #4628).
+
+When an output doesn't match its master, a file with `.err` extension will be produced under the `TreeDump` folder in the `ReactUWPTestAppTreeDump` artifact. The content of the `.err` file will usually just say:
+
+```txt
+Tree dump file does not match master at C:\Program Files\WindowsApps\ReactUWPTestApp_1.0.0.0_x64__kc2bncckyf4ap\Assets\TreeDump\masters\ControlStyleRoundBorder.json - See output at C:\Users\VssAdministrator\AppData\Local\Packages\ReactUWPTestApp_kc2bncckyf4ap\LocalState\TreeDump\ControlStyleRoundBorder.json
+```
+
+![Errors](img/e2e-errors.png)
+
+Find the corresponding `.json` file in that folder and compare it to its master. The masters live in [e2etest\windows\ReactUWPTestApp\Assets\TreeDump\masters](https://github.com/microsoft/react-native-windows/tree/master/packages/E2ETest/windows/ReactUWPTestApp/Assets/TreeDump/masters).
+
+Sometimes you'll have an element in your test that produces output that should not be used for comparison. You can manually edit the generated json and set the output that you want to ignore to the `<ANYTHING>` value:
+
+```json
+...
+"Windows.UI.Xaml.Button":
+{
+  "Text": "<ANYTHING>",
+  ...
+}
+...
+```
+
+## run_wdio
+
+WDIO is not really built to be run within Azure DevOps, so I wrote a utility called `run_wdio` to adapt it to something that can run in ADO.
+It can be found in [\packages\e2etest\run_wdio.js](https://github.com/microsoft/react-native-windows/blob/master/packages/E2ETest/run_wdio.js)
+Its main features (which WDIO lacks) are:
+
+* reports success/failure to ADO so that test failures will break the CI
+* supports test selection on the command line
+* supports test metadata and filtering (e.g. you can mark a test as "do not run in the lab")
+* prints out the list of tests and test cases that failed in the CI console
+
+
+## Debugging E2E Tests in CI
+If you have access to the AzureDevOps pipeline you'll be able to see test failures and debug crashes.
+Here are the artifacts that are produced during the build:
+- error screenshots of the app when a test failed
+- test run XML - this contains some information like the name of the wdio test that failed and the JS stack
+- tree dump outputs - you can compare these to the masters to see if there is a the difference responsible for the test failing. 
+- crash dumps of the e2e test app (ReactUWPTestApp)
+
+You can access these by going to the AzureDevOps run for your PR and clicking on the artifacts link:
+
+![Artifacts](img/e2e-artifacts.png)
+
+Then you can access crash dumps under the `ReactUWPTestAppTreeDump\CrashDumps` folder.
+![CrashDumps](img/e2e-crashdumps.png)
+
+You can get the symbols from the `appxsym` (just download it and rename it to `.zip`):
+![SymbolsPackage](img/e2e-syms.png)
+
+ The `ReactUWPTestAppTreeDump` folder will also contain any tree dump outputs that were produced that did not match the masters.

packages/E2ETest/README.md

@@ -1,3 +1,5 @@
 # E2ETest project
 
-This package is not published, and is just used to verify a standalone app
+This package is not published, and is just used to verify a standalone app
+
+For information on how to run and debug this project, see [e2e-testing](https://github.com/microsoft/react-native-windows/blob/master/docs/e2e-testing.md).

packages/E2ETest/app/ControlStyleTestPage.tsx

@@ -3,8 +3,7 @@
  * Licensed under the MIT License.
  */
 
- // Bug:4596 Switch
-import { /* Switch, */ CheckBox, TextInput, View, StyleSheet, Button } from 'react-native';
+ import { Switch, CheckBox, TextInput, View, StyleSheet, Button } from 'react-native';
 import { DatePicker, Picker } from 'react-native-windows';
 import React, { useState } from 'react';
 import { SHOWBORDER_ON_CONTROLSTYLE, TREE_DUMP_RESULT } from './Consts';
@@ -58,10 +57,8 @@ export function ControlStyleTestPage() {
     <View>
       <View testID={'ControlStyleView'}>
         {
-        /*
-        // Bug:4596 Switch
         <Switch style={showRoundBorder? styles.roundBorder :styles.regularBorder} thumbColor='blue'/> 
-        */}
+        }
         <CheckBox style={showRoundBorder? styles.roundBorder :styles.regularBorder} />
         <TextInput style={showRoundBorder? styles.roundBorder :styles.regularBorder}
           placeholder='TextBox'

packages/E2ETest/package.json

@@ -25,7 +25,7 @@
     "prompt-sync": "^4.2.0",
     "react": "16.11.0",
     "react-native": "0.62.2",
-    "react-native-windows": "0.0.0-master.62"
+    "react-native-windows": "0.0.0-master.65"
   },
   "devDependencies": {
     "@babel/core": "^7.8.4",

packages/E2ETest/run_wdio.js

@@ -4,10 +4,11 @@ const xml2js = require('xml2js');
 const parser = new xml2js.Parser({ attrkey: 'ATTR' });
 const child_process = require('child_process');
 const prompt = require('prompt-sync')();
+const chalk = require('chalk');
 
 const specFolder = 'wdio/test';
 
-function GetMetadata(specPath) {
+function getMetadata(specPath) {
   const contents = fs.readFileSync(specPath);
   const metadataTag = '// @metadata ';
   const metadataStart = contents.indexOf(metadataTag);
@@ -29,31 +30,31 @@ const filters = {
 
 // Returns true if the spec is to run.
 // Specs marked SkipCI are excluded from CI (identified by environment variables in the ADO lab)
-function FilterSpec(specPath) {
-  const metadata = GetMetadata(specPath);
-  for (let i = 0; i < metadata.length; i++) {
-    if (filters[metadata[i]](specPath)) {
+function filterSpec(specPath) {
+  const metadata = getMetadata(specPath);
+  for (const metadataElement of metadata) {
+    if (filters[metadataElement](specPath)) {
       return false;
     }
   }
   return true;
 }
 
-function SelectSpecs(folder) {
+function selectSpecs(folder) {
   let specs = [];
   if (process.argv.length > 2) {
     specs = process.argv.splice(2).map(spec => spec + '.spec.ts');
   } else {
     specs = fs.readdirSync(folder).filter(x => x.endsWith('.spec.ts'));
   }
-  specs = specs.map(spec => path.join(folder, spec)).filter(FilterSpec);
+  specs = specs.map(spec => path.join(folder, spec)).filter(filterSpec);
   return specs;
 }
 
-let opts = SelectSpecs(specFolder);
+let opts = selectSpecs(specFolder);
 console.log(`Selected tests: ${opts}`);
 
-function OverrideHyperV() {
+function ensureRunningInHyperV() {
   const baseboardMfr = child_process
     .execSync('powershell.exe (gwmi Win32_BaseBoard).Manufacturer')
     .toString()
@@ -69,51 +70,99 @@ function OverrideHyperV() {
   }
 }
 
-OverrideHyperV();
-
 const Launcher = require('@wdio/cli').default;
 
 const wdio = new Launcher('wdio.conf.js', { specs: opts });
 
 function parseLog(logfile) {
   const xmlString = fs.readFileSync(logfile);
-  let name;
+  let failures = {};
   parser.parseString(xmlString, (err, res) => {
     if (!res.testsuites) {
-      name = 'something went wrong';
+      console.error(`Something went wrong processing file ${logfile}`);
     } else {
-      const attr = res.testsuites.testsuite[0].ATTR;
-      if (attr.errors > 0 || attr.failures > 0) {
-        name = attr.name;
+      for (const testsuite of res.testsuites.testsuite) {
+        const attr = testsuite.ATTR;
+
+        if (attr.errors > 0 || attr.failures > 0) {
+          const name = attr.name;
+          failures[name] = {};
+
+          for (const testcase of testsuite.testcase) {
+            if (testcase.error && testcase.error[0].ATTR) {
+              failures[name].testcase = testcase.ATTR.name;
+              failures[name].error = testcase.error[0].ATTR.message;
+              const systemErr = testcase['system-err'][0];
+              const stack = systemErr.substr(
+                systemErr.indexOf('\n    at ') + 1
+              );
+              failures[name].stack = stack;
+            }
+          }
+        }
       }
     }
   });
-  return name;
+
+  return failures;
 }
 
 function parseLogs() {
   const reportsDir = path.join(__dirname, 'reports');
   const logs = fs.readdirSync(reportsDir).filter(x => x.endsWith('.log'));
   const names = logs
     .map(x => parseLog(path.join(reportsDir, x)))
-    .filter(x => x != null);
+    .filter(x => x != null && x != '');
   return names;
 }
 
-function Process(code) {
+function printFailedTests(ft) {
+  for (const key in ft) {
+    console.log(chalk.redBright(key));
+    console.log(
+      '  ',
+      chalk.underline('testcase'),
+      chalk.bold(ft[key].testcase)
+    );
+    console.log('  ', chalk.underline('error'), chalk.bold(ft[key].error));
+    console.log('  ', chalk.underline('stack'));
+    console.log(ft[key].stack);
+  }
+}
+
+function doProcess(code) {
   const failedTests = parseLogs();
-  for (let i = 0; i < failedTests.length; i++) {
-    console.log(`Failed test: ${failedTests[i]}`);
+  for (const failedTest of failedTests) {
+    console.log('Failed tests: ');
+    printFailedTests(failedTest);
   }
   process.exit(code);
 }
 
-wdio.run().then(
-  code => {
-    Process(code);
-  },
-  error => {
-    console.error('Launcher failed to start the test', error.stacktrace);
-    process.exit(1);
-  }
-);
+function runWdio() {
+  ensureRunningInHyperV();
+
+  wdio.run().then(
+    code => {
+      try {
+        doProcess(code);
+      } catch (e) {
+        console.log(e);
+        // any exception that isn't handled is an error
+        process.exit(3);
+      }
+    },
+    error => {
+      console.error('Launcher failed to start the test', error.stacktrace);
+      process.exit(1);
+    }
+  );
+}
+
+try {
+  runWdio();
+} catch (e) {
+  console.log(e);
+  // any exception that isn't handled is an error
+  process.exit(2);
+}

packages/E2ETest/windows/ReactUWPTestApp.sln

@@ -43,6 +43,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ReactNative.Manag
 EndProject
 Global
 	GlobalSection(SharedMSBuildProjectFiles) = preSolution
+		..\..\..\vnext\ReactWindowsCore\ReactWindowsCore.vcxitems*{11c084a3-a57c-4296-a679-cac17b603144}*SharedItemsImports = 4
 		..\..\..\vnext\JSI\Shared\JSI.Shared.vcxitems*{a62d504a-16b8-41d2-9f19-e2e86019e5e4}*SharedItemsImports = 4
 		..\..\..\vnext\Chakra\Chakra.vcxitems*{f7d32bd0-2749-483e-9a0d-1635ef7e3136}*SharedItemsImports = 4
 		..\..\..\vnext\JSI\Shared\JSI.Shared.vcxitems*{f7d32bd0-2749-483e-9a0d-1635ef7e3136}*SharedItemsImports = 4