Py3.8: Transparently use a Python virtual environment under the hood

.gitignore

@@ -42,3 +42,4 @@ tests/unit/nvda.ini
 source/locale/*/cldr.dic
 .vscode
 .vs
+.venv

.gitmodules

@@ -15,28 +15,9 @@
 	path = include/sonic
 	url = https://github.com/waywardgeek/sonic.git
 	ignore = untracked
-[submodule "include/pyserial"]
-	path = include/pyserial
-	url = https://github.com/pyserial/pyserial.git
-	ignore = untracked
-[submodule "include/comtypes"]
-	path = include/comtypes
-	url = https://github.com/nvaccess/comtypes-bin
-[submodule "include/scons"]
-	path = include/scons
-	url = https://github.com/SCons/scons
 [submodule "include/ia2"]
 	path = include/ia2
 	url = https://github.com/LinuxA11y/IAccessible2.git
-[submodule "include/wxPython"]
-	path = include/wxPython
-	url = https://github.com/nvaccess/wxPython-bin.git
-[submodule "include/configobj"]
-	path = include/configobj
-	url = https://github.com/DiffSK/configobj.git
-[submodule "include/py2exe"]
-	path = include/py2exe
-	url = https://github.com/nvaccess/py2exe-bin
 [submodule "include/javaAccessBridge32"]
 	path = include/javaAccessBridge32
 	url = https://github.com/nvaccess/javaAccessBridge32-bin.git

appveyor.yml

@@ -6,12 +6,13 @@ branches:
   - master
   - beta
   - rc
+  - py3.8
   - /try-.*/
   # For tags, branch = tag if GitHub couldn't work out the base branch.
   - /release-.*/
 
 environment:
- PY_PYTHON: 3.7-32
+ PY_PYTHON: 3.8-32
  encFileKey:
   secure: ekOvuyywHuDdGZmRmoj+b3jfrq39A2xlx4RD5ZUGd/8=
  mozillaSymsAuthToken:
@@ -95,12 +96,15 @@ build_script:
      Set-AppveyorBuildVariable "sconsOutTargets" $sconsOutTargets
      Set-AppveyorBuildVariable "sconsArgs" $sconsArgs
  - 'echo scons args: %sconsArgs%'
- - py scons.py source %sconsArgs%
+ - scons source %sconsArgs%
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
  # We don't need launcher to run checkPot, so run the checkPot before launcher.
- - py scons.py checkPot %sconsArgs%
+ - scons checkPot %sconsArgs%
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
  # The pot gets built by tests, but we don't actually need it as a build artifact.
  - 'echo scons output targets: %sconsOutTargets%'
- - py scons.py %sconsOutTargets% %sconsArgs%
+ - scons %sconsOutTargets% %sconsArgs%
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
  # Build symbol store.
  - ps: |
      foreach ($syms in
@@ -115,12 +119,10 @@ build_script:
      }
  - cd symbols
  - 7z a -tzip -r ..\output\symbols.zip *.dl_ *.ex_ *.pd_
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
  - cd ..
 
 before_test:
- # install required packages
- - py -m pip install --upgrade pip
- - py -m pip install -r tests/system/requirements.txt -r tests/lint/lintInstall/requirements.txt
  - mkdir testOutput
  - mkdir testOutput\unit
  - mkdir testOutput\system
@@ -153,7 +155,7 @@ test_script:
      $errorCode=0
      $outDir = (Resolve-Path .\testOutput\unit\)
      $unitTestsXml = "$outDir\unitTests.xml"
-     py -m nose -sv --with-xunit --xunit-file="$unitTestsXml" ./tests/unit
+     .\rununittests.bat --with-xunit --xunit-file="$unitTestsXml"
      if($LastExitCode -ne 0) {
       $errorCode=$LastExitCode
       Add-AppveyorMessage "Unit test failure"
@@ -168,12 +170,14 @@ test_script:
     if($env:APPVEYOR_PULL_REQUEST_NUMBER) {
       $lintOutput = (Resolve-Path .\testOutput\lint\)
       $lintSource = (Resolve-Path .\tests\lint\)
-      git fetch -q origin $env:APPVEYOR_REPO_BRANCH
-      $prDiff = "$lintOutput\prDiff.patch"
-      git diff -U0 FETCH_HEAD...HEAD > $prDiff
-      $flake8Config = "$lintSource\flake8.ini"
-      $flake8Output = "$lintOutput\PR-Flake8.txt"
-      type "$prDiff" | py -Xutf8 -m flake8 --diff --output-file="$flake8Output" --tee --config="$flake8Config"
+      # When Appveyor runs for a pr,
+      # the build is made from a new temporary commit,
+      # resulting from the pr branch being merged into its base branch.
+      # Therefore to create a diff for linting, we must fetch the head of the base branch.
+      # In a PR, APPVEYOR_REPO_BRANCH points to the head of the base branch. 
+     git fetch -q origin $env:APPVEYOR_REPO_BRANCH
+     $flake8Output = "$lintOutput\PR-Flake8.txt"
+      .\runlint.bat FETCH_HEAD "$flake8Output" 
       if($LastExitCode -ne 0) {
        $errorCode=$LastExitCode
        Add-AppveyorMessage "PR introduces Flake8 errors"
@@ -191,19 +195,14 @@ test_script:
  - ps: |
      $testOutput = (Resolve-Path .\testOutput\)
      $systemTestOutput = (Resolve-Path "$testOutput\system")
-     $testSource = ".\tests\system\robot"
-
-     $robotArgs = '--argumentfile', '.\tests\system\robotArgs.robot'
-     # useInstalledNVDA must come after args file to override the whichNVDA value
-     $useInstalledNVDA = '--variable', 'whichNVDA:installed'
-
-     py -m robot $robotArgs $useInstalledNVDA $testSource
-     Compress-Archive -Path "$systemTestOutput\*" -DestinationPath "$testOutput\systemTestResult.zip"
-     Push-AppveyorArtifact "$testOutput\systemTestResult.zip"
+
+     .\runsystemtests.bat --variable whichNVDA:installed
      if($LastExitCode -ne 0) {
       $errorCode=$LastExitCode
       Add-AppveyorMessage "System test failure"
      }
+     Compress-Archive -Path "$systemTestOutput\*" -DestinationPath "$testOutput\systemTestResult.zip"
+     Push-AppveyorArtifact "$testOutput\systemTestResult.zip"
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path "$systemTestOutput\systemTests.xml"))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
@@ -273,6 +272,9 @@ deploy_script:
 on_finish:
   # add a message to point to the NVDA build, to make testing PR's easier.
   - ps: |
+      # Save an exact list of the actual Python packages and their versions that got installed, to aide in reproducing a build
+      .\venvUtils\exportPackageList.bat installed_python_packages.txt
+      Push-AppveyorArtifact installed_python_packages.txt
       $appVeyorUrl = "https://ci.appveyor.com"
       $exe = Get-ChildItem -Name output\*.exe
       if($?){

devDocs/buildSystemNotes.md

@@ -0,0 +1,74 @@
+# Build System Notes
+A Python virtual environment is used transparently by the NVDA build system,
+and all Python dependencies are installed into this environment using `pip`.
+
+NVDA's build system commands will handle all aspects of the virtual environment.
+Developers should not create or activate the virtual environment manually, unless
+working on the build system itself.
+
+For the documentation on how to _use_ the build system (E.G. building,
+running NVDA or tests) see the main repository readme file.
+
+## How the build system works
+
+The virtual environment system used is `venv`.
+Dependencies are installed with `pip` via the `requirements.txt` file.
+Version numbers for dependencies should be used to lock in a version.
+
+The virtual environment is recreated if it is outdated, either due to:
+- Python version.
+- `pip` requirements.
+
+The user is consulted before modifying / removing a virtual environment that can't be identified
+as having been created by NVDA's build system.
+
+### Entry points to the build system
+
+These are the only files expected to be executed directly by a user/developer:
+- `scons.bat`
+- `runnvda.bat`
+- `rununittests.bat`
+- `runsystemtests.bat`
+- `runlint.bat`
+
+**Note:** The `runnvda.bat` script intentionally uses `pyw.exe` to run NVDA as
+this is the more common and expected way to run NVDA.
+Run NVDA with `py.exe` in order to have standard output/error output to the console.
+This is particularly useful if there is an error in NVDA before logging is initialised.
+To do this, modify the `runnvda.bat` file.
+
+**Note:** Executing `source/nvda.pyw` outside of a virtual environment will produce an error message
+and early termination.
+
+### Main implementation files:
+The following files contain the main implementation of the virtual environment setup.
+
+#### `venvUtils/ensureAndActivate.bat`
+   - Activates the virtual environment.
+   - If necessary, creates and configures it first. 
+   - The virtual environment is left active. 
+#### `venvUtils/venvCmd.bat`
+  - Uses `ensureAndActivate.bat` to run a command within the context
+   of the virtual environment. 
+  - The virtual environment is deactivated after the command
+   completes.
+  - All entry point scripts depend on this.
+#### `venvUtils/ensureVenv.py`
+- Does the actual work to create and configure the virtual
+   environment.
+
+## Motivation for using virtual environments
+
+Ensures the build environment is clean, and there are no conflicts with other installed packages.
+
+NVDA and its build system have many Python dependencies.
+Using `pip` and a virtual environment means:
+- Updating is easier than git submodules.
+  E.G. wxPython no longer has to be pre-built and stored in our bin repo.
+- Developers need to sync/update their submodules less often.
+- More consistency for dependencies.
+- IDE's can be configured more easily.
+- No conflict between NVDA dependencies and Python packages already installed globally on the
+  developer's system.
+- Don't interfere with the developer's system. Installing packages globally may break things
+  outside of NVDA.

devDocs/sconscript

@@ -19,10 +19,6 @@ Import("env", "outputDir", "sourceDir", "t2tBuildConf")
 
 env = env.Clone()
 
-devDocsInstall = env.SConscript("devDocsInstall/sconscript", exports=["env"])
-env.Depends(devDocsInstall, env.Dir("devDocs/devDocsInstall/requirements.txt"))
-env.Alias("devDocsInstall", devDocsInstall)
-
 devDocsOutputDir=outputDir.Dir('devDocs')
 
 #Build the developer guide and move it to the output directory
@@ -69,7 +65,6 @@ sphinxAPIDocs = env.Command(
 		] + [f"{sourceDir}\\{f}" for f in ignorePaths]
 	]
 )
-env.Depends(sphinxAPIDocs, devDocsInstall)
 sphinxHtml = env.Command(
 	"_build",
 	sphinxAPIDocs,

nvdaHelper/espeak/sconscript

@@ -1,5 +1,3 @@
-from include.scons.SCons.Util import CLVar
-
 Import([
 	'thirdPartyEnv',
 	'sourceDir',

nvdaHelper/liblouis/sconscript

@@ -17,8 +17,8 @@ import re
 import glob
 from SCons.Tool.MSCommon.vc import find_vc_pdir
 import typing
-from include.scons.SCons.Environment import Environment
-from include.scons.SCons.Environment import Base
+from SCons.Environment import Environment
+from SCons.Environment import Base
 
 Import([
 	"thirdPartyEnv",