Py3.8: Transparently use a Python virtual environment under the hood

.gitignore

@@ -42,3 +42,4 @@ tests/unit/nvda.ini
 source/locale/*/cldr.dic
 .vscode
 .vs
+.venv

.gitmodules

@@ -15,28 +15,9 @@
 	path = include/sonic
 	url = https://github.com/waywardgeek/sonic.git
 	ignore = untracked
-[submodule "include/pyserial"]
-	path = include/pyserial
-	url = https://github.com/pyserial/pyserial.git
-	ignore = untracked
-[submodule "include/comtypes"]
-	path = include/comtypes
-	url = https://github.com/nvaccess/comtypes-bin
-[submodule "include/scons"]
-	path = include/scons
-	url = https://github.com/SCons/scons
 [submodule "include/ia2"]
 	path = include/ia2
 	url = https://github.com/LinuxA11y/IAccessible2.git
-[submodule "include/wxPython"]
-	path = include/wxPython
-	url = https://github.com/nvaccess/wxPython-bin.git
-[submodule "include/configobj"]
-	path = include/configobj
-	url = https://github.com/DiffSK/configobj.git
-[submodule "include/py2exe"]
-	path = include/py2exe
-	url = https://github.com/nvaccess/py2exe-bin
 [submodule "include/javaAccessBridge32"]
 	path = include/javaAccessBridge32
 	url = https://github.com/nvaccess/javaAccessBridge32-bin.git

appveyor.yml

@@ -6,12 +6,13 @@ branches:
   - master
   - beta
   - rc
+  - py3.8
   - /try-.*/
   # For tags, branch = tag if GitHub couldn't work out the base branch.
   - /release-.*/
 
 environment:
- PY_PYTHON: 3.7-32
+ PY_PYTHON: 3.8-32
  encFileKey:
   secure: ekOvuyywHuDdGZmRmoj+b3jfrq39A2xlx4RD5ZUGd/8=
  mozillaSymsAuthToken:
@@ -59,6 +60,14 @@ init:
 clone_depth: 1
 
 install:
+ # Activate the NvDA build system Python virtual environment, creating / updating it if necessary.
+ - .\venvUtils\ensureAndActivate.bat
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
+ # Save an exact list of the actual Python packages and their versions that got installed, to aide in reproducing a build
+ - py -m pip freeze >installed_python_packages.txt
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
+ - appveyor PushArtifact  installed_python_packages.txt
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
  - cd appveyor
  # Decrypt files.
  - ps: |
@@ -95,12 +104,15 @@ build_script:
      Set-AppveyorBuildVariable "sconsOutTargets" $sconsOutTargets
      Set-AppveyorBuildVariable "sconsArgs" $sconsArgs
  - 'echo scons args: %sconsArgs%'
- - py scons.py source %sconsArgs%
+ - scons source %sconsArgs%
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
  # We don't need launcher to run checkPot, so run the checkPot before launcher.
- - py scons.py checkPot %sconsArgs%
+ - scons checkPot %sconsArgs%
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
  # The pot gets built by tests, but we don't actually need it as a build artifact.
  - 'echo scons output targets: %sconsOutTargets%'
- - py scons.py %sconsOutTargets% %sconsArgs%
+ - scons %sconsOutTargets% %sconsArgs%
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
  # Build symbol store.
  - ps: |
      foreach ($syms in
@@ -115,12 +127,10 @@ build_script:
      }
  - cd symbols
  - 7z a -tzip -r ..\output\symbols.zip *.dl_ *.ex_ *.pd_
+ - if ERRORLEVEL 1 exit %ERRORLEVEL%
  - cd ..
 
 before_test:
- # install required packages
- - py -m pip install --upgrade pip
- - py -m pip install -r tests/system/requirements.txt -r tests/lint/lintInstall/requirements.txt
  - mkdir testOutput
  - mkdir testOutput\unit
  - mkdir testOutput\system
@@ -153,7 +163,7 @@ test_script:
      $errorCode=0
      $outDir = (Resolve-Path .\testOutput\unit\)
      $unitTestsXml = "$outDir\unitTests.xml"
-     py -m nose -sv --with-xunit --xunit-file="$unitTestsXml" ./tests/unit
+     .\rununittests.bat --with-xunit --xunit-file="$unitTestsXml"
      if($LastExitCode -ne 0) {
       $errorCode=$LastExitCode
       Add-AppveyorMessage "Unit test failure"
@@ -168,12 +178,14 @@ test_script:
     if($env:APPVEYOR_PULL_REQUEST_NUMBER) {
       $lintOutput = (Resolve-Path .\testOutput\lint\)
       $lintSource = (Resolve-Path .\tests\lint\)
-      git fetch -q origin $env:APPVEYOR_REPO_BRANCH
-      $prDiff = "$lintOutput\prDiff.patch"
-      git diff -U0 FETCH_HEAD...HEAD > $prDiff
-      $flake8Config = "$lintSource\flake8.ini"
-      $flake8Output = "$lintOutput\PR-Flake8.txt"
-      type "$prDiff" | py -Xutf8 -m flake8 --diff --output-file="$flake8Output" --tee --config="$flake8Config"
+      # When Appveyor runs for a pr,
+      # the build is made from a new temporary commit,
+      # resulting from the pr branch being merged into its base branch.
+      # Therefore to create a diff for linting, we must fetch the head of the base branch.
+      # In a PR, APPVEYOR_REPO_BRANCH points to the head of the base branch. 
+     git fetch -q origin $env:APPVEYOR_REPO_BRANCH
+     $flake8Output = "$lintOutput\PR-Flake8.txt"
+      .\runlint.bat FETCH_HEAD "$flake8Output" 
       if($LastExitCode -ne 0) {
        $errorCode=$LastExitCode
        Add-AppveyorMessage "PR introduces Flake8 errors"
@@ -191,19 +203,14 @@ test_script:
  - ps: |
      $testOutput = (Resolve-Path .\testOutput\)
      $systemTestOutput = (Resolve-Path "$testOutput\system")
-     $testSource = ".\tests\system\robot"
-
-     $robotArgs = '--argumentfile', '.\tests\system\robotArgs.robot'
-     # useInstalledNVDA must come after args file to override the whichNVDA value
-     $useInstalledNVDA = '--variable', 'whichNVDA:installed'
-
-     py -m robot $robotArgs $useInstalledNVDA $testSource
-     Compress-Archive -Path "$systemTestOutput\*" -DestinationPath "$testOutput\systemTestResult.zip"
-     Push-AppveyorArtifact "$testOutput\systemTestResult.zip"
+
+     .\runsystemtests.bat --variable whichNVDA:installed
      if($LastExitCode -ne 0) {
       $errorCode=$LastExitCode
       Add-AppveyorMessage "System test failure"
      }
+     Compress-Archive -Path "$systemTestOutput\*" -DestinationPath "$testOutput\systemTestResult.zip"
+     Push-AppveyorArtifact "$testOutput\systemTestResult.zip"
      $wc = New-Object 'System.Net.WebClient'
      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path "$systemTestOutput\systemTests.xml"))
      if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }

devDocs/buildSystemNotes.md

@@ -0,0 +1,24 @@
+A Python virtual environment is used transparently by the NVDA build system, and all Python dependencies are installed into this environment using `pip`.
+
+NVDA's build system commands will handle all aspects of the virtual environment.
+Developers should not create or activate the virtual environment manually, unless working on the build system itself.
+
+* To build NVDA, SCons should continue to be used in the usual way. E.g. executing scons.bat in the root of the repository. Running `py -m SCons`  is no longer supported, and scons.py has also been removed. 
+* To run NVDA from source, rather than executing `source/nvda.pyw` directly, the developer should now use `runnvda.bat` in the root of the repository. `runnvda.bat` uses `pythonw.exe` internally to execute NVDA. If you do try to execute `source/nvda.pyw`, a message box will alert you this is no longer supported.
+* To perform unit tests, execute `rununittests.bat [<extra unittest discover options>]`
+* To perform system tests: execute `runsystemtests.bat [<extra robot options>]`
+* To perform linting, execute `runlint.bat <base branch>`
+
+Behind the scenes, the above batch files (`scons`, `runnvda`, `rununittests`, `runsystemtests` and `runlint`) ensure that the Python virtual environment is created and up to date, activates the environment, runs the command and then deactivates. All transparently. A developer should not have to know about the Python virtual environment at all.
+The first time one of these commands are run, the virtual environment will be created, and all required Python dependencies will be installed with `pip`. You can see the entire list of packages and their exact versions that `pip` will use, in `requirements.txt` in the root of the repository.
+
+`venvUtils/ensureVenv.py` contains the logic to check for, create and update the virtual environment.
+If a previous virtual environment exists but has a miss-matching Python version or pip package requirements have changed, The virtual environment is recreated with the updated version of Python and packages.
+If a virtual environment is found but does not seem to be ours, the user is asked if it should be overwritten or not.
+This script also detects if it is being run from an existing 3rd party Python virtual environment and aborts if so. thus, it is impossible to execute SCons or NVDA from source within another Python virtual environment.
+
+`venvUtils/ensureAndActivate.bat` can be used to ensure the virtual environment exists and is up to date, and then activates it in the current shell, ready for other commands to be executed in the context of NVDA's build system Python virtual environment. this would never normally be executed by itself, though appVeyor uses it at the beginning of its execution and leaves the environment active for the remainder of its execution.
+
+`venvUtils/venvCmd.bat` is a script that runs a single command within the context of the NVDA build system Python virtual environment. It ensures and activates the environment, executes the command, and then deactivates the environment. this script is what all the high-level batch files use internally. 
+
+SConstruct, and `source/nvda.pyw` both contain logic that detects the NVDA build system Python virtual environment, and abort if it is not active.

devDocs/sconscript

@@ -19,10 +19,6 @@ Import("env", "outputDir", "sourceDir", "t2tBuildConf")
 
 env = env.Clone()
 
-devDocsInstall = env.SConscript("devDocsInstall/sconscript", exports=["env"])
-env.Depends(devDocsInstall, env.Dir("devDocs/devDocsInstall/requirements.txt"))
-env.Alias("devDocsInstall", devDocsInstall)
-
 devDocsOutputDir=outputDir.Dir('devDocs')
 
 #Build the developer guide and move it to the output directory
@@ -69,7 +65,6 @@ sphinxAPIDocs = env.Command(
 		] + [f"{sourceDir}\\{f}" for f in ignorePaths]
 	]
 )
-env.Depends(sphinxAPIDocs, devDocsInstall)
 sphinxHtml = env.Command(
 	"_build",
 	sphinxAPIDocs,

nvdaHelper/espeak/sconscript

@@ -1,5 +1,3 @@
-from include.scons.SCons.Util import CLVar
-
 Import([
 	'thirdPartyEnv',
 	'sourceDir',

nvdaHelper/liblouis/sconscript

@@ -17,8 +17,8 @@ import re
 import glob
 from SCons.Tool.MSCommon.vc import find_vc_pdir
 import typing
-from include.scons.SCons.Environment import Environment
-from include.scons.SCons.Environment import Base
+from SCons.Environment import Environment
+from SCons.Environment import Base
 
 Import([
 	"thirdPartyEnv",