Add python level tests for datascience python files (#5302)

For #4804 

<!--
  If an item below does not apply to you, then go ahead and check it off as "done" and strikethrough the text, e.g.:
    - [x] ~Has unit tests & system/integration tests~
-->
- [x] Pull request represents a single change (i.e. not fixing disparate/unrelated things in a single PR)
- [x] Title summarizes what is changing
- [x] Has a [news entry](https://github.com/Microsoft/vscode-python/tree/master/news) file (remember to thank yourself!)
- [ ] Has sufficient logging.
- [ ] Has telemetry for enhancements.
- [x] Unit tests & system/integration tests are added/updated
- [ ] [Test plan](https://github.com/Microsoft/vscode-python/blob/master/.github/test_plan.md) is updated as appropriate
- [ ] [`package-lock.json`](https://github.com/Microsoft/vscode-python/blob/master/package-lock.json) has been regenerated by running `npm install` (if dependencies have changed)
- [ ] The wiki is updated with any design decisions/details.

.vscode/launch.json

@@ -2,6 +2,21 @@
 {
     "version": "0.1.0",
     "configurations": [
+        {
+            "name": "Python: Current File with iPython",
+            "type": "python",
+            "request": "launch",
+            "module": "IPython",
+            "console": "integratedTerminal",
+            "args": ["${file}"] // Additional args should be prefixed with a '--' first.
+        },
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        },
         {
             "name": "Extension",
             "type": "extensionHost",
@@ -179,4 +194,4 @@
             ]
         }
     ]
-}
+}

build/ci/templates/test_phases.yml

@@ -172,6 +172,21 @@ steps:
     displayName: 'pip install functional requirements'
     condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true'))
 
+  # Install the requirements for ipython tests.
+  #
+  # This task will only run if variable `NeedsIPythonReqs` is true.
+  #
+  # Example command line (windows pwsh):
+  # > python -m pip install numpy
+  # > python -m pip install --upgrade -r build/ipython-test-requirements.txt
+  # > python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
+  - bash: |
+      python -m pip install -U pip
+      python -m pip install numpy
+      python -m pip install --upgrade -r ./build/ipython-test-requirements.txt
+    displayName: 'pip install ipython requirements'
+    condition: and(succeeded(), eq(variables['NeedsIPythonReqs'], 'true'))
+
   # Run the Python unit tests in our codebase. Produces a JUnit-style log file that
   # will be uploaded after all tests are complete.
   #
@@ -197,6 +212,31 @@ steps:
       buildPlatform: '$(Agent.Os)-Py$(pythonVersion)'
       buildConfiguration: 'UnitTests'
 
+  # Run the Python IPython tests in our codebase. Produces a JUnit-style log file that
+  # will be uploaded after all tests are complete.
+  #
+  # This task only runs if the string 'pythonIPythonTests' exists in variable `TestsToRun`.
+  #
+  # Example command line (windows pwsh):
+  # > python -m pip install -m -U pip
+  # > python -m pip install -U -r build/test-requirements.txt
+  # > python pythonFiles/tests/run_all.py --color=yes --junit-xml=python-tests-junit.xml
+  - bash: |
+      python -m IPython pythonFiles/tests/run_all.py -- --color=yes --junit-xml=$COMMON_TESTRESULTSDIRECTORY/ipython-tests-junit.xml
+    displayName: 'Python ipython tests'
+    condition: and(succeeded(), contains(variables['TestsToRun'], 'pythonIPythonTests'))
+
+  # Upload the test results to Azure DevOps to facilitate test reporting in their UX.
+  - task: PublishTestResults@2
+    displayName: 'Publish IPython test results'
+    condition: contains(variables['TestsToRun'], 'pythonIPythonTests')
+    inputs:
+      testResultsFiles: 'ipython-tests-junit.xml'
+      searchFolder: '$(Common.TestResultsDirectory)'
+      testRunTitle: 'pythonIPythonTests-$(Agent.Os)-Py$(pythonVersion)'
+      buildPlatform: '$(Agent.Os)-Py$(pythonVersion)'
+      buildConfiguration: 'UnitTests'
+
   # Run the News tool tests.
   #
   # This task only runs if the string 'pythonInternalTools' exists in variable `TestsToRun`

build/ci/vscode-python-ci.yaml

@@ -32,6 +32,7 @@ jobs:
       # Each member of this list may contain these values:
         # NeedsPythonTestReqs: [true|false] - install the test-requirements prior to running tests. False if not set.
         # NeedsPythonFunctionalReqs: [true|false] - install the functional-requirements prior to running tests. False if not set.
+        # NeedsIPythonReqs: [true|false] - install the ipython-test-requirements prior to running tests. False if not set.
         # PythonVersion: 'M.m' - the Python version to run. DefaultPythonVersion if not set.
         # NodeVersion: 'x.y.z' - Node version to use. DefaultNodeVersion if not set.
         # SkipXvfb: [true|false] - skip initialization of xvfb prior to running system tests on Linux. False if not set
@@ -42,70 +43,82 @@ jobs:
       'Win-Py3.7 Unit':
         PythonVersion: '3.7'
         VMImageName: 'vs2017-win2016'
-        TestsToRun: 'testUnitTests, pythonUnitTests'
+        TestsToRun: 'testUnitTests, pythonUnitTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
       'Linux-Py3.7 Unit':
         PythonVersion: '3.7'
         VMImageName: 'ubuntu-16.04'
-        TestsToRun: 'testUnitTests, pythonUnitTests'
+        TestsToRun: 'testUnitTests, pythonUnitTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
       'Mac-Py3.7 Unit':
         PythonVersion: '3.7'
         VMImageName: 'macos-10.13'
-        TestsToRun: 'testUnitTests, pythonUnitTests'
+        TestsToRun: 'testUnitTests, pythonUnitTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
       'Win-Py3.6 Unit':
         PythonVersion: '3.6'
         VMImageName: 'vs2017-win2016'
-        TestsToRun: 'pythonUnitTests'
+        TestsToRun: 'pythonUnitTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
       'Linux-Py3.6 Unit':
         PythonVersion: '3.6'
         VMImageName: 'ubuntu-16.04'
-        TestsToRun: 'pythonUnitTests'
+        TestsToRun: 'pythonUnitTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
       'Mac-Py3.6 Unit':
         PythonVersion: '3.6'
         VMImageName: 'macos-10.13'
-        TestsToRun: 'pythonUnitTests'
+        TestsToRun: 'pythonUnitTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
 
       'Win-Py3.7 Venv':
         VMImageName: 'vs2017-win2016'
         PythonVersion: '3.7'
-        TestsToRun: 'venvTests'
+        TestsToRun: 'venvTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
         # This is for the venvTests to use, not needed if you don't run venv tests...
         PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
       'Linux-Py3.7 Venv':
         VMImageName: 'ubuntu-16.04'
         PythonVersion: '3.7'
-        TestsToRun: 'venvTests'
+        TestsToRun: 'venvTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
         PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
       'Mac-Py3.7 Venv':
         VMImageName: 'macos-10.13'
         PythonVersion: '3.7'
-        TestsToRun: 'venvTests'
+        TestsToRun: 'venvTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
         PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
       'Win-Py3.6 Venv':
         VMImageName: 'vs2017-win2016'
         PythonVersion: '3.6'
-        TestsToRun: 'venvTests'
+        TestsToRun: 'venvTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
         PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
       'Linux-Py3.6 Venv':
         VMImageName: 'ubuntu-16.04'
         PythonVersion: '3.6'
-        TestsToRun: 'venvTests'
+        TestsToRun: 'venvTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
         PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
       'Mac-Py3.6 Venv':
         VMImageName: 'macos-10.13'
         PythonVersion: '3.6'
-        TestsToRun: 'venvTests'
+        TestsToRun: 'venvTests, pythonIPythonTests'
         NeedsPythonTestReqs: true
+        NeedsIPythonReqs: true
         PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
 
       # SingleWorkspace Tests

build/ipython-test-requirements.txt

@@ -0,0 +1,4 @@
+# List of requirements for ipython tests
+numpy
+pandas
+ipython

news/3 Code Health/4804.md

@@ -0,0 +1 @@
+Add tests for variable explorer python files.

pythonFiles/datascience/getJupyterVariableDataFrameInfo.py

@@ -3,7 +3,7 @@
 import pandas as _VSCODE_pd
 
 # _VSCode_sub_supportsDataExplorer will contain our list of data explorer supported types
-_VSCode_supportsDataExplorer = _VSCode_sub_supportsDataExplorer
+_VSCode_supportsDataExplorer = "['list', 'Series', 'dict', 'ndarray', 'DataFrame']"
 
 # In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable
 # Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable
@@ -58,6 +58,8 @@
         _VSCODE_colobj['name'] = _VSCODE_column_name
         _VSCODE_colobj['type'] = str(_VSCODE_column_type)
         _VSCODE_columns.append(_VSCODE_colobj)
+        del _VSCODE_column_name
+        del _VSCODE_column_type
 
     del _VSCODE_columnNames
     del _VSCODE_columnTypes

pythonFiles/datascience/getJupyterVariableList.py

@@ -2,19 +2,26 @@
 # Tested on 2.7 and 3.6
 from sys import getsizeof as _VSCODE_getsizeof
 import json as _VSCODE_json
+from IPython import get_ipython
 
-# _VSCode_sub_supportsDataExplorer will contain our list of data explorer supported types
-_VSCode_supportsDataExplorer = _VSCode_sub_supportsDataExplorer
+# _VSCode_supportsDataExplorer will contain our list of data explorer supported types
+_VSCode_supportsDataExplorer = "['list', 'Series', 'dict', 'ndarray', 'DataFrame']"
 
 # who_ls is a Jupyter line magic to fetch currently defined vars
-_VSCode_JupyterVars = %who_ls
+_VSCode_JupyterVars = get_ipython().run_line_magic('who_ls', '')
 
-print(_VSCODE_json.dumps([{'name': var,
-                               'type': type(eval(var)).__name__,
-                               'size': _VSCODE_getsizeof(var),
-                               'supportsDataExplorer': type(eval(var)).__name__ in _VSCode_supportsDataExplorer
-                              } for var in _VSCode_JupyterVars]))
+_VSCode_output = []
+for var in _VSCode_JupyterVars:
+    try:
+        _VSCode_type = type(eval(var))
+        _VSCode_output.append({'name': var, 'type': _VSCode_type.__name__, 'size': _VSCODE_getsizeof(var), 'supportsDataExplorer': _VSCode_type.__name__ in _VSCode_supportsDataExplorer })
+        del _VSCode_type
+    except:
+        pass
 
+print(_VSCODE_json.dumps(_VSCode_output))
+
+del _VSCode_output
 del _VSCode_supportsDataExplorer
 del _VSCode_JupyterVars
 del _VSCODE_json

pythonFiles/tests/__main__.py

@@ -11,7 +11,7 @@
 TEST_ROOT = os.path.dirname(__file__)
 SRC_ROOT = os.path.dirname(TEST_ROOT)
 PROJECT_ROOT = os.path.dirname(SRC_ROOT)
-DATASCIENCE_ROOT = os.path.join(SRC_ROOT, 'datascience')
+IPYTHON_ROOT = os.path.join(SRC_ROOT, 'ipython')
 TESTING_TOOLS_ROOT = os.path.join(SRC_ROOT, 'testing_tools')
 
 
@@ -30,7 +30,7 @@ def parse_args():
 
 
 def main(pytestargs, markers=None):
-    sys.path.insert(1, DATASCIENCE_ROOT)
+    sys.path.insert(1, IPYTHON_ROOT)
     sys.path.insert(1, TESTING_TOOLS_ROOT)
 
     pytestargs = [