diff --git a/build.cmd b/build.cmd
index d2c350ba..e967bd4b 100644
--- a/build.cmd
+++ b/build.cmd
@@ -157,7 +157,7 @@ if /i [%1] == [DbgWinPy2.7] (
:Build
:: Install dotnet SDK version, see https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script
echo Installing dotnet SDK ...
-powershell -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Version 2.1.200 -InstallDir ./cli"
+powershell -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Version 2.1.701 -InstallDir ./cli"
set _dotnetRoot=%__currentScriptDir%cli
@@ -339,6 +339,8 @@ echo "Running tests ... "
echo "#################################"
call "%PythonExe%" -m pip install --upgrade nose pytest graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0"
if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq )
+:: Run azureml-dataprep tests only in pyhon 3.7 as its an optional dependency
+if %PythonVersion% == 3.7 ( call "%PythonExe%" -m pip install --upgrade azureml-dataprep )
call "%PythonExe%" -m pip install --upgrade "%__currentScriptDir%target\%WheelFile%"
call "%PythonExe%" -m pip install "scikit-learn==0.19.2"
diff --git a/build.sh b/build.sh
index 78de9ff8..4cb9becb 100755
--- a/build.sh
+++ b/build.sh
@@ -166,7 +166,7 @@ if [ ${__buildDotNetBridge} = true ]
then
# Install dotnet SDK version, see https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script
echo "Installing dotnet SDK ... "
- curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin -Version 2.1.200 -InstallDir ./cli
+ curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin -Version 2.1.701 -InstallDir ./cli
# Build managed code
echo "Building managed code ... "
@@ -266,7 +266,10 @@ then
elif [ ${PythonVersion} = 3.6 ] && [ "$(uname -s)" = "Darwin" ]
then
"${PythonExe}" -m pip install --upgrade pytest-remotedata
- fi
+ elif [ ${PythonVersion} = 3.7 ]
+ then
+ "${PythonExe}" -m pip install --upgrade azureml-dataprep
+ fi
"${PythonExe}" -m pip install --upgrade "${Wheel}"
"${PythonExe}" -m pip install "scikit-learn==0.19.2"
diff --git a/build/libs_linux.txt b/build/libs_linux.txt
index 15c3395e..e7be43ad 100644
--- a/build/libs_linux.txt
+++ b/build/libs_linux.txt
@@ -9,4 +9,6 @@ lib_lightgbm.so
libtensorflow.so
libtensorflow_framework.so
System.Drawing.Common.dll
+Microsoft.DataPrep.dll
+Microsoft.DPrep.*
Microsoft.ML.*
diff --git a/build/libs_mac.txt b/build/libs_mac.txt
index 2be6a809..3168c07d 100644
--- a/build/libs_mac.txt
+++ b/build/libs_mac.txt
@@ -9,4 +9,6 @@ lib_lightgbm.dylib
libtensorflow.dylib
libtensorflow_framework.dylib
System.Drawing.Common.dll
+Microsoft.DataPrep.dll
+Microsoft.DPrep.*
Microsoft.ML.*
diff --git a/build/libs_win.txt b/build/libs_win.txt
index dda6dcd6..69dc4537 100644
--- a/build/libs_win.txt
+++ b/build/libs_win.txt
@@ -10,4 +10,6 @@ MklImports.dll
SymSgdNative.dll
tensorflow.dll
System.Drawing.Common.dll
+Microsoft.DataPrep.dll
+Microsoft.DPrep.*
Microsoft.ML.*
diff --git a/src/DotNetBridge/DotNetBridge.csproj b/src/DotNetBridge/DotNetBridge.csproj
index 9e89c206..b7afdc3e 100644
--- a/src/DotNetBridge/DotNetBridge.csproj
+++ b/src/DotNetBridge/DotNetBridge.csproj
@@ -1,6 +1,6 @@
- netstandard2.0
+ netcoreapp2.1
true
x64
CORECLR
@@ -42,5 +42,6 @@
+
diff --git a/src/DotNetBridge/RunGraph.cs b/src/DotNetBridge/RunGraph.cs
index 09617aa6..f79cff9d 100644
--- a/src/DotNetBridge/RunGraph.cs
+++ b/src/DotNetBridge/RunGraph.cs
@@ -8,6 +8,7 @@
using System.Globalization;
using System.IO;
using System.Linq;
+using Microsoft.DataPrep.Common;
using Microsoft.ML;
using Microsoft.ML.CommandLine;
using Microsoft.ML.Data;
@@ -146,7 +147,8 @@ private static void RunGraphCore(EnvironmentBlock* penv, IHostEnvironment env, s
var extension = Path.GetExtension(path);
if (extension == ".txt")
dv = TextLoader.LoadFile(host, new TextLoader.Options(), new MultiFileSource(path));
-
+ else if(extension == ".dprep")
+ dv = DataFlow.FromDPrepFile(path).ToDataView();
else
dv = new BinaryLoader(host, new BinaryLoader.Arguments(), path);
}
diff --git a/src/NativeBridge/UnixInterface.h b/src/NativeBridge/UnixInterface.h
index ab53f428..0a7c1155 100644
--- a/src/NativeBridge/UnixInterface.h
+++ b/src/NativeBridge/UnixInterface.h
@@ -144,15 +144,20 @@ class UnixMlNetInterface
{
}
- FNGETTER EnsureGetter(const char *nimbuslibspath, const char *coreclrpath)
+ FNGETTER EnsureGetter(const char *mlnetpath, const char *coreclrpath, const char *dpreppath)
{
if (_getter != nullptr)
return _getter;
- std::string libsroot(nimbuslibspath);
+ std::string libsroot(mlnetpath);
std::string coreclrdir(coreclrpath);
+ if (strlen(dpreppath) == 0)
+ {
+ dpreppath = mlnetpath;
+ }
+ std::string dprepdir(dpreppath);
- ICLRRuntimeHost2* host = EnsureClrHost(libsroot.c_str(), coreclrdir.c_str());
+ ICLRRuntimeHost2* host = EnsureClrHost(libsroot.c_str(), coreclrdir.c_str(), dprepdir.c_str());
if (host == nullptr)
return nullptr;
@@ -246,7 +251,7 @@ class UnixMlNetInterface
closedir(dir);
}
- ICLRRuntimeHost2* EnsureClrHost(const char * libsRoot, const char * coreclrDirRoot)
+ ICLRRuntimeHost2* EnsureClrHost(const char * libsRoot, const char * coreclrDirRoot, const char * dprepDirRoot)
{
if (_host != nullptr)
return _host;
@@ -284,7 +289,7 @@ class UnixMlNetInterface
// TRUSTED_PLATFORM_ASSEMBLIES
tpaList.c_str(),
// APP_PATHS
- libsRoot,
+ dprepDirRoot,
// AppDomainCompatSwitch
W("UseLatestBehaviorWhenTFMNotSpecified")
};
diff --git a/src/NativeBridge/WinInterface.h b/src/NativeBridge/WinInterface.h
index 2fecf434..4f5238db 100644
--- a/src/NativeBridge/WinInterface.h
+++ b/src/NativeBridge/WinInterface.h
@@ -161,7 +161,7 @@ class WinMlNetInterface
FindClose(findHandle);
}
- ICLRRuntimeHost2* EnsureClrHost(const wchar_t * libsRoot, const wchar_t * coreclrDirRoot)
+ ICLRRuntimeHost2* EnsureClrHost(const wchar_t * libsRoot, const wchar_t * coreclrDirRoot, const wchar_t * dprepDirRoot)
{
if (_host != nullptr)
return _host;
@@ -228,7 +228,7 @@ class WinMlNetInterface
// TRUSTED_PLATFORM_ASSEMBLIES
tpaList.c_str(),
// APP_PATHS
- libsRoot,
+ dprepDirRoot,
// AppDomainCompatSwitch
W("UseLatestBehaviorWhenTFMNotSpecified")
};
@@ -267,26 +267,29 @@ class WinMlNetInterface
}
public:
- FNGETTER EnsureGetter(const char *nimbuslibspath, const char *coreclrpath)
+ FNGETTER EnsureGetter(const char *mlnetpath, const char *coreclrpath, const char *dpreppath)
{
if (_getter != nullptr)
return _getter;
- std::wstring libsdir = Utf8ToUtf16le(nimbuslibspath);
+ std::wstring libsdir = Utf8ToUtf16le(mlnetpath);
ConvertToWinPath(libsdir);
- std::wstring coreclrdir;
- if (strlen(coreclrpath) != 0)
+ std::wstring coreclrdir = Utf8ToUtf16le(coreclrpath);
+ ConvertToWinPath(coreclrdir);
+
+ std::wstring dprepdir;
+ if (strlen(dpreppath) != 0)
{
- coreclrdir = Utf8ToUtf16le(coreclrpath);
- ConvertToWinPath(coreclrdir);
+ dprepdir = Utf8ToUtf16le(dpreppath);
+ ConvertToWinPath(dprepdir);
}
else
{
- coreclrdir = libsdir;
+ dprepdir = libsdir;
}
- ICLRRuntimeHost2* host = EnsureClrHost(libsdir.c_str(), coreclrdir.c_str());
+ ICLRRuntimeHost2* host = EnsureClrHost(libsdir.c_str(), coreclrdir.c_str(), dprepdir.c_str());
if (host == nullptr)
return nullptr;
diff --git a/src/NativeBridge/dllmain.cpp b/src/NativeBridge/dllmain.cpp
index 3f521f87..c656f6d0 100644
--- a/src/NativeBridge/dllmain.cpp
+++ b/src/NativeBridge/dllmain.cpp
@@ -9,8 +9,9 @@
#define PARAM_SEED "seed"
#define PARAM_GRAPH "graph"
#define PARAM_VERBOSE "verbose"
-#define PARAM_NIMBUSML_PATH "nimbusmlPath"
+#define PARAM_MLNET_PATH "mlnetPath"
#define PARAM_DOTNETCLR_PATH "dotnetClrPath"
+#define PARAM_DPREP_PATH "dprepPath"
#define PARAM_DATA "data"
@@ -44,14 +45,14 @@ static MlNetInterface *g_mlnetInterface = nullptr;
static GENERICEXEC g_exec = nullptr;
// Ensure that we have the DotNetBridge managed code entry point.
-GENERICEXEC EnsureExec(const char *nimbuslibspath, const char *coreclrpath)
+GENERICEXEC EnsureExec(const char *mlnetpath, const char *coreclrpath, const char *dpreppath)
{
if (g_mlnetInterface == nullptr)
g_mlnetInterface = new MlNetInterface();
if (g_exec == nullptr)
{
- FNGETTER getter = g_mlnetInterface->EnsureGetter(nimbuslibspath, coreclrpath);
+ FNGETTER getter = g_mlnetInterface->EnsureGetter(mlnetpath, coreclrpath, dpreppath);
if (getter != nullptr)
g_exec = (GENERICEXEC)getter(FnIdGenericExec);
}
@@ -70,20 +71,23 @@ bp::dict pxCall(bp::dict& params)
try
{
bp::extract graph(params[PARAM_GRAPH]);
- bp::extract nimbusmlPath(params[PARAM_NIMBUSML_PATH]);
+ bp::extract mlnetPath(params[PARAM_MLNET_PATH]);
bp::extract dotnetClrPath(params[PARAM_DOTNETCLR_PATH]);
+ bp::extract dprepPath(params[PARAM_DPREP_PATH]);
bp::extract verbose(params[PARAM_VERBOSE]);
std::int32_t i_verbose = std::int32_t(verbose);
- std::string s_nimbusmlPath = std::string(nimbusmlPath);
+ std::string s_mlnetPath = std::string(mlnetPath);
std::string s_dotnetClrPath = std::string(dotnetClrPath);
+ std::string s_dprepPath = std::string(dprepPath);
std::string s_graph = std::string(graph);
- const char *nimbuslibspath = s_nimbusmlPath.c_str();
+ const char *mlnetpath = s_mlnetPath.c_str();
const char *coreclrpath = s_dotnetClrPath.c_str();
+ const char *dpreppath = s_dprepPath.c_str();
- GENERICEXEC exec = EnsureExec(nimbuslibspath, coreclrpath);
+ GENERICEXEC exec = EnsureExec(mlnetpath, coreclrpath, dpreppath);
if (exec == nullptr)
- throw std::invalid_argument("Failed to communicate with the managed library. Path searched: "
- + s_nimbusmlPath + " and " + s_dotnetClrPath);
+ throw std::invalid_argument("Failed to communicate with the managed library. Paths searched: "
+ + s_mlnetPath + " and " + s_dotnetClrPath);
int seed = 42;
if (params.has_key(PARAM_SEED))
diff --git a/src/Platforms/build.csproj b/src/Platforms/build.csproj
index 99150a1e..cb7f2445 100644
--- a/src/Platforms/build.csproj
+++ b/src/Platforms/build.csproj
@@ -3,7 +3,7 @@
dummy
Exe
- netcoreapp2.0
+ netcoreapp2.1
x64
DbgWinPy3.7;DbgWinPy3.6;DbgWinPy3.5;DbgWinPy2.7;RlsWinPy3.7;RlsWinPy3.6;RlsWinPy3.5;RlsWinPy2.7;DbgLinPy3.7;DbgLinPy3.6;DbgLinPy3.5;DbgLinPy2.7;RlsLinPy3.7;RlsLinPy3.6;RlsLinPy3.5;RlsLinPy2.7;RlsMacPy3.7;RlsMacPy3.6
$(ProjectDir)..\..\x64\$(Configuration)\Platform\
@@ -21,6 +21,7 @@
+
diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj
index ebf6b8c4..a97e8b14 100644
--- a/src/python/nimbusml.pyproj
+++ b/src/python/nimbusml.pyproj
@@ -12,9 +12,10 @@
{888888a0-9f3d-457c-b088-3a5042f75d52}
Standard Python launcher
nimbusml
- Global|VisualStudio|Py3.7
+ Global|VisualStudio|MinePy37
..\..\dependencies\Python3.7\python.exe
False
+ nimbusml\tests\dprep\test_dprep.py
@@ -581,6 +582,8 @@
+
+
@@ -735,6 +738,7 @@
+
@@ -1136,7 +1140,7 @@
-
+
\ No newline at end of file
diff --git a/src/python/nimbusml/__init__.py b/src/python/nimbusml/__init__.py
index e2bfa49c..5d332555 100644
--- a/src/python/nimbusml/__init__.py
+++ b/src/python/nimbusml/__init__.py
@@ -20,6 +20,7 @@
from .internal.utils.data_roles import Role
from .internal.utils.data_schema import DataSchema
from .internal.utils.data_stream import BinaryDataStream
+from .internal.utils.data_stream import DprepDataStream
from .internal.utils.data_stream import FileDataStream
from .internal.utils.utils import run_tests
from .pipeline import Pipeline
diff --git a/src/python/nimbusml/internal/utils/data_stream.py b/src/python/nimbusml/internal/utils/data_stream.py
index ede031d9..7d490bc6 100644
--- a/src/python/nimbusml/internal/utils/data_stream.py
+++ b/src/python/nimbusml/internal/utils/data_stream.py
@@ -5,6 +5,8 @@
"""
Owns nimbusml's containers.
"""
+import os
+import tempfile
from shutil import copyfile
from .data_roles import DataRoles
@@ -467,3 +469,34 @@ def clone(self):
"Method clone was not overwritten for class '{0}'".format(
type(self)))
return BinaryDataStream(self._filename)
+
+
+class DprepDataStream(BinaryDataStream):
+ """
+ Defines a data view over dprep file.
+ """
+
+ def __init__(self, dataflow=None, filename=None):
+ if dataflow is None and filename is None:
+ raise ValueError('Both dataflow object and filename are None')
+ super(DprepDataStream, self).__init__(DataSchema(""))
+ if dataflow is not None:
+ (fd, filename) = tempfile.mkstemp(suffix='.dprep')
+ fl = os.fdopen(fd, "wt")
+ fl.write(dataflow.to_json())
+ fl.close()
+ self._filename = filename
+
+ def __repr__(self):
+ return "DprepDataStream('{2}',\n '{0}',\n {1})".format(
+ self._schema, self._roles, self._filename.replace('\\', '\\\\'))
+
+ def clone(self):
+ """
+ Copy/clone the object.
+ """
+ if not isinstance(self, DprepDataStream):
+ raise NotImplementedError(
+ "Method clone was not overwritten for class '{0}'".format(
+ type(self)))
+ return DprepDataStream(self._filename)
\ No newline at end of file
diff --git a/src/python/nimbusml/internal/utils/entrypoints.py b/src/python/nimbusml/internal/utils/entrypoints.py
index 8d9ef085..0a292866 100644
--- a/src/python/nimbusml/internal/utils/entrypoints.py
+++ b/src/python/nimbusml/internal/utils/entrypoints.py
@@ -18,12 +18,13 @@
from scipy.sparse import csr_matrix
from nimbusml.utils import signature
+from .data_stream import DprepDataStream
from .data_stream import BinaryDataStream
from .data_stream import FileDataStream
from .dataframes import resolve_dataframe, resolve_csr_matrix, pd_concat, \
resolve_output
from .utils import try_set, set_clr_environment_vars, get_clr_path, \
- get_nimbusml_libs
+ get_mlnet_path, get_dprep_path
from ..libs.pybridge import px_call
@@ -399,7 +400,7 @@ def remove_multi_level_index(c):
concatenated = True
elif isinstance(X, FileDataStream):
self.inputs['file'] = X.filename
- elif isinstance(X, BinaryDataStream):
+ elif isinstance(X, BinaryDataStream) or isinstance(X, DprepDataStream):
if 'input_data' in self.inputs:
self.inputs['input_data'] = X._filename
elif 'data' in self.inputs:
@@ -439,17 +440,16 @@ def remove_multi_level_index(c):
with open(input_graphfilename, 'w') as f:
f.write(self.nimbusml_runnable_graph)
- nimbusml_path = os.path.join(os.path.dirname(__file__), "..", "libs")
- nimbusml_path = os.path.abspath(nimbusml_path)
call_parameters['verbose'] = try_set(verbose, False, six.integer_types)
call_parameters['graph'] = try_set(
'graph = {%s} %s' %
(str(self), code), False, str)
- # Set paths to ML.NET libs (in nimbusml) and to .NET Core CLR libs
- call_parameters['nimbusmlPath'] = try_set(get_nimbusml_libs(), True, str)
+ # Set paths to .NET Core CLR, ML.NET and DataPrep libs
set_clr_environment_vars()
- call_parameters['dotnetClrPath'] = try_set(get_clr_path(), True, str)
+ call_parameters['dotnetClrPath'] = try_set(get_clr_path(), False, str)
+ call_parameters['mlnetPath'] = try_set(get_mlnet_path(), False, str)
+ call_parameters['dprepPath'] = try_set(get_dprep_path(), False, str)
if random_state:
call_parameters['seed'] = try_set(random_state, False, six.integer_types)
diff --git a/src/python/nimbusml/internal/utils/utils.py b/src/python/nimbusml/internal/utils/utils.py
index a63452b6..b8aab001 100644
--- a/src/python/nimbusml/internal/utils/utils.py
+++ b/src/python/nimbusml/internal/utils/utils.py
@@ -301,12 +301,12 @@ def set_clr_environment_vars():
def get_clr_path():
"""
- Return path to .NET CLR libs.
+ Return path to .NET CLR binaries.
Use dotnetcore2 package if Python 3.x, otherwise look for libs bundled with
NimbusML.
"""
if six.PY2:
- return get_nimbusml_libs()
+ return get_mlnet_path()
else:
from dotnetcore2 import runtime as clr_runtime
libs_root = os.path.join(clr_runtime._get_bin_folder(), 'shared',
@@ -329,9 +329,20 @@ def get_clr_path():
"found in {}.".format(libs_root))
return clr_path
-def get_nimbusml_libs():
+def get_dprep_path():
"""
- Return path to NimbusML libs (the ML.NET binaries).
+ Return path to DataPrep binaries if its installed, None otherwise
+ """
+ try:
+ from azureml.dataprep.api.engineapi.engine import _get_engine_path
+ return os.path.dirname(_get_engine_path())
+ except ImportError:
+ pass
+ return ''
+
+def get_mlnet_path():
+ """
+ Return path to ML.NET binaries.
"""
return os.path.abspath(os.path.join(os.path.dirname(__file__), '..',
'libs'))
diff --git a/src/python/nimbusml/tests/dprep/__init__.py b/src/python/nimbusml/tests/dprep/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/python/nimbusml/tests/dprep/test_dprep.py b/src/python/nimbusml/tests/dprep/test_dprep.py
new file mode 100644
index 00000000..c8ebbbdb
--- /dev/null
+++ b/src/python/nimbusml/tests/dprep/test_dprep.py
@@ -0,0 +1,55 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------------------------
+
+import unittest
+
+import numpy as np
+import os
+import sys
+from nimbusml import Pipeline, FileDataStream, BinaryDataStream, DprepDataStream
+from nimbusml.datasets import get_dataset
+from nimbusml.preprocessing.normalization import MinMaxScaler
+from sklearn.utils.testing import assert_true, assert_array_equal
+
+def is_nan(x):
+ return (x is np.nan or x != x)
+
+def assert_2d_array_equal(actual, desired):
+ if len(actual) != len(desired):
+ assert_true(False, "arrays are of different lengths.")
+
+ for i in range(len(actual)):
+ if len(actual[i]) != len(desired[i]):
+ assert_true(False, "arrays are of different lengths.")
+ for y in range(len(actual[i])):
+ if is_nan(actual[i][y]) and is_nan(desired[i][y]):
+ continue
+ assert_true(actual[i][y] == desired[i][y])
+
+@unittest.skipIf(os.name == "posix" or sys.version_info[:2] != (3, 7), "azureml-dataprep is not installed.")
+class TestDprep(unittest.TestCase):
+
+ def test_fit_transform(self):
+ import azureml.dataprep as dprep
+
+ path = get_dataset('infert').as_filepath()
+ dflow = dprep.auto_read_file(path=path)
+ dprep_data = DprepDataStream(dflow)
+ file_data = FileDataStream.read_csv(path)
+
+ xf = MinMaxScaler(columns={'in': 'induced', 'sp': 'spontaneous'})
+ pipe = Pipeline([xf])
+ transformed_data = pipe.fit_transform(file_data)
+ transformed_data1 = pipe.fit_transform(dprep_data)
+
+ assert_array_equal(
+ transformed_data.columns,
+ transformed_data1.columns)
+ assert_2d_array_equal(
+ transformed_data.values,
+ transformed_data1.values)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/src/python/setup.py b/src/python/setup.py
index 80f47621..2a8a05b6 100644
--- a/src/python/setup.py
+++ b/src/python/setup.py
@@ -115,6 +115,7 @@
'nose>=1.3', 'pytest>=4.4.0',
'graphviz', 'imageio',
],
+ 'dprep': ['azureml-dataprep'],
'utils': ['graphviz', 'imageio'],
},
diff --git a/src/python/setup.py.in b/src/python/setup.py.in
index 07f92fe1..3ddce586 100644
--- a/src/python/setup.py.in
+++ b/src/python/setup.py.in
@@ -115,6 +115,7 @@ setup(
'nose>=1.3', 'pytest>=4.4.0',
'graphviz', 'imageio',
],
+ 'dprep': ['azureml-dataprep'],
'utils': ['graphviz', 'imageio'],
},