Pull from upstream

.gitignore

@@ -328,3 +328,5 @@ ASALocalRun/
 
 # MSBuild Binary and Structured Log
 *.binlog
+# Ignore external test datasets.
+/test/data/external/

CONTRIBUTING.md

@@ -24,8 +24,8 @@ All commits in a pull request will be squashed to a single commit with the origi
 
 # Contributing
 
-See [Contributing](Documentation/project-docs/contributing.md) for information about coding styles, source structure, making pull requests, and more.
+See [Contributing](docs/project-docs/contributing.md) for information about coding styles, source structure, making pull requests, and more.
 
 # Developers
 
-See the [Developer Guide](Documentation/project-docs/developer-guide.md) for details about developing in this repo.
+See the [Developer Guide](docs/project-docs/developer-guide.md) for details about developing in this repo.

Directory.Build.props

@@ -63,25 +63,29 @@
     <VersionSuffix Condition="'$(IncludeBuildNumberInPackageVersion)' == 'true'">$(VersionSuffix)-$(BuildNumberMajor)-$(BuildNumberMinor)</VersionSuffix>
   </PropertyGroup>
 
+  <!-- SourceLink properties used by dotnet/buildtools - need to be set before importing $(ToolsDir)versioning.props -->
+  <PropertyGroup>
+    <UseSourceLink>true</UseSourceLink>
+    <GitHubRepositoryName>machinelearning</GitHubRepositoryName>
+  </PropertyGroup>
+
+  <!--
+  Source code control properties used by the .NET Core SDK to inject SCC info into the NuGet package.
+  In future versions, these will be used for SourceLink and to generate AssemblyInfo.
+  -->
+  <PropertyGroup>
+    <PrivateRepositoryUrl>https://github.com/dotnet/$(GitHubRepositoryName)</PrivateRepositoryUrl>
+    <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <SourceRevisionId>$(LatestCommit)</SourceRevisionId>
+  </PropertyGroup>
+
   <!--
   Ensure to import versioning.props before overwriting BaseIntermediateOutputPath since
   the source link file exists in the root bin/obj folder.
    -->
   <Import Project="$(ToolsDir)versioning.props" 
           Condition="Exists('$(ToolsDir)versioning.props') and '$(DisableImportVersioningProps)' != 'true'" />
-
-  <!-- NuGet package properties -->
-  <PropertyGroup>
-    <Authors>Microsoft</Authors>
-    <IncludeSymbols>true</IncludeSymbols>
-  </PropertyGroup>
-
-  <!-- SourceLink properties -->
-  <PropertyGroup>
-    <UseSourceLink>true</UseSourceLink>
-    <GitHubRepositoryName>machinelearning</GitHubRepositoryName>
-  </PropertyGroup>
-
+
   <!-- Language configuration -->
   <PropertyGroup>
     <LangVersion>latest</LangVersion> <!-- default to allowing all language features -->

DotnetCLIVersion.txt

@@ -1 +1 @@
-2.1.2
+2.1.200

Microsoft.ML.sln

@@ -58,13 +58,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Sweeper", "src
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{E20AF96D-3F66-4065-8A89-BEE479D74536}"
 	ProjectSection(SolutionItems) = preProject
-		Documentation\README.md = Documentation\README.md
+		docs\README.md = docs\README.md
+		docs\release-notes\0.1\release-0.1.md = docs\release-notes\0.1\release-0.1.md
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "project-docs", "project-docs", "{52794B40-AB8A-41AF-9EF7-799C80D6E0BC}"
 	ProjectSection(SolutionItems) = preProject
-		Documentation\project-docs\contributing.md = Documentation\project-docs\contributing.md
-		Documentation\project-docs\developer-guide.md = Documentation\project-docs\developer-guide.md
+		docs\project-docs\contributing.md = docs\project-docs\contributing.md
+		docs\project-docs\developer-guide.md = docs\project-docs\developer-guide.md
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{76F579E4-B9D2-4A0C-A511-EEFA4B2B829F}"
@@ -76,15 +77,33 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "building", "building", "{DB751004-5D49-4B88-B78F-29CA9887087D}"
 	ProjectSection(SolutionItems) = preProject
-		Documentation\building\unix-instructions.md = Documentation\building\unix-instructions.md
-		Documentation\building\windows-instructions.md = Documentation\building\windows-instructions.md
+		docs\building\unix-instructions.md = docs\building\unix-instructions.md
+		docs\building\windows-instructions.md = docs\building\windows-instructions.md
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "specs", "specs", "{2DEFC784-F2B5-44EA-ABBB-0DCF3E689DAC}"
 	ProjectSection(SolutionItems) = preProject
-		Documentation\specs\mvp.md = Documentation\specs\mvp.md
+		docs\specs\mvp.md = docs\specs\mvp.md
 	EndProjectSection
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pkg", "pkg", "{D3D38B03-B557-484D-8348-8BADEE4DF592}"
+	ProjectSection(SolutionItems) = preProject
+		pkg\Directory.Build.props = pkg\Directory.Build.props
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML", "Microsoft.ML", "{DEC8F776-49F7-4D87-836C-FE4DC057D08C}"
+	ProjectSection(SolutionItems) = preProject
+		pkg\Microsoft.ML\Microsoft.ML.nupkgproj = pkg\Microsoft.ML\Microsoft.ML.nupkgproj
+		pkg\Microsoft.ML\Microsoft.ML.symbols.nupkgproj = pkg\Microsoft.ML\Microsoft.ML.symbols.nupkgproj
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML.Parquet", "Microsoft.ML.Parquet", "{6C95FC87-F5F2-4EEF-BB97-567F2F5DD141}"
+	ProjectSection(SolutionItems) = preProject
+		pkg\Microsoft.ML.Parquet\Microsoft.ML.Parquet.nupkgproj = pkg\Microsoft.ML.Parquet\Microsoft.ML.Parquet.nupkgproj
+	EndProjectSection
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks", "test\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -179,6 +198,10 @@ Global
 		{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -210,6 +233,9 @@ Global
 		{52794B40-AB8A-41AF-9EF7-799C80D6E0BC} = {E20AF96D-3F66-4065-8A89-BEE479D74536}
 		{DB751004-5D49-4B88-B78F-29CA9887087D} = {E20AF96D-3F66-4065-8A89-BEE479D74536}
 		{2DEFC784-F2B5-44EA-ABBB-0DCF3E689DAC} = {E20AF96D-3F66-4065-8A89-BEE479D74536}
+		{DEC8F776-49F7-4D87-836C-FE4DC057D08C} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
+		{6C95FC87-F5F2-4EEF-BB97-567F2F5DD141} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

README.md

@@ -14,27 +14,33 @@ Along with these ML capabilities this first release of ML.NET also brings the fi
 
 ## Installation
 
+[![NuGet Status](https://img.shields.io/nuget/v/Microsoft.ML.svg?style=flat)](https://www.nuget.org/packages/Microsoft.ML/)
+
 ML.NET runs on Windows, Linux, and macOS - any platform where 64 bit [.NET Core](https://github.com/dotnet/core) or later is available.
 
-The current release is 0.1. Check out the [release notes](Documentation/release-notes/0.1/release-0.1.md).
+The current release is 0.1. Check out the [release notes](docs/release-notes/0.1/release-0.1.md).
 
 First ensure you have installed [.NET Core 2.0](https://www.microsoft.com/net/learn/get-started) or later. ML.NET also works on the .NET Framework. Note that ML.NET currently must run in a 64 bit process.
 
-Once you have an app, you can install ML.NET NuGet from the .NET Core CLI using:
+Once you have an app, you can install the ML.NET NuGet package from the .NET Core CLI using:
 ```
 dotnet add package Microsoft.ML
 ```
 
-or from the package manager:
+or from the NuGet package manager:
 ```
 Install-Package Microsoft.ML
 ```
 
-Or alternatively you can add the Microsoft.ML package from within Visual Studio's NuGet package manager.
+Or alternatively you can add the Microsoft.ML package from within Visual Studio's NuGet package manager or via [Paket](https://github.com/fsprojects/Paket).
+
+Daily NuGet builds of the project are also available in our MyGet feed:
+
+> [https://dotnet.myget.org/F/dotnet-core/api/v3/index.json](https://dotnet.myget.org/F/dotnet-core/api/v3/index.json)
 
 ## Building
 
-To build ML.NET from source please visit our [developers guide](Documentation/project-docs/developer-guide.md).
+To build ML.NET from source please visit our [developers guide](docs/project-docs/developer-guide.md).
 
 |    | x64 Debug | x64 Release |
 |:---|----------------:|------------------:|

build.cmd

@@ -1,2 +1,2 @@
-@call %~dp0run.cmd build %*
+@call "%~dp0run.cmd" build %*
 @exit /b %ERRORLEVEL%

build.proj

@@ -8,7 +8,7 @@
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Directory.Build.props))\Directory.Build.props" />
 
   <Import Project="$(ToolsDir)VersionTools.targets" Condition="Exists('$(ToolsDir)VersionTools.targets')" />
-
+  <UsingTask TaskName="DownloadFilesFromUrl" AssemblyFile="$(ToolsDir)Microsoft.DotNet.Build.Tasks.dll"/>
   <PropertyGroup>
     <!-- To disable the restoration of packages, set RestoreDuringBuild=false or pass /p:RestoreDuringBuild=false.-->
     <RestoreDuringBuild Condition="'$(RestoreDuringBuild)'==''">true</RestoreDuringBuild>
@@ -33,6 +33,7 @@
       RestoreProjects;
       BuildNative;
       $(TraversalBuildDependsOn);
+      DownloadExternalTestFiles;
       RunTests;
     </TraversalBuildDependsOn>
   </PropertyGroup>
@@ -56,13 +57,26 @@
     <ItemGroup>
       <PkgProject Include="pkg\**\*.nupkgproj" />
     </ItemGroup>
-    
+
     <MSBuild Projects="@(PkgProject)"
              Targets="Restore" />
     <MSBuild Projects="@(PkgProject)"
              Targets="Pack" />
   </Target>
 
+  <ItemGroup>
+    <TestFile Include="$(MSBuildThisFileDirectory)/test/data/external/winequality-white.csv"
+      Url="https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv"
+      DestinationFile="$(MSBuildThisFileDirectory)test/data/external/winequality-white.csv" />
+  </ItemGroup>
+
+  <Target Name="DownloadExternalTestFiles" Inputs="@(TestFile)" Outputs="%(TestFile.DestinationFile)">
+    <Message Importance="High" Text="Downloading external test files... %(TestFile.DestinationFile)" />
+    <DownloadFilesFromUrl Items="@(TestFile)"
+                         DestinationDir="test/data/external"
+                         TreatErrorsAsWarnings="true"/>
+  </Target>
+
   <Target Name="RunTests" Condition="'$(RunTests)'=='true'">
     <MSBuild Projects="test\run-tests.proj"
              Targets="RunTests" />

build.sh

@@ -10,4 +10,4 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-$DIR/run.sh build "$@"
+"$DIR/run.sh" build "$@"

build/BranchInfo.props

@@ -1,8 +1,8 @@
 <Project>
   <PropertyGroup>
     <MajorVersion>0</MajorVersion>
-    <MinorVersion>1</MinorVersion>
+    <MinorVersion>3</MinorVersion>
     <PatchVersion>0</PatchVersion>
     <PreReleaseLabel>preview</PreReleaseLabel>
   </PropertyGroup>
-</Project>
+</Project>

build/publish.proj

@@ -1,14 +1,50 @@
-<Project DefaultTargets="Publish" >
+<Project>
   <Import Project="$(MSBuildThisFileDirectory)..\Directory.Build.props" />
 
+  <UsingTask TaskName="ExecWithRetriesForNuGetPush" AssemblyFile="$(ToolsDir)Microsoft.DotNet.Build.Tasks.dll" />
+
   <PropertyGroup>
     <PublishSymbolsPackage>Microsoft.SymbolUploader.Build.Task</PublishSymbolsPackage>
-    <EnablePublishSymbols>false</EnablePublishSymbols>
+    <EnablePublishSymbols Condition="'$(EnablePublishSymbols)'==''" >true</EnablePublishSymbols>
+    <NuGetPushTimeoutSeconds Condition="'$(NuGetPushTimeoutSeconds)' == ''">600</NuGetPushTimeoutSeconds>
   </PropertyGroup>
 
   <Import Project="$(PackagesDir)\$(PublishSymbolsPackage.ToLower())\$(PublishSymbolsPackageVersion)\build\PublishSymbols.targets" />
 
-  <Target Name="Publish" Condition="'$(EnablePublishSymbols)'=='true'" DependsOnTargets="SetupPublishSymbols">
+  <Target Name="PublishPackages">
+    <Error Condition="'$(NuGetFeedUrl)' == ''" Text="Missing required property NuGetFeedUrl" />
+    <Error Condition="'$(NuGetApiKey)' == ''" Text="Missing required property NuGetApiKey" />
+
+    <ItemGroup>
+      <NuGetPackages Include="$(PackageOutputPath)**\*.nupkg"
+                     Exclude="$(PackageOutputPath)**\*.symbols.*nupkg" />
+
+      <!--
+      IgnorableErrorMessages applies to the "ExectWithRetriesForNuGetPush" task.
+      There's a very special failure scenario that we want to ignore.  That scenario is
+      when NuGet hits a timeout on one "push" attempt, and then gets a "Forbidden" response
+      because the package "already exists" on the next response.  This indicates that the
+      timeout occurred, but the push was actually successful.
+      -->
+      <IgnorableErrorMessages Include="Overwriting existing packages is forbidden according to the package retention settings for this feed.">
+        <ConditionalErrorMessage>Pushing took too long</ConditionalErrorMessage>
+      </IgnorableErrorMessages>
+    </ItemGroup>
+
+    <Message Text="Pushing ML.NET packages to $(NuGetFeedUrl)" />
+
+    <PropertyGroup>
+      <DotnetToolCommand>$(ToolsDir)dotnetcli/dotnet</DotnetToolCommand>
+      <NuGetPushCommand>$(DotnetToolCommand) nuget push --source $(NuGetFeedUrl) --api-key $(NuGetApiKey) --timeout $(NuGetPushTimeoutSeconds)</NuGetPushCommand>
+    </PropertyGroup>
+
+    <ExecWithRetriesForNuGetPush Command="$(NuGetPushCommand) %(NuGetPackages.Identity)"
+                                 IgnoredErrorMessagesWithConditional="@(IgnorableErrorMessages)" />
+  </Target>
+
+  <Target Name="PublishSymbolPackages"
+          Condition="'$(EnablePublishSymbols)'=='true'"
+          DependsOnTargets="SetupPublishSymbols">
     <Message Text="Attempting to Publish Symbols" />
     <Error Condition="!Exists('$(PackageOutputPath)')" Text="'PackageOutputPath' folder '$(PackageOutputPath)' does not exist."/>
     <Error Condition="'$(SymbolServerPath)'==''" Text="Missing property SymbolServerPath" />
@@ -18,7 +54,7 @@
 
   <Target Name="SetupPublishSymbols">
     <PropertyGroup>
-      <SymbolExpirationInDays Condition="'$(SymbolExpirationInDays)'=='' and '$(SymbolExpirationDate)'==''">1</SymbolExpirationInDays>
+      <SymbolExpirationInDays Condition="'$(SymbolExpirationInDays)'=='' and '$(SymbolExpirationDate)'==''">180</SymbolExpirationInDays>
       <ConvertPortablePdbsToWindowsPdbs>true</ConvertPortablePdbsToWindowsPdbs>
     </PropertyGroup>
 

build/vsts-ci.yml

@@ -6,6 +6,9 @@ phases:
     _PREVIEW_VSTS_DOCKER_IMAGE: microsoft/dotnet-buildtools-prereqs:centos-7-d485f41-20173404063424
     BuildConfig: Release
     OfficialBuildId: $(BUILD.BUILDNUMBER)
+    DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+    DOTNET_MULTILEVEL_LOOKUP: 0
   queue:
     name: DotNetCore-Test
     demands:
@@ -27,6 +30,9 @@ phases:
   variables:
     BuildConfig: Release
     OfficialBuildId: $(BUILD.BUILDNUMBER)
+    DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+    DOTNET_MULTILEVEL_LOOKUP: 0
   queue:
     name: DotNetCore-Build
     demands:
@@ -48,6 +54,9 @@ phases:
   variables:
     BuildConfig: Release
     OfficialBuildId: $(BUILD.BUILDNUMBER)
+    DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+    DOTNET_MULTILEVEL_LOOKUP: 0
     _SignType: real
     _UseEsrpSigning: true
     _TeamName: DotNetCore
@@ -96,12 +105,15 @@ phases:
   variables:
     BuildConfig: Release
     OfficialBuildId: $(BUILD.BUILDNUMBER)
+    DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+    DOTNET_MULTILEVEL_LOOKUP: 0
     _SignType: real
     _UseEsrpSigning: true
     _TeamName: DotNetCore
+    _NuGetFeedUrl: https://dotnet.myget.org/F/dotnet-core/api/v2/package
     _SymwebSymbolServerPath: https://microsoft.artifacts.visualstudio.com/DefaultCollection
     _MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection
-    _SymbolExpirationInDays: 30
   queue:
     name: DotNetCore-Build
     demands: 
@@ -145,26 +157,25 @@ phases:
       msbuildVersion: 15.0
     continueOnError: false
 
-  - task: NuGetCommand@2
-    displayName: Publish Packages to VSTS Feed
+  - task: MSBuild@1
+    displayName: Publish Packages to MyGet Feed
     inputs:
-      command: push
-      packagesToPush: $(Build.SourcesDirectory)/bin/packages/**/*.nupkg;!$(Build.SourcesDirectory)/bin/packages/**/*.symbols.nupkg
-      nuGetFeedType: internal
-      feedPublish: MachineLearning
+      solution: build/publish.proj
+      msbuildArguments: /t:PublishPackages /p:NuGetFeedUrl=$(_NuGetFeedUrl) /p:NuGetApiKey=$(dotnet-myget-org-api-key)
+      msbuildVersion: 15.0
 
   - task: MSBuild@1
     displayName: Publish Symbols to SymWeb Symbol Server
     inputs:
       solution: build/publish.proj
-      msbuildArguments: /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(SymwebSymbolServerPAT) /p:SymbolExpirationInDays=$(_SymbolExpirationInDays)
+      msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(SymwebSymbolServerPAT)
       msbuildVersion: 15.0
     continueOnError: true
 
   - task: MSBuild@1
     displayName: Publish Symbols to Msdl Symbol Server
     inputs:
       solution: build/publish.proj
-      msbuildArguments: /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(MsdlSymbolServerPAT) /p:SymbolExpirationInDays=$(_SymbolExpirationInDays)
+      msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(MsdlSymbolServerPAT)
       msbuildVersion: 15.0
     continueOnError: true