From 749d821e6765228bcb4ea9d867120ce4d8ed3999 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 13 Sep 2018 08:48:05 -0700 Subject: [PATCH 001/244] Recursively expand directories before rename and enable folder rename functional tests --- .../MoveRenameFolderTests.cs | 52 ++++++++++--- .../WorkingDirectoryTests.cs | 4 +- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 58 +++++++-------- ProjFS.Mac/PrjFSKext/public/Message.h | 1 + ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 74 ++++++++++++++++++- 5 files changed, 142 insertions(+), 47 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs index 7e5c70ba53..2f0071a048 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs @@ -1,5 +1,6 @@ using GVFS.FunctionalTests.FileSystemRunners; using GVFS.FunctionalTests.Should; +using GVFS.Tests.Should; using NUnit.Framework; using System.IO; @@ -58,10 +59,8 @@ public void RenameFolderShouldFail() this.Enlistment.GetVirtualPathTo(Path.Combine(oldFolderName, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void ChangeUnhydratedFolderName() { @@ -79,10 +78,8 @@ public void ChangeUnhydratedFolderName() this.Enlistment.GetVirtualPathTo(Path.Combine(newFolderName, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void MoveUnhydratedFolderToNewFolder() { @@ -104,10 +101,8 @@ public void MoveUnhydratedFolderToNewFolder() this.Enlistment.GetVirtualPathTo(Path.Combine(movedFolderPath, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void MoveUnhydratedFolderToFullFolderInDotGitFolder() { @@ -156,10 +151,8 @@ public void MoveFullFolderToFullFolderInDotGitFolder() Path.Combine(movedFolderPath, testFileName).ShouldBeAFile(this.fileSystem).WithContents(fileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void MoveAndRenameUnhydratedFolderToNewFolder() { @@ -181,14 +174,12 @@ public void MoveAndRenameUnhydratedFolderToNewFolder() this.Enlistment.GetVirtualPathTo(Path.Combine(movedFolderPath, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void MoveFolderWithUnhydratedAndFullContents() { - string testFileName = "MoveFolderWithUnhydratedAndFullContents.cs"; + string testFileName = "MoveFolderWithUnhydratedAndFullContents.cpp"; string oldFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "MoveFolderWithUnhydratedAndFullContents"); string newFile = "TestFile.txt"; @@ -214,5 +205,44 @@ public void MoveFolderWithUnhydratedAndFullContents() this.Enlistment.GetVirtualPathTo(Path.Combine(oldFolderName, newFile)).ShouldNotExistOnDisk(this.fileSystem); this.Enlistment.GetVirtualPathTo(Path.Combine(movedFolderPath, newFile)).ShouldBeAFile(this.fileSystem).WithContents(newFileContents); } + + // MacOnly because renames of partial folders are blocked on Windows + [TestCase] + [Category(Categories.MacOnly)] + public void MoveFolderWithUnexpandedChildFolders() + { + string oldFolderPath = this.Enlistment.GetVirtualPathTo("Test_EPF_MoveRenameFileTests"); + string newFolderName = "Test_EPF_MoveRenameFileTests_Renamed"; + string newFolderPath = this.Enlistment.GetVirtualPathTo(newFolderName); + this.fileSystem.MoveDirectory(oldFolderPath, newFolderPath); + oldFolderPath.ShouldNotExistOnDisk(this.fileSystem); + + newFolderPath.ShouldBeADirectory(this.fileSystem); + this.Enlistment.GetVirtualPathTo(newFolderName, "ChangeNestedUnhydratedFileNameCase", "Program.cs") + .ShouldBeAFile(this.fileSystem) + .WithContents(MoveRenameFileTests.TestFileContents); + + this.Enlistment.GetVirtualPathTo(newFolderName, "ChangeUnhydratedFileName", "Program.cs") + .ShouldBeAFile(this.fileSystem) + .WithContents(MoveRenameFileTests.TestFileContents); + + this.Enlistment.GetVirtualPathTo(newFolderName, "MoveUnhydratedFileToDotGitFolder", "Program.cs") + .ShouldBeAFile(this.fileSystem) + .WithContents(MoveRenameFileTests.TestFileContents); + + // Test moving a folder with a very deep unhydrated child tree + oldFolderPath = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests"); + + // But expand the folder we will be renaming (so that only the children have not been expanded) + oldFolderPath.ShouldBeADirectory(this.fileSystem).WithDirectories().ShouldContain(dir => dir.Name.Equals("1")); + + newFolderName = "Test_EPF_WorkingDirectoryTests_Renamed"; + newFolderPath = this.Enlistment.GetVirtualPathTo(newFolderName); + this.fileSystem.MoveDirectory(oldFolderPath, newFolderPath); + oldFolderPath.ShouldNotExistOnDisk(this.fileSystem); + this.Enlistment.GetVirtualPathTo(newFolderName, "1", "2", "3", "4", "ReadDeepProjectedFile.cpp") + .ShouldBeAFile(this.fileSystem) + .WithContents(WorkingDirectoryTests.TestFileContents); + } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index d5088205ea..0190434625 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -16,8 +16,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] public class WorkingDirectoryTests : TestsWithEnlistmentPerFixture { - private const int CurrentPlaceholderVersion = 1; - private const string TestFileContents = + public const string TestFileContents = @"// dllmain.cpp : Defines the entry point for the DLL application. #include ""stdafx.h"" @@ -41,6 +40,7 @@ LPVOID lpReserved } "; + private const int CurrentPlaceholderVersion = 1; private FileSystemRunner fileSystem; diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index acce65bce3..ab53da185b 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -222,6 +222,7 @@ void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType re case MessageType_UtoK_StartVirtualizationInstance: case MessageType_UtoK_StopVirtualizationInstance: case MessageType_KtoU_EnumerateDirectory: + case MessageType_KtoU_RecursivelyEnumerateDirectory: case MessageType_KtoU_HydrateFile: case MessageType_KtoU_NotifyFileModified: case MessageType_KtoU_NotifyFilePreDelete: @@ -276,9 +277,27 @@ static int HandleVnodeOperation( goto CleanupAndReturn; } + if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) + { + if (!TrySendRequestAndWaitForResponse( + root, + VDIR == vnodeType ? MessageType_KtoU_NotifyDirectoryPreDelete : MessageType_KtoU_NotifyFilePreDelete, + currentVnode, + pid, + procname, + &kauthResult, + kauthError)) + { + goto CleanupAndReturn; + } + } + if (VDIR == vnodeType) { - if (ActionBitIsSet( + bool deleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); + + if (deleteAction || + ActionBitIsSet( action, KAUTH_VNODE_LIST_DIRECTORY | KAUTH_VNODE_SEARCH | @@ -286,11 +305,14 @@ static int HandleVnodeOperation( KAUTH_VNODE_READ_ATTRIBUTES | KAUTH_VNODE_READ_EXTATTRIBUTES)) { - if (FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) + // Recursively expand directory on delete to ensure child placeholders are created before rename operations + if (deleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { if (!TrySendRequestAndWaitForResponse( root, - MessageType_KtoU_EnumerateDirectory, + deleteAction ? + MessageType_KtoU_RecursivelyEnumerateDirectory : + MessageType_KtoU_EnumerateDirectory, currentVnode, pid, procname, @@ -301,39 +323,9 @@ static int HandleVnodeOperation( } } } - - if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) - { - if (!TrySendRequestAndWaitForResponse( - root, - MessageType_KtoU_NotifyDirectoryPreDelete, - currentVnode, - pid, - procname, - &kauthResult, - kauthError)) - { - goto CleanupAndReturn; - } - } } else { - if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) - { - if (!TrySendRequestAndWaitForResponse( - root, - MessageType_KtoU_NotifyFilePreDelete, - currentVnode, - pid, - procname, - &kauthResult, - kauthError)) - { - goto CleanupAndReturn; - } - } - if (ActionBitIsSet( action, KAUTH_VNODE_READ_ATTRIBUTES | diff --git a/ProjFS.Mac/PrjFSKext/public/Message.h b/ProjFS.Mac/PrjFSKext/public/Message.h index a4a200fb8e..43a99ab066 100644 --- a/ProjFS.Mac/PrjFSKext/public/Message.h +++ b/ProjFS.Mac/PrjFSKext/public/Message.h @@ -14,6 +14,7 @@ typedef enum // Messages from kernel to user mode MessageType_KtoU_EnumerateDirectory, + MessageType_KtoU_RecursivelyEnumerateDirectory, MessageType_KtoU_HydrateFile, MessageType_KtoU_NotifyFileModified, diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index a858a7fe36..5fb1afec6c 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -54,6 +56,7 @@ static errno_t RegisterVirtualizationRootPath(const char* path); static void HandleKernelRequest(Message requestSpec, void* messageMemory); static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path); +static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleFileNotification( const MessageHeader* request, @@ -491,6 +494,12 @@ static void HandleKernelRequest(Message request, void* messageMemory) result = HandleEnumerateDirectoryRequest(requestHeader, request.path); break; } + + case MessageType_KtoU_RecursivelyEnumerateDirectory: + { + result = HandleRecursivelyEnumerateDirectoryRequest(requestHeader, request.path); + break; + } case MessageType_KtoU_HydrateFile: { @@ -579,7 +588,7 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) { - // TODO: how should we handle this scenario where the provider thinks it succeeded, but we were unable to + // TODO(Mac): how should we handle this scenario where the provider thinks it succeeded, but we were unable to // update placeholder metadata? return PrjFS_Result_EIOError; } @@ -588,6 +597,68 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request return callbackResult; } +static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path) +{ +#ifdef DEBUG + std::cout << "PrjFSLib.HandleRecursivelyEnumerateDirectoryRequest: " << path << std::endl; +#endif + + DIR* directory = nullptr; + PrjFS_Result result = PrjFS_Result_Success; + std::list directoryRelativePaths; + directoryRelativePaths.push_back(path); + + // Walk each directory, expanding those that are found to be empty + char pathBuffer[PrjFSMaxPath]; + std::list::iterator relativePathIter = directoryRelativePaths.begin(); + while(relativePathIter != directoryRelativePaths.end()) + { + CombinePaths(s_virtualizationRootFullPath.c_str(), relativePathIter->c_str(), pathBuffer); + + // TODO(Mac): how should we handle scenarios where we were unable to fully expand the directory and + // its children? + if (IsBitSetInFileFlags(pathBuffer, FileFlags_IsEmpty)) + { + PrjFS_Result result = HandleEnumerateDirectoryRequest(request, relativePathIter->c_str()); + if (result != PrjFS_Result_Success) + { + goto CleanupAndReturn; + } + } + + DIR* directory = opendir(pathBuffer); + if (nullptr == directory) + { + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + dirent* dirEntry = readdir(directory); + while(dirEntry != nullptr) + { + if (dirEntry->d_type == DT_DIR && + 0 != strncmp(".", dirEntry->d_name, sizeof(dirEntry->d_name)) && + 0 != strncmp("..", dirEntry->d_name, sizeof(dirEntry->d_name))) + { + CombinePaths(relativePathIter->c_str(), dirEntry->d_name, pathBuffer); + directoryRelativePaths.push_back(pathBuffer); + } + + dirEntry = readdir(directory); + } + + ++relativePathIter; + } + +CleanupAndReturn: + if (directory != nullptr) + { + closedir(directory); + } + + return result; +} + static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG @@ -815,6 +886,7 @@ static inline PrjFS_NotificationType KUMessageTypeToNotificationType(MessageType case MessageType_UtoK_StartVirtualizationInstance: case MessageType_UtoK_StopVirtualizationInstance: case MessageType_KtoU_EnumerateDirectory: + case MessageType_KtoU_RecursivelyEnumerateDirectory: case MessageType_KtoU_HydrateFile: case MessageType_Response_Success: case MessageType_Response_Fail: From 2c2d116711e613752d556378ecd3f9b3d5ca03e7 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 18 Sep 2018 15:19:38 -0700 Subject: [PATCH 002/244] Catch and log exceptions thrown by ConvertDirectoryToVirtualizationRoot --- GVFS/GVFS.Common/FileSystem/IKernelDriver.cs | 3 +- GVFS/GVFS.Platform.Mac/ProjFSKext.cs | 3 +- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 35 +++++++++++++++++--- GVFS/GVFS/CommandLine/CloneVerb.cs | 12 +++++-- GVFS/GVFS/CommandLine/DehydrateVerb.cs | 11 +++++- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index 270019da43..35fbdd1c11 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -1,4 +1,5 @@ using GVFS.Common.Tracing; +using System; namespace GVFS.Common.FileSystem { @@ -8,7 +9,7 @@ public interface IKernelDriver string DriverLogFolderName { get; } bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error); string FlushDriverLogs(); - bool TryPrepareFolderForCallbacks(string folderPath, out string error); + bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception); bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error); } } diff --git a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs index 9adc1d68c3..ede5623fc4 100644 --- a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs +++ b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs @@ -46,8 +46,9 @@ public bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error) return true; } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) { + exception = null; error = string.Empty; Result result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(folderPath); if (result != Result.Success) diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 544b5678a1..7df193d964 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -30,6 +30,7 @@ public class ProjFSFilter : IKernelDriver private const string FilterLoggerSessionName = "Microsoft-Windows-ProjFS-Filter-Log"; private const string ProjFSNativeLibFileName = "ProjectedFSLib.dll"; + private const string ProjFSManagedLibFileName = "ProjectedFSLib.Managed.dll"; private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; @@ -320,14 +321,40 @@ public string FlushDriverLogs() return sb.ToString(); } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) { + exception = null; error = string.Empty; Guid virtualizationInstanceGuid = Guid.NewGuid(); - HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); - if (result != HResult.Ok) + + try + { + HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); + if (result != HResult.Ok) + { + error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); + return false; + } + } + catch (FileNotFoundException e) + { + exception = e; + + if (e.FileName.Equals(ProjFSManagedLibFileName, StringComparison.OrdinalIgnoreCase)) + { + error = $"Failed to load {ProjFSManagedLibFileName}. Ensure that ProjFS is installed and enabled"; + } + else + { + error = $"FileNotFoundException while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; + } + + return false; + } + catch (Exception e) { - error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); + exception = e; + error = $"Exception while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; return false; } diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 00a429c076..5d86b909fc 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -609,10 +609,18 @@ private Result CreateClone( } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed + Exception exception; string prepFileSystemError; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError)) + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception)) { - tracer.RelatedError(prepFileSystemError); + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(prepFileSystemError), prepFileSystemError); + if (exception != null) + { + metadata.Add("Exception", exception.ToString()); + } + + tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); return new Result(prepFileSystemError); } diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index 71342283a4..906745f3c9 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -221,9 +221,18 @@ private void Mount(ITracer tracer) private void PrepareSrcFolder(ITracer tracer, GVFSEnlistment enlistment) { + Exception exception; string error; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error)) + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error, out exception)) { + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(error), error); + if (exception != null) + { + metadata.Add("Exception", exception.ToString()); + } + + tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); } } From f2bda4254a57e3218a0dc1c74da6f8543a76a96e Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 18 Sep 2018 15:53:38 -0700 Subject: [PATCH 003/244] Catch exceptions one level higher --- GVFS/GVFS.Common/FileSystem/IKernelDriver.cs | 3 +- GVFS/GVFS.Platform.Mac/ProjFSKext.cs | 3 +- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 35 +++----------------- GVFS/GVFS/CommandLine/CloneVerb.cs | 27 +++++++++------ GVFS/GVFS/CommandLine/DehydrateVerb.cs | 22 +++++++----- 5 files changed, 36 insertions(+), 54 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index 35fbdd1c11..270019da43 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -1,5 +1,4 @@ using GVFS.Common.Tracing; -using System; namespace GVFS.Common.FileSystem { @@ -9,7 +8,7 @@ public interface IKernelDriver string DriverLogFolderName { get; } bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error); string FlushDriverLogs(); - bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception); + bool TryPrepareFolderForCallbacks(string folderPath, out string error); bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error); } } diff --git a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs index ede5623fc4..9adc1d68c3 100644 --- a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs +++ b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs @@ -46,9 +46,8 @@ public bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error) return true; } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error) { - exception = null; error = string.Empty; Result result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(folderPath); if (result != Result.Success) diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 7df193d964..544b5678a1 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -30,7 +30,6 @@ public class ProjFSFilter : IKernelDriver private const string FilterLoggerSessionName = "Microsoft-Windows-ProjFS-Filter-Log"; private const string ProjFSNativeLibFileName = "ProjectedFSLib.dll"; - private const string ProjFSManagedLibFileName = "ProjectedFSLib.Managed.dll"; private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; @@ -321,40 +320,14 @@ public string FlushDriverLogs() return sb.ToString(); } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error) { - exception = null; error = string.Empty; Guid virtualizationInstanceGuid = Guid.NewGuid(); - - try - { - HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); - if (result != HResult.Ok) - { - error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); - return false; - } - } - catch (FileNotFoundException e) - { - exception = e; - - if (e.FileName.Equals(ProjFSManagedLibFileName, StringComparison.OrdinalIgnoreCase)) - { - error = $"Failed to load {ProjFSManagedLibFileName}. Ensure that ProjFS is installed and enabled"; - } - else - { - error = $"FileNotFoundException while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; - } - - return false; - } - catch (Exception e) + HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); + if (result != HResult.Ok) { - exception = e; - error = $"Exception while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; + error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); return false; } diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 5d86b909fc..6cf233ba2a 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -609,22 +609,29 @@ private Result CreateClone( } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed - Exception exception; - string prepFileSystemError; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception)) + Result prepForCallbacksResult = new Result(true); + try { - EventMetadata metadata = new EventMetadata(); - metadata.Add(nameof(prepFileSystemError), prepFileSystemError); - if (exception != null) + string prepFileSystemError; + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError)) { - metadata.Add("Exception", exception.ToString()); + prepForCallbacksResult = new Result(prepFileSystemError); } - + } + catch (Exception e) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Exception", e.ToString()); tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); - return new Result(prepFileSystemError); + prepForCallbacksResult = new Result($"Failed to prepare \"{enlistment.WorkingDirectoryRoot}\" for callbacks, exception: {e.Message}"); } - return new Result(true); + if (!prepForCallbacksResult.Success) + { + tracer.RelatedError($"TryPrepareFolderForCallbacks failed, error: {prepForCallbacksResult.ErrorMessage}"); + } + + return prepForCallbacksResult; } private void CreateGitScript(GVFSEnlistment enlistment) diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index 906745f3c9..a82e7cd130 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -221,19 +221,23 @@ private void Mount(ITracer tracer) private void PrepareSrcFolder(ITracer tracer, GVFSEnlistment enlistment) { - Exception exception; - string error; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error, out exception)) + try { - EventMetadata metadata = new EventMetadata(); - metadata.Add(nameof(error), error); - if (exception != null) + string error; + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error)) { - metadata.Add("Exception", exception.ToString()); + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(error), error); + tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); + this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); } - + } + catch (Exception e) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Exception", e.ToString()); tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); - this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); + this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + e.Message); } } From 6bc7066de5e3e7a609eb7f5c1f68bbb1aa286f22 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Thu, 20 Sep 2018 15:02:26 -0700 Subject: [PATCH 004/244] Handle read only files when hydrating with the mirror provider by only requesting Read access --- MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs index 07ddf3955d..40efddbe9e 100644 --- a/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs @@ -81,7 +81,7 @@ protected FileSystemResult HydrateFile(string relativePath, int bufferSize, Func return FileSystemResult.EFileNotFound; } - using (FileStream fs = new FileStream(fullPathInMirror, FileMode.Open)) + using (FileStream fs = new FileStream(fullPathInMirror, FileMode.Open, FileAccess.Read)) { long remainingData = fs.Length; byte[] buffer = new byte[bufferSize]; From 3aac4138b4c6e08b86ae736fa501f42f30d5b19c Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Tue, 11 Sep 2018 11:33:30 -0400 Subject: [PATCH 005/244] Tests demonstrating issue with newline in Git commands --- .../Tests/GitCommands/GitCommandsTests.cs | 11 +++++++++++ .../Tests/GitCommands/UpdateIndexTests.cs | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index f079e54648..f7f5a6d475 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -381,6 +381,17 @@ public void AddFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBac this.FileShouldHaveContents(newFileContents, newFilePath); } + [TestCase] + [Ignore("Currently failing regression test for how GVFS handles newlines in git command line")] + public void CommitWithNewlinesInMessage() + { + this.ValidateGitCommand("checkout -b tests/functional/commit_with_uncommon_arguments"); + this.CreateFile(); + this.ValidateGitCommand("status"); + this.ValidateGitCommand("add ."); + this.RunGitCommand("commit -m \"Message that contains \na\nnew\nline\""); + } + [TestCase] public void CaseOnlyRenameFileAndChangeBranches() { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index 5ca28eaa43..df6871f24c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -63,6 +63,17 @@ public void UpdateIndexRemoveAddFileOpenForWrite() GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "update-index --add Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt"); } + [TestCase] + [Category(Categories.MacTODO.M4)] + [Ignore("Currently failing regression test for how GVFS handles newlines in git command line")] + public void UpdateIndexWithCacheInfo() + { + // Update Protocol.md with the contents from blob 583f1... + string command = $"update-index --cacheinfo 100644 \"583f1a56db7cc884d54534c5d9c56b93a1e00a2b\n\" Protocol.md"; + + this.ValidateGitCommand(command); + } + protected override void CreateEnlistment() { base.CreateEnlistment(); From fb44c71f86bfb1304bed0321930f76be11651f2b Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Fri, 7 Sep 2018 10:37:22 -0400 Subject: [PATCH 006/244] Tweak the protocol GVFS used to communicate over named pipes GVFS currently sends a text based data across a named pipe with lines / messages separated by newline characters. This prohibits sending messages that include a newline as part of the message content. The main current manifestation of this problem is that GVFS does not handle git command lines that contain a newline character. There is another problem that GVFS needs to send paths across the named pipe (for ModifiedPathsList queries), and these paths can include newline characters on certain filesystems (macOS, Linux). Additionally, the protocol that GVFS uses to communicate across the named pipe is currently not consistent. It usually uses a text based stream, but for the ModifiedPathsList response, it uses null bytes to seperate entries in a list. To address these issues, this change tweaks the protocol used to communicate via the named pipe to use the 0x3 byte to indicate the end of a line / message (this is the End of text ASCII code). --- .../GVFS.Common/NamedPipes/NamedPipeClient.cs | 13 ++- .../GVFS.Common/NamedPipes/NamedPipeServer.cs | 45 +++++---- .../NamedPipes/NamedPipeStreamReader.cs | 92 +++++++++++++++++++ .../NamedPipes/NamedPipeStreamWriter.cs | 24 +++++ .../NamedPipes/StreamWriterExtensions.cs | 16 ---- .../Tests/GitCommands/GitCommandsTests.cs | 1 - .../Tests/GitCommands/UpdateIndexTests.cs | 1 - GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj | 7 +- GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj | 7 +- GVFS/GVFS.ReadObjectHook/main.cpp | 6 +- GVFS/GVFS.VirtualFileSystemHook/main.cpp | 5 +- 11 files changed, 164 insertions(+), 53 deletions(-) create mode 100644 GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs create mode 100644 GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriter.cs delete mode 100644 GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs index 69e27b111c..18745788fa 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs @@ -8,8 +8,8 @@ public class NamedPipeClient : IDisposable { private string pipeName; private NamedPipeClientStream clientStream; - private StreamReader reader; - private StreamWriter writer; + private NamedPipeStreamReader reader; + private NamedPipeStreamWriter writer; public NamedPipeClient(string pipeName) { @@ -37,8 +37,8 @@ public bool Connect(int timeoutMilliseconds = 3000) return false; } - this.reader = new StreamReader(this.clientStream); - this.writer = new StreamWriter(this.clientStream); + this.reader = new NamedPipeStreamReader(this.clientStream); + this.writer = new NamedPipeStreamWriter(this.clientStream); return true; } @@ -68,8 +68,7 @@ public void SendRequest(string message) try { - this.writer.WritePlatformIndependentLine(message); - this.writer.Flush(); + this.writer.WriteMessage(message); } catch (IOException e) { @@ -81,7 +80,7 @@ public string ReadRawResponse() { try { - string response = this.reader.ReadLine(); + string response = this.reader.ReadMessage(); if (response == null) { throw new BrokenPipeException("Unable to read from pipe", null); diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs index d6eebe7111..83ac690e3d 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs @@ -88,7 +88,7 @@ private void OpenListeningPipe() private void OnNewConnection(IAsyncResult ar) { if (!this.isStopping) - { + { this.OnNewConnection(ar, createNewThreadIfSynchronous: true); } } @@ -137,14 +137,14 @@ private void OnNewConnection(IAsyncResult ar, bool createNewThreadIfSynchronous) if (!connectionBroken) { - try - { - this.handleConnection(new Connection(pipe, () => this.isStopping)); - } - catch (Exception e) - { - this.LogErrorAndExit("Unhandled exception in connection handler", e); - } + try + { + this.handleConnection(new Connection(pipe, this.tracer, () => this.isStopping)); + } + catch (Exception e) + { + this.LogErrorAndExit("Unhandled exception in connection handler", e); + } } } } @@ -174,16 +174,18 @@ private void LogErrorAndExit(string message, Exception e) public class Connection { private NamedPipeServerStream serverStream; - private StreamReader reader; - private StreamWriter writer; + private NamedPipeStreamReader reader; + private NamedPipeStreamWriter writer; + private ITracer tracer; private Func isStopping; - public Connection(NamedPipeServerStream serverStream, Func isStopping) + public Connection(NamedPipeServerStream serverStream, ITracer tracer, Func isStopping) { this.serverStream = serverStream; + this.tracer = tracer; this.isStopping = isStopping; - this.reader = new StreamReader(this.serverStream); - this.writer = new StreamWriter(this.serverStream); + this.reader = new NamedPipeStreamReader(this.serverStream); + this.writer = new NamedPipeStreamWriter(this.serverStream); } public bool IsConnected @@ -200,10 +202,17 @@ public string ReadRequest() { try { - return this.reader.ReadLine(); + return this.reader.ReadMessage(); } - catch (IOException) + catch (IOException e) { + EventMetadata metadata = new EventMetadata(); + metadata.Add("ExceptionMessage", e.Message); + metadata.Add("StackTrace", e.StackTrace); + this.tracer.RelatedError( + metadata: metadata, + message: $"Error reading message from NamedPipe: {e.Message}"); + return null; } } @@ -212,9 +221,7 @@ public bool TrySendResponse(string message) { try { - this.writer.WritePlatformIndependentLine(message); - this.writer.Flush(); - + this.writer.WriteMessage(message); return true; } catch (IOException) diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs new file mode 100644 index 0000000000..17ced2e205 --- /dev/null +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace GVFS.Common.NamedPipes +{ + /// + /// Implements the NamedPipe protocol as described in NamedPipeServer. + /// + public class NamedPipeStreamReader + { + private const int DefaultBufferSize = 1024; + private const byte TerminatorByte = 0x3; + + private int bufferSize; + private byte[] buffer; + private Stream stream; + + public NamedPipeStreamReader(Stream stream, int bufferSize) + { + this.stream = stream; + this.bufferSize = bufferSize; + this.buffer = new byte[this.bufferSize]; + } + + public NamedPipeStreamReader(Stream stream) + : this(stream, DefaultBufferSize) + { + } + + /// + /// Read a message from the stream. + /// + /// The message read from the stream, or null if the end of the input stream has been reached. + public string ReadMessage() + { + int bytesRead = this.Read(); + if (bytesRead == 0) + { + // The end of the stream has been reached - return null to indicate this. + return null; + } + + // If we have read in the entire message in the first read (mainline scenario), + // then just process the data directly from the buffer. + if (this.buffer[bytesRead - 1] == TerminatorByte) + { + return Encoding.UTF8.GetString(this.buffer, 0, bytesRead - 1); + } + + // We need to process multiple chunks - collect data from multiple chunks + // into a single list + List bytes = new List(this.bufferSize * 2); + + while (true) + { + bool encounteredTerminatorByte = this.buffer[bytesRead - 1] == TerminatorByte; + int lengthToCopy = encounteredTerminatorByte ? bytesRead - 1 : bytesRead; + + bytes.AddRange(new ArraySegment(this.buffer, 0, lengthToCopy)); + if (encounteredTerminatorByte) + { + break; + } + + bytesRead = this.Read(); + + if (bytesRead == 0) + { + // We have read a partial message (the last byte received does not indicate that + // this was the end of the message), but the stream has been closed. Throw an exception + // and let upper layer deal with this condition. + + throw new IOException("Incomplete message read from stream. The end of the stream was reached without the expected terminating byte."); + } + } + + return Encoding.UTF8.GetString(bytes.ToArray()); + } + + /// + /// Read the next chunk of bytes from the stream. + /// + /// The number of bytes read. + private int Read() + { + return this.stream.Read(this.buffer, 0, this.buffer.Length); + } + } +} diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriter.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriter.cs new file mode 100644 index 0000000000..f3f11ecbe1 --- /dev/null +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriter.cs @@ -0,0 +1,24 @@ +using System.IO; +using System.Text; + +namespace GVFS.Common.NamedPipes +{ + public class NamedPipeStreamWriter + { + private const byte TerminatorByte = 0x3; + private const string TerminatorByteString = "\x3"; + private Stream stream; + + public NamedPipeStreamWriter(Stream stream) + { + this.stream = stream; + } + + public void WriteMessage(string message) + { + byte[] byteBuffer = Encoding.UTF8.GetBytes(message + TerminatorByteString); + this.stream.Write(byteBuffer, 0, byteBuffer.Length); + this.stream.Flush(); + } + } +} diff --git a/GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs b/GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs deleted file mode 100644 index 58db185d66..0000000000 --- a/GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System.IO; - -namespace GVFS.Common.NamedPipes -{ - public static class StreamWriterExtensions - { - public const int Foo = 0; - - public static void WritePlatformIndependentLine(this StreamWriter writer, string value) - { - // WriteLine is not platform independent as on some platforms it terminates lines with \r\n - // and on others it uses \n - writer.Write(value + "\n"); - } - } -} diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index f7f5a6d475..0b3ccccce0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -382,7 +382,6 @@ public void AddFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBac } [TestCase] - [Ignore("Currently failing regression test for how GVFS handles newlines in git command line")] public void CommitWithNewlinesInMessage() { this.ValidateGitCommand("checkout -b tests/functional/commit_with_uncommon_arguments"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index df6871f24c..d729822d0f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -65,7 +65,6 @@ public void UpdateIndexRemoveAddFileOpenForWrite() [TestCase] [Category(Categories.MacTODO.M4)] - [Ignore("Currently failing regression test for how GVFS handles newlines in git command line")] public void UpdateIndexWithCacheInfo() { // Update Protocol.md with the contents from blob 583f1... diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj index ddde816183..dccf8bdd31 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj @@ -59,8 +59,11 @@ Common\NamedPipes\NamedPipeClient.cs - - Common\NamedPipes\StreamWriterExtensions.cs + + Common\NamedPipes\NamedPipeStreamReader.cs + + + Common\NamedPipes\NamedPipeStreamWriter.cs Common\NativeMethods.Shared.cs diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj index 692c8832ae..05629c39e3 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj @@ -83,8 +83,11 @@ Common\NamedPipes\NamedPipeClient.cs - - Common\NamedPipes\StreamWriterExtensions.cs + + Common\NamedPipes\NamedPipeStreamReader.cs + + + Common\NamedPipes\NamedPipeStreamWriter.cs Common\NativeMethods.Shared.cs diff --git a/GVFS/GVFS.ReadObjectHook/main.cpp b/GVFS/GVFS.ReadObjectHook/main.cpp index 5fcb05af4f..7a3f608874 100644 --- a/GVFS/GVFS.ReadObjectHook/main.cpp +++ b/GVFS/GVFS.ReadObjectHook/main.cpp @@ -18,8 +18,8 @@ #define DLO_REQUEST_LENGTH (4 + SHA1_LENGTH + 1) // Expected response: -// "S\n" -> Success -// "F\n" -> Failure +// "S\x3" -> Success +// "F\x3" -> Failure #define DLO_RESPONSE_LENGTH 2 enum ReadObjectHookErrorReturnCode @@ -33,7 +33,7 @@ int DownloadSHA(PIPE_HANDLE pipeHandle, const char *sha1) // Format: "DLO|<40 character SHA>" // Example: "DLO|920C34DCDDFC8F07AC4704C8C0D087D6F2095729" char request[DLO_REQUEST_LENGTH+1]; - if (snprintf(request, DLO_REQUEST_LENGTH+1, "DLO|%s\n", sha1) != DLO_REQUEST_LENGTH) + if (snprintf(request, DLO_REQUEST_LENGTH+1, "DLO|%s\x3", sha1) != DLO_REQUEST_LENGTH) { die(ReturnCode::InvalidSHA, "First argument must be a 40 character SHA, actual value: %s\n", sha1); } diff --git a/GVFS/GVFS.VirtualFileSystemHook/main.cpp b/GVFS/GVFS.VirtualFileSystemHook/main.cpp index d0c1aea189..60498900ec 100644 --- a/GVFS/GVFS.VirtualFileSystemHook/main.cpp +++ b/GVFS/GVFS.VirtualFileSystemHook/main.cpp @@ -31,7 +31,7 @@ int main(int argc, char *argv[]) int error = 0; bool success = WriteToPipe( pipeHandle, - "MPL|1\n", + "MPL|1\x3", messageLength, &bytesWritten, &error); @@ -49,6 +49,7 @@ int main(int argc, char *argv[]) int lastError; bool finishedReading = false; bool firstRead = true; + do { char *pMessage = &message[0]; @@ -80,7 +81,7 @@ int main(int argc, char *argv[]) messageLength -= 2; } - if (*(pMessage + messageLength - 1) == '\n') + if (*(pMessage + messageLength - 1) == '\x3') { finishedReading = true; messageLength -= 1; From 659c2d4ba5e2059f0372226f8824778c0ba80e8a Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Fri, 21 Sep 2018 15:33:23 -0400 Subject: [PATCH 007/244] Add automated tests around NamedPipeStream protocol --- .../NamedPipeStreamReaderWriterTests.cs | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs diff --git a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs new file mode 100644 index 0000000000..003ca048fe --- /dev/null +++ b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs @@ -0,0 +1,108 @@ +using GVFS.Common.NamedPipes; +using GVFS.Tests.Should; +using NUnit.Framework; +using System.IO; + +namespace GVFS.UnitTests.Common +{ + [TestFixture] + public class NamedPipeStreamReaderWriterTests + { + private const int BufferSize = 256; + + private MemoryStream stream; + private NamedPipeStreamWriter streamWriter; + private NamedPipeStreamReader streamReader; + + [SetUp] + public void Setup() + { + this.stream = new MemoryStream(); + this.streamWriter = new NamedPipeStreamWriter(this.stream); + this.streamReader = new NamedPipeStreamReader(this.stream, BufferSize); + } + + [Test] + [Description("Verify that we can transmit multiple messages")] + public void CanWriteAndReadMessages() + { + string firstMessage = @"This is a new message"; + this.TestTransmitMessage(firstMessage); + + string secondMessage = @"This is another message"; + this.TestTransmitMessage(secondMessage); + + string thirdMessage = @"This is the third message in a series of messages"; + this.TestTransmitMessage(thirdMessage); + + string longMessage = new string('T', 1024 * 5); + this.TestTransmitMessage(longMessage); + } + + [Test] + [Description("Verify that we can transmit a message that contains content that is the size of a NamedPipeStreamReader's buffer")] + public void CanSendBufferSizedContent() + { + string longMessage = new string('T', BufferSize); + this.TestTransmitMessage(longMessage); + } + + [Test] + [Description("Verify that we can transmit message that is the same size a NamedPipeStreamReader's buffer")] + public void CanSendBufferSizedMessage() + { + int numBytesInMessageTerminator = 1; + string longMessage = new string('T', BufferSize - numBytesInMessageTerminator); + this.TestTransmitMessage(longMessage); + } + + [Test] + [Description("Verify that the expected exception is thrown if message is not terminated with expected byte.")] + public void ReadingPartialMessgeThrows() + { + byte[] bytes = System.Text.Encoding.ASCII.GetBytes("This is a partial message"); + + this.stream.Write(bytes, 0, bytes.Length); + this.stream.Seek(0, SeekOrigin.Begin); + + Assert.Throws(() => this.streamReader.ReadMessage()); + } + + [Test] + [Description("Verify that we can transmit message that is larger than the buffer")] + public void CanSendMultiBufferSizedMessage() + { + string longMessage = new string('T', BufferSize * 3); + this.TestTransmitMessage(longMessage); + } + + [Test] + [Description("Verify that we can transmit message that newline characters")] + public void CanSendNewLines() + { + string messageWithNewLines = "This is a \nstringwith\nnewlines"; + this.TestTransmitMessage(messageWithNewLines); + } + + private void TestTransmitMessage(string message) + { + long pos = this.ReadStreamPosition(); + this.streamWriter.WriteMessage(message); + + this.SetStreamPosition(pos); + + string readMessage = this.streamReader.ReadMessage(); + readMessage.ShouldEqual(message, "The message read from the stream reader is not the same as the message that was sent."); + } + + private long ReadStreamPosition() + { + return this.stream.Position; + } + + private void SetStreamPosition(long position) + { + this.stream.Seek(position, SeekOrigin.Begin); + } + } +} From d67eac1a4d0f7e280d5a7d348434fd93de8a4693 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 21 Sep 2018 16:52:24 -0700 Subject: [PATCH 008/244] PR Feedback: Move exception handling into TryPrepareFolderForCallbacks --- GVFS/GVFS.Common/FileSystem/IKernelDriver.cs | 3 +- GVFS/GVFS.Platform.Mac/ProjFSKext.cs | 3 +- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 51 ++++++++++++++++---- GVFS/GVFS/CommandLine/CloneVerb.cs | 27 ++++------- GVFS/GVFS/CommandLine/DehydrateVerb.cs | 22 ++++----- 5 files changed, 65 insertions(+), 41 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index 270019da43..35fbdd1c11 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -1,4 +1,5 @@ using GVFS.Common.Tracing; +using System; namespace GVFS.Common.FileSystem { @@ -8,7 +9,7 @@ public interface IKernelDriver string DriverLogFolderName { get; } bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error); string FlushDriverLogs(); - bool TryPrepareFolderForCallbacks(string folderPath, out string error); + bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception); bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error); } } diff --git a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs index 9adc1d68c3..ede5623fc4 100644 --- a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs +++ b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs @@ -46,8 +46,9 @@ public bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error) return true; } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) { + exception = null; error = string.Empty; Result result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(folderPath); if (result != Result.Success) diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 544b5678a1..3d7b4ce926 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -30,6 +30,7 @@ public class ProjFSFilter : IKernelDriver private const string FilterLoggerSessionName = "Microsoft-Windows-ProjFS-Filter-Log"; private const string ProjFSNativeLibFileName = "ProjectedFSLib.dll"; + private const string ProjFSManagedLibFileName = "ProjectedFSLib.Managed.dll"; private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; @@ -320,18 +321,34 @@ public string FlushDriverLogs() return sb.ToString(); } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) { - error = string.Empty; - Guid virtualizationInstanceGuid = Guid.NewGuid(); - HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); - if (result != HResult.Ok) + exception = null; + try { - error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); - return false; + return this.TryPrepareFolderForCallbacksImpl(folderPath, out error); } + catch (FileNotFoundException e) + { + exception = e; - return true; + if (e.FileName.Equals(ProjFSManagedLibFileName, StringComparison.OrdinalIgnoreCase)) + { + error = $"Failed to load {ProjFSManagedLibFileName}. Ensure that ProjFS is installed and enabled"; + } + else + { + error = $"FileNotFoundException while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; + } + + return false; + } + catch (Exception e) + { + exception = e; + error = $"Exception while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; + return false; + } } // TODO 1050199: Once the service is an optional component, GVFS should only attempt to attach @@ -562,7 +579,23 @@ private static EventMetadata CreateEventMetadata(Exception e = null) private static ProcessResult CallPowershellCommand(string command) { return ProcessHelper.Run("powershell.exe", "-NonInteractive -NoProfile -Command \"& { " + command + " }\""); - } + } + + // Using an Impl method allows TryPrepareFolderForCallbacks to catch any ProjFS dependency related exceptions + // thrown in the process of calling this method. + private bool TryPrepareFolderForCallbacksImpl(string folderPath, out string error) + { + error = string.Empty; + Guid virtualizationInstanceGuid = Guid.NewGuid(); + HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); + if (result != HResult.Ok) + { + error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); + return false; + } + + return true; + } private static class NativeMethods { diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 6cf233ba2a..5d86b909fc 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -609,29 +609,22 @@ private Result CreateClone( } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed - Result prepForCallbacksResult = new Result(true); - try + Exception exception; + string prepFileSystemError; + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception)) { - string prepFileSystemError; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError)) + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(prepFileSystemError), prepFileSystemError); + if (exception != null) { - prepForCallbacksResult = new Result(prepFileSystemError); + metadata.Add("Exception", exception.ToString()); } - } - catch (Exception e) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Exception", e.ToString()); - tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); - prepForCallbacksResult = new Result($"Failed to prepare \"{enlistment.WorkingDirectoryRoot}\" for callbacks, exception: {e.Message}"); - } - if (!prepForCallbacksResult.Success) - { - tracer.RelatedError($"TryPrepareFolderForCallbacks failed, error: {prepForCallbacksResult.ErrorMessage}"); + tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); + return new Result(prepFileSystemError); } - return prepForCallbacksResult; + return new Result(true); } private void CreateGitScript(GVFSEnlistment enlistment) diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index a82e7cd130..906745f3c9 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -221,23 +221,19 @@ private void Mount(ITracer tracer) private void PrepareSrcFolder(ITracer tracer, GVFSEnlistment enlistment) { - try + Exception exception; + string error; + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error, out exception)) { - string error; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error)) + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(error), error); + if (exception != null) { - EventMetadata metadata = new EventMetadata(); - metadata.Add(nameof(error), error); - tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); - this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); + metadata.Add("Exception", exception.ToString()); } - } - catch (Exception e) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Exception", e.ToString()); + tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); - this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + e.Message); + this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); } } From 71f5406f918f9145260042adf3b60c661609cae8 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 21 Sep 2018 17:15:23 -0700 Subject: [PATCH 009/244] PR Feedback: Move delete action check and use a queue for processing directories --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 9 +++---- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 25 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index ab53da185b..d46aa411f8 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -294,17 +294,16 @@ static int HandleVnodeOperation( if (VDIR == vnodeType) { - bool deleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); - - if (deleteAction || - ActionBitIsSet( + if (ActionBitIsSet( action, KAUTH_VNODE_LIST_DIRECTORY | KAUTH_VNODE_SEARCH | KAUTH_VNODE_READ_SECURITY | KAUTH_VNODE_READ_ATTRIBUTES | - KAUTH_VNODE_READ_EXTATTRIBUTES)) + KAUTH_VNODE_READ_EXTATTRIBUTES | + KAUTH_VNODE_DELETE)) { + bool deleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); // Recursively expand directory on delete to ensure child placeholders are created before rename operations if (deleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 5fb1afec6c..eb67b38dc1 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -605,21 +606,21 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead DIR* directory = nullptr; PrjFS_Result result = PrjFS_Result_Success; - std::list directoryRelativePaths; - directoryRelativePaths.push_back(path); + std::queue directoryRelativePaths; + directoryRelativePaths.push(path); // Walk each directory, expanding those that are found to be empty char pathBuffer[PrjFSMaxPath]; - std::list::iterator relativePathIter = directoryRelativePaths.begin(); - while(relativePathIter != directoryRelativePaths.end()) + while (!directoryRelativePaths.empty()) { - CombinePaths(s_virtualizationRootFullPath.c_str(), relativePathIter->c_str(), pathBuffer); + string directoryRelativePath(directoryRelativePaths.front()); + directoryRelativePaths.pop(); + + CombinePaths(s_virtualizationRootFullPath.c_str(), directoryRelativePath.c_str(), pathBuffer); - // TODO(Mac): how should we handle scenarios where we were unable to fully expand the directory and - // its children? if (IsBitSetInFileFlags(pathBuffer, FileFlags_IsEmpty)) { - PrjFS_Result result = HandleEnumerateDirectoryRequest(request, relativePathIter->c_str()); + PrjFS_Result result = HandleEnumerateDirectoryRequest(request, directoryRelativePath.c_str()); if (result != PrjFS_Result_Success) { goto CleanupAndReturn; @@ -634,20 +635,18 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead } dirent* dirEntry = readdir(directory); - while(dirEntry != nullptr) + while (dirEntry != nullptr) { if (dirEntry->d_type == DT_DIR && 0 != strncmp(".", dirEntry->d_name, sizeof(dirEntry->d_name)) && 0 != strncmp("..", dirEntry->d_name, sizeof(dirEntry->d_name))) { - CombinePaths(relativePathIter->c_str(), dirEntry->d_name, pathBuffer); - directoryRelativePaths.push_back(pathBuffer); + CombinePaths(directoryRelativePath.c_str(), dirEntry->d_name, pathBuffer); + directoryRelativePaths.emplace(pathBuffer); } dirEntry = readdir(directory); } - - ++relativePathIter; } CleanupAndReturn: From eab7934d187c2ce82a9e7775b23727d2373b15d0 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 24 Sep 2018 10:12:18 -0700 Subject: [PATCH 010/244] Track both file type and file mode --- .../MacFileSystemVirtualizer.cs | 8 ++--- .../Projection/MockGitIndexProjection.cs | 8 ++--- .../MacFileSystemVirtualizerTests.cs | 13 +++---- .../GitIndexProjection.GitIndexEntry.cs | 2 +- .../GitIndexProjection.GitIndexParser.cs | 35 +++++++++++++++++-- .../Projection/GitIndexProjection.cs | 31 +++++++++++----- 6 files changed, 71 insertions(+), 26 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 3d22f7f34c..c8d954bc7a 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -80,13 +80,13 @@ public override FileSystemResult WritePlaceholderFile( string sha) { // TODO(Mac): Add functional tests that validate file mode is set correctly - ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); + ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); Result result = this.virtualizationInstance.WritePlaceholderFile( relativePath, PlaceholderVersionId, ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), (ulong)endOfFile, - fileMode); + (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask)); return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } @@ -114,14 +114,14 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( // TODO(Mac): Add functional tests that include: // - Mode + content changes between commits // - Mode only changes (without any change to content, see issue #223) - ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); + ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( relativePath, PlaceholderVersionId, ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), (ulong)endOfFile, - fileMode, + (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask), (UpdateType)updateFlags, out failureCause); failureReason = (UpdateFailureReason)failureCause; diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs index 904946610a..9bd7c5f5a6 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs @@ -36,7 +36,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) this.PlaceholdersCreated = new ConcurrentHashSet(); this.ExpandedFolders = new ConcurrentHashSet(); - this.MockFileModes = new ConcurrentDictionary(); + this.MockFileTypesAndModes = new ConcurrentDictionary(); this.unblockGetProjectedItems = new ManualResetEvent(true); this.waitForGetProjectedItems = new ManualResetEvent(true); @@ -56,7 +56,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) public ConcurrentHashSet ExpandedFolders { get; } - public ConcurrentDictionary MockFileModes { get; } + public ConcurrentDictionary MockFileTypesAndModes { get; } public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } @@ -161,10 +161,10 @@ public override bool TryGetProjectedItemsFromMemory(string folderPath, out List< return false; } - public override ushort GetFilePathMode(string path) + public override ushort GetFileTypeAndMode(string path) { ushort result; - if (this.MockFileModes.TryGetValue(path, out result)) + if (this.MockFileTypesAndModes.TryGetValue(path, out result)) { return result; } diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 675ae8a938..96cb89f606 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -21,6 +21,7 @@ namespace GVFS.UnitTests.Platform.Mac [TestFixture] public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { + private static readonly ushort RegularFileType = 0x8000; private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); private static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); @@ -98,7 +99,7 @@ public void UpdatePlaceholderIfNeeded() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileMode644); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -189,7 +190,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileMode644); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -219,7 +220,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileMode644); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -250,9 +251,9 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", FileMode644); - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", FileMode664); - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", FileMode755); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", (ushort)(RegularFileType | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", (ushort)(RegularFileType | FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", (ushort)(RegularFileType | FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 1da6d66426..612931242c 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -27,7 +27,7 @@ public GitIndexEntry() public byte[] Sha { get; } = new byte[20]; public bool SkipWorktree { get; set; } - public ushort FileMode { get; set; } + public ushort FileTypeAndMode { get; set; } public GitIndexParser.MergeStage MergeState { get; set; } public int ReplaceIndex { get; set; } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 682f9ec3ba..cd0cc20315 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -16,6 +16,10 @@ internal partial class GitIndexParser private const ushort ExtendedBit = 0x4000; private const ushort SkipWorktreeBit = 0x4000; + private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); + private static readonly ushort FileMode664 = Convert.ToUInt16("644", 8); + private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); + private Stream indexStream; private byte[] page; private int nextByteIndex; @@ -181,8 +185,35 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func // 3-bit unused // 9-bit unix permission. Only 0755 and 0644 are valid for regular files. (Legacy repos can also contain 664) // Symbolic links and gitlinks have value 0 in this field. - ushort mode = this.ReadUInt16(); - this.resuableParsedIndexEntry.FileMode = (ushort)(mode & 0x1FF); + ushort typeAndMode = this.ReadUInt16(); + + ushort type = (ushort)(typeAndMode & FileTypeMask); + ushort mode = (ushort)(typeAndMode & FileModeMask); + + switch ((FileType)type) + { + case FileType.Regular: + if (mode != FileMode755 && mode != FileMode644 && mode != FileMode664) + { + throw new InvalidDataException($"Invalid file mode {Convert.ToString(mode, 8)} found for regular file in index"); + } + + break; + + case FileType.SymLink: + case FileType.GitLink: + if (mode != 0) + { + throw new InvalidDataException($"Invalid file mode {Convert.ToString(mode, 8)} found for link file({type:X}) in index"); + } + + break; + + default: + throw new InvalidDataException($"Invalid object type {type:X} found in index"); + } + + this.resuableParsedIndexEntry.FileTypeAndMode = typeAndMode; this.Skip(12); } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 66b1396156..8c1dbe9245 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -22,7 +22,11 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject { public const string ProjectionIndexBackupName = "GVFS_projection"; - protected static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); + // Bitmasks for extracting file type and mode from the ushort stored in the index + public const ushort FileTypeMask = 0xF000; + public const ushort FileModeMask = 0x1FF; + + protected static readonly ushort Regular644File = (ushort)((ushort)FileType.Regular | Convert.ToUInt16("644", 8)); private const int IndexFileStreamBufferSize = 512 * 1024; @@ -54,7 +58,7 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject // nonDefaultFileModes is only populated when the platform supports file mode // On platforms that support file modes, file paths that are not in nonDefaultFileModes have mode 644 - private Dictionary nonDefaultFileModes = new Dictionary(StringComparer.OrdinalIgnoreCase); + private Dictionary nonDefaultFileTypesAndModes = new Dictionary(StringComparer.OrdinalIgnoreCase); private BlobSizes blobSizes; private PlaceholderListDatabase placeholderList; @@ -118,6 +122,15 @@ protected GitIndexProjection() { } + public enum FileType : ushort + { + Invalid = 0, + + Regular = 0x8000, + SymLink = 0xA000, + GitLink = 0xE000, + } + public int EstimatedPlaceholderCount { get @@ -356,11 +369,11 @@ public virtual bool TryGetProjectedItemsFromMemory(string folderPath, out List

")); } From f0185654fee7453647637e81ff5e1b550acdca72 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 24 Sep 2018 11:16:04 -0700 Subject: [PATCH 011/244] Add symlink projection code --- .../FileSystem/PhysicalFileSystem.cs | 4 +- GVFS/GVFS.Common/Git/GVFSGitObjects.cs | 1 + GVFS/GVFS.Common/NativeMethods.cs | 2 +- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 8 +- .../MacFileSystemVirtualizer.cs | 164 ++++++++++++++++-- ...skLayout12to13Upgrade_FolderPlaceholder.cs | 2 +- .../MacFileSystemVirtualizerTests.cs | 9 +- .../Background/FileSystemTask.cs | 8 +- .../FileSystemCallbacks.cs | 6 + .../GitIndexProjection.GitIndexParser.cs | 2 +- .../PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs | 14 +- .../VirtualizationInstance.cs | 24 +++ ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 55 ++++++ ProjFS.Mac/PrjFSLib/PrjFSLib.h | 10 ++ 14 files changed, 276 insertions(+), 33 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs b/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs index 9d0b23998d..cbe88f8516 100644 --- a/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs +++ b/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs @@ -103,9 +103,9 @@ public virtual void DeleteDirectory(string path, bool recursive = false) RecursiveDelete(path); } - public virtual bool IsSymlink(string path) + public virtual bool IsSymLink(string path) { - return (this.GetAttributes(path) & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && NativeMethods.IsSymlink(path); + return (this.GetAttributes(path) & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && NativeMethods.IsSymLink(path); } public virtual IEnumerable ItemsInDirectory(string path) diff --git a/GVFS/GVFS.Common/Git/GVFSGitObjects.cs b/GVFS/GVFS.Common/Git/GVFSGitObjects.cs index 6c77a211d3..6f3335f61a 100644 --- a/GVFS/GVFS.Common/Git/GVFSGitObjects.cs +++ b/GVFS/GVFS.Common/Git/GVFSGitObjects.cs @@ -28,6 +28,7 @@ public enum RequestSource FileStreamCallback, GVFSVerb, NamedPipeMessage, + SymLinkCreation, } protected GVFSContext Context { get; private set; } diff --git a/GVFS/GVFS.Common/NativeMethods.cs b/GVFS/GVFS.Common/NativeMethods.cs index 4f19f1e252..65074fdb05 100644 --- a/GVFS/GVFS.Common/NativeMethods.cs +++ b/GVFS/GVFS.Common/NativeMethods.cs @@ -127,7 +127,7 @@ public static uint GetWindowsBuildNumber() return versionInfo.BuildNumber; } - public static bool IsSymlink(string path) + public static bool IsSymLink(string path) { using (SafeFileHandle output = CreateFile( path, diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index d3ea1f93be..e29598bfbb 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -361,14 +361,14 @@ private void PerformIOBeforePlaceholderDatabaseUpgradeTest() this.fileSystem.DeleteDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Tests\\Properties")); string junctionTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirJunction"); - string symlinkTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirSymlink"); + string symLinkTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirSymLink"); Directory.CreateDirectory(junctionTarget); - Directory.CreateDirectory(symlinkTarget); + Directory.CreateDirectory(symLinkTarget); string junctionLink = Path.Combine(this.Enlistment.RepoRoot, "DirJunction"); - string symlink = Path.Combine(this.Enlistment.RepoRoot, "DirLink"); + string symLink = Path.Combine(this.Enlistment.RepoRoot, "DirLink"); ProcessHelper.Run("CMD.exe", "/C mklink /J " + junctionLink + " " + junctionTarget); - ProcessHelper.Run("CMD.exe", "/C mklink /D " + symlink + " " + symlinkTarget); + ProcessHelper.Run("CMD.exe", "/C mklink /D " + symLink + " " + symLinkTarget); string target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.UnitTests"); string link = Path.Combine(this.Enlistment.RepoRoot, "UnitTests"); diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index c8d954bc7a..610133d856 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text; using System.Threading; namespace GVFS.Platform.Mac @@ -81,14 +82,43 @@ public override FileSystemResult WritePlaceholderFile( { // TODO(Mac): Add functional tests that validate file mode is set correctly ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - Result result = this.virtualizationInstance.WritePlaceholderFile( - relativePath, - PlaceholderVersionId, - ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), - (ulong)endOfFile, - (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask)); + GitIndexProjection.FileType fileType = (GitIndexProjection.FileType)(fileTypeAndMode & GitIndexProjection.FileTypeMask); - return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + if (fileType == GitIndexProjection.FileType.Regular) + { + Result result = this.virtualizationInstance.WritePlaceholderFile( + relativePath, + PlaceholderVersionId, + ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), + (ulong)endOfFile, + (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask)); + + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + else if (fileType == GitIndexProjection.FileType.SymLink) + { + string symLinkContents; + if (this.TryGetSymLinkObjectContents(sha, out symLinkContents)) + { + Result result = this.virtualizationInstance.WriteSymLink(relativePath, symLinkContents); + + this.FileSystemCallbacks.OnFileSymLinkCreated(relativePath); + + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(sha), sha); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.WritePlaceholderFile)}: Failed to read contents of symlink object"); + return new FileSystemResult(FSResult.IOError, 0); + } + else + { + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(fileType), fileType); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.WritePlaceholderFile)}: Unsupported fileType"); + return new FileSystemResult(FSResult.IOError, 0); + } } public override FileSystemResult WritePlaceholderDirectory(string relativePath) @@ -115,17 +145,54 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( // - Mode + content changes between commits // - Mode only changes (without any change to content, see issue #223) ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); + GitIndexProjection.FileType fileType = (GitIndexProjection.FileType)(fileTypeAndMode & GitIndexProjection.FileTypeMask); + + if (fileType == GitIndexProjection.FileType.Regular) + { + Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( + relativePath, + PlaceholderVersionId, + ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), + (ulong)endOfFile, + (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask), + (UpdateType)updateFlags, + out failureCause); + + failureReason = (UpdateFailureReason)failureCause; + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + else if (fileType == GitIndexProjection.FileType.SymLink) + { + string symLinkContents; + if (this.TryGetSymLinkObjectContents(shaContentId, out symLinkContents)) + { + Result result = this.virtualizationInstance.ReplacePlaceholderFileWithSymLink( + relativePath, + symLinkContents, + (UpdateType)updateFlags, + out failureCause); + + this.FileSystemCallbacks.OnFileSymLinkCreated(relativePath); + + failureReason = (UpdateFailureReason)failureCause; + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } - Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( - relativePath, - PlaceholderVersionId, - ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), - (ulong)endOfFile, - (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask), - (UpdateType)updateFlags, - out failureCause); - failureReason = (UpdateFailureReason)failureCause; - return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(shaContentId), shaContentId); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Failed to read contents of symlink object"); + failureReason = UpdateFailureReason.NoFailure; + return new FileSystemResult(FSResult.IOError, 0); + } + else + { + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(fileType), fileType); + metadata.Add(nameof(fileTypeAndMode), fileTypeAndMode); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Unsupported fileType"); + failureReason = UpdateFailureReason.NoFailure; + return new FileSystemResult(FSResult.IOError, 0); + } } protected override bool TryStart(out string error) @@ -165,6 +232,67 @@ private static byte[] ToVersionIdByteArray(byte[] version) return bytes; } + private bool TryGetSymLinkObjectContents(string sha, out string contents) + { + contents = null; + StringBuilder objectContents = new StringBuilder(); + + try + { + if (!this.GitObjects.TryCopyBlobContentStream( + sha, + CancellationToken.None, + GVFSGitObjects.RequestSource.SymLinkCreation, + (stream, blobLength) => + { + // TODO(Mac): Find a better solution than reading from the stream one byte at at time + byte[] buffer = new byte[4096]; + uint bufferIndex = 0; + int nextByte = stream.ReadByte(); + while (nextByte != -1) + { + while (bufferIndex < buffer.Length && nextByte != -1) + { + buffer[bufferIndex] = (byte)nextByte; + nextByte = stream.ReadByte(); + ++bufferIndex; + } + + while (bufferIndex < buffer.Length) + { + buffer[bufferIndex] = 0; + ++bufferIndex; + } + + objectContents.Append(System.Text.Encoding.ASCII.GetString(buffer)); + + if (bufferIndex == buffer.Length) + { + bufferIndex = 0; + } + } + })) + { + EventMetadata metadata = this.CreateEventMetadata(); + metadata.Add(nameof(sha), sha); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkObjectContents)}: TryCopyBlobContentStream failed"); + + return false; + } + } + catch (GetFileStreamException e) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e); + metadata.Add(nameof(sha), sha); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkObjectContents)}: TryCopyBlobContentStream caught GetFileStreamException"); + + return false; + } + + contents = objectContents.ToString(); + return true; + } + private Result OnGetFileStream( ulong commandId, string relativePath, @@ -258,7 +386,7 @@ private Result OnGetFileStream( { activity.RelatedError(metadata, $"{nameof(this.OnGetFileStream)}: TryCopyBlobContentStream failed"); - // TODO: Is this the correct Result to return? + // TODO(Mac): Is this the correct Result to return? return Result.EFileNotFound; } } diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs index 28653438ca..e2d88bf09c 100644 --- a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs @@ -75,7 +75,7 @@ public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) private static IEnumerable GetFolderPlaceholdersFromDisk(ITracer tracer, PhysicalFileSystem fileSystem, string path) { - if (!fileSystem.IsSymlink(path)) + if (!fileSystem.IsSymLink(path)) { foreach (string directory in fileSystem.EnumerateDirectories(path)) { diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 96cb89f606..6d4d19d6d0 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -99,7 +99,8 @@ public void UpdatePlaceholderIfNeeded() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); + string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)(RegularFileType | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -109,7 +110,7 @@ public void UpdatePlaceholderIfNeeded() mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - "test.txt", + filePath, DateTime.Now, DateTime.Now, DateTime.Now, @@ -126,7 +127,7 @@ public void UpdatePlaceholderIfNeeded() mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - "test.txt", + filePath, DateTime.Now, DateTime.Now, DateTime.Now, @@ -146,7 +147,7 @@ public void UpdatePlaceholderIfNeeded() // TODO: The result should probably be VirtualizationInvalidOperation but for now it's IOError virtualizer .UpdatePlaceholderIfNeeded( - "test.txt", + filePath, DateTime.Now, DateTime.Now, DateTime.Now, diff --git a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs index b5644bf868..ff8950b4ff 100644 --- a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs +++ b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs @@ -29,7 +29,8 @@ public enum OperationType OnFolderFirstWrite, OnIndexWriteWithoutProjectionChange, OnPlaceholderCreationsBlockedForGit, - OnFileHardLinkCreated + OnFileHardLinkCreated, + OnFileSymLinkCreated } public OperationType Operation { get; } @@ -52,6 +53,11 @@ public static FileSystemTask OnFileHardLinkCreated(string newLinkRelativePath) return new FileSystemTask(OperationType.OnFileHardLinkCreated, newLinkRelativePath, oldVirtualPath: null); } + public static FileSystemTask OnFileSymLinkCreated(string newLinkRelativePath) + { + return new FileSystemTask(OperationType.OnFileSymLinkCreated, newLinkRelativePath, oldVirtualPath: null); + } + public static FileSystemTask OnFileDeleted(string virtualPath) { return new FileSystemTask(OperationType.OnFileDeleted, virtualPath, oldVirtualPath: null); diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index a9ea5200eb..b8ea79d833 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -428,6 +428,11 @@ public virtual void OnFileHardLinkCreated(string newLinkRelativePath) this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileHardLinkCreated(newLinkRelativePath)); } + public virtual void OnFileSymLinkCreated(string newLinkRelativePath) + { + this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileSymLinkCreated(newLinkRelativePath)); + } + public void OnFileDeleted(string relativePath) { this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileDeleted(relativePath)); @@ -615,6 +620,7 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate case FileSystemTask.OperationType.OnFileCreated: case FileSystemTask.OperationType.OnFailedPlaceholderDelete: case FileSystemTask.OperationType.OnFileHardLinkCreated: + case FileSystemTask.OperationType.OnFileSymLinkCreated: metadata.Add("virtualPath", gitUpdate.VirtualPath); result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath); break; diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index cd0cc20315..ca5f88dade 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -210,7 +210,7 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func break; default: - throw new InvalidDataException($"Invalid object type {type:X} found in index"); + throw new InvalidDataException($"Invalid file type {type:X} found in index"); } this.resuableParsedIndexEntry.FileTypeAndMode = typeAndMode; diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs index 6edba146fe..786426139c 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs @@ -32,8 +32,13 @@ public static extern Result WritePlaceholderFile( ulong fileSize, ushort fileMode); + [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_WriteSymLink")] + public static extern Result WriteSymLink( + string relativePath, + string symLinkContents); + [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_UpdatePlaceholderFileIfNeeded")] - public static extern Result UpdatePlaceholderFileIfNeeded( + public static extern Result UpdatePlaceholderFileIfNeeded( string relativePath, [MarshalAs(UnmanagedType.LPArray, SizeConst = PlaceholderIdLength)] byte[] providerId, @@ -44,6 +49,13 @@ public static extern Result UpdatePlaceholderFileIfNeeded( UpdateType updateType, ref UpdateFailureCause failureCause); + [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_ReplacePlaceholderFileWithSymLink")] + public static extern Result ReplacePlaceholderFileWithSymLink( + string relativePath, + string symLinkContents, + UpdateType updateType, + ref UpdateFailureCause failureCause); + [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_DeleteFile")] public static extern Result DeleteFile( string relativePath, diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs index 3b7cf74b47..8a2ef8a72b 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs @@ -108,6 +108,13 @@ public virtual Result WritePlaceholderFile( fileMode); } + public virtual Result WriteSymLink( + string relativePath, + string symLinkContents) + { + return Interop.PrjFSLib.WriteSymLink(relativePath, symLinkContents); + } + public virtual Result UpdatePlaceholderIfNeeded( string relativePath, byte[] providerId, @@ -137,6 +144,23 @@ public virtual Result UpdatePlaceholderIfNeeded( return result; } + public virtual Result ReplacePlaceholderFileWithSymLink( + string relativePath, + string symLinkContents, + UpdateType updateFlags, + out UpdateFailureCause failureCause) + { + UpdateFailureCause updateFailureCause = UpdateFailureCause.NoFailure; + Result result = Interop.PrjFSLib.ReplacePlaceholderFileWithSymLink( + relativePath, + symLinkContents, + updateFlags, + ref updateFailureCause); + + failureCause = updateFailureCause; + return result; + } + public virtual Result CompleteCommand( ulong commandId, Result result) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index a858a7fe36..755d0fc9b8 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -355,6 +355,38 @@ PrjFS_Result PrjFS_WritePlaceholderFile( return PrjFS_Result_EIOError; } +PrjFS_Result PrjFS_WriteSymLink( + _In_ const char* relativePath, + _In_ const char* symLinkContents) +{ +#ifdef DEBUG + std::cout + << "PrjFS_WriteSymLink(" + << relativePath << ", " + << symLinkContents << ")" << std::endl; +#endif + + if (nullptr == relativePath || nullptr == symLinkContents) + { + return PrjFS_Result_EInvalidArgs; + } + + char fullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); + + if(symlink(symLinkContents, fullPath)) + { + goto CleanupAndFail; + } + + return PrjFS_Result_Success; + +CleanupAndFail: + + return PrjFS_Result_EIOError; + +} + PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( _In_ const char* relativePath, _In_ unsigned char providerId[PrjFS_PlaceholderIdLength], @@ -387,6 +419,29 @@ PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( return PrjFS_WritePlaceholderFile(relativePath, providerId, contentId, fileSize, fileMode); } +PrjFS_Result PrjFS_ReplacePlaceholderFileWithSymLink( + _In_ const char* relativePath, + _In_ const char* symLinkContents, + _In_ PrjFS_UpdateType updateFlags, + _Out_ PrjFS_UpdateFailureCause* failureCause) +{ +#ifdef DEBUG + std::cout + << "PrjFS_ReplacePlaceholderFileWithSymLink(" + << relativePath << ", " + << symLinkContents << ", " + << std::hex << updateFlags << std::dec << ")" << std::endl; +#endif + + PrjFS_Result result = PrjFS_DeleteFile(relativePath, updateFlags, failureCause); + if (result != PrjFS_Result_Success) + { + return result; + } + + return PrjFS_WriteSymLink(relativePath, symLinkContents); +} + PrjFS_Result PrjFS_DeleteFile( _In_ const char* relativePath, _In_ PrjFS_UpdateType updateFlags, diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.h b/ProjFS.Mac/PrjFSLib/PrjFSLib.h index 6e7a49a0b7..5c7eff9838 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.h +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.h @@ -85,6 +85,10 @@ extern "C" PrjFS_Result PrjFS_WritePlaceholderFile( _In_ unsigned long fileSize, _In_ uint16_t fileMode); +extern "C" PrjFS_Result PrjFS_WriteSymLink( + _In_ const char* relativePath, + _In_ const char* symLinkContents); + typedef enum { PrjFS_UpdateType_Invalid = 0x00000000, @@ -111,6 +115,12 @@ extern "C" PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( _In_ PrjFS_UpdateType updateFlags, _Out_ PrjFS_UpdateFailureCause* failureCause); +extern "C" PrjFS_Result PrjFS_ReplacePlaceholderFileWithSymLink( + _In_ const char* relativePath, + _In_ const char* symLinkContents, + _In_ PrjFS_UpdateType updateFlags, + _Out_ PrjFS_UpdateFailureCause* failureCause); + extern "C" PrjFS_Result PrjFS_DeleteFile( _In_ const char* relativePath, _In_ PrjFS_UpdateType updateFlags, From 64feb7c04c66bcd920ca15e2926e63c340e5d479 Mon Sep 17 00:00:00 2001 From: Yehezkel Bernat Date: Tue, 25 Sep 2018 17:46:25 +0300 Subject: [PATCH 012/244] trivial: Fix a typo in error messages --- GVFS/GVFS/CommandLine/ServiceVerb.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index f440206aca..13b4a56892 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -51,11 +51,11 @@ public override void Execute() int optionCount = new[] { this.MountAll, this.UnmountAll, this.List }.Count(flag => flag); if (optionCount == 0) { - this.ReportErrorAndExit("Error: You must specify an argument. Run 'gvfs serivce --help' for details."); + this.ReportErrorAndExit($"Error: You must specify an argument. Run 'gvfs {ServiceVerbName} --help' for details."); } else if (optionCount > 1) { - this.ReportErrorAndExit("Error: You cannot specify multiple arguments. Run 'gvfs serivce --help' for details."); + this.ReportErrorAndExit($"Error: You cannot specify multiple arguments. Run 'gvfs {ServiceVerbName} --help' for details."); } string errorMessage; From 3f8fd9e44a4b31fa328f2abd3bb6bd7358f38659 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 22 Aug 2018 12:38:04 -0600 Subject: [PATCH 013/244] Add tests for removing entries from the modified files database --- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 42 ++++--- .../ModifiedPathsTests.cs | 105 ++++++++++++++---- 2 files changed, 108 insertions(+), 39 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index d3ea1f93be..eb1ce473d9 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -324,24 +324,30 @@ public void MountCreatesModifiedPathsDatabase() this.Enlistment.MountGVFS(); this.Enlistment.UnmountGVFS(); - string expectedModifiedPaths = @"A .gitattributes -A developer/me/ -A developer/me/JLANGE9._prerazzle -A developer/me/StateSwitch.Save -A tools/x86/remote.exe -A tools/x86/runelevated.exe -A tools/amd64/remote.exe -A tools/amd64/runelevated.exe -A tools/perllib/MS/TraceLogging.dll -A tools/managed/v2.0/midldd.CheckedInExe -A tools/managed/v4.0/sdapi.dll -A tools/managed/v2.0/midlpars.dll -A tools/managed/v2.0/RPCDataSupport.dll -A tools/managed/v2.0/MidlStaticAnalysis.dll -A tools/perllib/MS/Somefile.txt -"; - - modifiedPathsDatabasePath.ShouldBeAFile(this.fileSystem).WithContents(expectedModifiedPaths); + string[] expectedModifiedPaths = + { + "A .gitattributes", + "A developer/me/", + "A developer/me/JLANGE9._prerazzle", + "A developer/me/StateSwitch.Save", + "A tools/x86/remote.exe", + "A tools/x86/runelevated.exe", + "A tools/amd64/remote.exe", + "A tools/amd64/runelevated.exe", + "A tools/perllib/MS/TraceLogging.dll", + "A tools/managed/v2.0/midldd.CheckedInExe", + "A tools/managed/v4.0/sdapi.dll", + "A tools/managed/v2.0/midlpars.dll", + "A tools/managed/v2.0/RPCDataSupport.dll", + "A tools/managed/v2.0/MidlStaticAnalysis.dll", + "A tools/perllib/MS/Somefile.txt", + }; + + modifiedPathsDatabasePath.ShouldBeAFile(this.fileSystem); + this.fileSystem.ReadAllText(modifiedPathsDatabasePath) + .Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) + .OrderBy(x => x) + .ShouldMatchInOrder(expectedModifiedPaths.OrderBy(x => x)); this.ValidatePersistedVersionMatchesCurrentVersion(); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index c57bffccd2..62fa2a9f96 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -3,7 +3,9 @@ using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; using NUnit.Framework; +using System; using System.IO; +using System.Linq; namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase { @@ -23,26 +25,64 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase private static readonly string FileToCreateOutsideRepo = $"{nameof(ModifiedPathsTests)}_outsideRepo.txt"; private static readonly string FolderToCreateOutsideRepo = $"{nameof(ModifiedPathsTests)}_outsideFolder"; private static readonly string FolderToDelete = "Scripts"; - private static readonly string ExpectedModifiedFilesContentsAfterRemount = -$@"A .gitattributes -A {GVFSHelpers.ConvertPathToGitFormat(FileToAdd)} -A {GVFSHelpers.ConvertPathToGitFormat(FileToUpdate)} -A {FileToDelete} -A {GVFSHelpers.ConvertPathToGitFormat(FileToRename)} -A {GVFSHelpers.ConvertPathToGitFormat(RenameFileTarget)} -A {FolderToCreate}/ -A {FolderToRename}/ -A {RenameFolderTarget}/ -A {RenameNewDotGitFileTarget} -A {FileToCreateOutsideRepo} -A {FolderToCreateOutsideRepo}/ -A {FolderToDelete}/CreateCommonAssemblyVersion.bat -A {FolderToDelete}/CreateCommonCliAssemblyVersion.bat -A {FolderToDelete}/CreateCommonVersionHeader.bat -A {FolderToDelete}/RunFunctionalTests.bat -A {FolderToDelete}/RunUnitTests.bat -A {FolderToDelete}/ -"; + private static readonly string[] ExpectedModifiedFilesContentsAfterRemount = + { + $"A .gitattributes", + $"A {GVFSHelpers.ConvertPathToGitFormat(FileToAdd)}", + $"A {GVFSHelpers.ConvertPathToGitFormat(FileToUpdate)}", + $"A {FileToDelete}", + $"A {GVFSHelpers.ConvertPathToGitFormat(FileToRename)}", + $"A {GVFSHelpers.ConvertPathToGitFormat(RenameFileTarget)}", + $"A {FolderToCreate}/", + $"A {FolderToRename}/", + $"A {RenameFolderTarget}/", + $"A {RenameNewDotGitFileTarget}", + $"A {FileToCreateOutsideRepo}", + $"A {FolderToCreateOutsideRepo}/", + $"A {FolderToDelete}/CreateCommonAssemblyVersion.bat", + $"A {FolderToDelete}/CreateCommonCliAssemblyVersion.bat", + $"A {FolderToDelete}/CreateCommonVersionHeader.bat", + $"A {FolderToDelete}/RunFunctionalTests.bat", + $"A {FolderToDelete}/RunUnitTests.bat", + $"A {FolderToDelete}/", + }; + + [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + public void DeletedTempFileIsRemovedFromModifiedFiles(FileSystemRunner fileSystem) + { + string tempFile = this.CreateFile(fileSystem, "temp.txt"); + fileSystem.DeleteFile(tempFile); + tempFile.ShouldNotExistOnDisk(fileSystem); + + this.Enlistment.UnmountGVFS(); + this.ValidateModifiedPathsDoNotContain(fileSystem, "temp.txt"); + } + + [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + public void DeletedTempFolderIsRemovedFromModifiedFiles(FileSystemRunner fileSystem) + { + string tempFolder = this.CreateDirectory(fileSystem, "Temp"); + fileSystem.DeleteDirectory(tempFolder); + tempFolder.ShouldNotExistOnDisk(fileSystem); + + this.Enlistment.UnmountGVFS(); + this.ValidateModifiedPathsDoNotContain(fileSystem, "Temp/"); + } + + [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + public void DeletedTempFolderDeletesFilesFromModifiedFiles(FileSystemRunner fileSystem) + { + string tempFolder = this.CreateDirectory(fileSystem, "Temp"); + string tempFile1 = this.CreateFile(fileSystem, Path.Combine("Temp", "temp1.txt")); + string tempFile2 = this.CreateFile(fileSystem, Path.Combine("Temp", "temp2.txt")); + fileSystem.DeleteDirectory(tempFolder); + tempFolder.ShouldNotExistOnDisk(fileSystem); + tempFile1.ShouldNotExistOnDisk(fileSystem); + tempFile2.ShouldNotExistOnDisk(fileSystem); + + this.Enlistment.UnmountGVFS(); + this.ValidateModifiedPathsDoNotContain(fileSystem, "Temp/", "Temp/temp1.txt", "Temp/temp2.txt"); + } [Category(Categories.MacTODO.M2)] [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] @@ -114,7 +154,8 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) modifiedPathsDatabase.ShouldBeAFile(fileSystem); using (StreamReader reader = new StreamReader(File.Open(modifiedPathsDatabase, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) { - reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContentsAfterRemount); + reader.ReadToEnd().Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).OrderBy(x => x) + .ShouldMatchInOrder(ExpectedModifiedFilesContentsAfterRemount.OrderBy(x => x)); } } @@ -160,5 +201,27 @@ A LinkToFileOutsideSrc.txt reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContentsAfterHardlinks); } } + + private string CreateDirectory(FileSystemRunner fileSystem, string relativePath) + { + string tempFolder = this.Enlistment.GetVirtualPathTo(relativePath); + fileSystem.CreateDirectory(tempFolder); + tempFolder.ShouldBeADirectory(fileSystem); + return tempFolder; + } + + private string CreateFile(FileSystemRunner fileSystem, string relativePath) + { + string tempFile = this.Enlistment.GetVirtualPathTo(relativePath); + fileSystem.WriteAllText(tempFile, $"Contents for the {relativePath} file"); + tempFile.ShouldBeAFile(fileSystem); + return tempFile; + } + + private void ValidateModifiedPathsDoNotContain(FileSystemRunner fileSystem, params string[] paths) + { + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, paths.Select(x => $"A {x}" + Environment.NewLine).ToArray()); + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, paths.Select(x => $"D {x}" + Environment.NewLine).ToArray()); + } } } From a0f9b408c228b89fb7d6e5000d914bd14fa6408f Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 21 Aug 2018 11:00:38 -0600 Subject: [PATCH 014/244] Add TryRemove method to the ModifiedPathsDatabase --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 99 ++++++++++++++++++----- 1 file changed, 80 insertions(+), 19 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 043a1e3bc4..409b36a3c4 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -75,38 +75,99 @@ public bool TryAdd(string path, bool isFolder, out bool isRetryable) } catch (IOException e) { - if (this.Tracer != null) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", "ModifiedPathsDatabase"); - metadata.Add(nameof(entry), entry); - metadata.Add(nameof(isFolder), isFolder); - metadata.Add("Exception", e.ToString()); - this.Tracer.RelatedWarning(metadata, $"IOException caught while processing {nameof(this.TryAdd)}"); - } - + this.TraceWarning(isFolder, entry, e, nameof(this.TryAdd)); return false; } catch (Exception e) { - if (this.Tracer != null) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", "ModifiedPathsDatabase"); - metadata.Add(nameof(entry), entry); - metadata.Add(nameof(isFolder), isFolder); - metadata.Add("Exception", e.ToString()); - this.Tracer.RelatedError(metadata, $"Exception caught while processing {nameof(this.TryAdd)}"); - } + this.TraceError(isFolder, entry, e, nameof(this.TryAdd)); + isRetryable = false; + return false; + } + + return true; + } + return true; + } + + public bool TryRemove(string path, bool isFolder, out bool isRetryable) + { + isRetryable = true; + string entry = this.NormalizeEntryString(path, isFolder); + if (this.modifiedPaths.Contains(entry)) + { + isRetryable = true; + try + { + this.WriteRemoveEntry(entry, () => this.modifiedPaths.TryRemove(entry)); + } + catch (IOException e) + { + this.TraceWarning(isFolder, entry, e, nameof(this.TryRemove)); + return false; + } + catch (Exception e) + { + this.TraceError(isFolder, entry, e, nameof(this.TryRemove)); isRetryable = false; return false; } + + return true; } return true; } + public void WriteAllEntriesAndFlush() + { + try + { + this.WriteAndReplaceDataFile(this.GenerateDataLines); + } + catch (Exception e) + { + throw new FileBasedCollectionException(e); + } + } + + private static EventMetadata CreateEventMetadata(bool isFolder, string entry, Exception e) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", "ModifiedPathsDatabase"); + metadata.Add(nameof(entry), entry); + metadata.Add(nameof(isFolder), isFolder); + metadata.Add("Exception", e.ToString()); + return metadata; + } + + private IEnumerable GenerateDataLines() + { + foreach (string entry in this.modifiedPaths) + { + yield return this.FormatAddLine(entry); + } + } + + private void TraceWarning(bool isFolder, string entry, Exception e, string method) + { + if (this.Tracer != null) + { + EventMetadata metadata = CreateEventMetadata(isFolder, entry, e); + this.Tracer.RelatedWarning(metadata, $"{e.GetType().Name} caught while processing {method}"); + } + } + + private void TraceError(bool isFolder, string entry, Exception e, string method) + { + if (this.Tracer != null) + { + EventMetadata metadata = CreateEventMetadata(isFolder, entry, e); + this.Tracer.RelatedError(metadata, $"{e.GetType().Name} caught while processing {method}"); + } + } + private bool TryParseAddLine(string line, out string key, out string value, out string error) { key = line; From ae5f9bace1a4da28fea769761d3cfc80c610753e Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Sep 2018 14:28:36 -0600 Subject: [PATCH 015/244] Handle the file system callbacks to track created then deleted files --- .../FileSystemCallbacks.cs | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index a9ea5200eb..d954ae6c4a 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -33,6 +33,7 @@ public class FileSystemCallbacks : IDisposable, IHeartBeatMetadataProvider private GVFSContext context; private GVFSGitObjects gitObjects; private ModifiedPathsDatabase modifiedPaths; + private ConcurrentHashSet newlyCreatedFileAndFolderPaths; private ConcurrentDictionary placeHolderCreationCount; private BackgroundFileSystemTaskRunner backgroundFileSystemTaskRunner; private FileSystemVirtualizer fileSystemVirtualizer; @@ -75,6 +76,7 @@ public FileSystemCallbacks( this.fileSystemVirtualizer = fileSystemVirtualizer; this.placeHolderCreationCount = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + this.newlyCreatedFileAndFolderPaths = new ConcurrentHashSet(StringComparer.OrdinalIgnoreCase); string error; if (!ModifiedPathsDatabase.TryLoadOrCreate( @@ -364,6 +366,8 @@ public virtual void OnIndexFileChange() this.GitIndexProjection.InvalidateProjection(); this.InvalidateGitStatusCache(); } + + this.newlyCreatedFileAndFolderPaths.Clear(); } public void InvalidateGitStatusCache() @@ -400,6 +404,7 @@ public void OnExcludeFileChanged() public void OnFileCreated(string relativePath) { + this.newlyCreatedFileAndFolderPaths.Add(relativePath); this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileCreated(relativePath)); } @@ -435,6 +440,7 @@ public void OnFileDeleted(string relativePath) public void OnFolderCreated(string relativePath) { + this.newlyCreatedFileAndFolderPaths.Add(relativePath); this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderCreated(relativePath)); } @@ -639,7 +645,15 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate case FileSystemTask.OperationType.OnFileDeleted: metadata.Add("virtualPath", gitUpdate.VirtualPath); - result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: false); + } + else + { + result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath); + } + break; case FileSystemTask.OperationType.OnFileOverwritten: @@ -736,7 +750,15 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate case FileSystemTask.OperationType.OnFolderDeleted: metadata.Add("virtualPath", gitUpdate.VirtualPath); - result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: true); + } + else + { + result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); + } + break; case FileSystemTask.OperationType.OnFolderFirstWrite: @@ -768,6 +790,26 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate return result; } + private FileSystemTaskResult TryRemoveModifiedPath(string virtualPath, bool isFolder) + { + string fullPathToItem = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, virtualPath); + if ((isFolder && this.context.FileSystem.DirectoryExists(fullPathToItem)) || + (!isFolder && this.context.FileSystem.FileExists(fullPathToItem))) + { + return FileSystemTaskResult.Success; + } + + if (!this.modifiedPaths.TryRemove(virtualPath, isFolder, out bool isRetryable)) + { + return isRetryable ? FileSystemTaskResult.RetryableError : FileSystemTaskResult.FatalError; + } + + this.newlyCreatedFileAndFolderPaths.TryRemove(virtualPath); + + this.InvalidateGitStatusCache(); + return FileSystemTaskResult.Success; + } + private FileSystemTaskResult TryAddModifiedPath(string virtualPath, bool isFolder) { if (!this.modifiedPaths.TryAdd(virtualPath, isFolder, out bool isRetryable)) From b50dd1a0396042229940beb790493ad61799364f Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 6 Sep 2018 13:03:05 -0600 Subject: [PATCH 016/244] Add handling of rename of files for modified paths cleanup --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index d954ae6c4a..de5541d4fc 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -425,6 +425,7 @@ public void OnFileConvertedToFull(string relativePath) public virtual void OnFileRenamed(string oldRelativePath, string newRelativePath) { + this.newlyCreatedFileAndFolderPaths.Add(newRelativePath); this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileRenamed(oldRelativePath, newRelativePath)); } @@ -631,7 +632,14 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate result = FileSystemTaskResult.Success; if (!string.IsNullOrEmpty(gitUpdate.OldVirtualPath) && !IsPathInsideDotGit(gitUpdate.OldVirtualPath)) { - result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.OldVirtualPath); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) + { + result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: false); + } + else + { + result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.OldVirtualPath); + } } if (result == FileSystemTaskResult.Success && @@ -844,7 +852,7 @@ private FileSystemTaskResult AddModifiedPathAndRemoveFromPlaceholderList(string private FileSystemTaskResult PostBackgroundOperation() { - this.modifiedPaths.ForceFlush(); + this.modifiedPaths.WriteAllEntriesAndFlush(); this.gitStatusCache.RefreshAsynchronously(); return this.GitIndexProjection.CloseIndex(); } From 848e6d38fb4840aa1f715fa5b821763f9211a949 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 7 Sep 2018 12:37:43 -0600 Subject: [PATCH 017/244] Handle cleaning up modified paths when a folder is renamed --- .../FileSystemCallbacks.cs | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index de5541d4fc..c640d41a0a 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -691,10 +691,20 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); if (result == FileSystemTaskResult.Success) { + if (!this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + this.newlyCreatedFileAndFolderPaths.Add(gitUpdate.VirtualPath); + } + + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) + { + this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); + } + Queue relativeFolderPaths = new Queue(); relativeFolderPaths.Enqueue(gitUpdate.VirtualPath); - // Add all the files in the renamed folder to the always_exclude file + // Remove old paths from modified paths if in the newly created list while (relativeFolderPaths.Count > 0) { string folderPath = relativeFolderPaths.Dequeue(); @@ -705,14 +715,20 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate foreach (DirectoryItemInfo itemInfo in this.context.FileSystem.ItemsInDirectory(Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, folderPath))) { string itemVirtualPath = Path.Combine(folderPath, itemInfo.Name); - if (itemInfo.IsDirectory) + string oldItemVirtualPath = gitUpdate.OldVirtualPath + itemVirtualPath.Substring(gitUpdate.VirtualPath.Length); + if (!this.newlyCreatedFileAndFolderPaths.Contains(itemVirtualPath)) { - relativeFolderPaths.Enqueue(itemVirtualPath); + this.newlyCreatedFileAndFolderPaths.Add(itemVirtualPath); } - else + + if (this.newlyCreatedFileAndFolderPaths.Contains(oldItemVirtualPath)) { - string oldItemVirtualPath = gitUpdate.OldVirtualPath + itemVirtualPath.Substring(gitUpdate.VirtualPath.Length); - result = this.TryAddModifiedPath(itemVirtualPath, isFolder: false); + this.TryRemoveModifiedPath(oldItemVirtualPath, isFolder: itemInfo.IsDirectory); + } + + if (itemInfo.IsDirectory) + { + relativeFolderPaths.Enqueue(itemVirtualPath); } } } From c189b2c356dcd3e279cbfd7024a43c66935fde18 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 7 Sep 2018 22:09:03 -0600 Subject: [PATCH 018/244] Add null check when creating trace metadata using and exception --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 409b36a3c4..9456131151 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -138,7 +138,11 @@ private static EventMetadata CreateEventMetadata(bool isFolder, string entry, Ex metadata.Add("Area", "ModifiedPathsDatabase"); metadata.Add(nameof(entry), entry); metadata.Add(nameof(isFolder), isFolder); - metadata.Add("Exception", e.ToString()); + if (e != null) + { + metadata.Add("Exception", e.ToString()); + } + return metadata; } From 1772519160fefa052275c7238875f918f4fbbf7b Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 11 Sep 2018 13:45:24 -0600 Subject: [PATCH 019/244] Fix broken tests --- .../Tests/EnlistmentPerFixture/GitFilesTests.cs | 10 ++-------- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 2 +- .../Tests/GitCommands/GitCommandsTests.cs | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index eca57a1c29..88dbf17097 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -88,7 +88,6 @@ public void RenameEmptyFolderTest() string renamedFolderName = "folder3b"; string[] expectedModifiedEntries = { - folderName + "/", renamedFolderName + "/", }; @@ -110,19 +109,14 @@ public void RenameFolderTest() string[] fileNames = { "a", "b", "c" }; string[] expectedModifiedEntries = { - renamedFolderName + "/" + fileNames[0], - renamedFolderName + "/" + fileNames[1], - renamedFolderName + "/" + fileNames[2], - folderName + "/" + fileNames[0], - folderName + "/" + fileNames[1], - folderName + "/" + fileNames[2], + renamedFolderName + "/", }; this.Enlistment.GetVirtualPathTo(folderName).ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.CreateDirectory(this.Enlistment.GetVirtualPathTo(folderName)); foreach (string fileName in fileNames) { - string filePath = folderName + "\\" + fileName; + string filePath = Path.Combine(folderName, fileName); this.fileSystem.CreateEmptyFile(this.Enlistment.GetVirtualPathTo(filePath)); this.Enlistment.GetVirtualPathTo(filePath).ShouldBeAFile(this.fileSystem); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 62fa2a9f96..14fe1a5bed 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -109,7 +109,7 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) string folderToRenameTarget = this.Enlistment.GetVirtualPathTo(RenameFolderTarget); fileSystem.MoveDirectory(folderToRename, folderToRenameTarget); - // Moving the new folder out of the repo should not change the modified paths + // Moving the new folder out of the repo should not change the modified paths file string folderTargetOutsideSrc = Path.Combine(this.Enlistment.EnlistmentRoot, RenameFolderTarget); folderTargetOutsideSrc.ShouldNotExistOnDisk(fileSystem); fileSystem.MoveDirectory(folderToRenameTarget, folderTargetOutsideSrc); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 0b3ccccce0..c623f5ea72 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -1041,7 +1041,7 @@ private void BasicCommit(Action fileSystemAction, string addCommand, [CallerMemb fileSystemAction(); this.ValidateGitCommand("status"); this.ValidateGitCommand(addCommand); - this.RunGitCommand("commit -m \"BasicCommit for {test}\""); + this.RunGitCommand($"commit -m \"BasicCommit for {test}\""); } private void SwitchBranch(Action fileSystemAction, [CallerMemberName]string test = GitCommandsTests.UnknownTestName) From 9f8944e1686bb880c7e50c6d0e3eb15453f4a025 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 12 Sep 2018 09:06:46 -0600 Subject: [PATCH 020/244] Simplify checking the newly created files and removing from modified paths --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index c640d41a0a..b41a2c6341 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -691,14 +691,10 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); if (result == FileSystemTaskResult.Success) { - if (!this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) - { - this.newlyCreatedFileAndFolderPaths.Add(gitUpdate.VirtualPath); - } - + this.newlyCreatedFileAndFolderPaths.Add(gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) { - this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); + result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); } Queue relativeFolderPaths = new Queue(); @@ -716,14 +712,11 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate { string itemVirtualPath = Path.Combine(folderPath, itemInfo.Name); string oldItemVirtualPath = gitUpdate.OldVirtualPath + itemVirtualPath.Substring(gitUpdate.VirtualPath.Length); - if (!this.newlyCreatedFileAndFolderPaths.Contains(itemVirtualPath)) - { - this.newlyCreatedFileAndFolderPaths.Add(itemVirtualPath); - } + this.newlyCreatedFileAndFolderPaths.Add(itemVirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(oldItemVirtualPath)) { - this.TryRemoveModifiedPath(oldItemVirtualPath, isFolder: itemInfo.IsDirectory); + result = this.TryRemoveModifiedPath(oldItemVirtualPath, isFolder: itemInfo.IsDirectory); } if (itemInfo.IsDirectory) From f7e268a3fedbbdadf0de1f634459794a0d194def Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 20 Sep 2018 10:46:06 -0400 Subject: [PATCH 021/244] Fix broken tests for cleaning modified paths --- .../EnlistmentPerTestCase/ModifiedPathsTests.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 14fe1a5bed..49981b86a5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -34,7 +34,6 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase $"A {GVFSHelpers.ConvertPathToGitFormat(FileToRename)}", $"A {GVFSHelpers.ConvertPathToGitFormat(RenameFileTarget)}", $"A {FolderToCreate}/", - $"A {FolderToRename}/", $"A {RenameFolderTarget}/", $"A {RenameNewDotGitFileTarget}", $"A {FileToCreateOutsideRepo}", @@ -162,11 +161,12 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] public void ModifiedPathsCorrectAfterHardLinking(FileSystemRunner fileSystem) { - const string ExpectedModifiedFilesContentsAfterHardlinks = -@"A .gitattributes -A LinkToReadme.md -A LinkToFileOutsideSrc.txt -"; + string[] expectedModifiedFilesContentsAfterHardlinks = + { + "A .gitattributes", + "A LinkToReadme.md", + "A LinkToFileOutsideSrc.txt", + }; // Create a link from src\LinkToReadme.md to src\Readme.md string existingFileInRepoPath = this.Enlistment.GetVirtualPathTo("Readme.md"); @@ -198,7 +198,8 @@ A LinkToFileOutsideSrc.txt modifiedPathsDatabase.ShouldBeAFile(fileSystem); using (StreamReader reader = new StreamReader(File.Open(modifiedPathsDatabase, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) { - reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContentsAfterHardlinks); + reader.ReadToEnd().Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).OrderBy(x => x) + .ShouldMatchInOrder(expectedModifiedFilesContentsAfterHardlinks.OrderBy(x => x)); } } From a0023aa659997d6abc9cb697d7730ba6fa770101 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 21 Sep 2018 13:42:23 -0600 Subject: [PATCH 022/244] Remove extra return statments and adding to new paths during rename --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 4 ---- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 1 - 2 files changed, 5 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 9456131151..ffde4f6927 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -84,8 +84,6 @@ public bool TryAdd(string path, bool isFolder, out bool isRetryable) isRetryable = false; return false; } - - return true; } return true; @@ -113,8 +111,6 @@ public bool TryRemove(string path, bool isFolder, out bool isRetryable) isRetryable = false; return false; } - - return true; } return true; diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index b41a2c6341..4931aa1b37 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -425,7 +425,6 @@ public void OnFileConvertedToFull(string relativePath) public virtual void OnFileRenamed(string oldRelativePath, string newRelativePath) { - this.newlyCreatedFileAndFolderPaths.Add(newRelativePath); this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileRenamed(oldRelativePath, newRelativePath)); } From dcf925fe61d618caea768fa7d8f7d1527f54e0eb Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 24 Sep 2018 15:36:55 -0600 Subject: [PATCH 023/244] Add PreDelete handler in BG operation to handle Mac --- .../MacFileSystemVirtualizer.cs | 2 +- .../WindowsFileSystemVirtualizer.cs | 2 +- .../Background/FileSystemTask.cs | 14 ++++- .../FileSystem/FileSystemVirtualizer.cs | 20 ++++++- .../FileSystemCallbacks.cs | 59 ++++++++++++++++--- 5 files changed, 84 insertions(+), 13 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 3d22f7f34c..127173d0e0 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -352,7 +352,7 @@ private Result OnPreDelete(string relativePath, bool isDirectory) } else { - this.OnWorkingDirectoryFileOrFolderDeleted(relativePath, isDirectory); + this.OnWorkingDirectoryFileOrFolderDeleteNotification(relativePath, isDirectory, isPreDelete: true); } } catch (Exception e) diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index 42c1f109a2..de1963b994 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -1296,7 +1296,7 @@ private void NotifyFileHandleClosedFileModifiedOrDeletedHandler( } else { - this.OnWorkingDirectoryFileOrFolderDeleted(virtualPath, isDirectory); + this.OnWorkingDirectoryFileOrFolderDeleteNotification(virtualPath, isDirectory, isPreDelete: false); } } } diff --git a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs index b5644bf868..eaab198a4e 100644 --- a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs +++ b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs @@ -29,7 +29,9 @@ public enum OperationType OnFolderFirstWrite, OnIndexWriteWithoutProjectionChange, OnPlaceholderCreationsBlockedForGit, - OnFileHardLinkCreated + OnFileHardLinkCreated, + OnFilePreDelete, + OnFolderPreDelete, } public OperationType Operation { get; } @@ -57,6 +59,11 @@ public static FileSystemTask OnFileDeleted(string virtualPath) return new FileSystemTask(OperationType.OnFileDeleted, virtualPath, oldVirtualPath: null); } + public static FileSystemTask OnFilePreDelete(string virtualPath) + { + return new FileSystemTask(OperationType.OnFilePreDelete, virtualPath, oldVirtualPath: null); + } + public static FileSystemTask OnFileOverwritten(string virtualPath) { return new FileSystemTask(OperationType.OnFileOverwritten, virtualPath, oldVirtualPath: null); @@ -97,6 +104,11 @@ public static FileSystemTask OnFolderDeleted(string virtualPath) return new FileSystemTask(OperationType.OnFolderDeleted, virtualPath, oldVirtualPath: null); } + public static FileSystemTask OnFolderPreDelete(string virtualPath) + { + return new FileSystemTask(OperationType.OnFolderPreDelete, virtualPath, oldVirtualPath: null); + } + public static FileSystemTask OnIndexWriteWithoutProjectionChange() { return new FileSystemTask(OperationType.OnIndexWriteWithoutProjectionChange, virtualPath: null, oldVirtualPath: null); diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index c77e94669d..70cac75d11 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -185,7 +185,7 @@ protected void OnDotGitFileOrFolderDeleted(string relativePath) } } - protected void OnWorkingDirectoryFileOrFolderDeleted(string relativePath, bool isDirectory) + protected void OnWorkingDirectoryFileOrFolderDeleteNotification(string relativePath, bool isDirectory, bool isPreDelete) { if (isDirectory) { @@ -193,12 +193,26 @@ protected void OnWorkingDirectoryFileOrFolderDeleted(string relativePath, bool i GitCommandLineParser gitCommand = new GitCommandLineParser(this.Context.Repository.GVFSLock.GetLockedGitCommand()); if (!gitCommand.IsValidGitCommand) { - this.FileSystemCallbacks.OnFolderDeleted(relativePath); + if (isPreDelete) + { + this.FileSystemCallbacks.OnFolderPreDelete(relativePath); + } + else + { + this.FileSystemCallbacks.OnFolderDeleted(relativePath); + } } } else { - this.FileSystemCallbacks.OnFileDeleted(relativePath); + if (isPreDelete) + { + this.FileSystemCallbacks.OnFilePreDelete(relativePath); + } + else + { + this.FileSystemCallbacks.OnFileDeleted(relativePath); + } } this.FileSystemCallbacks.InvalidateGitStatusCache(); diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 4931aa1b37..44a0aaf62d 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -438,6 +438,11 @@ public void OnFileDeleted(string relativePath) this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileDeleted(relativePath)); } + public void OnFilePreDelete(string relativePath) + { + this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFilePreDelete(relativePath)); + } + public void OnFolderCreated(string relativePath) { this.newlyCreatedFileAndFolderPaths.Add(relativePath); @@ -454,6 +459,11 @@ public void OnFolderDeleted(string relativePath) this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderDeleted(relativePath)); } + public void OnFolderPreDelete(string relativePath) + { + this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderPreDelete(relativePath)); + } + public void OnPlaceholderFileCreated(string relativePath, string sha, string triggeringProcessImageFileName) { this.GitIndexProjection.OnPlaceholderFileCreated(relativePath, sha); @@ -650,6 +660,27 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; + case FileSystemTask.OperationType.OnFilePreDelete: + metadata.Add("virtualPath", gitUpdate.VirtualPath); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + string fullPathToFolder = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, gitUpdate.VirtualPath); + if (!this.context.FileSystem.FileExists(fullPathToFolder)) + { + result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: false); + } + else + { + result = FileSystemTaskResult.Success; + } + } + else + { + result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath); + } + + break; + case FileSystemTask.OperationType.OnFileDeleted: metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) @@ -764,6 +795,27 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; + case FileSystemTask.OperationType.OnFolderPreDelete: + metadata.Add("virtualPath", gitUpdate.VirtualPath); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + string fullPathToFolder = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, gitUpdate.VirtualPath); + if (!this.context.FileSystem.DirectoryExists(fullPathToFolder)) + { + result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: true); + } + else + { + result = FileSystemTaskResult.Success; + } + } + else + { + result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); + } + + break; + case FileSystemTask.OperationType.OnFolderDeleted: metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) @@ -808,13 +860,6 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate private FileSystemTaskResult TryRemoveModifiedPath(string virtualPath, bool isFolder) { - string fullPathToItem = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, virtualPath); - if ((isFolder && this.context.FileSystem.DirectoryExists(fullPathToItem)) || - (!isFolder && this.context.FileSystem.FileExists(fullPathToItem))) - { - return FileSystemTaskResult.Success; - } - if (!this.modifiedPaths.TryRemove(virtualPath, isFolder, out bool isRetryable)) { return isRetryable ? FileSystemTaskResult.RetryableError : FileSystemTaskResult.FatalError; From 45159dbf5e41c2352e660605ea62ee03b333a2b2 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 25 Sep 2018 09:14:37 -0600 Subject: [PATCH 024/244] Add comment explaining only one of predelete or delete should be queued --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 44a0aaf62d..95618e10e3 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -661,6 +661,9 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; case FileSystemTask.OperationType.OnFilePreDelete: + // This code assumes that the current implementations of FileSystemVirtualizer will call either + // the PreDelete or the Delete not both so if a new implementation starts calling both + // this will need to be cleaned up to not duplicate the work that is being done. metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) { @@ -682,6 +685,9 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; case FileSystemTask.OperationType.OnFileDeleted: + // This code assumes that the current implementations of FileSystemVirtualizer will call either + // the PreDelete or the Delete not both so if a new implementation starts calling both + // this will need to be cleaned up to not duplicate the work that is being done. metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) { @@ -796,6 +802,9 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; case FileSystemTask.OperationType.OnFolderPreDelete: + // This code assumes that the current implementations of FileSystemVirtualizer will call either + // the PreDelete or the Delete not both so if a new implementation starts calling both + // this will need to be cleaned up to not duplicate the work that is being done. metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) { @@ -817,6 +826,9 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; case FileSystemTask.OperationType.OnFolderDeleted: + // This code assumes that the current implementations of FileSystemVirtualizer will call either + // the PreDelete or the Delete not both so if a new implementation starts calling both + // this will need to be cleaned up to not duplicate the work that is being done. metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) { From 472904c45ef03f8757bdf151924652802e73d7f0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 25 Sep 2018 12:21:17 -0600 Subject: [PATCH 025/244] Fix case only folder rename test --- .../EnlistmentPerFixture/GitFilesTests.cs | 20 ++++++++++++++----- .../FileSystemCallbacks.cs | 12 ++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 88dbf17097..bfbdcc4576 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -132,22 +132,32 @@ public void RenameFolderTest() [Category(Categories.MacTODO.M2)] public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() { - string[] expectedModifiedPathsEntries = + if (this.fileSystem is PowerShellRunner) { - "Folder/", - "Folder/testfile", + Assert.Ignore("Powershell does not support case only renames."); + } + + string[] expectedModifiedPathsEntriesAfterCreate = + { + "A Folder/", + "A Folder/testfile", + }; + + string[] expectedModifiedPathsEntriesAfterRename = + { + "A folder/", }; this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "Folder")); this.fileSystem.CreateEmptyFile(Path.Combine(this.Enlistment.RepoRoot, "Folder", "testfile")); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntries); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntriesAfterCreate); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntries); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntriesAfterRename); } [TestCase, Order(7)] diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 95618e10e3..d7c0d57b4c 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -718,20 +718,22 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate metadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath); metadata.Add("virtualPath", gitUpdate.VirtualPath); + if (!string.IsNullOrEmpty(gitUpdate.OldVirtualPath) && + this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) + { + result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); + } + // An empty destination path means the folder was renamed to somewhere outside of the repo // Note that only full folders can be moved\renamed, and so there will already be a recursive // sparse-checkout entry for the virtualPath of the folder being moved (meaning that no // additional work is needed for any files\folders inside the folder being moved) - if (!string.IsNullOrEmpty(gitUpdate.VirtualPath)) + if (result == FileSystemTaskResult.Success && !string.IsNullOrEmpty(gitUpdate.VirtualPath)) { result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); if (result == FileSystemTaskResult.Success) { this.newlyCreatedFileAndFolderPaths.Add(gitUpdate.VirtualPath); - if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) - { - result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); - } Queue relativeFolderPaths = new Queue(); relativeFolderPaths.Enqueue(gitUpdate.VirtualPath); From 243f6214d7334a69df31fee1ffa3563e5e45f3af Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 25 Sep 2018 16:03:35 -0600 Subject: [PATCH 026/244] Fix test that moved new folder outside the repo --- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 49981b86a5..e3e9de6a51 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -34,7 +34,6 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase $"A {GVFSHelpers.ConvertPathToGitFormat(FileToRename)}", $"A {GVFSHelpers.ConvertPathToGitFormat(RenameFileTarget)}", $"A {FolderToCreate}/", - $"A {RenameFolderTarget}/", $"A {RenameNewDotGitFileTarget}", $"A {FileToCreateOutsideRepo}", $"A {FolderToCreateOutsideRepo}/", @@ -108,7 +107,7 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) string folderToRenameTarget = this.Enlistment.GetVirtualPathTo(RenameFolderTarget); fileSystem.MoveDirectory(folderToRename, folderToRenameTarget); - // Moving the new folder out of the repo should not change the modified paths file + // Moving the new folder out of the repo will remove it from the modified paths file string folderTargetOutsideSrc = Path.Combine(this.Enlistment.EnlistmentRoot, RenameFolderTarget); folderTargetOutsideSrc.ShouldNotExistOnDisk(fileSystem); fileSystem.MoveDirectory(folderToRenameTarget, folderTargetOutsideSrc); From 355fd062cdc290ea12211df99e70b0a71e662ed5 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 25 Sep 2018 13:36:04 -0700 Subject: [PATCH 027/244] Add new unit and functional tests --- .../FileSystemRunners/BashRunner.cs | 56 ++++-- .../EnlistmentPerFixture/SymbolicLinkTests.cs | 163 ++++++++++++++++++ .../NamedPipeStreamReaderWriterTests.cs | 2 + .../Mock/Mac/MockVirtualizationInstance.cs | 22 +++ .../MacFileSystemVirtualizerTests.cs | 105 ++++++++++- 5 files changed, 331 insertions(+), 17 deletions(-) create mode 100644 GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index cd1a79337a..af35b87372 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -53,6 +53,14 @@ public BashRunner() } } + private enum FileType + { + Invalid, + File, + Directory, + SymLink + } + protected override string FileName { get @@ -86,15 +94,22 @@ public static void DeleteDirectoryWithUnlimitedRetries(string path) } } - public override bool FileExists(string path) + public bool IsSymbolicLink(string path) { - string bashPath = this.ConvertWinPathToBashPath(path); + return this.FileExistsOnDisk(path, FileType.SymLink); + } - string command = string.Format("-c \"[ -f {0} ] && echo {1} || echo {2}\"", bashPath, ShellRunner.SuccessOutput, ShellRunner.FailureOutput); + public void CreateSymbolicLink(string newLinkFilePath, string existingFilePath) + { + string existingFileBashPath = this.ConvertWinPathToBashPath(existingFilePath); + string newLinkBashPath = this.ConvertWinPathToBashPath(newLinkFilePath); - string output = this.RunProcess(command).Trim(); + this.RunProcess(string.Format("-c \"ln -s -F {0} {1}\"", existingFileBashPath, newLinkBashPath)); + } - return output.Equals(ShellRunner.SuccessOutput, StringComparison.InvariantCulture); + public override bool FileExists(string path) + { + return this.FileExistsOnDisk(path, FileType.File); } public override string MoveFile(string sourcePath, string targetPath) @@ -187,11 +202,7 @@ public override void WriteAllTextShouldFail(string path, string c public override bool DirectoryExists(string path) { - string bashPath = this.ConvertWinPathToBashPath(path); - - string output = this.RunProcess(string.Format("-c \"[ -d {0} ] && echo {1} || echo {2}\"", bashPath, ShellRunner.SuccessOutput, ShellRunner.FailureOutput)).Trim(); - - return output.Equals(ShellRunner.SuccessOutput, StringComparison.InvariantCulture); + return this.FileExistsOnDisk(path, FileType.Directory); } public override void MoveDirectory(string sourcePath, string targetPath) @@ -267,6 +278,31 @@ public override void DeleteDirectory_ShouldBeBlockedByProcess(string path) Assert.Fail("Unlike the other runners, bash.exe does not check folder handle before recusively deleting"); } + private bool FileExistsOnDisk(string path, FileType type) + { + string checkArgument = string.Empty; + switch (type) + { + case FileType.File: + checkArgument = "-f"; + break; + case FileType.Directory: + checkArgument = "-d"; + break; + case FileType.SymLink: + checkArgument = "-h"; + break; + default: + Assert.Fail($"{nameof(FileExistsOnDisk)} does not support {nameof(FileType)} {type}"); + break; + } + + string bashPath = this.ConvertWinPathToBashPath(path); + string command = $"-c \"[ {checkArgument} {bashPath} ] && echo {ShellRunner.SuccessOutput} || echo {ShellRunner.FailureOutput}\""; + string output = this.RunProcess(command).Trim(); + return output.Equals(ShellRunner.SuccessOutput, StringComparison.InvariantCulture); + } + private string ConvertWinPathToBashPath(string winPath) { string bashPath = string.Concat("/", winPath); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs new file mode 100644 index 0000000000..04179c6711 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs @@ -0,0 +1,163 @@ +using System.IO; +using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.Should; +using GVFS.FunctionalTests.Tools; +using GVFS.Tests.Should; +using NUnit.Framework; + +namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture +{ + // MacOnly until issue #297 (add SymLink support for Windows) is complete + [Category(Categories.MacOnly)] + [TestFixture] + public class SymbolicLinkTests : TestsWithEnlistmentPerFixture + { + private const string TestFolderName = "Test_EPF_SymbolicLinks"; + + // FunctionalTests/20180925_SymLinksPart1 files + private const string TestFileName = "TestFile.txt"; + private const string TestFileContents = "This is a real file"; + private const string TestFile2Name = "TestFile2.txt"; + private const string TestFile2Contents = "This is the second real file"; + private const string ChildFolderName = "ChildDir"; + private const string ChildLinkName = "LinkToFileInFolder"; + private const string GrandChildLinkName = "LinkToFileInParentFolder"; + + // FunctionalTests/20180925_SymLinksPart2 files + // Note: In this branch ChildLinkName has been changed to point to TestFile2Name + private const string GrandChildFileName = "TestFile3.txt"; + private const string GrandChildFileContents = "This is the third file"; + private const string GrandChildLinkNowAFileContents = "This was a link but is now a file"; + + private BashRunner bashRunner; + public SymbolicLinkTests() + { + this.bashRunner = new BashRunner(); + } + + [TestCase, Order(1)] + public void CheckoutBranchWithSymLinks() + { + GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout FunctionalTests/20180925_SymLinksPart1"); + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart1", + "nothing to commit, working tree clean"); + + string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); + + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); + testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + + string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); + this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); + childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); + + string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); + this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeTrue($"{grandChildLinkPath} should be a symlink"); + grandChildLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + } + + [TestCase, Order(2)] + public void CheckoutBranchWhereSymLinksChangeContentsAndTransitionToFile() + { + GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout FunctionalTests/20180925_SymLinksPart2"); + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart2", + "nothing to commit, working tree clean"); + + // testFilePath and testFile2Path are unchanged from FunctionalTests/20180925_SymLinksPart2 + string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); + + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); + testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + + // In this branch childLinkPath has been changed to point to testFile2Path + string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); + this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); + childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + + // grandChildLinkPath should now be a file + string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); + this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeFalse($"{grandChildLinkPath} should not be a symlink"); + grandChildLinkPath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildLinkNowAFileContents); + + // There should also be a new file in the child folder + string newGrandChildFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildFileName)); + newGrandChildFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); + this.bashRunner.IsSymbolicLink(newGrandChildFilePath).ShouldBeFalse($"{newGrandChildFilePath} should not be a symlink"); + } + + [TestCase, Order(3)] + public void CheckoutBranchWhereFilesTransitionToSymLinks() + { + GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout FunctionalTests/20180925_SymLinksPart3"); + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart3", + "nothing to commit, working tree clean"); + + // In this branch testFilePath has been changed to point to newGrandChildFilePath + string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeTrue($"{testFilePath} should be a symlink"); + + // The rest of the files are unchanged from FunctionalTests/20180925_SymLinksPart2 + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); + testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + + string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); + this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); + childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + + string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); + this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeFalse($"{grandChildLinkPath} should not be a symlink"); + grandChildLinkPath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildLinkNowAFileContents); + + string newGrandChildFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildFileName)); + newGrandChildFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); + this.bashRunner.IsSymbolicLink(newGrandChildFilePath).ShouldBeFalse($"{newGrandChildFilePath} should not be a symlink"); + } + + [TestCase, Order(4)] + public void GitStatusReportsSymLinkChanges() + { + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart3", + "nothing to commit, working tree clean"); + + string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeTrue($"{testFilePath} should be a symlink"); + + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); + testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + + // Update testFilePath's symlink to point to testFile2Path + this.bashRunner.CreateSymbolicLink(testFilePath, testFile2Path); + + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeTrue($"{testFilePath} should be a symlink"); + + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart3", + $"modified: {TestFolderName}/{TestFileName}"); + } + } +} diff --git a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs index 003ca048fe..7e630fea6a 100644 --- a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs +++ b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs @@ -1,5 +1,6 @@ using GVFS.Common.NamedPipes; using GVFS.Tests.Should; +using GVFS.UnitTests.Category; using NUnit.Framework; using System.IO; @@ -58,6 +59,7 @@ public void CanSendBufferSizedMessage() [Test] [Description("Verify that the expected exception is thrown if message is not terminated with expected byte.")] + [Category(CategoryConstants.ExceptionExpected)] public void ReadingPartialMessgeThrows() { byte[] bytes = System.Text.Encoding.ASCII.GetBytes("This is a partial message"); diff --git a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs index 57ae1b7204..082565d229 100644 --- a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs +++ b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs @@ -15,6 +15,7 @@ public MockVirtualizationInstance() { this.commandCompleted = new AutoResetEvent(false); this.CreatedPlaceholders = new ConcurrentDictionary(); + this.CreatedSymLinks = new ConcurrentHashSet(); this.WriteFileReturnResult = Result.Success; } @@ -28,6 +29,8 @@ public MockVirtualizationInstance() public ConcurrentDictionary CreatedPlaceholders { get; private set; } + public ConcurrentHashSet CreatedSymLinks { get; } + public override EnumerateDirectoryCallback OnEnumerateDirectory { get; set; } public override GetFileStreamCallback OnGetFileStream { get; set; } @@ -79,6 +82,14 @@ public override Result WritePlaceholderFile( return Result.Success; } + public override Result WriteSymLink( + string relativePath, + string symLinkContents) + { + this.CreatedSymLinks.Add(relativePath); + return Result.Success; + } + public override Result UpdatePlaceholderIfNeeded( string relativePath, byte[] providerId, @@ -92,6 +103,17 @@ public override Result UpdatePlaceholderIfNeeded( return this.UpdatePlaceholderIfNeededResult; } + public override Result ReplacePlaceholderFileWithSymLink( + string relativePath, + string symLinkContents, + UpdateType updateFlags, + out UpdateFailureCause failureCause) + { + this.CreatedSymLinks.Add(relativePath); + failureCause = this.UpdatePlaceholderIfNeededFailureCause; + return this.UpdatePlaceholderIfNeededResult; + } + public override Result CompleteCommand( ulong commandId, Result result) diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 6d4d19d6d0..9be03ae2cb 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -10,6 +10,7 @@ using GVFS.UnitTests.Virtual; using GVFS.Virtualization; using GVFS.Virtualization.FileSystem; +using GVFS.Virtualization.Projection; using NUnit.Framework; using PrjFSLib.Mac; using System; @@ -21,7 +22,6 @@ namespace GVFS.UnitTests.Platform.Mac [TestFixture] public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { - private static readonly ushort RegularFileType = 0x8000; private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); private static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); @@ -100,7 +100,7 @@ public void UpdatePlaceholderIfNeeded() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)(RegularFileType | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -163,6 +163,97 @@ public void UpdatePlaceholderIfNeeded() } } + [TestCase] + public void WritePlaceholderForSymLink() + { + using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) + using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) + using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + new MockBlobSizes(), + gitIndexProjection, + backgroundTaskRunner, + virtualizer)) + { + string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)GitIndexProjection.FileType.SymLink); + string error; + fileSystemCallbacks.TryStart(out error).ShouldEqual(true); + + virtualizer.WritePlaceholderFile( + filePath, + endOfFile: 0, + sha: string.Empty).ShouldEqual(new FileSystemResult(FSResult.Ok, (int)Result.Success)); + + mockVirtualization.CreatedPlaceholders.ShouldBeEmpty(); + mockVirtualization.CreatedSymLinks.Count.ShouldEqual(1); + mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(filePath)); + + // Creating a symlink should schedule a background task + backgroundTaskRunner.Count.ShouldEqual(1); + backgroundTaskRunner.BackgroundTasks[0].Operation.ShouldEqual(GVFS.Virtualization.Background.FileSystemTask.OperationType.OnFileSymLinkCreated); + backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(filePath); + + fileSystemCallbacks.Stop(); + } + } + + [TestCase] + public void UpdatePlaceholderToSymLink() + { + using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) + using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) + using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + new MockBlobSizes(), + gitIndexProjection, + backgroundTaskRunner, + virtualizer)) + { + string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)GitIndexProjection.FileType.SymLink); + string error; + fileSystemCallbacks.TryStart(out error).ShouldEqual(true); + + UpdateFailureReason failureReason = UpdateFailureReason.NoFailure; + + mockVirtualization.UpdatePlaceholderIfNeededResult = Result.Success; + mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; + virtualizer + .UpdatePlaceholderIfNeeded( + filePath, + DateTime.Now, + DateTime.Now, + DateTime.Now, + DateTime.Now, + 0, + 15, + string.Empty, + UpdatePlaceholderType.AllowReadOnly, + out failureReason) + .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.UpdatePlaceholderIfNeededResult)); + failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.UpdatePlaceholderIfNeededFailureCause); + + mockVirtualization.CreatedSymLinks.Count.ShouldEqual(1); + mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(filePath)); + + // Creating a symlink should schedule a background task + backgroundTaskRunner.Count.ShouldEqual(1); + backgroundTaskRunner.BackgroundTasks[0].Operation.ShouldEqual(GVFS.Virtualization.Background.FileSystemTask.OperationType.OnFileSymLinkCreated); + backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(filePath); + + fileSystemCallbacks.Stop(); + } + } + [TestCase] public void ClearNegativePathCacheIsNoOp() { @@ -191,7 +282,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -221,7 +312,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -252,9 +343,9 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", (ushort)(RegularFileType | FileMode644)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", (ushort)(RegularFileType | FileMode664)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", (ushort)(RegularFileType | FileMode755)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); From 42a399121a47bb07862afcbd6acdbe1a58621ef4 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 26 Sep 2018 12:55:14 -0400 Subject: [PATCH 028/244] Fix a flaky test involving the background prefetch thread --- .../Tests/EnlistmentPerFixture/PrefetchVerbTests.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs index cb0a2a4a67..bcfa9cbd31 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs @@ -131,10 +131,13 @@ private void PostFetchJobShouldComplete() string objectDir = this.Enlistment.GetObjectRoot(this.fileSystem); string postFetchLock = Path.Combine(objectDir, "post-fetch.lock"); - while (this.fileSystem.FileExists(postFetchLock)) + // Wait first, to hopefully ensure the background thread has + // started before we check for the lock file. + do { Thread.Sleep(500); } + while (this.fileSystem.FileExists(postFetchLock)); ProcessResult midxResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "midx --read --pack-dir=\"" + objectDir + "/pack\""); midxResult.ExitCode.ShouldEqual(0); From 6c38d6c1d702f231f937d73a95c206da4448c6ce Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 27 Sep 2018 14:07:23 -0600 Subject: [PATCH 029/244] Add wait for background operations to complete in modified paths test --- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index e3e9de6a51..ee4c1e060f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -193,6 +193,8 @@ public void ModifiedPathsCorrectAfterHardLinking(FileSystemRunner fileSystem) fileSystem.CreateHardLink(hardLinkOutsideRepoToFileInRepoPath, secondFileInRepoPath); hardLinkOutsideRepoToFileInRepoPath.ShouldBeAFile(fileSystem).WithContents(contents); + this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + string modifiedPathsDatabase = Path.Combine(this.Enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); using (StreamReader reader = new StreamReader(File.Open(modifiedPathsDatabase, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) From 25ac9c543b449632a52593c65fccedbea8a30685 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 27 Sep 2018 09:14:47 -0700 Subject: [PATCH 030/244] PR Feedback: Cleanup and flag link as in root --- .../FileSystemRunners/BashRunner.cs | 2 +- .../MacFileSystemVirtualizer.cs | 99 ++++++++++++------- .../Mock/Mac/MockVirtualizationInstance.cs | 4 +- .../Projection/MockGitIndexProjection.cs | 10 +- .../MacFileSystemVirtualizerTests.cs | 16 +-- .../Projection/FileTypeAndMode.cs | 78 +++++++++++++++ .../GitIndexProjection.GitIndexEntry.cs | 2 +- .../GitIndexProjection.GitIndexParser.cs | 31 +++--- .../Projection/GitIndexProjection.cs | 35 ++----- .../PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs | 4 +- .../VirtualizationInstance.cs | 8 +- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 22 +++-- ProjFS.Mac/PrjFSLib/PrjFSLib.h | 4 +- 13 files changed, 202 insertions(+), 113 deletions(-) create mode 100644 GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index af35b87372..747345cd56 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -58,7 +58,7 @@ private enum FileType Invalid, File, Directory, - SymLink + SymLink, } protected override string FileName diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index c1fa1f3409..e351f4f751 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -17,6 +17,8 @@ public class MacFileSystemVirtualizer : FileSystemVirtualizer { public static readonly byte[] PlaceholderVersionId = ToVersionIdByteArray(new byte[] { PlaceholderVersion }); + private const int SymLinkTargetBufferSize = 4096; + private const string ClassName = nameof(MacFileSystemVirtualizer); private VirtualizationInstance virtualizationInstance; @@ -81,26 +83,25 @@ public override FileSystemResult WritePlaceholderFile( string sha) { // TODO(Mac): Add functional tests that validate file mode is set correctly - ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - GitIndexProjection.FileType fileType = (GitIndexProjection.FileType)(fileTypeAndMode & GitIndexProjection.FileTypeMask); + FileTypeAndMode fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - if (fileType == GitIndexProjection.FileType.Regular) + if (fileTypeAndMode.Type == FileTypeAndMode.FileType.Regular) { Result result = this.virtualizationInstance.WritePlaceholderFile( relativePath, PlaceholderVersionId, ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), (ulong)endOfFile, - (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask)); + fileTypeAndMode.Mode); return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } - else if (fileType == GitIndexProjection.FileType.SymLink) + else if (fileTypeAndMode.Type == FileTypeAndMode.FileType.SymLink) { - string symLinkContents; - if (this.TryGetSymLinkObjectContents(sha, out symLinkContents)) + string symLinkTarget; + if (this.TryGetSymLinkTarget(sha, out symLinkTarget)) { - Result result = this.virtualizationInstance.WriteSymLink(relativePath, symLinkContents); + Result result = this.virtualizationInstance.WriteSymLink(relativePath, symLinkTarget); this.FileSystemCallbacks.OnFileSymLinkCreated(relativePath); @@ -115,7 +116,7 @@ public override FileSystemResult WritePlaceholderFile( else { EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add(nameof(fileType), fileType); + metadata.Add("FileType", fileTypeAndMode.Type); this.Context.Tracer.RelatedError(metadata, $"{nameof(this.WritePlaceholderFile)}: Unsupported fileType"); return new FileSystemResult(FSResult.IOError, 0); } @@ -144,31 +145,30 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( // TODO(Mac): Add functional tests that include: // - Mode + content changes between commits // - Mode only changes (without any change to content, see issue #223) - ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - GitIndexProjection.FileType fileType = (GitIndexProjection.FileType)(fileTypeAndMode & GitIndexProjection.FileTypeMask); + FileTypeAndMode fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - if (fileType == GitIndexProjection.FileType.Regular) + if (fileTypeAndMode.Type == FileTypeAndMode.FileType.Regular) { Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( relativePath, PlaceholderVersionId, ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), (ulong)endOfFile, - (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask), + fileTypeAndMode.Mode, (UpdateType)updateFlags, out failureCause); failureReason = (UpdateFailureReason)failureCause; return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } - else if (fileType == GitIndexProjection.FileType.SymLink) + else if (fileTypeAndMode.Type == FileTypeAndMode.FileType.SymLink) { - string symLinkContents; - if (this.TryGetSymLinkObjectContents(shaContentId, out symLinkContents)) + string symLinkTarget; + if (this.TryGetSymLinkTarget(shaContentId, out symLinkTarget)) { Result result = this.virtualizationInstance.ReplacePlaceholderFileWithSymLink( relativePath, - symLinkContents, + symLinkTarget, (UpdateType)updateFlags, out failureCause); @@ -187,8 +187,8 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( else { EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add(nameof(fileType), fileType); - metadata.Add(nameof(fileTypeAndMode), fileTypeAndMode); + metadata.Add("FileType", fileTypeAndMode.Type); + metadata.Add("FileMode", fileTypeAndMode.Mode); this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Unsupported fileType"); failureReason = UpdateFailureReason.NoFailure; return new FileSystemResult(FSResult.IOError, 0); @@ -232,11 +232,16 @@ private static byte[] ToVersionIdByteArray(byte[] version) return bytes; } - private bool TryGetSymLinkObjectContents(string sha, out string contents) + ///

+ /// Gets the target of the symbolic link. + /// + /// SHA of the loose object containing the target path of the symbolic link + /// Target path of the symbolic link + private bool TryGetSymLinkTarget(string sha, out string symLinkTarget) { - contents = null; - StringBuilder objectContents = new StringBuilder(); + symLinkTarget = null; + string symLinkBlobContents = null; try { if (!this.GitObjects.TryCopyBlobContentStream( @@ -245,9 +250,10 @@ private bool TryGetSymLinkObjectContents(string sha, out string contents) GVFSGitObjects.RequestSource.SymLinkCreation, (stream, blobLength) => { - // TODO(Mac): Find a better solution than reading from the stream one byte at at time - byte[] buffer = new byte[4096]; + byte[] buffer = new byte[SymLinkTargetBufferSize]; uint bufferIndex = 0; + + // TODO(Mac): Find a better solution than reading from the stream one byte at at time int nextByte = stream.ReadByte(); while (nextByte != -1) { @@ -258,38 +264,51 @@ private bool TryGetSymLinkObjectContents(string sha, out string contents) ++bufferIndex; } - while (bufferIndex < buffer.Length) + if (bufferIndex < buffer.Length) { buffer[bufferIndex] = 0; - ++bufferIndex; - } - - objectContents.Append(System.Text.Encoding.ASCII.GetString(buffer)); - - if (bufferIndex == buffer.Length) + symLinkBlobContents = Encoding.UTF8.GetString(buffer); + } + else { - bufferIndex = 0; + buffer[bufferIndex - 1] = 0; + + EventMetadata metadata = this.CreateEventMetadata(); + metadata.Add(nameof(sha), sha); + metadata.Add("bufferContents", Encoding.UTF8.GetString(buffer)); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: SymLink target exceeds buffer size"); + + throw new GetSymLinkTargetException("SymLink target exceeds buffer size");; } } })) { EventMetadata metadata = this.CreateEventMetadata(); metadata.Add(nameof(sha), sha); - this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkObjectContents)}: TryCopyBlobContentStream failed"); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream failed"); return false; } } - catch (GetFileStreamException e) + catch (GetSymLinkTargetException e) { EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e); metadata.Add(nameof(sha), sha); - this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkObjectContents)}: TryCopyBlobContentStream caught GetFileStreamException"); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream caught GetSymLinkTargetException"); return false; } + catch (DecoderFallbackException e) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e); + metadata.Add(nameof(sha), sha); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream caught DecoderFallbackException"); + + return false; + } + + symLinkTarget = symLinkBlobContents; - contents = objectContents.ToString(); return true; } @@ -664,5 +683,13 @@ public GetFileStreamException(string message, Result result) public Result Result { get; } } + + private class GetSymLinkTargetException : Exception + { + public GetSymLinkTargetException(string message) + : base(message) + { + } + } } } diff --git a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs index 082565d229..156ae8382e 100644 --- a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs +++ b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs @@ -84,7 +84,7 @@ public override Result WritePlaceholderFile( public override Result WriteSymLink( string relativePath, - string symLinkContents) + string symLinkTarget) { this.CreatedSymLinks.Add(relativePath); return Result.Success; @@ -105,7 +105,7 @@ public override Result UpdatePlaceholderIfNeeded( public override Result ReplacePlaceholderFileWithSymLink( string relativePath, - string symLinkContents, + string symLinkTarget, UpdateType updateFlags, out UpdateFailureCause failureCause) { diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs index 9bd7c5f5a6..ca877baa86 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs @@ -36,7 +36,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) this.PlaceholdersCreated = new ConcurrentHashSet(); this.ExpandedFolders = new ConcurrentHashSet(); - this.MockFileTypesAndModes = new ConcurrentDictionary(); + this.MockFileTypesAndModes = new ConcurrentDictionary(); this.unblockGetProjectedItems = new ManualResetEvent(true); this.waitForGetProjectedItems = new ManualResetEvent(true); @@ -56,7 +56,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) public ConcurrentHashSet ExpandedFolders { get; } - public ConcurrentDictionary MockFileTypesAndModes { get; } + public ConcurrentDictionary MockFileTypesAndModes { get; } public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } @@ -161,15 +161,15 @@ public override bool TryGetProjectedItemsFromMemory(string folderPath, out List< return false; } - public override ushort GetFileTypeAndMode(string path) + public override FileTypeAndMode GetFileTypeAndMode(string path) { - ushort result; + FileTypeAndMode result; if (this.MockFileTypesAndModes.TryGetValue(path, out result)) { return result; } - return 0; + return new FileTypeAndMode(0); } public override List GetProjectedItems( diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 9be03ae2cb..81c52e43f7 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -100,7 +100,7 @@ public void UpdatePlaceholderIfNeeded() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, FileTypeAndMode.Regular644File); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -180,7 +180,7 @@ public void WritePlaceholderForSymLink() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)GitIndexProjection.FileType.SymLink); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode((ushort)FileTypeAndMode.FileType.SymLink)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -219,7 +219,7 @@ public void UpdatePlaceholderToSymLink() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)GitIndexProjection.FileType.SymLink); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode((ushort)FileTypeAndMode.FileType.SymLink)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -282,7 +282,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileTypeAndMode.Regular644File); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -312,7 +312,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileTypeAndMode.Regular644File); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -343,9 +343,9 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode664)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode755)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", FileTypeAndMode.Regular644File); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", new FileTypeAndMode((ushort)((ushort)FileTypeAndMode.FileType.Regular | FileMode664))); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", new FileTypeAndMode((ushort)((ushort)FileTypeAndMode.FileType.Regular | FileMode755))); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); diff --git a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs new file mode 100644 index 0000000000..0afab8cc26 --- /dev/null +++ b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs @@ -0,0 +1,78 @@ +using System; +namespace GVFS.Virtualization.Projection +{ + public struct FileTypeAndMode + { + public static readonly ushort FileMode755; + public static readonly ushort FileMode664; + public static readonly ushort FileMode644; + public static readonly FileTypeAndMode Regular644File; + + // Bitmasks for extracting file type and mode from the ushort stored in the index + private const ushort FileTypeMask = 0xF000; + private const ushort FileModeMask = 0x1FF; + + private readonly ushort fileTypeAndMode; + + static FileTypeAndMode() + { + FileMode755 = Convert.ToUInt16("755", 8); + FileMode664 = Convert.ToUInt16("664", 8); + FileMode644 = Convert.ToUInt16("644", 8); + + Regular644File = new FileTypeAndMode((ushort)((ushort)FileType.Regular | FileMode644)); + } + + public FileTypeAndMode(ushort typeAndModeInIndexFormat) + { + this.fileTypeAndMode = typeAndModeInIndexFormat; + } + + public enum FileType : ushort + { + Invalid = 0, + + Regular = 0x8000, + SymLink = 0xA000, + GitLink = 0xE000, + } + + public FileType Type => (FileType)(this.fileTypeAndMode & FileTypeMask); + public ushort Mode => (ushort)(this.fileTypeAndMode & FileModeMask); + + public static bool operator ==(FileTypeAndMode lhs, FileTypeAndMode rhs) + { + return lhs.Equals(rhs); + } + + public static bool operator !=(FileTypeAndMode lhs, FileTypeAndMode rhs) + { + return !lhs.Equals(rhs); + } + + public override bool Equals(object obj) + { + if (obj is FileTypeAndMode) + { + return this.Equals((FileTypeAndMode)obj); + } + + return false; + } + + public override int GetHashCode() + { + return this.fileTypeAndMode; + } + + public bool Equals(FileTypeAndMode otherFileTypeAndMode) + { + return this.fileTypeAndMode == otherFileTypeAndMode.fileTypeAndMode; + } + + public string GetModeAsOctalString() + { + return Convert.ToString(this.Mode, 8); + } + } +} diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 612931242c..8cd38e283d 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -27,7 +27,7 @@ public GitIndexEntry() public byte[] Sha { get; } = new byte[20]; public bool SkipWorktree { get; set; } - public ushort FileTypeAndMode { get; set; } + public FileTypeAndMode TypeAndMode { get; set; } public GitIndexParser.MergeStage MergeState { get; set; } public int ReplaceIndex { get; set; } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index ca5f88dade..3753abb7c5 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -16,10 +16,6 @@ internal partial class GitIndexParser private const ushort ExtendedBit = 0x4000; private const ushort SkipWorktreeBit = 0x4000; - private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); - private static readonly ushort FileMode664 = Convert.ToUInt16("644", 8); - private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); - private Stream indexStream; private byte[] page; private int nextByteIndex; @@ -185,35 +181,36 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func // 3-bit unused // 9-bit unix permission. Only 0755 and 0644 are valid for regular files. (Legacy repos can also contain 664) // Symbolic links and gitlinks have value 0 in this field. - ushort typeAndMode = this.ReadUInt16(); + ushort indexFormatTypeAndMode = this.ReadUInt16(); - ushort type = (ushort)(typeAndMode & FileTypeMask); - ushort mode = (ushort)(typeAndMode & FileModeMask); + FileTypeAndMode typeAndMode = new FileTypeAndMode(indexFormatTypeAndMode); - switch ((FileType)type) + switch (typeAndMode.Type) { - case FileType.Regular: - if (mode != FileMode755 && mode != FileMode644 && mode != FileMode664) + case FileTypeAndMode.FileType.Regular: + if (typeAndMode.Mode != FileTypeAndMode.FileMode755 && + typeAndMode.Mode != FileTypeAndMode.FileMode644 && + typeAndMode.Mode != FileTypeAndMode.FileMode664) { - throw new InvalidDataException($"Invalid file mode {Convert.ToString(mode, 8)} found for regular file in index"); + throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for regular file in index"); } break; - case FileType.SymLink: - case FileType.GitLink: - if (mode != 0) + case FileTypeAndMode.FileType.SymLink: + case FileTypeAndMode.FileType.GitLink: + if (typeAndMode.Mode != 0) { - throw new InvalidDataException($"Invalid file mode {Convert.ToString(mode, 8)} found for link file({type:X}) in index"); + throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for link file({typeAndMode.Type:X}) in index"); } break; default: - throw new InvalidDataException($"Invalid file type {type:X} found in index"); + throw new InvalidDataException($"Invalid file type {typeAndMode.Type:X} found in index"); } - this.resuableParsedIndexEntry.FileTypeAndMode = typeAndMode; + this.resuableParsedIndexEntry.TypeAndMode = typeAndMode; this.Skip(12); } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 8c1dbe9245..deaa26d1e4 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -22,12 +22,6 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject { public const string ProjectionIndexBackupName = "GVFS_projection"; - // Bitmasks for extracting file type and mode from the ushort stored in the index - public const ushort FileTypeMask = 0xF000; - public const ushort FileModeMask = 0x1FF; - - protected static readonly ushort Regular644File = (ushort)((ushort)FileType.Regular | Convert.ToUInt16("644", 8)); - private const int IndexFileStreamBufferSize = 512 * 1024; private const UpdatePlaceholderType FolderPlaceholderDeleteFlags = @@ -56,9 +50,9 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject // Cache of folder paths (in Windows format) to folder data private ConcurrentDictionary projectionFolderCache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - // nonDefaultFileModes is only populated when the platform supports file mode - // On platforms that support file modes, file paths that are not in nonDefaultFileModes have mode 644 - private Dictionary nonDefaultFileTypesAndModes = new Dictionary(StringComparer.OrdinalIgnoreCase); + // nonDefaultFileTypesAndModes is only populated when the platform supports file mode + // On platforms that support file modes, file paths that are not in nonDefaultFileTypesAndModes are regular files with mode 644 + private Dictionary nonDefaultFileTypesAndModes = new Dictionary(StringComparer.OrdinalIgnoreCase); private BlobSizes blobSizes; private PlaceholderListDatabase placeholderList; @@ -122,15 +116,6 @@ protected GitIndexProjection() { } - public enum FileType : ushort - { - Invalid = 0, - - Regular = 0x8000, - SymLink = 0xA000, - GitLink = 0xE000, - } - public int EstimatedPlaceholderCount { get @@ -369,7 +354,7 @@ public virtual bool TryGetProjectedItemsFromMemory(string folderPath, out List

Date: Tue, 25 Sep 2018 14:35:00 -0400 Subject: [PATCH 031/244] Set config to useHttpPath=true Hostname is no longer sufficent for VSTS authentication. VSTS now requires dev.azure.com/account to determine the tenant. By setting useHttpPath, credential managers will get the path which contains the account as the first parameter. They can then use this information for auth appropriately. --- GVFS/GVFS.Common/Git/GitConfigSetting.cs | 1 + GVFS/GVFS.Common/Git/GitProcess.cs | 2 +- GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs | 2 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitConfigSetting.cs b/GVFS/GVFS.Common/Git/GitConfigSetting.cs index 12ea0eb23d..fc836d2348 100644 --- a/GVFS/GVFS.Common/Git/GitConfigSetting.cs +++ b/GVFS/GVFS.Common/Git/GitConfigSetting.cs @@ -6,6 +6,7 @@ public class GitConfigSetting { public const string CoreVirtualizeObjectsName = "core.virtualizeobjects"; public const string CoreVirtualFileSystemName = "core.virtualfilesystem"; + public const string CredentialUseHttpPath = "credential.useHttpPath"; public GitConfigSetting(string name, params string[] values) { diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 47ad144e83..5c3bba3cde 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -112,7 +112,7 @@ public virtual bool TryGetCredentials( using (ITracer activity = tracer.StartActivity("TryGetCredentials", EventLevel.Informational)) { Result gitCredentialOutput = this.InvokeGitAgainstDotGitFolder( - "credential fill", + "-c " + GitConfigSetting.CredentialUseHttpPath + "=true credential fill", stdin => stdin.Write("url=" + repoUrl + "\n\n"), parseStdOutLine: null); diff --git a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs index abe2bac1ac..bc22fd7c13 100644 --- a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs +++ b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs @@ -194,7 +194,7 @@ private MockGitProcess GetGitProcess() int revocations = 0; gitProcess.SetExpectedCommandResult( - "credential fill", + "-c credential.useHttpPath=true credential fill", () => new GitProcess.Result("username=username\r\npassword=password" + revocations + "\r\n", string.Empty, GitProcess.Result.SuccessCode)); gitProcess.SetExpectedCommandResult( diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 325e2dceeb..d001699c35 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -92,6 +92,7 @@ public static bool TrySetRequiredGitConfigSettings(Enlistment enlistment) { GitConfigSetting.CoreVirtualizeObjectsName, "true" }, { GitConfigSetting.CoreVirtualFileSystemName, Paths.ConvertPathToGitFormat(GVFSConstants.DotGit.Hooks.VirtualFileSystemPath) }, { "core.hookspath", expectedHooksPath }, + { GitConfigSetting.CredentialUseHttpPath, "true" }, { "credential.validate", "false" }, { "diff.autoRefreshIndex", "false" }, { "gc.auto", "0" }, From ae21413746629f3d7fa0902557017ec92e1882f5 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 14:32:28 -0600 Subject: [PATCH 032/244] Check modified paths for parent directories before adding --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 20 +++- .../EnlistmentPerFixture/GitFilesTests.cs | 2 - .../Common/ModifiedPathsDatabaseTests.cs | 95 ++++++++++++++----- 3 files changed, 89 insertions(+), 28 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index ffde4f6927..338c1f4ce2 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -67,7 +69,7 @@ public bool TryAdd(string path, bool isFolder, out bool isRetryable) { isRetryable = true; string entry = this.NormalizeEntryString(path, isFolder); - if (!this.modifiedPaths.Contains(entry)) + if (!this.modifiedPaths.Contains(entry) && !this.ContainsParentDirectory(entry)) { try { @@ -183,6 +185,22 @@ private bool TryParseRemoveLine(string line, out string key, out string error) return true; } + private bool ContainsParentDirectory(string modifiedPath) + { + string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + StringBuilder parentFolder = new StringBuilder(); + foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) + { + parentFolder.Append(pathPart + "/"); + if (this.modifiedPaths.Contains(parentFolder.ToString())) + { + return true; + } + } + + return false; + } + private string NormalizeEntryString(string virtualPath, bool isFolder) { // TODO(Mac) This can be optimized if needed diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index bfbdcc4576..8a0a0c3f64 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -77,7 +77,6 @@ public void CreateFileInFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName); } [TestCase, Order(4)] @@ -140,7 +139,6 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() string[] expectedModifiedPathsEntriesAfterCreate = { "A Folder/", - "A Folder/testfile", }; string[] expectedModifiedPathsEntriesAfterRename = diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index 3134842008..663a76bb08 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -22,31 +22,31 @@ A dir1/dir2/file3.txt [TestCase] public void ParsesExistingDataCorrectly() { - ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(ExistingEntries); - mpd.Count.ShouldEqual(3); - mpd.Contains("file.txt", isFolder: false).ShouldBeTrue(); - mpd.Contains("dir/file2.txt", isFolder: false).ShouldBeTrue(); - mpd.Contains("dir1/dir2/file3.txt", isFolder: false).ShouldBeTrue(); + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(ExistingEntries); + modifiedPathsDatabase.Count.ShouldEqual(3); + modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir/file2.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/dir2/file3.txt", isFolder: false).ShouldBeTrue(); } [TestCase] public void AddsDefaultEntry() { - ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(initialContents: string.Empty); - mpd.Count.ShouldEqual(1); - mpd.Contains(DefaultEntry, isFolder: false).ShouldBeTrue(); + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(initialContents: string.Empty); + modifiedPathsDatabase.Count.ShouldEqual(1); + modifiedPathsDatabase.Contains(DefaultEntry, isFolder: false).ShouldBeTrue(); } [TestCase] public void BadDataFailsToLoad() { - ConfigurableFileSystem fs = new ConfigurableFileSystem(); - fs.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream("This is bad data!\r\n")); + ConfigurableFileSystem configurableFileSystem = new ConfigurableFileSystem(); + configurableFileSystem.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream("This is bad data!\r\n")); string error; - ModifiedPathsDatabase mpd; - ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, fs, out mpd, out error).ShouldBeFalse(); - mpd.ShouldBeNull(); + ModifiedPathsDatabase modifiedPathsDatabase; + ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, configurableFileSystem, out modifiedPathsDatabase, out error).ShouldBeFalse(); + modifiedPathsDatabase.ShouldBeNull(); } [TestCase] @@ -74,6 +74,51 @@ public void DirectorySeparatorAddedForFolder() TestAddingPath(pathToAdd: Path.Combine("dir", "subdir"), pathInList: Path.Combine("dir", "subdir") + Path.DirectorySeparatorChar, isFolder: true); } + [TestCase] + public void EntryNotAddedIfParentDirectoryExists() + { + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(initialContents: "A dir/\r\n"); + modifiedPathsDatabase.Count.ShouldEqual(1); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + + // Try adding a file for the directory that is in the modified paths + modifiedPathsDatabase.TryAdd("dir/file.txt", isFolder: false, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(1); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + + // Try adding a directory for the directory that is in the modified paths + modifiedPathsDatabase.TryAdd("dir/dir2", isFolder: true, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(1); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + + // Try adding a file for a directory that is not in the modified paths + modifiedPathsDatabase.TryAdd("dir2/file.txt", isFolder: false, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(2); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue(); + + // Try adding a directory for a the directory that is not in the modified paths + modifiedPathsDatabase.TryAdd("dir2/dir", isFolder: true, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(3); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue(); + + // Try adding a file in a subdirectory that is in the modified paths + modifiedPathsDatabase.TryAdd("dir2/dir/file.txt", isFolder: false, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(3); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue(); + + // Try adding a directory for a subdirectory that is in the modified paths + modifiedPathsDatabase.TryAdd("dir2/dir/dir3", isFolder: true, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(3); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue(); + } + private static void TestAddingPath(string path, bool isFolder = false) { TestAddingPath(path, path, isFolder); @@ -81,13 +126,13 @@ private static void TestAddingPath(string path, bool isFolder = false) private static void TestAddingPath(string pathToAdd, string pathInList, bool isFolder = false) { - ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(initialContents: $"A {DefaultEntry}\r\n"); + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(initialContents: $"A {DefaultEntry}\r\n"); bool isRetryable; - mpd.TryAdd(pathToAdd, isFolder, out isRetryable); - mpd.Count.ShouldEqual(2); - mpd.Contains(pathInList, isFolder).ShouldBeTrue(); - mpd.Contains(ToGitPathSeparators(pathInList), isFolder).ShouldBeTrue(); - mpd.GetAllModifiedPaths().ShouldContainSingle(x => string.Compare(x, ToGitPathSeparators(pathInList), StringComparison.OrdinalIgnoreCase) == 0); + modifiedPathsDatabase.TryAdd(pathToAdd, isFolder, out isRetryable); + modifiedPathsDatabase.Count.ShouldEqual(2); + modifiedPathsDatabase.Contains(pathInList, isFolder).ShouldBeTrue(); + modifiedPathsDatabase.Contains(ToGitPathSeparators(pathInList), isFolder).ShouldBeTrue(); + modifiedPathsDatabase.GetAllModifiedPaths().ShouldContainSingle(x => string.Compare(x, ToGitPathSeparators(pathInList), StringComparison.OrdinalIgnoreCase) == 0); } private static string ToGitPathSeparators(string path) @@ -102,14 +147,14 @@ private static string ToPathSeparators(string path) private static ModifiedPathsDatabase CreateModifiedPathsDatabase(string initialContents) { - ConfigurableFileSystem fs = new ConfigurableFileSystem(); - fs.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream(initialContents)); + ConfigurableFileSystem configurableFileSystem = new ConfigurableFileSystem(); + configurableFileSystem.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream(initialContents)); string error; - ModifiedPathsDatabase mpd; - ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, fs, out mpd, out error).ShouldBeTrue(); - mpd.ShouldNotBeNull(); - return mpd; + ModifiedPathsDatabase modifiedPathsDatabase; + ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, configurableFileSystem, out modifiedPathsDatabase, out error).ShouldBeTrue(); + modifiedPathsDatabase.ShouldNotBeNull(); + return modifiedPathsDatabase; } } } From 8c8c06f9b7f198d986d20c29481e0066260e14e3 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 28 Sep 2018 11:59:51 -0600 Subject: [PATCH 033/244] Fix test the will no longer have child entries of a folder in the modified paths --- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index eb1ce473d9..1f39f0285e 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -328,8 +328,6 @@ public void MountCreatesModifiedPathsDatabase() { "A .gitattributes", "A developer/me/", - "A developer/me/JLANGE9._prerazzle", - "A developer/me/StateSwitch.Save", "A tools/x86/remote.exe", "A tools/x86/runelevated.exe", "A tools/amd64/remote.exe", From d4ae8e113ab56b77028a9125c4f5c3b17b65d751 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 28 Sep 2018 11:40:39 -0700 Subject: [PATCH 034/244] PR Feedback: Code cleanup --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 18 ++++++++++++------ ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 1 - 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 13e4cacb59..dbf425d533 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -266,6 +266,8 @@ static int HandleVnodeOperation( uint32_t currentVnodeFileFlags; int pid; char procname[MAXCOMLEN + 1]; + bool isDeleteAction = false; + bool isDirectory = false; // TODO(Mac): Issue #271 - Reduce reliance on vn_getpath // Call vn_getpath first when the cache is hottest to increase the chances @@ -289,11 +291,16 @@ static int HandleVnodeOperation( goto CleanupAndReturn; } - if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) + isDeleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); + isDirectory = VDIR == vnodeType; + + if (isDeleteAction) { if (!TrySendRequestAndWaitForResponse( root, - VDIR == vnodeType ? MessageType_KtoU_NotifyDirectoryPreDelete : MessageType_KtoU_NotifyFilePreDelete, + isDirectory ? + MessageType_KtoU_NotifyDirectoryPreDelete : + MessageType_KtoU_NotifyFilePreDelete, currentVnode, vnodePath, pid, @@ -305,7 +312,7 @@ static int HandleVnodeOperation( } } - if (VDIR == vnodeType) + if (isDirectory) { if (ActionBitIsSet( action, @@ -316,13 +323,12 @@ static int HandleVnodeOperation( KAUTH_VNODE_READ_EXTATTRIBUTES | KAUTH_VNODE_DELETE)) { - bool deleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); // Recursively expand directory on delete to ensure child placeholders are created before rename operations - if (deleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) + if (isDeleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { if (!TrySendRequestAndWaitForResponse( root, - deleteAction ? + isDeleteAction ? MessageType_KtoU_RecursivelyEnumerateDirectory : MessageType_KtoU_EnumerateDirectory, currentVnode, diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index eb67b38dc1..62f0175fd3 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include From 63bd3e0b55dfd449694106aba5f3fdaf1f8a602e Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 28 Sep 2018 11:55:57 -0700 Subject: [PATCH 035/244] PR Feedback: Remove explicit std namespace usage --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 82 +++++++++++++++++++------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 62f0175fd3..be777c7a3d 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -26,10 +26,25 @@ #define STRINGIFY(s) #s -using std::endl; using std::cerr; -using std::unordered_map; using std::set; using std::string; +using std::cerr; +using std::cout; +using std::dec; +using std::endl; +using std::extent; +using std::hex; +using std::is_pod; +using std::lock_guard; +using std::make_pair; +using std::move; using std::mutex; -typedef std::lock_guard mutex_lock; +using std::oct; +using std::pair; +using std::queue; +using std::set; +using std::string; +using std::unordered_map; + +typedef lock_guard mutex_lock; // Structs struct _PrjFS_FileHandle @@ -74,14 +89,14 @@ static const char* NotificationTypeToString(PrjFS_NotificationType notificationT // State static io_connect_t s_kernelServiceConnection = IO_OBJECT_NULL; -static std::string s_virtualizationRootFullPath; +static string s_virtualizationRootFullPath; static PrjFS_Callbacks s_callbacks; static dispatch_queue_t s_messageQueueDispatchQueue; static dispatch_queue_t s_kernelRequestHandlingConcurrentQueue; // Map of relative path -> set of pending message IDs for that path, plus mutex to protect it. static unordered_map> s_PendingRequestMessageIDs; -static std::mutex s_PendingRequestMessageMutex; +static mutex s_PendingRequestMessageMutex; // The full API is defined in the header, but only the minimal set of functions needed @@ -96,13 +111,13 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( _In_ unsigned int poolThreadCount) { #ifdef DEBUG - std::cout + cout << "PrjFS_StartVirtualizationInstance(" << virtualizationRootFullPath << ", " << callbacks.EnumerateDirectory << ", " << callbacks.GetFileStream << ", " << callbacks.NotifyOperation << ", " - << poolThreadCount << ")" << std::endl; + << poolThreadCount << ")" << endl; #endif if (nullptr == virtualizationRootFullPath || @@ -174,7 +189,7 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( IOReturn result = IODataQueueDequeue(dataQueue.queueMemory, messageMemory, &dequeuedSize); if (kIOReturnSuccess != result || dequeuedSize != messageSize) { - cerr << "Unexpected result dequeueing message - result 0x" << std::hex << result << " dequeued " << dequeuedSize << "/" << messageSize << " bytes\n"; + cerr << "Unexpected result dequeueing message - result 0x" << hex << result << " dequeued " << dequeuedSize << "/" << messageSize << " bytes\n"; abort(); } @@ -191,8 +206,8 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( if (file_messages_found == s_PendingRequestMessageIDs.end()) { // Not handling this file/dir yet - std::pair inserted = - s_PendingRequestMessageIDs.insert(std::make_pair(string(message.path), set{ message.messageHeader->messageId })); + pair inserted = + s_PendingRequestMessageIDs.insert(make_pair(string(message.path), set{ message.messageHeader->messageId })); assert(inserted.second); } else @@ -220,7 +235,7 @@ PrjFS_Result PrjFS_ConvertDirectoryToVirtualizationRoot( _In_ const char* virtualizationRootFullPath) { #ifdef DEBUG - std::cout << "PrjFS_ConvertDirectoryToVirtualizationRoot(" << virtualizationRootFullPath << ")" << std::endl; + cout << "PrjFS_ConvertDirectoryToVirtualizationRoot(" << virtualizationRootFullPath << ")" << endl; #endif if (nullptr == virtualizationRootFullPath) @@ -253,7 +268,7 @@ PrjFS_Result PrjFS_WritePlaceholderDirectory( _In_ const char* relativePath) { #ifdef DEBUG - std::cout << "PrjFS_WritePlaceholderDirectory(" << relativePath << ")" << std::endl; + cout << "PrjFS_WritePlaceholderDirectory(" << relativePath << ")" << endl; #endif if (nullptr == relativePath) @@ -289,13 +304,13 @@ PrjFS_Result PrjFS_WritePlaceholderFile( _In_ uint16_t fileMode) { #ifdef DEBUG - std::cout + cout << "PrjFS_WritePlaceholderFile(" << relativePath << ", " << (int)providerId[0] << ", " << (int)contentId[0] << ", " << fileSize << ", " - << std::oct << fileMode << std::dec << ")" << std::endl; + << oct << fileMode << dec << ")" << endl; #endif if (nullptr == relativePath) @@ -368,14 +383,14 @@ PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( _Out_ PrjFS_UpdateFailureCause* failureCause) { #ifdef DEBUG - std::cout + cout << "PrjFS_UpdatePlaceholderFileIfNeeded(" << relativePath << ", " << (int)providerId[0] << ", " << (int)contentId[0] << ", " << fileSize << ", " - << std::oct << fileMode << std::dec << ", " - << std::hex << updateFlags << std::dec << ")" << std::endl; + << oct << fileMode << dec << ", " + << hex << updateFlags << dec << ")" << endl; #endif // TODO(Mac): Check if the contentId or fileMode have changed before proceeding @@ -396,10 +411,10 @@ PrjFS_Result PrjFS_DeleteFile( _Out_ PrjFS_UpdateFailureCause* failureCause) { #ifdef DEBUG - std::cout + cout << "PrjFS_DeleteFile(" << relativePath << ", " - << std::hex << updateFlags << std::dec << ")" << std::endl; + << hex << updateFlags << dec << ")" << endl; #endif // TODO(Mac): Populate failure cause appropriately @@ -435,13 +450,13 @@ PrjFS_Result PrjFS_WriteFileContents( _In_ unsigned int byteCount) { #ifdef DEBUG - std::cout + cout << "PrjFS_WriteFile(" << fileHandle->file << ", " << (int)((char*)bytes)[0] << ", " << (int)((char*)bytes)[1] << ", " << (int)((char*)bytes)[2] << ", " - << byteCount << ")" << std::endl; + << byteCount << ")" << endl; #endif if (nullptr == fileHandle->file || @@ -550,13 +565,13 @@ static void HandleKernelRequest(Message request, void* messageMemory) ? MessageType_Response_Success : MessageType_Response_Fail; - std::set messageIDs; + set messageIDs; { mutex_lock lock(s_PendingRequestMessageMutex); unordered_map>::iterator fileMessageIDsFound = s_PendingRequestMessageIDs.find(request.path); assert(fileMessageIDsFound != s_PendingRequestMessageIDs.end()); - messageIDs = std::move(fileMessageIDsFound->second); + messageIDs = move(fileMessageIDsFound->second); s_PendingRequestMessageIDs.erase(fileMessageIDsFound); } @@ -572,7 +587,7 @@ static void HandleKernelRequest(Message request, void* messageMemory) static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleEnumerateDirectoryRequest: " << path << std::endl; + cout << "PrjFSLib.HandleEnumerateDirectoryRequest: " << path << endl; #endif PrjFS_Result callbackResult = s_callbacks.EnumerateDirectory( @@ -600,12 +615,12 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleRecursivelyEnumerateDirectoryRequest: " << path << std::endl; + cout << "PrjFSLib.HandleRecursivelyEnumerateDirectoryRequest: " << path << endl; #endif DIR* directory = nullptr; PrjFS_Result result = PrjFS_Result_Success; - std::queue directoryRelativePaths; + queue directoryRelativePaths; directoryRelativePaths.push(path); // Walk each directory, expanding those that are found to be empty @@ -660,7 +675,7 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleHydrateFileRequest: " << path << std::endl; + cout << "PrjFSLib.HandleHydrateFileRequest: " << path << endl; #endif char fullPath[PrjFSMaxPath]; @@ -734,9 +749,10 @@ static PrjFS_Result HandleFileNotification( PrjFS_NotificationType notificationType) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleFileNotification: " << path - << " notificationType: " << NotificationTypeToString(notificationType) - << " isDirectory: " << isDirectory << std::endl; + cout + << "PrjFSLib.HandleFileNotification: " << path + << " notificationType: " << NotificationTypeToString(notificationType) + << " isDirectory: " << isDirectory << endl; #endif char fullPath[PrjFSMaxPath]; @@ -772,7 +788,7 @@ static bool InitializeEmptyPlaceholder(const char* fullPath, TPlaceholder* data, data->header.magicNumber = PlaceholderMagicNumber; data->header.formatVersion = PlaceholderFormatVersion; - static_assert(std::is_pod(), "TPlaceholder must be a POD struct"); + static_assert(is_pod(), "TPlaceholder must be a POD struct"); if (AddXAttr(fullPath, xattrName, data, sizeof(TPlaceholder))) { return true; @@ -898,8 +914,8 @@ static errno_t SendKernelMessageResponse(uint64_t messageId, MessageType respons IOReturn callResult = IOConnectCallScalarMethod( s_kernelServiceConnection, ProviderSelector_KernelMessageResponse, - inputs, std::extent::value, // scalar inputs - nullptr, nullptr); // no outputs + inputs, extent::value, // scalar inputs + nullptr, nullptr); // no outputs return callResult == kIOReturnSuccess ? 0 : EBADMSG; } From 7302b1f7194ede68b530ab5f669d4cdd46406b99 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 28 Sep 2018 13:56:31 -0700 Subject: [PATCH 036/244] Clean up std namespace usage --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 0f95354aef..82e273fb46 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -378,10 +378,10 @@ PrjFS_Result PrjFS_WriteSymLink( _In_ const char* symLinkTarget) { #ifdef DEBUG - std::cout + cout << "PrjFS_WriteSymLink(" << relativePath << ", " - << symLinkTarget << ")" << std::endl; + << symLinkTarget << ")" << endl; #endif if (nullptr == relativePath || nullptr == symLinkTarget) @@ -446,11 +446,11 @@ PrjFS_Result PrjFS_ReplacePlaceholderFileWithSymLink( _Out_ PrjFS_UpdateFailureCause* failureCause) { #ifdef DEBUG - std::cout + cout << "PrjFS_ReplacePlaceholderFileWithSymLink(" << relativePath << ", " << symLinkTarget << ", " - << std::hex << updateFlags << std::dec << ")" << std::endl; + << hex << updateFlags << dec << ")" << endl; #endif PrjFS_Result result = PrjFS_DeleteFile(relativePath, updateFlags, failureCause); From f97a47cd7f054554ca502c5e9bd0dc2d0e9585fc Mon Sep 17 00:00:00 2001 From: John Briggs Date: Mon, 1 Oct 2018 09:30:08 -0400 Subject: [PATCH 037/244] Fix large repo and mac build badges Links were broken when a project was renamed in Azure DevOps. --- Readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index e049af181d..133b937547 100644 --- a/Readme.md +++ b/Readme.md @@ -4,14 +4,14 @@ |Branch|Unit Tests|Functional Tests|Large Repo Perf|Large Repo Build| |:--:|:--:|:--:|:--:|:--:| -|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=master)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=master)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179&branchName=master)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7180&branchName=master)| -|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7180&branchName=releases%2Fshipped)| +|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=master)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=master)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=master)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7179&branchName=master)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=master)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7180&branchName=master)| +|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7179&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7180&branchName=releases%2Fshipped)| ## Mac |Branch|Unit Tests|Functional Tests| |:--:|:--:|:--:| -|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=master)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7376&branchName=master)| -|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7376&branchName=releases%2Fshipped)| +|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=master)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=master)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7376&branchName=master)| +|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7376&branchName=releases%2Fshipped)| ## What is VFS for Git? From c04e0ff1892a4314d3a388650e6097a7398bc490 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 11:24:13 -0700 Subject: [PATCH 038/244] PR Feedback: Eliminate masking and simplify FileTypeAndMode --- .../MacFileSystemVirtualizerTests.cs | 31 ++++--- .../Projection/FileTypeAndMode.cs | 81 ++++++++----------- .../Projection/GitIndexProjection.cs | 11 ++- 3 files changed, 54 insertions(+), 69 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 81c52e43f7..8835cbe33f 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -22,10 +22,7 @@ namespace GVFS.UnitTests.Platform.Mac [TestFixture] public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { - private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); - private static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); - private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); - + private static readonly FileTypeAndMode Regular644FileTypeAndMode = new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode644); private static readonly Dictionary MappedResults = new Dictionary() { { Result.Success, FSResult.Ok }, @@ -100,7 +97,7 @@ public void UpdatePlaceholderIfNeeded() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, FileTypeAndMode.Regular644File); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -180,7 +177,7 @@ public void WritePlaceholderForSymLink() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode((ushort)FileTypeAndMode.FileType.SymLink)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -219,7 +216,7 @@ public void UpdatePlaceholderToSymLink() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode((ushort)FileTypeAndMode.FileType.SymLink)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -282,7 +279,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileTypeAndMode.Regular644File); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -291,7 +288,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() gitIndexProjection.EnumerationInMemory = false; mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode644); + kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); fileSystemCallbacks.Stop(); } } @@ -312,7 +309,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileTypeAndMode.Regular644File); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -321,7 +318,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() gitIndexProjection.EnumerationInMemory = true; mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode644); + kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); gitIndexProjection.ExpandedFolders.ShouldMatchInOrder("test"); fileSystemCallbacks.Stop(); } @@ -343,9 +340,9 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", FileTypeAndMode.Regular644File); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", new FileTypeAndMode((ushort)((ushort)FileTypeAndMode.FileType.Regular | FileMode664))); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", new FileTypeAndMode((ushort)((ushort)FileTypeAndMode.FileType.Regular | FileMode755))); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -354,11 +351,11 @@ public void OnEnumerateDirectorySetsFileModes() gitIndexProjection.EnumerationInMemory = true; mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test644.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode644); + kvp => kvp.Key.Equals(Path.Combine("test", "test644.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test664.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode664); + kvp => kvp.Key.Equals(Path.Combine("test", "test664.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode664); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test755.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode755); + kvp => kvp.Key.Equals(Path.Combine("test", "test755.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode755); fileSystemCallbacks.Stop(); } } diff --git a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs index 0afab8cc26..eaea06fd72 100644 --- a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs +++ b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs @@ -3,73 +3,58 @@ namespace GVFS.Virtualization.Projection { public struct FileTypeAndMode { - public static readonly ushort FileMode755; - public static readonly ushort FileMode664; - public static readonly ushort FileMode644; - public static readonly FileTypeAndMode Regular644File; + public static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); + public static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); + public static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); // Bitmasks for extracting file type and mode from the ushort stored in the index private const ushort FileTypeMask = 0xF000; private const ushort FileModeMask = 0x1FF; - private readonly ushort fileTypeAndMode; - - static FileTypeAndMode() - { - FileMode755 = Convert.ToUInt16("755", 8); - FileMode664 = Convert.ToUInt16("664", 8); - FileMode644 = Convert.ToUInt16("644", 8); - - Regular644File = new FileTypeAndMode((ushort)((ushort)FileType.Regular | FileMode644)); - } + // Values used in the index file to indicate the type of the file + private const ushort RegularFileIndexEntry = 0x8000; + private const ushort SymLinkFileIndexEntry = 0xA000; + private const ushort GitLinkFileIndexEntry = 0xE000; public FileTypeAndMode(ushort typeAndModeInIndexFormat) { - this.fileTypeAndMode = typeAndModeInIndexFormat; - } - - public enum FileType : ushort - { - Invalid = 0, - - Regular = 0x8000, - SymLink = 0xA000, - GitLink = 0xE000, - } - - public FileType Type => (FileType)(this.fileTypeAndMode & FileTypeMask); - public ushort Mode => (ushort)(this.fileTypeAndMode & FileModeMask); - - public static bool operator ==(FileTypeAndMode lhs, FileTypeAndMode rhs) - { - return lhs.Equals(rhs); - } - - public static bool operator !=(FileTypeAndMode lhs, FileTypeAndMode rhs) - { - return !lhs.Equals(rhs); - } - - public override bool Equals(object obj) - { - if (obj is FileTypeAndMode) + switch (typeAndModeInIndexFormat & FileTypeMask) { - return this.Equals((FileTypeAndMode)obj); + case RegularFileIndexEntry: + this.Type = FileType.Regular; + break; + case SymLinkFileIndexEntry: + this.Type = FileType.SymLink; + break; + case GitLinkFileIndexEntry: + this.Type = FileType.GitLink; + break; + default: + this.Type = FileType.Invalid; + break; } - return false; + this.Mode = (ushort)(typeAndModeInIndexFormat & FileModeMask); } - public override int GetHashCode() + public FileTypeAndMode(FileType type, ushort mode) { - return this.fileTypeAndMode; + this.Type = type; + this.Mode = mode; } - public bool Equals(FileTypeAndMode otherFileTypeAndMode) + public enum FileType : short { - return this.fileTypeAndMode == otherFileTypeAndMode.fileTypeAndMode; + Invalid, + + Regular, + SymLink, + GitLink, } + public FileType Type { get; } + public ushort Mode { get; } + public string GetModeAsOctalString() { return Convert.ToString(this.Mode, 8); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index deaa26d1e4..211c2ce5b5 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -37,6 +37,8 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject private const int ExternalLockReleaseTimeoutMs = 50; + private static readonly FileTypeAndMode Regular644FileTypeAndMode = new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode644); + private char[] gitPathSeparatorCharArray = new char[] { GVFSConstants.GitPathSeparator }; private GVFSContext context; @@ -371,7 +373,7 @@ public virtual FileTypeAndMode GetFileTypeAndMode(string filePath) return fileTypeAndMode; } - return FileTypeAndMode.Regular644File; + return Regular644FileTypeAndMode; } finally { @@ -663,9 +665,10 @@ private void AddItemFromIndexEntry(GitIndexEntry indexEntry) if (GVFSPlatform.Instance.FileSystem.SupportsFileMode) { - // TODO(Mac): Test if performance could be improved by reduce this to a single check - // (e.g. by defaulting FileMode to 644 and eliminating the SupportsFileMode check) - if (indexEntry.TypeAndMode != FileTypeAndMode.Regular644File) + // TODO(Mac): Test if performance could be improved by eliminating the SupportsFileMode check + // (e.g. by defaulting FileMode to Regular 644 and eliminating the SupportsFileMode check) + if (indexEntry.TypeAndMode.Type != FileTypeAndMode.FileType.Regular || + indexEntry.TypeAndMode.Mode != FileTypeAndMode.FileMode644) { // TODO(Mac): The line below causes a conversion from LazyUTF8String to .NET string. // Measure the perf and memory overhead of performing this conversion, and determine if we need From fd1cfc6d1552a6e2588639ba2903c45e79757a11 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 28 Sep 2018 10:09:46 -0400 Subject: [PATCH 039/244] CloneVerb: Force download of commit and trees on failed initial checkout --- GVFS/GVFS/CommandLine/CloneVerb.cs | 24 ++++++++++++++++++++++++ GVFS/GVFS/CommandLine/GVFSVerb.cs | 27 ++++++++++++++------------- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 5d86b909fc..9078e688ca 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -567,6 +567,30 @@ private Result CreateClone( } GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); + if (forceCheckoutResult.HasErrors && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0) + { + // It is possible to have the above TryDownloadCommit() fail because we + // already have the commit and root tree we intend to check out, but + // don't have a tree further down the working directory. If we fail + // checkout here, it may be due to not having these trees and the + // read-object hook is not available yet. Force downloading the commit + // again and retry the checkout. + + if (!this.TryDownloadCommit( + refs.GetTipCommitId(branch), + enlistment, + objectRequestor, + gitObjects, + gitRepo, + out errorMessage, + ignoreIfRootExists: false)) + { + return new Result(errorMessage); + } + + forceCheckoutResult = git.ForceCheckout(branch); + } + if (forceCheckoutResult.HasErrors) { string[] errorLines = forceCheckoutResult.Errors.Split('\n'); diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 325e2dceeb..38ee3e1e65 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -176,15 +176,15 @@ protected bool ShowStatusWhileRunning( } protected bool ShowStatusWhileRunning( - Func action, - string message, + Func action, + string message, bool suppressGvfsLogMessage = false) { string gvfsLogEnlistmentRoot = null; if (!suppressGvfsLogMessage) { string errorMessage; - GVFSPlatform.Instance.TryGetGVFSEnlistmentRoot(this.EnlistmentRootPathParameter, out gvfsLogEnlistmentRoot, out errorMessage); + GVFSPlatform.Instance.TryGetGVFSEnlistmentRoot(this.EnlistmentRootPathParameter, out gvfsLogEnlistmentRoot, out errorMessage); } return this.ShowStatusWhileRunning(action, message, gvfsLogEnlistmentRoot); @@ -261,7 +261,7 @@ protected GVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlistment, } return gvfsConfig; - } + } protected void ValidateClientVersions(ITracer tracer, GVFSEnlistment enlistment, GVFSConfig gvfsConfig, bool showWarnings) { @@ -401,9 +401,10 @@ protected bool TryDownloadCommit( GitObjectsHttpRequestor objectRequestor, GVFSGitObjects gitObjects, GitRepo repo, - out string error) + out string error, + bool ignoreIfRootExists = true) { - if (!repo.CommitAndRootTreeExists(commitId)) + if (!ignoreIfRootExists || !repo.CommitAndRootTreeExists(commitId)) { if (!gitObjects.TryDownloadCommit(commitId)) { @@ -740,7 +741,7 @@ public ForExistingEnlistment(bool validateOrigin = true) : base(validateOrigin) MetaName = "Enlistment Root Path", HelpText = "Full or relative path to the GVFS enlistment root")] public override string EnlistmentRootPathParameter { get; set; } - + public sealed override void Execute() { this.ValidatePathParameter(this.EnlistmentRootPathParameter); @@ -855,7 +856,7 @@ private void EnsureLocalCacheIsHealthy( if (File.Exists(alternatesFilePath)) { try - { + { using (Stream stream = fileSystem.OpenFileStream( alternatesFilePath, FileMode.Open, @@ -922,7 +923,7 @@ private void EnsureLocalCacheIsHealthy( gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); } - + string localCacheKey; LocalCacheResolver localCacheResolver = new LocalCacheResolver(enlistment); if (!localCacheResolver.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( @@ -1009,7 +1010,7 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) if (GVFSPlatform.Instance.IsUnderConstruction) { hooksPath = "hooksUnderConstruction"; - } + } else { hooksPath = ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSHooksExecutableName); @@ -1017,8 +1018,8 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) { this.ReportErrorAndExit("Could not find " + GVFSPlatform.Instance.Constants.GVFSHooksExecutableName); } - } - + } + GVFSEnlistment enlistment = null; try { @@ -1030,7 +1031,7 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) { enlistment = GVFSEnlistment.CreateWithoutRepoUrlFromDirectory(enlistmentRootPath, gitBinPath, hooksPath); } - + if (enlistment == null) { this.ReportErrorAndExit( From 6f79da131a0ff87c681a8d790b5304268bfa15b0 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 13:10:03 -0700 Subject: [PATCH 040/244] Enhance SymbolicLinkTests and fix bug in ModifiedPathsShouldContain --- .../EnlistmentPerFixture/SymbolicLinkTests.cs | 47 ++++++++++++++++++- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 11 +++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs index 04179c6711..3b32e037b0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs @@ -29,6 +29,12 @@ public class SymbolicLinkTests : TestsWithEnlistmentPerFixture private const string GrandChildFileContents = "This is the third file"; private const string GrandChildLinkNowAFileContents = "This was a link but is now a file"; + // FunctionalTests/20180925_SymLinksPart3 files + private const string ChildFolder2Name = "ChildDir2"; + + // FunctionalTests/20180925_SymLinksPart4 files + // Note: In this branch ChildLinkName has been changed to a directory and ChildFolder2Name has been changed to a link to ChildFolderName + private BashRunner bashRunner; public SymbolicLinkTests() { @@ -48,18 +54,22 @@ public void CheckoutBranchWithSymLinks() string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFileName); string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFile2Name); string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeTrue($"{grandChildLinkPath} should be a symlink"); grandChildLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolderName + "/" + GrandChildLinkName); } [TestCase, Order(2)] @@ -76,15 +86,18 @@ public void CheckoutBranchWhereSymLinksChangeContentsAndTransitionToFile() string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFileName); string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFile2Name); // In this branch childLinkPath has been changed to point to testFile2Path string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); // grandChildLinkPath should now be a file string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); @@ -95,6 +108,7 @@ public void CheckoutBranchWhereSymLinksChangeContentsAndTransitionToFile() string newGrandChildFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildFileName)); newGrandChildFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); this.bashRunner.IsSymbolicLink(newGrandChildFilePath).ShouldBeFalse($"{newGrandChildFilePath} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolderName + "/" + GrandChildFileName); } [TestCase, Order(3)] @@ -111,6 +125,13 @@ public void CheckoutBranchWhereFilesTransitionToSymLinks() string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); testFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeTrue($"{testFilePath} should be a symlink"); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFileName); + + // There should be a new ChildFolder2Name directory + string childFolder2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolder2Name)); + this.bashRunner.IsSymbolicLink(childFolder2Path).ShouldBeFalse($"{childFolder2Path} should not be a symlink"); + childFolder2Path.ShouldBeADirectory(this.bashRunner); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolder2Name); // The rest of the files are unchanged from FunctionalTests/20180925_SymLinksPart2 string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); @@ -120,6 +141,7 @@ public void CheckoutBranchWhereFilesTransitionToSymLinks() string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeFalse($"{grandChildLinkPath} should not be a symlink"); @@ -131,12 +153,33 @@ public void CheckoutBranchWhereFilesTransitionToSymLinks() } [TestCase, Order(4)] + public void CheckoutBranchWhereSymLinkTransistionsToFolderAndFolderTransitionsToSymlink() + { + GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout FunctionalTests/20180925_SymLinksPart4"); + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart4", + "nothing to commit, working tree clean"); + + // In this branch ChildLinkName has been changed to a directory and ChildFolder2Name has been changed to a link to ChildFolderName + string linkNowADirectoryPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); + this.bashRunner.IsSymbolicLink(linkNowADirectoryPath).ShouldBeFalse($"{linkNowADirectoryPath} should not be a symlink"); + linkNowADirectoryPath.ShouldBeADirectory(this.bashRunner); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); + + string directoryNowALinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolder2Name)); + this.bashRunner.IsSymbolicLink(directoryNowALinkPath).ShouldBeTrue($"{directoryNowALinkPath} should be a symlink"); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolder2Name); + } + + [TestCase, Order(5)] public void GitStatusReportsSymLinkChanges() { GitHelpers.CheckGitCommandAgainstGVFSRepo( this.Enlistment.RepoRoot, "status", - "On branch FunctionalTests/20180925_SymLinksPart3", + "On branch FunctionalTests/20180925_SymLinksPart4", "nothing to commit, working tree clean"); string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); @@ -156,7 +199,7 @@ public void GitStatusReportsSymLinkChanges() GitHelpers.CheckGitCommandAgainstGVFSRepo( this.Enlistment.RepoRoot, "status", - "On branch FunctionalTests/20180925_SymLinksPart3", + "On branch FunctionalTests/20180925_SymLinksPart4", $"modified: {TestFolderName}/{TestFileName}"); } } diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 95e988c182..2d41a0db05 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -7,7 +7,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Reflection; namespace GVFS.FunctionalTests.Tools { @@ -19,6 +18,8 @@ public static class GVFSHelpers public static readonly string PlaceholderListFile = Path.Combine("databases", "PlaceholderList.dat"); public static readonly string RepoMetadataName = Path.Combine("databases", "RepoMetadata.dat"); + private const string ModifedPathsLinePrefix = "A "; + private const string DiskLayoutMajorVersionKey = "DiskLayoutVersion"; private const string DiskLayoutMinorVersionKey = "DiskLayoutMinorVersion"; private const string LocalCacheRootKey = "LocalCacheRoot"; @@ -111,8 +112,12 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin { string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); - GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldContain( - gitPaths.Select(path => path + ModifiedPathsNewLine).ToArray()); + string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); + string[] modifedPathLines = modifedPathsContents.Split(ModifiedPathsNewLine); + foreach (string gitPath in gitPaths) + { + modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLinePrefix + gitPath)); + } } public static void ModifiedPathsShouldNotContain(FileSystemRunner fileSystem, string dotGVFSRoot, params string[] gitPaths) From 413f29ee37d033ba2ffcf550705644f89561f8cf Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 13:20:41 -0700 Subject: [PATCH 041/244] Fix .NET Framework compilation issue --- GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 2d41a0db05..ba7dc5ad59 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -4,9 +4,9 @@ using Microsoft.Data.Sqlite; using Newtonsoft.Json; using NUnit.Framework; +using System; using System.Collections.Generic; using System.IO; -using System.Linq; namespace GVFS.FunctionalTests.Tools { @@ -113,7 +113,7 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); - string[] modifedPathLines = modifedPathsContents.Split(ModifiedPathsNewLine); + string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); foreach (string gitPath in gitPaths) { modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLinePrefix + gitPath)); From 8eddf8b19522f60722a0d0ce67778f687fb8d6cb Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 1 Oct 2018 10:34:35 -0600 Subject: [PATCH 042/244] Cleanup ContainsParentDirectory and add checks in tests for unexpected paths --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 12 ++++---- .../EnlistmentPerFixture/GitFilesTests.cs | 29 +++++++++++-------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 338c1f4ce2..5a7e34c43c 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -187,12 +185,12 @@ private bool TryParseRemoveLine(string line, out string key, out string error) private bool ContainsParentDirectory(string modifiedPath) { - string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); - StringBuilder parentFolder = new StringBuilder(); - foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) + string[] pathParts = modifiedPath.Split(new char[] { GVFSConstants.GitPathSeparator }, StringSplitOptions.RemoveEmptyEntries); + string parentFolder = string.Empty; + for (int i = 0; i < pathParts.Length - 1; i++) { - parentFolder.Append(pathPart + "/"); - if (this.modifiedPaths.Contains(parentFolder.ToString())) + parentFolder += pathParts[i] + GVFSConstants.GitPathSeparatorString; + if (this.modifiedPaths.Contains(parentFolder)) { return true; } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 8a0a0c3f64..5352f7fc86 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -77,6 +77,7 @@ public void CreateFileInFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName); } [TestCase, Order(4)] @@ -97,6 +98,7 @@ public void RenameEmptyFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedEntries); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); } [TestCase, Order(5)] @@ -111,6 +113,17 @@ public void RenameFolderTest() renamedFolderName + "/", }; + string[] unexpectedModifiedEntries = + { + renamedFolderName + "/" + fileNames[0], + renamedFolderName + "/" + fileNames[1], + renamedFolderName + "/" + fileNames[2], + folderName + "/", + folderName + "/" + fileNames[0], + folderName + "/" + fileNames[1], + folderName + "/" + fileNames[2], + }; + this.Enlistment.GetVirtualPathTo(folderName).ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.CreateDirectory(this.Enlistment.GetVirtualPathTo(folderName)); foreach (string fileName in fileNames) @@ -125,6 +138,7 @@ public void RenameFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedEntries); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, unexpectedModifiedEntries); } [TestCase, Order(6)] @@ -136,26 +150,17 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() Assert.Ignore("Powershell does not support case only renames."); } - string[] expectedModifiedPathsEntriesAfterCreate = - { - "A Folder/", - }; - - string[] expectedModifiedPathsEntriesAfterRename = - { - "A folder/", - }; - this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "Folder")); this.fileSystem.CreateEmptyFile(Path.Combine(this.Enlistment.RepoRoot, "Folder", "testfile")); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntriesAfterCreate); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A Folder/"); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntriesAfterRename); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A folder/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A folder/testfile"); } [TestCase, Order(7)] From 24cf13c29d77d1bb64e9d1ffcb43ddcb05ccddf4 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 14:41:05 -0700 Subject: [PATCH 043/244] Fix functional test failures due to change in ModifiedPathsShouldContain behavior --- .../Tests/EnlistmentPerFixture/GitFilesTests.cs | 6 +++--- .../Tests/GitCommands/GitCommandsTests.cs | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index bfbdcc4576..1d444c8576 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -139,13 +139,13 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() string[] expectedModifiedPathsEntriesAfterCreate = { - "A Folder/", - "A Folder/testfile", + "Folder/", + "Folder/testfile", }; string[] expectedModifiedPathsEntriesAfterRename = { - "A folder/", + "folder/", }; this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "Folder")); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index c623f5ea72..31ac5cb08c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -980,15 +980,17 @@ public void EditFileNeedingUtf8Encoding() { this.ValidateGitCommand("checkout -b tests/functional/EditFileNeedingUtf8Encoding"); this.ValidateGitCommand("status"); + string virtualFile = Path.Combine(this.Enlistment.RepoRoot, EncodingFileFolder, EncodingFilename); string controlFile = Path.Combine(this.ControlGitRepo.RootPath, EncodingFileFolder, EncodingFilename); + string relativeGitPath = EncodingFileFolder + "/" + EncodingFilename; string contents = virtualFile.ShouldBeAFile(this.FileSystem).WithContents(); string expectedContents = controlFile.ShouldBeAFile(this.FileSystem).WithContents(); contents.ShouldEqual(expectedContents); // Confirm that the entry is not in the the modified paths database - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, EncodingFilename); + GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); this.ValidateGitCommand("status"); this.AppendAllText(ContentWhenEditingFile, virtualFile); @@ -997,7 +999,7 @@ public void EditFileNeedingUtf8Encoding() this.ValidateGitCommand("status"); // Confirm that the entry was added to the modified paths database - GVFSHelpers.ModifiedPathsShouldContain(this.FileSystem, this.Enlistment.DotGVFSRoot, EncodingFilename); + GVFSHelpers.ModifiedPathsShouldContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); } [TestCase] From 169deabab412bf8935a81def9d9e6ea8b81d65d9 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 15:19:11 -0700 Subject: [PATCH 044/244] Clean up MacFileSystemVirtualizerTests --- .../Mock/Mac/MockVirtualizationInstance.cs | 7 ++ .../MacFileSystemVirtualizerTests.cs | 114 +++++++++++------- 2 files changed, 78 insertions(+), 43 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs index 156ae8382e..532bbcbbf6 100644 --- a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs +++ b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs @@ -15,6 +15,7 @@ public MockVirtualizationInstance() { this.commandCompleted = new AutoResetEvent(false); this.CreatedPlaceholders = new ConcurrentDictionary(); + this.UpdatedPlaceholders = new ConcurrentDictionary(); this.CreatedSymLinks = new ConcurrentHashSet(); this.WriteFileReturnResult = Result.Success; } @@ -28,6 +29,7 @@ public MockVirtualizationInstance() public UpdateFailureCause DeleteFileUpdateFailureCause { get; set; } public ConcurrentDictionary CreatedPlaceholders { get; private set; } + public ConcurrentDictionary UpdatedPlaceholders { get; private set; } public ConcurrentHashSet CreatedSymLinks { get; } @@ -100,6 +102,11 @@ public override Result UpdatePlaceholderIfNeeded( out UpdateFailureCause failureCause) { failureCause = this.UpdatePlaceholderIfNeededFailureCause; + if (failureCause == UpdateFailureCause.NoFailure) + { + this.UpdatedPlaceholders[relativePath] = fileMode; + } + return this.UpdatePlaceholderIfNeededResult; } diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 8835cbe33f..353a82e26b 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -52,19 +52,20 @@ public void DeleteFile() using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) { + const string DeleteTestFileName = "deleteMe.txt"; UpdateFailureReason failureReason = UpdateFailureReason.NoFailure; mockVirtualization.DeleteFileResult = Result.Success; mockVirtualization.DeleteFileUpdateFailureCause = UpdateFailureCause.NoFailure; virtualizer - .DeleteFile("test.txt", UpdatePlaceholderType.AllowReadOnly, out failureReason) + .DeleteFile(DeleteTestFileName, UpdatePlaceholderType.AllowReadOnly, out failureReason) .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.DeleteFileResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.DeleteFileUpdateFailureCause); mockVirtualization.DeleteFileResult = Result.EFileNotFound; mockVirtualization.DeleteFileUpdateFailureCause = UpdateFailureCause.NoFailure; virtualizer - .DeleteFile("test.txt", UpdatePlaceholderType.AllowReadOnly, out failureReason) + .DeleteFile(DeleteTestFileName, UpdatePlaceholderType.AllowReadOnly, out failureReason) .ShouldEqual(new FileSystemResult(FSResult.FileOrPathNotFound, (int)mockVirtualization.DeleteFileResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.DeleteFileUpdateFailureCause); @@ -74,7 +75,7 @@ public void DeleteFile() // TODO: The result should probably be VirtualizationInvalidOperation but for now it's IOError mockVirtualization.DeleteFileUpdateFailureCause = UpdateFailureCause.DirtyData; virtualizer - .DeleteFile("test.txt", UpdatePlaceholderType.AllowReadOnly, out failureReason) + .DeleteFile(DeleteTestFileName, UpdatePlaceholderType.AllowReadOnly, out failureReason) .ShouldEqual(new FileSystemResult(FSResult.IOError, (int)mockVirtualization.DeleteFileResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.DeleteFileUpdateFailureCause); } @@ -83,9 +84,10 @@ public void DeleteFile() [TestCase] public void UpdatePlaceholderIfNeeded() { + const string UpdatePlaceholderFileName = "testUpdatePlaceholder.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { UpdatePlaceholderFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -96,8 +98,7 @@ public void UpdatePlaceholderIfNeeded() backgroundTaskRunner, virtualizer)) { - string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd(UpdatePlaceholderFileName, Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -107,7 +108,7 @@ public void UpdatePlaceholderIfNeeded() mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - filePath, + UpdatePlaceholderFileName, DateTime.Now, DateTime.Now, DateTime.Now, @@ -119,12 +120,14 @@ public void UpdatePlaceholderIfNeeded() out failureReason) .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.UpdatePlaceholderIfNeededResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.UpdatePlaceholderIfNeededFailureCause); + mockVirtualization.UpdatedPlaceholders.ShouldContain(path => path.Key.Equals(UpdatePlaceholderFileName) && path.Value == FileTypeAndMode.FileMode644); + mockVirtualization.UpdatedPlaceholders.Clear(); mockVirtualization.UpdatePlaceholderIfNeededResult = Result.EFileNotFound; mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - filePath, + UpdatePlaceholderFileName, DateTime.Now, DateTime.Now, DateTime.Now, @@ -144,7 +147,7 @@ public void UpdatePlaceholderIfNeeded() // TODO: The result should probably be VirtualizationInvalidOperation but for now it's IOError virtualizer .UpdatePlaceholderIfNeeded( - filePath, + UpdatePlaceholderFileName, DateTime.Now, DateTime.Now, DateTime.Now, @@ -163,9 +166,10 @@ public void UpdatePlaceholderIfNeeded() [TestCase] public void WritePlaceholderForSymLink() { + const string WriteSymLinkFileName = "testWriteSymLink.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { WriteSymLinkFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -176,24 +180,24 @@ public void WritePlaceholderForSymLink() backgroundTaskRunner, virtualizer)) { - string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(WriteSymLinkFileName, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); virtualizer.WritePlaceholderFile( - filePath, + WriteSymLinkFileName, endOfFile: 0, sha: string.Empty).ShouldEqual(new FileSystemResult(FSResult.Ok, (int)Result.Success)); mockVirtualization.CreatedPlaceholders.ShouldBeEmpty(); mockVirtualization.CreatedSymLinks.Count.ShouldEqual(1); - mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(filePath)); + mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(WriteSymLinkFileName)); // Creating a symlink should schedule a background task backgroundTaskRunner.Count.ShouldEqual(1); backgroundTaskRunner.BackgroundTasks[0].Operation.ShouldEqual(GVFS.Virtualization.Background.FileSystemTask.OperationType.OnFileSymLinkCreated); - backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(filePath); + backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(WriteSymLinkFileName); fileSystemCallbacks.Stop(); } @@ -202,9 +206,10 @@ public void WritePlaceholderForSymLink() [TestCase] public void UpdatePlaceholderToSymLink() { + const string PlaceholderToLinkFileName = "testUpdatePlaceholderToLink.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { PlaceholderToLinkFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -215,8 +220,7 @@ public void UpdatePlaceholderToSymLink() backgroundTaskRunner, virtualizer)) { - string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(PlaceholderToLinkFileName, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -226,7 +230,7 @@ public void UpdatePlaceholderToSymLink() mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - filePath, + PlaceholderToLinkFileName, DateTime.Now, DateTime.Now, DateTime.Now, @@ -238,14 +242,14 @@ public void UpdatePlaceholderToSymLink() out failureReason) .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.UpdatePlaceholderIfNeededResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.UpdatePlaceholderIfNeededFailureCause); - + mockVirtualization.UpdatedPlaceholders.Count.ShouldEqual(0, "UpdatePlaceholderIfNeeded should not be called when converting a placeholder to a link"); mockVirtualization.CreatedSymLinks.Count.ShouldEqual(1); - mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(filePath)); + mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(PlaceholderToLinkFileName)); // Creating a symlink should schedule a background task backgroundTaskRunner.Count.ShouldEqual(1); backgroundTaskRunner.BackgroundTasks[0].Operation.ShouldEqual(GVFS.Virtualization.Background.FileSystemTask.OperationType.OnFileSymLinkCreated); - backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(filePath); + backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(PlaceholderToLinkFileName); fileSystemCallbacks.Stop(); } @@ -266,9 +270,15 @@ public void ClearNegativePathCacheIsNoOp() [TestCase] public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() { + const string TestFileName = "test.txt"; + const string TestFolderName = "testFolder"; + string testFilePath = Path.Combine(TestFolderName, TestFileName); using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + + // Don't include TestFolderName as MockGitIndexProjection returns the same list of files regardless of what folder name + // it is passed + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -279,16 +289,16 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd(testFilePath, Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); Guid enumerationGuid = Guid.NewGuid(); gitIndexProjection.EnumerationInMemory = false; - mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); + mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); fileSystemCallbacks.Stop(); } } @@ -296,9 +306,15 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() [TestCase] public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() { + const string TestFileName = "test.txt"; + const string TestFolderName = "testFolder"; + string testFilePath = Path.Combine(TestFolderName, TestFileName); using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + + // Don't include TestFolderName as MockGitIndexProjection returns the same list of files regardless of what folder name + // it is passed + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -309,17 +325,17 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd(testFilePath, Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); Guid enumerationGuid = Guid.NewGuid(); gitIndexProjection.EnumerationInMemory = true; - mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); + mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); - gitIndexProjection.ExpandedFolders.ShouldMatchInOrder("test"); + kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + gitIndexProjection.ExpandedFolders.ShouldMatchInOrder(TestFolderName); fileSystemCallbacks.Stop(); } } @@ -327,9 +343,19 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() [TestCase] public void OnEnumerateDirectorySetsFileModes() { + const string TestFile644Name = "test644.txt"; + const string TestFile664Name = "test664.txt"; + const string TestFile755Name = "test755.txt"; + const string TestFolderName = "testFolder"; + string TestFile644Path = Path.Combine(TestFolderName, TestFile644Name); + string TestFile664Path = Path.Combine(TestFolderName, TestFile664Name); + string TestFile755Path = Path.Combine(TestFolderName, TestFile755Name); using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test644.txt", "test664.txt", "test755.txt" })) + + // Don't include TestFolderName as MockGitIndexProjection returns the same list of files regardless of what folder name + // it is passed + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFile644Name, TestFile664Name, TestFile755Name })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -340,22 +366,22 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", Regular644FileTypeAndMode); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode664)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode755)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile644Path, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile664Path, new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile755Path, new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); Guid enumerationGuid = Guid.NewGuid(); gitIndexProjection.EnumerationInMemory = true; - mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); + mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test644.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(TestFile644Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test664.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode664); + kvp => kvp.Key.Equals(TestFile664Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode664); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test755.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode755); + kvp => kvp.Key.Equals(TestFile755Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode755); fileSystemCallbacks.Stop(); } } @@ -363,9 +389,10 @@ public void OnEnumerateDirectorySetsFileModes() [TestCase] public void OnGetFileStreamReturnsSuccessWhenFileStreamAvailable() { + const string TestFileName = "test.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -389,7 +416,7 @@ public void OnGetFileStreamReturnsSuccessWhenFileStreamAvailable() mockVirtualization.OnGetFileStream( commandId: 1, - relativePath: "test.txt", + relativePath: TestFileName, providerId: placeholderVersion, contentId: contentId, triggeringProcessId: 2, @@ -406,9 +433,10 @@ public void OnGetFileStreamReturnsSuccessWhenFileStreamAvailable() [Category(CategoryConstants.ExceptionExpected)] public void OnGetFileStreamReturnsErrorWhenWriteFileContentsFails() { + const string TestFileName = "test.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -432,7 +460,7 @@ public void OnGetFileStreamReturnsErrorWhenWriteFileContentsFails() mockVirtualization.OnGetFileStream( commandId: 1, - relativePath: "test.txt", + relativePath: TestFileName, providerId: placeholderVersion, contentId: contentId, triggeringProcessId: 2, From 558b7e2c0aa300b31c0898a1a6f65ddc9cded279 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 1 Oct 2018 15:58:16 -0600 Subject: [PATCH 045/244] Make checks on the modified paths be more strict --- .../EnlistmentPerFixture/GitFilesTests.cs | 6 ++--- .../ModifiedPathsTests.cs | 12 +++------ .../Tests/GitCommands/GitCommandsTests.cs | 6 +++-- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 25 ++++++++++++++++--- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 5352f7fc86..3313fb8f68 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -154,13 +154,13 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() this.fileSystem.CreateEmptyFile(Path.Combine(this.Enlistment.RepoRoot, "Folder", "testfile")); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A Folder/"); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "Folder/"); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A folder/"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A folder/testfile"); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "folder/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "folder/testfile"); } [TestCase, Order(7)] diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index ee4c1e060f..f8e42331e6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -53,7 +53,7 @@ public void DeletedTempFileIsRemovedFromModifiedFiles(FileSystemRunner fileSyste tempFile.ShouldNotExistOnDisk(fileSystem); this.Enlistment.UnmountGVFS(); - this.ValidateModifiedPathsDoNotContain(fileSystem, "temp.txt"); + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "temp.txt"); } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] @@ -64,7 +64,7 @@ public void DeletedTempFolderIsRemovedFromModifiedFiles(FileSystemRunner fileSys tempFolder.ShouldNotExistOnDisk(fileSystem); this.Enlistment.UnmountGVFS(); - this.ValidateModifiedPathsDoNotContain(fileSystem, "Temp/"); + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "Temp/"); } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] @@ -79,7 +79,7 @@ public void DeletedTempFolderDeletesFilesFromModifiedFiles(FileSystemRunner file tempFile2.ShouldNotExistOnDisk(fileSystem); this.Enlistment.UnmountGVFS(); - this.ValidateModifiedPathsDoNotContain(fileSystem, "Temp/", "Temp/temp1.txt", "Temp/temp2.txt"); + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "Temp/", "Temp/temp1.txt", "Temp/temp2.txt"); } [Category(Categories.MacTODO.M2)] @@ -219,11 +219,5 @@ private string CreateFile(FileSystemRunner fileSystem, string relativePath) tempFile.ShouldBeAFile(fileSystem); return tempFile; } - - private void ValidateModifiedPathsDoNotContain(FileSystemRunner fileSystem, params string[] paths) - { - GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, paths.Select(x => $"A {x}" + Environment.NewLine).ToArray()); - GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, paths.Select(x => $"D {x}" + Environment.NewLine).ToArray()); - } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index c623f5ea72..31ac5cb08c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -980,15 +980,17 @@ public void EditFileNeedingUtf8Encoding() { this.ValidateGitCommand("checkout -b tests/functional/EditFileNeedingUtf8Encoding"); this.ValidateGitCommand("status"); + string virtualFile = Path.Combine(this.Enlistment.RepoRoot, EncodingFileFolder, EncodingFilename); string controlFile = Path.Combine(this.ControlGitRepo.RootPath, EncodingFileFolder, EncodingFilename); + string relativeGitPath = EncodingFileFolder + "/" + EncodingFilename; string contents = virtualFile.ShouldBeAFile(this.FileSystem).WithContents(); string expectedContents = controlFile.ShouldBeAFile(this.FileSystem).WithContents(); contents.ShouldEqual(expectedContents); // Confirm that the entry is not in the the modified paths database - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, EncodingFilename); + GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); this.ValidateGitCommand("status"); this.AppendAllText(ContentWhenEditingFile, virtualFile); @@ -997,7 +999,7 @@ public void EditFileNeedingUtf8Encoding() this.ValidateGitCommand("status"); // Confirm that the entry was added to the modified paths database - GVFSHelpers.ModifiedPathsShouldContain(this.FileSystem, this.Enlistment.DotGVFSRoot, EncodingFilename); + GVFSHelpers.ModifiedPathsShouldContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); } [TestCase] diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 95e988c182..58abd79851 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -4,10 +4,10 @@ using Microsoft.Data.Sqlite; using Newtonsoft.Json; using NUnit.Framework; +using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Reflection; namespace GVFS.FunctionalTests.Tools { @@ -19,6 +19,9 @@ public static class GVFSHelpers public static readonly string PlaceholderListFile = Path.Combine("databases", "PlaceholderList.dat"); public static readonly string RepoMetadataName = Path.Combine("databases", "RepoMetadata.dat"); + private const string ModifedPathsLineAddPrefix = "A "; + private const string ModifedPathsLineDeletePrefix = "D "; + private const string DiskLayoutMajorVersionKey = "DiskLayoutVersion"; private const string DiskLayoutMinorVersionKey = "DiskLayoutMinorVersion"; private const string LocalCacheRootKey = "LocalCacheRoot"; @@ -111,15 +114,29 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin { string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); - GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldContain( - gitPaths.Select(path => path + ModifiedPathsNewLine).ToArray()); + string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); + string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); + foreach (string gitPath in gitPaths) + { + modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLineAddPrefix + gitPath)); + } } public static void ModifiedPathsShouldNotContain(FileSystemRunner fileSystem, string dotGVFSRoot, params string[] gitPaths) { string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); - GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldNotContain(ignoreCase: true, unexpectedSubstrings: gitPaths); + string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); + string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); + foreach (string gitPath in gitPaths) + { + modifedPathLines.ShouldNotContain( + path => + { + return path.Equals(ModifedPathsLineAddPrefix + gitPath, StringComparison.OrdinalIgnoreCase) || + path.Equals(ModifedPathsLineDeletePrefix + gitPath, StringComparison.OrdinalIgnoreCase); + }); + } } private static byte[] StringToShaBytes(string sha) From 5548b3d198f99ba5af9092506dfc854475a107e2 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 15:51:16 -0700 Subject: [PATCH 046/244] Make FileTypeAndMode an internal struct and fix StyleCop unit test issue --- .../MacFileSystemVirtualizer.cs | 27 ++++--- .../Projection/MockGitIndexProjection.cs | 19 +++-- .../MacFileSystemVirtualizerTests.cs | 77 ++++++++++++++----- .../Projection/FileTypeAndMode.cs | 63 --------------- .../GitIndexProjection.FileTypeAndMode.cs | 47 +++++++++++ .../GitIndexProjection.GitIndexParser.cs | 12 +-- .../Projection/GitIndexProjection.cs | 29 +++++-- 7 files changed, 160 insertions(+), 114 deletions(-) delete mode 100644 GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs create mode 100644 GVFS/GVFS.Virtualization/Projection/GitIndexProjection.FileTypeAndMode.cs diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index e351f4f751..4b20c7e4ae 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -83,20 +83,22 @@ public override FileSystemResult WritePlaceholderFile( string sha) { // TODO(Mac): Add functional tests that validate file mode is set correctly - FileTypeAndMode fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); + GitIndexProjection.FileType fileType; + ushort fileMode; + this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath, out fileType, out fileMode); - if (fileTypeAndMode.Type == FileTypeAndMode.FileType.Regular) + if (fileType == GitIndexProjection.FileType.Regular) { Result result = this.virtualizationInstance.WritePlaceholderFile( relativePath, PlaceholderVersionId, ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), (ulong)endOfFile, - fileTypeAndMode.Mode); + fileMode); return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } - else if (fileTypeAndMode.Type == FileTypeAndMode.FileType.SymLink) + else if (fileType == GitIndexProjection.FileType.SymLink) { string symLinkTarget; if (this.TryGetSymLinkTarget(sha, out symLinkTarget)) @@ -116,7 +118,8 @@ public override FileSystemResult WritePlaceholderFile( else { EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add("FileType", fileTypeAndMode.Type); + metadata.Add(nameof(fileType), fileType); + metadata.Add(nameof(fileMode), fileMode); this.Context.Tracer.RelatedError(metadata, $"{nameof(this.WritePlaceholderFile)}: Unsupported fileType"); return new FileSystemResult(FSResult.IOError, 0); } @@ -145,23 +148,25 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( // TODO(Mac): Add functional tests that include: // - Mode + content changes between commits // - Mode only changes (without any change to content, see issue #223) - FileTypeAndMode fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); + GitIndexProjection.FileType fileType; + ushort fileMode; + this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath, out fileType, out fileMode); - if (fileTypeAndMode.Type == FileTypeAndMode.FileType.Regular) + if (fileType == GitIndexProjection.FileType.Regular) { Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( relativePath, PlaceholderVersionId, ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), (ulong)endOfFile, - fileTypeAndMode.Mode, + fileMode, (UpdateType)updateFlags, out failureCause); failureReason = (UpdateFailureReason)failureCause; return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } - else if (fileTypeAndMode.Type == FileTypeAndMode.FileType.SymLink) + else if (fileType == GitIndexProjection.FileType.SymLink) { string symLinkTarget; if (this.TryGetSymLinkTarget(shaContentId, out symLinkTarget)) @@ -187,8 +192,8 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( else { EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add("FileType", fileTypeAndMode.Type); - metadata.Add("FileMode", fileTypeAndMode.Mode); + metadata.Add(nameof(fileType), fileType); + metadata.Add(nameof(fileMode), fileMode); this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Unsupported fileType"); failureReason = UpdateFailureReason.NoFailure; return new FileSystemResult(FSResult.IOError, 0); diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs index ca877baa86..c83730a735 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs @@ -36,7 +36,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) this.PlaceholdersCreated = new ConcurrentHashSet(); this.ExpandedFolders = new ConcurrentHashSet(); - this.MockFileTypesAndModes = new ConcurrentDictionary(); + this.MockFileTypesAndModes = new ConcurrentDictionary(); this.unblockGetProjectedItems = new ManualResetEvent(true); this.waitForGetProjectedItems = new ManualResetEvent(true); @@ -56,7 +56,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) public ConcurrentHashSet ExpandedFolders { get; } - public ConcurrentDictionary MockFileTypesAndModes { get; } + public ConcurrentDictionary MockFileTypesAndModes { get; } public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } @@ -161,15 +161,18 @@ public override bool TryGetProjectedItemsFromMemory(string folderPath, out List< return false; } - public override FileTypeAndMode GetFileTypeAndMode(string path) + public override void GetFileTypeAndMode(string path, out FileType fileType, out ushort fileMode) { - FileTypeAndMode result; - if (this.MockFileTypesAndModes.TryGetValue(path, out result)) + fileType = FileType.Invalid; + fileMode = 0; + + ushort mockFileTypeAndMode; + if (this.MockFileTypesAndModes.TryGetValue(path, out mockFileTypeAndMode)) { - return result; + FileTypeAndMode typeAndMode = new FileTypeAndMode(mockFileTypeAndMode); + fileType = typeAndMode.Type; + fileMode = typeAndMode.Mode; } - - return new FileTypeAndMode(0); } public override List GetProjectedItems( diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 353a82e26b..96c2d6e288 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -22,7 +22,6 @@ namespace GVFS.UnitTests.Platform.Mac [TestFixture] public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { - private static readonly FileTypeAndMode Regular644FileTypeAndMode = new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode644); private static readonly Dictionary MappedResults = new Dictionary() { { Result.Success, FSResult.Ok }, @@ -98,7 +97,10 @@ public void UpdatePlaceholderIfNeeded() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(UpdatePlaceholderFileName, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + UpdatePlaceholderFileName, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode644)); + string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -120,7 +122,7 @@ public void UpdatePlaceholderIfNeeded() out failureReason) .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.UpdatePlaceholderIfNeededResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.UpdatePlaceholderIfNeededFailureCause); - mockVirtualization.UpdatedPlaceholders.ShouldContain(path => path.Key.Equals(UpdatePlaceholderFileName) && path.Value == FileTypeAndMode.FileMode644); + mockVirtualization.UpdatedPlaceholders.ShouldContain(path => path.Key.Equals(UpdatePlaceholderFileName) && path.Value == GitIndexProjection.FileMode644); mockVirtualization.UpdatedPlaceholders.Clear(); mockVirtualization.UpdatePlaceholderIfNeededResult = Result.EFileNotFound; @@ -180,7 +182,9 @@ public void WritePlaceholderForSymLink() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(WriteSymLinkFileName, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + WriteSymLinkFileName, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.SymLink, fileMode: 0)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -220,7 +224,10 @@ public void UpdatePlaceholderToSymLink() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(PlaceholderToLinkFileName, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + PlaceholderToLinkFileName, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.SymLink, fileMode: 0)); + string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -289,7 +296,9 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(testFilePath, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFilePath, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -298,7 +307,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() gitIndexProjection.EnumerationInMemory = false; mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode644); fileSystemCallbacks.Stop(); } } @@ -325,7 +334,9 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(testFilePath, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFilePath, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -334,7 +345,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() gitIndexProjection.EnumerationInMemory = true; mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode644); gitIndexProjection.ExpandedFolders.ShouldMatchInOrder(TestFolderName); fileSystemCallbacks.Stop(); } @@ -347,9 +358,9 @@ public void OnEnumerateDirectorySetsFileModes() const string TestFile664Name = "test664.txt"; const string TestFile755Name = "test755.txt"; const string TestFolderName = "testFolder"; - string TestFile644Path = Path.Combine(TestFolderName, TestFile644Name); - string TestFile664Path = Path.Combine(TestFolderName, TestFile664Name); - string TestFile755Path = Path.Combine(TestFolderName, TestFile755Name); + string testFile644Path = Path.Combine(TestFolderName, TestFile644Name); + string testFile664Path = Path.Combine(TestFolderName, TestFile664Name); + string testFile755Path = Path.Combine(TestFolderName, TestFile755Name); using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) @@ -366,9 +377,15 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile644Path, Regular644FileTypeAndMode); - gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile664Path, new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode664)); - gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile755Path, new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode755)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFile644Path, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFile664Path, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFile755Path, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -377,11 +394,11 @@ public void OnEnumerateDirectorySetsFileModes() gitIndexProjection.EnumerationInMemory = true; mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(TestFile644Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(testFile644Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode644); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(TestFile664Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode664); + kvp => kvp.Key.Equals(testFile664Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode664); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(TestFile755Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode755); + kvp => kvp.Key.Equals(testFile755Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode755); fileSystemCallbacks.Stop(); } } @@ -470,5 +487,29 @@ public void OnGetFileStreamReturnsErrorWhenWriteFileContentsFails() fileSystemCallbacks.Stop(); } } + + private static ushort ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType fileType, ushort fileMode) + { + // Values used in the index file to indicate the type of the file + const ushort RegularFileIndexEntry = 0x8000; + const ushort SymLinkFileIndexEntry = 0xA000; + const ushort GitLinkFileIndexEntry = 0xE000; + + switch (fileType) + { + case GitIndexProjection.FileType.Regular: + return (ushort)(RegularFileIndexEntry | fileMode); + + case GitIndexProjection.FileType.SymLink: + return (ushort)(SymLinkFileIndexEntry | fileMode); + + case GitIndexProjection.FileType.GitLink: + return (ushort)(GitLinkFileIndexEntry | fileMode); + + default: + Assert.Fail($"Invalid fileType {fileType}"); + return 0; + } + } } } diff --git a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs deleted file mode 100644 index eaea06fd72..0000000000 --- a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System; -namespace GVFS.Virtualization.Projection -{ - public struct FileTypeAndMode - { - public static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); - public static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); - public static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); - - // Bitmasks for extracting file type and mode from the ushort stored in the index - private const ushort FileTypeMask = 0xF000; - private const ushort FileModeMask = 0x1FF; - - // Values used in the index file to indicate the type of the file - private const ushort RegularFileIndexEntry = 0x8000; - private const ushort SymLinkFileIndexEntry = 0xA000; - private const ushort GitLinkFileIndexEntry = 0xE000; - - public FileTypeAndMode(ushort typeAndModeInIndexFormat) - { - switch (typeAndModeInIndexFormat & FileTypeMask) - { - case RegularFileIndexEntry: - this.Type = FileType.Regular; - break; - case SymLinkFileIndexEntry: - this.Type = FileType.SymLink; - break; - case GitLinkFileIndexEntry: - this.Type = FileType.GitLink; - break; - default: - this.Type = FileType.Invalid; - break; - } - - this.Mode = (ushort)(typeAndModeInIndexFormat & FileModeMask); - } - - public FileTypeAndMode(FileType type, ushort mode) - { - this.Type = type; - this.Mode = mode; - } - - public enum FileType : short - { - Invalid, - - Regular, - SymLink, - GitLink, - } - - public FileType Type { get; } - public ushort Mode { get; } - - public string GetModeAsOctalString() - { - return Convert.ToString(this.Mode, 8); - } - } -} diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.FileTypeAndMode.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.FileTypeAndMode.cs new file mode 100644 index 0000000000..7c20ca2c58 --- /dev/null +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.FileTypeAndMode.cs @@ -0,0 +1,47 @@ +using System; +namespace GVFS.Virtualization.Projection +{ + public partial class GitIndexProjection + { + internal struct FileTypeAndMode + { + // Bitmasks for extracting file type and mode from the ushort stored in the index + private const ushort FileTypeMask = 0xF000; + private const ushort FileModeMask = 0x1FF; + + // Values used in the index file to indicate the type of the file + private const ushort RegularFileIndexEntry = 0x8000; + private const ushort SymLinkFileIndexEntry = 0xA000; + private const ushort GitLinkFileIndexEntry = 0xE000; + + public FileTypeAndMode(ushort typeAndModeInIndexFormat) + { + switch (typeAndModeInIndexFormat & FileTypeMask) + { + case RegularFileIndexEntry: + this.Type = FileType.Regular; + break; + case SymLinkFileIndexEntry: + this.Type = FileType.SymLink; + break; + case GitLinkFileIndexEntry: + this.Type = FileType.GitLink; + break; + default: + this.Type = FileType.Invalid; + break; + } + + this.Mode = (ushort)(typeAndModeInIndexFormat & FileModeMask); + } + + public FileType Type { get; } + public ushort Mode { get; } + + public string GetModeAsOctalString() + { + return Convert.ToString(this.Mode, 8); + } + } + } +} diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 3753abb7c5..4893ede463 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -187,18 +187,18 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func switch (typeAndMode.Type) { - case FileTypeAndMode.FileType.Regular: - if (typeAndMode.Mode != FileTypeAndMode.FileMode755 && - typeAndMode.Mode != FileTypeAndMode.FileMode644 && - typeAndMode.Mode != FileTypeAndMode.FileMode664) + case FileType.Regular: + if (typeAndMode.Mode != FileMode755 && + typeAndMode.Mode != FileMode644 && + typeAndMode.Mode != FileMode664) { throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for regular file in index"); } break; - case FileTypeAndMode.FileType.SymLink: - case FileTypeAndMode.FileType.GitLink: + case FileType.SymLink: + case FileType.GitLink: if (typeAndMode.Mode != 0) { throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for link file({typeAndMode.Type:X}) in index"); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 211c2ce5b5..c99808b744 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -22,6 +22,10 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject { public const string ProjectionIndexBackupName = "GVFS_projection"; + public static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); + public static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); + public static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); + private const int IndexFileStreamBufferSize = 512 * 1024; private const UpdatePlaceholderType FolderPlaceholderDeleteFlags = @@ -37,8 +41,6 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject private const int ExternalLockReleaseTimeoutMs = 50; - private static readonly FileTypeAndMode Regular644FileTypeAndMode = new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode644); - private char[] gitPathSeparatorCharArray = new char[] { GVFSConstants.GitPathSeparator }; private GVFSContext context; @@ -118,6 +120,15 @@ protected GitIndexProjection() { } + public enum FileType : short + { + Invalid, + + Regular, + SymLink, + GitLink, + } + public int EstimatedPlaceholderCount { get @@ -356,13 +367,16 @@ public virtual bool TryGetProjectedItemsFromMemory(string folderPath, out List

Date: Wed, 5 Sep 2018 13:26:42 -0600 Subject: [PATCH 047/244] Compress the entries in the modified paths database when mounting This will search the modified paths for a parent folder and if one exists it will remove the entry from the modified paths list --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 28 +++++++++++++++++++ .../Common/ModifiedPathsDatabaseTests.cs | 26 ++++++++++++++++- .../FileSystemCallbacks.cs | 3 ++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 5a7e34c43c..9fef5318ba 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -52,6 +52,34 @@ public static bool TryLoadOrCreate(ITracer tracer, string dataDirectory, Physica return true; } + public void Compress(ITracer tracer) + { + int startingCount = this.modifiedPaths.Count; + using (ITracer activity = tracer.StartActivity("Compress ModifiedPaths", EventLevel.Informational)) + { + foreach (var item in this.modifiedPaths) + { + int pathSeparatorIndex = item.IndexOf('/'); + while (pathSeparatorIndex >= 0 && pathSeparatorIndex < item.Length - 1) + { + string folder = item.Substring(0, pathSeparatorIndex + 1); + if (this.modifiedPaths.Contains(folder)) + { + this.modifiedPaths.TryRemove(item); + break; + } + + pathSeparatorIndex = item.IndexOf('/', pathSeparatorIndex + 1); + } + } + + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(startingCount), startingCount); + metadata.Add("EndCount", this.modifiedPaths.Count); + activity.Stop(metadata); + } + } + public bool Contains(string path, bool isFolder) { string entry = this.NormalizeEntryString(path, isFolder); diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index 663a76bb08..59b52e921b 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -1,6 +1,7 @@ using GVFS.Common; using GVFS.Tests.Should; using GVFS.UnitTests.Mock; +using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using NUnit.Framework; using System; @@ -17,6 +18,17 @@ public class ModifiedPathsDatabaseTests private const string ExistingEntries = @"A file.txt A dir/file2.txt A dir1/dir2/file3.txt +"; + private const string EntriesToCompress = @"A file.txt +A dir/file2.txt +A dir/dir3/dir4/ +A dir1/dir2/file3.txt +A dir/ +A dir1/dir2/ +A dir1/file.txt +A dir1/dir2/dir3/dir4/dir5/ +A dir/dir2/file3.txt +A dir/dir4/dir5/ "; [TestCase] @@ -119,7 +131,19 @@ public void EntryNotAddedIfParentDirectoryExists() modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue(); } - private static void TestAddingPath(string path, bool isFolder = false) + [TestCase] + public void CompressEntries() + { + ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(EntriesToCompress); + mpd.Compress(new MockTracer()); + mpd.Count.ShouldEqual(4); + mpd.Contains("file.txt", isFolder: false).ShouldBeTrue(); + mpd.Contains("dir/", isFolder: true).ShouldBeTrue(); + mpd.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); + mpd.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); + } + + private static void TestAddingPath(string path, bool isFolder = false) { TestAddingPath(path, path, isFolder); } diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index d7c0d57b4c..44bce1687d 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -89,6 +89,9 @@ public FileSystemCallbacks( throw new InvalidRepoException(error); } + this.modifiedPaths.Compress(this.context.Tracer); + this.modifiedPaths.WriteAllEntriesAndFlush(); + this.BlobSizes = blobSizes; this.BlobSizes.Initialize(); From d28270e8ea06f8f47a4ad827df3f6f04545663c9 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 15:15:26 -0600 Subject: [PATCH 048/244] Fix test for modified paths that was broken --- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index f8e42331e6..f2005745eb 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -37,11 +37,6 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase $"A {RenameNewDotGitFileTarget}", $"A {FileToCreateOutsideRepo}", $"A {FolderToCreateOutsideRepo}/", - $"A {FolderToDelete}/CreateCommonAssemblyVersion.bat", - $"A {FolderToDelete}/CreateCommonCliAssemblyVersion.bat", - $"A {FolderToDelete}/CreateCommonVersionHeader.bat", - $"A {FolderToDelete}/RunFunctionalTests.bat", - $"A {FolderToDelete}/RunUnitTests.bat", $"A {FolderToDelete}/", }; From 36fef4d4335fead8127d68cb2f48c4ecc2ebd9c0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 15:20:18 -0600 Subject: [PATCH 049/244] Use better names in the modified paths test --- .../Common/ModifiedPathsDatabaseTests.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index 59b52e921b..bc3106da5c 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -134,16 +134,16 @@ public void EntryNotAddedIfParentDirectoryExists() [TestCase] public void CompressEntries() { - ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(EntriesToCompress); - mpd.Compress(new MockTracer()); - mpd.Count.ShouldEqual(4); - mpd.Contains("file.txt", isFolder: false).ShouldBeTrue(); - mpd.Contains("dir/", isFolder: true).ShouldBeTrue(); - mpd.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); - mpd.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); - } + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress); + modifiedPathsDatabase.Compress(new MockTracer()); + modifiedPathsDatabase.Count.ShouldEqual(4); + modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); + } - private static void TestAddingPath(string path, bool isFolder = false) + private static void TestAddingPath(string path, bool isFolder = false) { TestAddingPath(path, path, isFolder); } From 2bbca795df85e63392955a754c9dbe4dd9ea40bb Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 15:24:23 -0600 Subject: [PATCH 050/244] Move code to remove modified path entries to the TryStart method --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 44bce1687d..6ff4d40f3e 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -89,9 +89,6 @@ public FileSystemCallbacks( throw new InvalidRepoException(error); } - this.modifiedPaths.Compress(this.context.Tracer); - this.modifiedPaths.WriteAllEntriesAndFlush(); - this.BlobSizes = blobSizes; this.BlobSizes.Initialize(); @@ -174,6 +171,9 @@ public static bool IsPathInsideDotGit(string relativePath) public bool TryStart(out string error) { + this.modifiedPaths.Compress(this.context.Tracer); + this.modifiedPaths.WriteAllEntriesAndFlush(); + if (!this.fileSystemVirtualizer.TryStart(this, out error)) { return false; From f0bcb3425f4928848d7f4c4918c84d7ac1a76208 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 16:09:52 -0600 Subject: [PATCH 051/244] Rename method and use Split method for building parent paths --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 26 ++++++++++++------- .../Common/ModifiedPathsDatabaseTests.cs | 2 +- .../FileSystemCallbacks.cs | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 9fef5318ba..6f40923b36 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -52,24 +54,30 @@ public static bool TryLoadOrCreate(ITracer tracer, string dataDirectory, Physica return true; } - public void Compress(ITracer tracer) + ///

+ /// This method will examine the modified paths to check if there is already a parent folder entry in + /// the modified paths. If there is a parent folder the entry does not need to be in the modified paths + /// and will be removed because the parent folder is recursive and covers any children. + /// + public void RemoveEntriesWithParentFolderEntry(ITracer tracer) { int startingCount = this.modifiedPaths.Count; using (ITracer activity = tracer.StartActivity("Compress ModifiedPaths", EventLevel.Informational)) { - foreach (var item in this.modifiedPaths) + StringBuilder parentFolder = new StringBuilder(); + foreach (string modifiedPath in this.modifiedPaths) { - int pathSeparatorIndex = item.IndexOf('/'); - while (pathSeparatorIndex >= 0 && pathSeparatorIndex < item.Length - 1) + string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + + parentFolder.Clear(); + foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) { - string folder = item.Substring(0, pathSeparatorIndex + 1); - if (this.modifiedPaths.Contains(folder)) + parentFolder.Append(pathPart + "/"); + if (this.modifiedPaths.Contains(parentFolder.ToString())) { - this.modifiedPaths.TryRemove(item); + this.modifiedPaths.TryRemove(modifiedPath); break; } - - pathSeparatorIndex = item.IndexOf('/', pathSeparatorIndex + 1); } } diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index bc3106da5c..ced7ba5857 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -135,7 +135,7 @@ public void EntryNotAddedIfParentDirectoryExists() public void CompressEntries() { ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress); - modifiedPathsDatabase.Compress(new MockTracer()); + modifiedPathsDatabase.RemoveEntriesWithParentFolder(new MockTracer()); modifiedPathsDatabase.Count.ShouldEqual(4); modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue(); diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 6ff4d40f3e..627c895a5f 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -171,7 +171,7 @@ public static bool IsPathInsideDotGit(string relativePath) public bool TryStart(out string error) { - this.modifiedPaths.Compress(this.context.Tracer); + this.modifiedPaths.RemoveEntriesWithParentFolderEntry(this.context.Tracer); this.modifiedPaths.WriteAllEntriesAndFlush(); if (!this.fileSystemVirtualizer.TryStart(this, out error)) From 02442473d9c1fba9d8361f7a4305e4dee06f59b0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 27 Sep 2018 12:28:43 -0600 Subject: [PATCH 052/244] Add more tests for modified paths --- .../EnlistmentPerTestCase/ModifiedPathsTests.cs | 13 ++++++++++--- .../Common/ModifiedPathsDatabaseTests.cs | 11 ++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index f2005745eb..f2f255180f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -133,9 +133,16 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) folderToCreateOutsideRepoTargetPath.ShouldBeADirectory(fileSystem); folderToCreateOutsideRepoPath.ShouldNotExistOnDisk(fileSystem); - string folderToDelete = this.Enlistment.GetVirtualPathTo(FolderToDelete); - fileSystem.DeleteDirectory(folderToDelete); - folderToDelete.ShouldNotExistOnDisk(fileSystem); + string folderToDeleteFullPath = this.Enlistment.GetVirtualPathTo(FolderToDelete); + fileSystem.WriteAllText(Path.Combine(folderToDeleteFullPath, "NewFile.txt"), "Contents for new file"); + string newFileToDelete = Path.Combine(folderToDeleteFullPath, "NewFileToDelete.txt"); + fileSystem.WriteAllText(newFileToDelete, "Contents for new file"); + fileSystem.DeleteFile(newFileToDelete); + fileSystem.WriteAllText(Path.Combine(folderToDeleteFullPath, "CreateCommonVersionHeader.bat"), "Changing the file contents"); + fileSystem.DeleteFile(Path.Combine(folderToDeleteFullPath, "RunUnitTests.bat")); + + fileSystem.DeleteDirectory(folderToDeleteFullPath); + folderToDeleteFullPath.ShouldNotExistOnDisk(fileSystem); // Remount this.Enlistment.UnmountGVFS(); diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index ced7ba5857..40bacf08f2 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -20,15 +20,19 @@ A dir/file2.txt A dir1/dir2/file3.txt "; private const string EntriesToCompress = @"A file.txt +D deleted.txt A dir/file2.txt A dir/dir3/dir4/ A dir1/dir2/file3.txt A dir/ +D deleted/ A dir1/dir2/ A dir1/file.txt A dir1/dir2/dir3/dir4/dir5/ A dir/dir2/file3.txt A dir/dir4/dir5/ +D dir/dir2/deleted.txt +A dir1/dir2 "; [TestCase] @@ -132,13 +136,14 @@ public void EntryNotAddedIfParentDirectoryExists() } [TestCase] - public void CompressEntries() + public void RemoveEntriesWithParentFolderEntry() { ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress); - modifiedPathsDatabase.RemoveEntriesWithParentFolder(new MockTracer()); - modifiedPathsDatabase.Count.ShouldEqual(4); + modifiedPathsDatabase.RemoveEntriesWithParentFolderEntry(new MockTracer()); + modifiedPathsDatabase.Count.ShouldEqual(5); modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/dir2", isFolder: false).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); } From c3f5aaec9726d1a17c5595e33e741749ee57d426 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 1 Oct 2018 23:07:21 -0600 Subject: [PATCH 053/244] Change to use ContainsParentDirectory for removing modified paths --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 6f40923b36..95fb18e733 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -62,22 +60,13 @@ public static bool TryLoadOrCreate(ITracer tracer, string dataDirectory, Physica public void RemoveEntriesWithParentFolderEntry(ITracer tracer) { int startingCount = this.modifiedPaths.Count; - using (ITracer activity = tracer.StartActivity("Compress ModifiedPaths", EventLevel.Informational)) + using (ITracer activity = tracer.StartActivity(nameof(this.RemoveEntriesWithParentFolderEntry), EventLevel.Informational)) { - StringBuilder parentFolder = new StringBuilder(); foreach (string modifiedPath in this.modifiedPaths) { - string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); - - parentFolder.Clear(); - foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) + if (this.ContainsParentDirectory(modifiedPath)) { - parentFolder.Append(pathPart + "/"); - if (this.modifiedPaths.Contains(parentFolder.ToString())) - { - this.modifiedPaths.TryRemove(modifiedPath); - break; - } + this.modifiedPaths.TryRemove(modifiedPath); } } From ec71c1454ecc0522d9e23573b79793e37e31b959 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 16:53:53 -0700 Subject: [PATCH 054/244] Windows: Use a faster comparison when enumeration filter strings don't have wildcards --- .../ActiveEnumeration.cs | 33 ++++++++++++++++--- .../WindowsFileSystemVirtualizer.cs | 4 +-- .../Virtualization/ActiveEnumerationTests.cs | 2 +- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs b/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs index 6caa10785d..55cb7c0ff5 100644 --- a/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs +++ b/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs @@ -6,10 +6,11 @@ namespace GVFS.Platform.Windows { public class ActiveEnumeration { - private static FileNamePatternMatcher doesPatternMatch = null; + private static FileNamePatternMatcher wildcardPatternMatcher = null; // Use our own enumerator to avoid having to dispose anything private ProjectedFileInfoEnumerator fileInfoEnumerator; + private FileNamePatternMatcher patternMatcher; private string filterString = null; @@ -36,12 +37,13 @@ public ProjectedFileInfo Current } /// - /// Sets the pattern matching delegate that will be used for file name comparisons + /// Sets the pattern matching delegate that will be used for file name comparisons when the filter + /// contains wildcards. /// /// FileNamePatternMatcher to be used by ActiveEnumeration - public static void SetPatternMatcher(FileNamePatternMatcher patternMatcher) + public static void SetWildcardPatternMatcher(FileNamePatternMatcher patternMatcher) { - doesPatternMatch = patternMatcher; + wildcardPatternMatcher = patternMatcher; } /// @@ -107,15 +109,31 @@ public string GetFilterString() return this.filterString; } + private static bool NameMatchsNoWildcardFilter(string name, string filter) + { + return string.Equals(name, filter, System.StringComparison.OrdinalIgnoreCase); + } + private void SaveFilter(string filter) { if (string.IsNullOrEmpty(filter)) { this.filterString = string.Empty; + this.patternMatcher = null; } else { this.filterString = filter; + + if (ProjFS.Utils.DoesNameContainWildCards(this.filterString)) + { + this.patternMatcher = wildcardPatternMatcher; + } + else + { + this.patternMatcher = NameMatchsNoWildcardFilter; + } + if (this.IsCurrentValid && this.IsCurrentHidden()) { this.MoveNext(); @@ -125,7 +143,12 @@ private void SaveFilter(string filter) private bool IsCurrentHidden() { - return !doesPatternMatch(this.Current.Name, this.GetFilterString()); + if (this.patternMatcher == null) + { + return false; + } + + return !this.patternMatcher(this.Current.Name, this.GetFilterString()); } private void ResetEnumerator() diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index de1963b994..a5d02a6f33 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -272,11 +272,11 @@ private void InitializeEnumerationPatternMatcher() if (projFSPatternMatchingWorks) { - ActiveEnumeration.SetPatternMatcher(Utils.IsFileNameMatch); + ActiveEnumeration.SetWildcardPatternMatcher(Utils.IsFileNameMatch); } else { - ActiveEnumeration.SetPatternMatcher(InternalFileNameMatchesFilter); + ActiveEnumeration.SetWildcardPatternMatcher(InternalFileNameMatchesFilter); } this.Context.Tracer.RelatedEvent( diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/ActiveEnumerationTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/ActiveEnumerationTests.cs index f74f42815c..28e7de98f1 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/ActiveEnumerationTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/ActiveEnumerationTests.cs @@ -23,7 +23,7 @@ public class ActiveEnumerationTests public ActiveEnumerationTests(PatternMatcherWrapper wrapper) { - ActiveEnumeration.SetPatternMatcher(wrapper.Matcher); + ActiveEnumeration.SetWildcardPatternMatcher(wrapper.Matcher); } public static object[] Runners From 47ad31a761115e5aee7b3e5263f57f0f5150783b Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 27 Sep 2018 15:18:37 -0700 Subject: [PATCH 055/244] Mac: Update MirrorProvider to support symbolic links --- .../MacFileSystemVirtualizer.cs | 63 +++++++++- .../MirrorProvider/FileSystemVirtualizer.cs | 117 ++++++++++++------ .../MirrorProvider/ProjectedFileInfo.cs | 17 ++- 3 files changed, 157 insertions(+), 40 deletions(-) diff --git a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs index 3d9396c82e..b8df167896 100644 --- a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs @@ -1,7 +1,9 @@ using PrjFSLib.Mac; using System; using System.IO; - +using System.Runtime.InteropServices; +using System.Text; + namespace MirrorProvider.Mac { public class MacFileSystemVirtualizer : FileSystemVirtualizer @@ -58,7 +60,7 @@ private Result OnEnumerateDirectory( foreach (ProjectedFileInfo child in this.GetChildItems(relativePath)) { - if (child.IsDirectory) + if (child.Type == ProjectedFileInfo.FileType.Directory) { Result result = this.virtualizationInstance.WritePlaceholderDirectory( Path.Combine(relativePath, child.Name)); @@ -69,6 +71,28 @@ private Result OnEnumerateDirectory( return result; } } + else if (child.Type == ProjectedFileInfo.FileType.SymLink) + { + string childRelativePath = Path.Combine(relativePath, child.Name); + + string symLinkTarget; + if (this.TryGetSymLinkTarget(childRelativePath, out symLinkTarget)) + { + Result result = this.virtualizationInstance.WriteSymLink( + childRelativePath, + symLinkTarget); + + if (result != Result.Success) + { + Console.WriteLine($"WriteSymLink failed: {result}"); + return result; + } + } + else + { + return Result.EIOError; + } + } else { // The MirrorProvider marks every file as executable (mode 755), but this is just a shortcut to avoid the pain of @@ -176,6 +200,35 @@ private void OnHardLinkCreated(string relativeNewLinkPath) { Console.WriteLine($"OnHardLinkCreated: {relativeNewLinkPath}"); } + + private bool TryGetSymLinkTarget(string relativePath, out string symLinkTarget) + { + symLinkTarget = null; + string fullPathInMirror = this.GetFullPathInMirror(relativePath); + + const ulong BufSize = 4096; + byte[] targetBuffer = new byte[BufSize]; + long bytesRead = ReadLink(fullPathInMirror, targetBuffer, BufSize); + if (bytesRead < 0) + { + Console.WriteLine($"GetSymLinkTarget failed: {Marshal.GetLastWin32Error()}"); + return false; + } + + targetBuffer[bytesRead] = 0; + symLinkTarget = Encoding.UTF8.GetString(targetBuffer); + + if (symLinkTarget.StartsWith(this.Enlistment.MirrorRoot, StringComparison.OrdinalIgnoreCase)) + { + // Link target is an absolute path inside the MirrorRoot. + // The target needs to be adjusted to point inside the src root + symLinkTarget = Path.Combine( + this.Enlistment.SrcRoot.TrimEnd(Path.DirectorySeparatorChar), + symLinkTarget.Substring(this.Enlistment.MirrorRoot.Length).TrimStart(Path.DirectorySeparatorChar)); + } + + return true; + } private static byte[] ToVersionIdByteArray(byte version) { @@ -184,5 +237,11 @@ private static byte[] ToVersionIdByteArray(byte version) return bytes; } + + [DllImport("libc", EntryPoint = "readlink", SetLastError = true)] + private static extern long ReadLink( + string path, + byte[] buf, + ulong bufsize); } } diff --git a/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs index 40efddbe9e..cb73aa0b01 100644 --- a/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs @@ -1,25 +1,30 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; - +using System.Linq; + namespace MirrorProvider { public abstract class FileSystemVirtualizer { - private Enlistment enlistment; + protected Enlistment Enlistment { get; private set; } public abstract bool TryConvertVirtualizationRoot(string directory, out string error); public virtual bool TryStartVirtualizationInstance(Enlistment enlistment, out string error) { - this.enlistment = enlistment; + this.Enlistment = enlistment; error = null; return true; } + protected string GetFullPathInMirror(string relativePath) + { + return Path.Combine(this.Enlistment.MirrorRoot, relativePath); + } + protected bool DirectoryExists(string relativePath) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); DirectoryInfo dirInfo = new DirectoryInfo(fullPathInMirror); return dirInfo.Exists; @@ -27,7 +32,7 @@ protected bool DirectoryExists(string relativePath) protected bool FileExists(string relativePath) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); FileInfo fileInfo = new FileInfo(fullPathInMirror); return fileInfo.Exists; @@ -35,18 +40,18 @@ protected bool FileExists(string relativePath) protected ProjectedFileInfo GetFileInfo(string relativePath) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); string fullParentPath = Path.GetDirectoryName(fullPathInMirror); string fileName = Path.GetFileName(relativePath); string actualCaseName; - if (this.DirectoryExists(fullParentPath, fileName, out actualCaseName)) + ProjectedFileInfo.FileType type; + if (this.FileOrDirectoryExists(fullParentPath, fileName, out actualCaseName, out type)) { - return new ProjectedFileInfo(actualCaseName, size: 0, isDirectory: true); - } - else if (this.FileExists(fullParentPath, fileName, out actualCaseName)) - { - return new ProjectedFileInfo(actualCaseName, size: new FileInfo(fullPathInMirror).Length, isDirectory: false); + return new ProjectedFileInfo( + actualCaseName, + size: (type == ProjectedFileInfo.FileType.File) ? new FileInfo(fullPathInMirror).Length : 0, + type: type); } return null; @@ -54,7 +59,7 @@ protected ProjectedFileInfo GetFileInfo(string relativePath) protected IEnumerable GetChildItems(string relativePath) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); DirectoryInfo dirInfo = new DirectoryInfo(fullPathInMirror); if (!dirInfo.Exists) @@ -62,20 +67,38 @@ protected IEnumerable GetChildItems(string relativePath) yield break; } - foreach (FileInfo file in dirInfo.GetFiles()) + foreach (FileSystemInfo fileSystemInfo in dirInfo.GetFileSystemInfos()) { - yield return new ProjectedFileInfo(file.Name, file.Length, isDirectory: false); - } + if ((fileSystemInfo.Attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) + { + // While not 100% accurate on all platforms, for simplicity assume that if the the file has reparse data it's a symlink + yield return new ProjectedFileInfo( + fileSystemInfo.Name, + size: 0, + type: ProjectedFileInfo.FileType.SymLink); + } + else if ((fileSystemInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) + { + yield return new ProjectedFileInfo( + fileSystemInfo.Name, + size: 0, + type: ProjectedFileInfo.FileType.Directory); + } + else + { + FileInfo fileInfo = fileSystemInfo as FileInfo; + yield return new ProjectedFileInfo( + fileInfo.Name, + fileInfo.Length, + ProjectedFileInfo.FileType.File); + } - foreach (DirectoryInfo subDirectory in dirInfo.GetDirectories()) - { - yield return new ProjectedFileInfo(subDirectory.Name, size: 0, isDirectory: true); } } protected FileSystemResult HydrateFile(string relativePath, int bufferSize, Func tryWriteBytes) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); if (!File.Exists(fullPathInMirror)) { return FileSystemResult.EFileNotFound; @@ -106,23 +129,47 @@ protected FileSystemResult HydrateFile(string relativePath, int bufferSize, Func return FileSystemResult.Success; } - private bool DirectoryExists(string fullParentPath, string directoryName, out string actualCaseName) + private bool FileOrDirectoryExists( + string fullParentPath, + string fileName, + out string actualCaseName, + out ProjectedFileInfo.FileType type) { - return this.NameExists(Directory.GetDirectories(fullParentPath), directoryName, out actualCaseName); - } + actualCaseName = null; + type = ProjectedFileInfo.FileType.Invalid; - private bool FileExists(string fullParentPath, string fileName, out string actualCaseName) - { - return this.NameExists(Directory.GetFiles(fullParentPath), fileName, out actualCaseName); - } + DirectoryInfo dirInfo = new DirectoryInfo(fullParentPath); + if (!dirInfo.Exists) + { + return false; + } - private bool NameExists(IEnumerable paths, string name, out string actualCaseName) - { - actualCaseName = - paths - .Select(path => Path.GetFileName(path)) - .FirstOrDefault(actualName => actualName.Equals(name, StringComparison.OrdinalIgnoreCase)); - return actualCaseName != null; + FileSystemInfo fileSystemInfo = + dirInfo + .GetFileSystemInfos() + .FirstOrDefault(fsInfo => fsInfo.Name.Equals(fileName, StringComparison.OrdinalIgnoreCase)); + + if (fileSystemInfo == null) + { + return false; + } + + actualCaseName = fileSystemInfo.Name; + + if ((fileSystemInfo.Attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) + { + type = ProjectedFileInfo.FileType.SymLink; + } + else if ((fileSystemInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) + { + type = ProjectedFileInfo.FileType.Directory; + } + else + { + type = ProjectedFileInfo.FileType.File; + } + + return true; } } } diff --git a/MirrorProvider/MirrorProvider/ProjectedFileInfo.cs b/MirrorProvider/MirrorProvider/ProjectedFileInfo.cs index 310447b69c..71269ad2eb 100644 --- a/MirrorProvider/MirrorProvider/ProjectedFileInfo.cs +++ b/MirrorProvider/MirrorProvider/ProjectedFileInfo.cs @@ -2,15 +2,26 @@ { public class ProjectedFileInfo { - public ProjectedFileInfo(string name, long size, bool isDirectory) + public ProjectedFileInfo(string name, long size, FileType type) { this.Name = name; this.Size = size; - this.IsDirectory = isDirectory; + this.Type = type; + } + + public enum FileType + { + Invalid, + + File, + Directory, + SymLink + } public string Name { get; } public long Size { get; } - public bool IsDirectory { get; } + public FileType Type { get; } + public bool IsDirectory => this.Type == FileType.Directory; } } From 355ead878bac3fade5f213aadf239332ff396224 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 2 Oct 2018 13:00:05 -0700 Subject: [PATCH 056/244] Variable renaming --- .../GVFS.Platform.Windows/ActiveEnumeration.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs b/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs index 55cb7c0ff5..675c6d1bce 100644 --- a/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs +++ b/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs @@ -6,11 +6,11 @@ namespace GVFS.Platform.Windows { public class ActiveEnumeration { - private static FileNamePatternMatcher wildcardPatternMatcher = null; + private static FileNamePatternMatcher doesWildcardPatternMatch = null; // Use our own enumerator to avoid having to dispose anything private ProjectedFileInfoEnumerator fileInfoEnumerator; - private FileNamePatternMatcher patternMatcher; + private FileNamePatternMatcher doesPatternMatch; private string filterString = null; @@ -43,7 +43,7 @@ public ProjectedFileInfo Current /// FileNamePatternMatcher to be used by ActiveEnumeration public static void SetWildcardPatternMatcher(FileNamePatternMatcher patternMatcher) { - wildcardPatternMatcher = patternMatcher; + doesWildcardPatternMatch = patternMatcher; } /// @@ -109,7 +109,7 @@ public string GetFilterString() return this.filterString; } - private static bool NameMatchsNoWildcardFilter(string name, string filter) + private static bool NameMatchesNoWildcardFilter(string name, string filter) { return string.Equals(name, filter, System.StringComparison.OrdinalIgnoreCase); } @@ -119,7 +119,7 @@ private void SaveFilter(string filter) if (string.IsNullOrEmpty(filter)) { this.filterString = string.Empty; - this.patternMatcher = null; + this.doesPatternMatch = null; } else { @@ -127,11 +127,11 @@ private void SaveFilter(string filter) if (ProjFS.Utils.DoesNameContainWildCards(this.filterString)) { - this.patternMatcher = wildcardPatternMatcher; + this.doesPatternMatch = doesWildcardPatternMatch; } else { - this.patternMatcher = NameMatchsNoWildcardFilter; + this.doesPatternMatch = NameMatchesNoWildcardFilter; } if (this.IsCurrentValid && this.IsCurrentHidden()) @@ -143,12 +143,12 @@ private void SaveFilter(string filter) private bool IsCurrentHidden() { - if (this.patternMatcher == null) + if (this.doesPatternMatch == null) { return false; } - return !this.patternMatcher(this.Current.Name, this.GetFilterString()); + return !this.doesPatternMatch(this.Current.Name, this.GetFilterString()); } private void ResetEnumerator() From f97b5cdcacfb6ea019ba092a532a619fd50c37da Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 1 Oct 2018 15:42:21 -0400 Subject: [PATCH 057/244] SharedCacheTests: add test for multi-enlistment checkout failures --- .../MultiEnlistmentTests/SharedCacheTests.cs | 27 +++++++++++++++++-- GVFS/GVFS/CommandLine/CloneVerb.cs | 4 +-- GVFS/GVFS/CommandLine/GVFSVerb.cs | 4 +-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index de37e90708..2a55c4c5e6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -96,7 +96,7 @@ public void CloneCleansUpStaleMetadataLock() enlistment1.Status().ShouldContain("Mount status: Ready"); enlistment2.Status().ShouldContain("Mount status: Ready"); } - + [TestCase] public void ParallelReadsInASharedCache() { @@ -265,7 +265,30 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() newObjectsRoot.ShouldContain(newCacheKey); newObjectsRoot.ShouldBeADirectory(this.fileSystem); - this.AlternatesFileShouldHaveGitObjectsRoot(enlistment); + this.AlternatesFileShouldHaveGitObjectsRoot(enlistment); + } + + [TestCase] + public void SecondCloneSucceedsWithMissingTrees() + { + string newCachePath = Path.Combine(this.localCacheParentPath, ".customGvfsCache2"); + GVFSFunctionalTestEnlistment enlistment1 = this.CreateNewEnlistment(localCacheRoot: newCachePath); + File.ReadAllText(Path.Combine(enlistment1.RepoRoot, WellKnownFile)); + + this.AlternatesFileShouldHaveGitObjectsRoot(enlistment1); + + string packDir = Path.Combine(enlistment1.GetObjectRoot(this.fileSystem), "pack"); + string[] packDirFiles = Directory.EnumerateFiles(packDir, "*", SearchOption.AllDirectories).ToArray(); + for (int i = 0; i < packDirFiles.Length; i++) + { + this.fileSystem.DeleteFile(Path.Combine(packDir, packDirFiles[i])); + } + + // Enumerate the root directory, populating the tip commit and root tree + this.fileSystem.EnumerateDirectory(enlistment1.RepoRoot); + + GVFSFunctionalTestEnlistment enlistment2 = this.CreateNewEnlistment(localCacheRoot: newCachePath); + this.fileSystem.EnumerateDirectory(enlistment2.RepoRoot); } // Override OnTearDownEnlistmentsDeleted rathern than using [TearDown] as the enlistments need to be unmounted before diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 9078e688ca..ef59eea865 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -572,7 +572,7 @@ private Result CreateClone( // It is possible to have the above TryDownloadCommit() fail because we // already have the commit and root tree we intend to check out, but // don't have a tree further down the working directory. If we fail - // checkout here, it may be due to not having these trees and the + // checkout here, its' because we don't have these trees and the // read-object hook is not available yet. Force downloading the commit // again and retry the checkout. @@ -583,7 +583,7 @@ private Result CreateClone( gitObjects, gitRepo, out errorMessage, - ignoreIfRootExists: false)) + checkLocalObjectCache: false)) { return new Result(errorMessage); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 38ee3e1e65..cb69014d1b 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -402,9 +402,9 @@ protected bool TryDownloadCommit( GVFSGitObjects gitObjects, GitRepo repo, out string error, - bool ignoreIfRootExists = true) + bool checkLocalObjectCache = true) { - if (!ignoreIfRootExists || !repo.CommitAndRootTreeExists(commitId)) + if (!checkLocalObjectCache || !repo.CommitAndRootTreeExists(commitId)) { if (!gitObjects.TryDownloadCommit(commitId)) { From 48f62a2535f5476a05b9b58013894281b0ac2560 Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Wed, 3 Oct 2018 12:18:20 -0400 Subject: [PATCH 058/244] Use correct repoUrl for credential This change allows us to reinitialize the GitProces with valid repository information information and pass it correctly to the credential call. --- .../Tests/EnlistmentPerTestCase/RepairTests.cs | 8 ++++++++ GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs index 34da63c006..b53d2caf85 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs @@ -13,6 +13,14 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase [Category(Categories.MacTODO.M4)] public class RepairTests : TestsWithEnlistmentPerTestCase { + [TestCase] + public void NoFixesNeeded() + { + this.Enlistment.UnmountGVFS(); + + this.Enlistment.Repair(); + } + [TestCase] public void FixesCorruptHeadSha() { diff --git a/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs b/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs index b8432ae197..58416e0cce 100644 --- a/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs +++ b/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs @@ -45,6 +45,7 @@ public override IssueType HasIssue(List messages) // At this point, we've confirmed that the repo url can be gotten, so we have to // reinitialize the GitProcess with a valid repo url for 'git credential fill' + string repoUrl = null; try { GVFSEnlistment enlistment = GVFSEnlistment.CreateFromDirectory( @@ -52,6 +53,7 @@ public override IssueType HasIssue(List messages) this.Enlistment.GitBinPath, this.Enlistment.GVFSHooksRoot); git = new GitProcess(enlistment); + repoUrl = enlistment.RepoUrl; } catch (InvalidRepoException) { @@ -61,7 +63,7 @@ public override IssueType HasIssue(List messages) string username; string password; - if (!git.TryGetCredentials(this.Tracer, this.Enlistment.RepoUrl, out username, out password)) + if (!git.TryGetCredentials(this.Tracer, repoUrl, out username, out password)) { messages.Add("Authentication failed. Run 'gvfs log' for more info."); messages.Add(".git\\config is valid and remote 'origin' is set, but may have a typo:"); From d2bbde892ea70b70c9e1b71fd9bebd15144df0e6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 3 Oct 2018 13:04:08 -0400 Subject: [PATCH 059/244] Remove SharedCacheTests.SecondCloneSucceedsWithMissingTrees This test attempted to cover a rare scenario, but is flaky due to something holding a handle on the pack-files we are trying to delete. --- .../MultiEnlistmentTests/SharedCacheTests.cs | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index 2a55c4c5e6..7ec39205a1 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -267,29 +267,6 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() this.AlternatesFileShouldHaveGitObjectsRoot(enlistment); } - - [TestCase] - public void SecondCloneSucceedsWithMissingTrees() - { - string newCachePath = Path.Combine(this.localCacheParentPath, ".customGvfsCache2"); - GVFSFunctionalTestEnlistment enlistment1 = this.CreateNewEnlistment(localCacheRoot: newCachePath); - File.ReadAllText(Path.Combine(enlistment1.RepoRoot, WellKnownFile)); - - this.AlternatesFileShouldHaveGitObjectsRoot(enlistment1); - - string packDir = Path.Combine(enlistment1.GetObjectRoot(this.fileSystem), "pack"); - string[] packDirFiles = Directory.EnumerateFiles(packDir, "*", SearchOption.AllDirectories).ToArray(); - for (int i = 0; i < packDirFiles.Length; i++) - { - this.fileSystem.DeleteFile(Path.Combine(packDir, packDirFiles[i])); - } - - // Enumerate the root directory, populating the tip commit and root tree - this.fileSystem.EnumerateDirectory(enlistment1.RepoRoot); - - GVFSFunctionalTestEnlistment enlistment2 = this.CreateNewEnlistment(localCacheRoot: newCachePath); - this.fileSystem.EnumerateDirectory(enlistment2.RepoRoot); - } // Override OnTearDownEnlistmentsDeleted rathern than using [TearDown] as the enlistments need to be unmounted before // localCacheParentPath can be deleted (as the SQLite blob sizes database cannot be deleted while GVFS is mounted) From 0bf8b3f2709acceebe915fb0b1eebf1b639d77df Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Wed, 3 Oct 2018 15:06:02 -0400 Subject: [PATCH 060/244] Fix diagnose message for cache server If not cache server is present display "None" --- GVFS/GVFS.Common/GVFSEnlistment.cs | 3 +-- GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSEnlistment.cs b/GVFS/GVFS.Common/GVFSEnlistment.cs index 345bfd8d46..f897dbfa8a 100644 --- a/GVFS/GVFS.Common/GVFSEnlistment.cs +++ b/GVFS/GVFS.Common/GVFSEnlistment.cs @@ -13,7 +13,6 @@ public partial class GVFSEnlistment : Enlistment public const string BlobSizesCacheName = "blobSizes"; private const string GitObjectCacheName = "gitObjects"; - private const string InvalidRepoUrl = "invalid://repoUrl"; private string gitVersion; private string gvfsVersion; @@ -90,7 +89,7 @@ public static GVFSEnlistment CreateWithoutRepoUrlFromDirectory(string directory, return null; } - return new GVFSEnlistment(enlistmentRoot, InvalidRepoUrl, gitBinRoot, gvfsHooksRoot); + return new GVFSEnlistment(enlistmentRoot, string.Empty, gitBinRoot, gvfsHooksRoot); } return null; diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index 138259e8c8..1d4188340f 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -53,7 +53,7 @@ protected override void Execute(GVFSEnlistment enlistment) this.WriteMessage(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); this.WriteMessage(string.Empty); this.WriteMessage("Enlistment root: " + enlistment.EnlistmentRoot); - this.WriteMessage("Cache Server: " + CacheServerResolver.GetUrlFromConfig(enlistment)); + this.WriteMessage("Cache Server: " + CacheServerResolver.GetCacheServerFromConfig(enlistment)); string localCacheRoot; string gitObjectsRoot; From e99c92f8fafa7d5a03e11fce1c480e7f2b04497d Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 3 Oct 2018 12:32:50 -0700 Subject: [PATCH 061/244] Mac: Prevent double hydration and directory expansion --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 280 ++++++++++++++++++------------- 1 file changed, 168 insertions(+), 112 deletions(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 82e273fb46..a901cfed26 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -9,8 +9,9 @@ #include #include #include -#include +#include #include +#include #include #include #include @@ -35,14 +36,16 @@ using std::hex; using std::is_pod; using std::lock_guard; using std::make_pair; +using std::make_shared; +using std::map; using std::move; using std::mutex; using std::oct; using std::pair; using std::queue; using std::set; +using std::shared_ptr; using std::string; -using std::unordered_map; typedef lock_guard mutex_lock; @@ -69,7 +72,7 @@ static void CombinePaths(const char* root, const char* relative, char (&combined static errno_t SendKernelMessageResponse(uint64_t messageId, MessageType responseType); static errno_t RegisterVirtualizationRootPath(const char* path); -static void HandleKernelRequest(Message requestSpec, void* messageMemory); +static void HandleKernelRequest(void* messageMemory, uint32_t messageSize); static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path); @@ -94,10 +97,27 @@ static PrjFS_Callbacks s_callbacks; static dispatch_queue_t s_messageQueueDispatchQueue; static dispatch_queue_t s_kernelRequestHandlingConcurrentQueue; -// Map of relative path -> set of pending message IDs for that path, plus mutex to protect it. -static unordered_map> s_PendingRequestMessageIDs; -static mutex s_PendingRequestMessageMutex; +struct CaseInsensitiveStringCompare +{ + bool operator() (const std::string& lhs, const std::string& rhs) const + { + return strcasecmp(lhs.c_str(), rhs.c_str()) < 0; + } +}; + +struct MutexAndUseCount +{ + shared_ptr mutex; + int useCount; +}; +// Map of relative path -> MutexAndUseCount for that path, plus mutex to protect the map itself. +typedef map PathToMutexMap; +PathToMutexMap s_inProgressExpansions; +mutex s_inProgressExpansionsMutex; + +static shared_ptr CheckoutPathMutex(const string& fullPath); +static void ReturnPathMutex(const string& fullPath, const shared_ptr& mutex); // The full API is defined in the header, but only the minimal set of functions needed // for the initial MirrorProvider implementation are listed here. Calling any other function @@ -192,37 +212,11 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( cerr << "Unexpected result dequeueing message - result 0x" << hex << result << " dequeued " << dequeuedSize << "/" << messageSize << " bytes\n"; abort(); } - - Message message = ParseMessageMemory(messageMemory, messageSize); - - // At the moment, we expect all messages to include a path - assert(message.path != nullptr); - - // Ensure we don't run more than one request handler at once for the same file - { - mutex_lock lock(s_PendingRequestMessageMutex); - typedef unordered_map>::iterator PendingMessageIterator; - PendingMessageIterator file_messages_found = s_PendingRequestMessageIDs.find(message.path); - if (file_messages_found == s_PendingRequestMessageIDs.end()) - { - // Not handling this file/dir yet - pair inserted = - s_PendingRequestMessageIDs.insert(make_pair(string(message.path), set{ message.messageHeader->messageId })); - assert(inserted.second); - } - else - { - // Already a handler running for this path, don't handle it again. - file_messages_found->second.insert(message.messageHeader->messageId); - continue; - } - } - dispatch_async( s_kernelRequestHandlingConcurrentQueue, ^{ - HandleKernelRequest(message, messageMemory); + HandleKernelRequest(messageMemory, messageSize); }); } }); @@ -436,6 +430,7 @@ PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( return result; } + // TODO(Mac): Ensure that races with hydration are handled properly return PrjFS_WritePlaceholderFile(relativePath, providerId, contentId, fileSize, fileMode); } @@ -482,6 +477,7 @@ PrjFS_Result PrjFS_DeleteFile( return PrjFS_Result_EInvalidArgs; } + // TODO(Mac): Ensure that races with hydration are handled properly // TODO(Mac): Ensure file is not full before proceeding char fullPath[PrjFSMaxPath]; @@ -554,10 +550,15 @@ static Message ParseMessageMemory(const void* messageMemory, uint32_t size) return Message { header, path }; } -static void HandleKernelRequest(Message request, void* messageMemory) +static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) { PrjFS_Result result = PrjFS_Result_EIOError; + Message request = ParseMessageMemory(messageMemory, messageSize); + + // At the moment, we expect all messages to include a path + assert(request.path != nullptr); + const MessageHeader* requestHeader = request.messageHeader; switch (requestHeader->messageType) { @@ -621,21 +622,8 @@ static void HandleKernelRequest(Message request, void* messageMemory) PrjFS_Result_Success == result ? MessageType_Response_Success : MessageType_Response_Fail; - - set messageIDs; - - { - mutex_lock lock(s_PendingRequestMessageMutex); - unordered_map>::iterator fileMessageIDsFound = s_PendingRequestMessageIDs.find(request.path); - assert(fileMessageIDsFound != s_PendingRequestMessageIDs.end()); - messageIDs = move(fileMessageIDsFound->second); - s_PendingRequestMessageIDs.erase(fileMessageIDsFound); - } - - for (uint64_t messageID : messageIDs) - { - SendKernelMessageResponse(messageID, responseType); - } + + SendKernelMessageResponse(requestHeader->messageId, responseType); } free(messageMemory); @@ -647,26 +635,44 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request cout << "PrjFSLib.HandleEnumerateDirectoryRequest: " << path << endl; #endif - PrjFS_Result callbackResult = s_callbacks.EnumerateDirectory( - 0 /* commandId */, - path, - request->pid, - request->procname); + char fullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), path, fullPath); + if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) + { + return PrjFS_Result_Success; + } - if (PrjFS_Result_Success == callbackResult) + PrjFS_Result result; + shared_ptr expansionMutex = CheckoutPathMutex(fullPath); { - char fullPath[PrjFSMaxPath]; - CombinePaths(s_virtualizationRootFullPath.c_str(), path, fullPath); + mutex_lock lock(*expansionMutex); + if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) + { + result = PrjFS_Result_Success; + goto CleanupAndReturn; + } + + result = s_callbacks.EnumerateDirectory( + 0 /* commandId */, + path, + request->pid, + request->procname); - if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) + if (PrjFS_Result_Success == result) { - // TODO(Mac): how should we handle this scenario where the provider thinks it succeeded, but we were unable to - // update placeholder metadata? - return PrjFS_Result_EIOError; + if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) + { + // TODO(Mac): how should we handle this scenario where the provider thinks it succeeded, but we were unable to + // update placeholder metadata? + result = PrjFS_Result_EIOError; + } } } - - return callbackResult; + +CleanupAndReturn: + ReturnPathMutex(fullPath, expansionMutex); + + return result; } static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path) @@ -689,13 +695,10 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead CombinePaths(s_virtualizationRootFullPath.c_str(), directoryRelativePath.c_str(), pathBuffer); - if (IsBitSetInFileFlags(pathBuffer, FileFlags_IsEmpty)) + PrjFS_Result result = HandleEnumerateDirectoryRequest(request, directoryRelativePath.c_str()); + if (result != PrjFS_Result_Success) { - PrjFS_Result result = HandleEnumerateDirectoryRequest(request, directoryRelativePath.c_str()); - if (result != PrjFS_Result_Success) - { - goto CleanupAndReturn; - } + goto CleanupAndReturn; } DIR* directory = opendir(pathBuffer); @@ -744,59 +747,81 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const return PrjFS_Result_EIOError; } - PrjFS_FileHandle fileHandle; - - // Mode "rb+" means: - // - The file must already exist - // - The handle is opened for reading and writing - // - We are allowed to seek to somewhere other than end of stream for writing - fileHandle.file = fopen(fullPath, "rb+"); - if (nullptr == fileHandle.file) - { - return PrjFS_Result_EIOError; - } - - // Seek back to the beginning so the provider can overwrite the empty contents - if (fseek(fileHandle.file, 0, 0)) + if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) { - fclose(fileHandle.file); - return PrjFS_Result_EIOError; + return PrjFS_Result_Success; } - PrjFS_Result callbackResult = s_callbacks.GetFileStream( - 0 /* comandId */, - path, - xattrData.providerId, - xattrData.contentId, - request->pid, - request->procname, - &fileHandle); - - // TODO: once we support async callbacks, we'll need to save off the fileHandle if the result is Pending + PrjFS_Result result; + PrjFS_FileHandle fileHandle; - if (fclose(fileHandle.file)) - { - // TODO: under what conditions can fclose fail? How do we recover? - return PrjFS_Result_EIOError; - } + shared_ptr hydrationMutex = CheckoutPathMutex(fullPath); - if (PrjFS_Result_Success == callbackResult) { - // TODO: validate that the total bytes written match the size that was reported on the placeholder in the first place - // Potential bugs if we don't: - // * The provider writes fewer bytes than expected. The hydrated is left with extra padding up to the original reported size. - // * The provider writes more bytes than expected. The write succeeds, but whatever tool originally opened the file may have already - // allocated the originally reported size, and now the contents appear truncated. + mutex_lock lock(*hydrationMutex); + if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) + { + result = PrjFS_Result_Success; + goto CleanupAndReturn; + } - if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) + // Mode "rb+" means: + // - The file must already exist + // - The handle is opened for reading and writing + // - We are allowed to seek to somewhere other than end of stream for writing + fileHandle.file = fopen(fullPath, "rb+"); + if (nullptr == fileHandle.file) { - // TODO: how should we handle this scenario where the provider thinks it succeeded, but we were unable to - // update placeholder metadata? - return PrjFS_Result_EIOError; + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + // Seek back to the beginning so the provider can overwrite the empty contents + if (fseek(fileHandle.file, 0, 0)) + { + fclose(fileHandle.file); + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + result = s_callbacks.GetFileStream( + 0 /* comandId */, + path, + xattrData.providerId, + xattrData.contentId, + request->pid, + request->procname, + &fileHandle); + + // TODO: once we support async callbacks, we'll need to save off the fileHandle if the result is Pending + + if (fclose(fileHandle.file)) + { + // TODO: under what conditions can fclose fail? How do we recover? + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + if (PrjFS_Result_Success == result) + { + // TODO: validate that the total bytes written match the size that was reported on the placeholder in the first place + // Potential bugs if we don't: + // * The provider writes fewer bytes than expected. The hydrated is left with extra padding up to the original reported size. + // * The provider writes more bytes than expected. The write succeeds, but whatever tool originally opened the file may have already + // allocated the originally reported size, and now the contents appear truncated. + + if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) + { + // TODO: how should we handle this scenario where the provider thinks it succeeded, but we were unable to + // update placeholder metadata? + result = PrjFS_Result_EIOError; + } } } - - return callbackResult; + +CleanupAndReturn: + ReturnPathMutex(fullPath, hydrationMutex); + return result; } static PrjFS_Result HandleFileNotification( @@ -1031,3 +1056,34 @@ static const char* NotificationTypeToString(PrjFS_NotificationType notificationT } } #endif + +static shared_ptr CheckoutPathMutex(const string& fullPath) +{ + mutex_lock lock(s_inProgressExpansionsMutex); + PathToMutexMap::iterator iter = s_inProgressExpansions.find(fullPath); + if (iter == s_inProgressExpansions.end()) + { + pair newEntry = s_inProgressExpansions.insert( + PathToMutexMap::value_type(fullPath, { make_shared(), 1 })); + assert(newEntry.second); + return newEntry.first->second.mutex; + } + else + { + iter->second.useCount++; + return iter->second.mutex; + } +} + +static void ReturnPathMutex(const string& fullPath, const shared_ptr& mutex) +{ + mutex_lock lock(s_inProgressExpansionsMutex); + PathToMutexMap::iterator iter = s_inProgressExpansions.find(fullPath); + assert(iter != s_inProgressExpansions.end()); + assert(iter->second.mutex.get() == mutex.get()); + iter->second.useCount--; + if (iter->second.useCount == 0) + { + s_inProgressExpansions.erase(iter); + } +} From c226d42e03c47cba09ebc49e64af83f3777c8c62 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Tue, 18 Sep 2018 12:01:29 -0400 Subject: [PATCH 062/244] 1. GVFS.Service checks for upgrade availability periodically. When an new release is available it will be downloaded in the background. 2. GVFS Git Hooks will nag user to install the upgrade. 3. GVFS Upgrade [--confirm] verb will enable user to manually trigger installation of the upgrade release. 4. Unit test cases 5. Functional test for upgrade reminder messaging --- GVFS.sln | 10 + GVFS/GVFS.Common/DiskLayoutUpgrade.cs | 2 +- GVFS/GVFS.Common/FileBasedLock.cs | 2 +- GVFS/GVFS.Common/FileSystem/IKernelDriver.cs | 1 + GVFS/GVFS.Common/GVFSConstants.cs | 16 +- GVFS/GVFS.Common/GVFSPlatform.cs | 7 +- GVFS/GVFS.Common/Git/GitProcess.cs | 26 + GVFS/GVFS.Common/Git/GitVersion.cs | 17 +- GVFS/GVFS.Common/InstallerPreRunChecker.cs | 212 ++++++++ GVFS/GVFS.Common/ProcessHelper.cs | 9 +- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 69 +++ GVFS/GVFS.Common/ProductUpgrader.cs | 477 ++++++++++++++++++ .../GVFSUpgradeReminderTests.cs | 88 ++++ GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj | 1 + GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj | 5 +- GVFS/GVFS.Hooks/Program.cs | 8 +- GVFS/GVFS.Installer/Setup.iss | 19 +- GVFS/GVFS.Platform.Mac/MacPlatform.cs | 5 + GVFS/GVFS.Platform.Mac/ProjFSKext.cs | 8 +- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 49 +- GVFS/GVFS.Platform.Windows/WindowsPlatform.cs | 9 + GVFS/GVFS.Service/GVFS.Service.csproj | 1 + GVFS/GVFS.Service/GvfsService.cs | 11 +- GVFS/GVFS.Service/ProductUpgradeTimer.cs | 83 +++ GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj | 1 + .../GVFS.UnitTests.Windows.csproj | 12 + .../Mock/MockInstallerPreRunChecker.cs | 129 +++++ .../Windows/Mock/MockProcessLauncher.cs | 45 ++ .../Windows/Mock/MockProductUpgrader.cs | 241 +++++++++ .../Windows/Mock/MockTextWriter.cs | 47 ++ .../Windows/Upgrader/ProductUpgraderTests.cs | 135 +++++ .../Upgrader/UpgradeOrchestratorTests.cs | 250 +++++++++ .../Windows/Upgrader/UpgradeTests.cs | 128 +++++ .../Windows/Upgrader/UpgradeVerbTests.cs | 247 +++++++++ .../Mock/Common/MockPlatform.cs | 5 + GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs | 9 +- GVFS/GVFS.Upgrader/App.config | 6 + GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj | 122 +++++ GVFS/GVFS.Upgrader/Program.cs | 16 + GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs | 22 + GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 389 ++++++++++++++ GVFS/GVFS.Upgrader/packages.config | 6 + GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 20 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 5 +- GVFS/GVFS/CommandLine/LogVerb.cs | 5 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 325 ++++++++++++ GVFS/GVFS/GVFS.Windows.csproj | 3 +- GVFS/GVFS/Program.cs | 13 +- Scripts/UninstallGVFS.bat | 2 + 49 files changed, 3281 insertions(+), 37 deletions(-) create mode 100644 GVFS/GVFS.Common/InstallerPreRunChecker.cs create mode 100644 GVFS/GVFS.Common/ProductUpgrader.Shared.cs create mode 100644 GVFS/GVFS.Common/ProductUpgrader.cs create mode 100644 GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs create mode 100644 GVFS/GVFS.Service/ProductUpgradeTimer.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockTextWriter.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/ProductUpgraderTests.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs create mode 100644 GVFS/GVFS.Upgrader/App.config create mode 100644 GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj create mode 100644 GVFS/GVFS.Upgrader/Program.cs create mode 100644 GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs create mode 100644 GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs create mode 100644 GVFS/GVFS.Upgrader/packages.config create mode 100644 GVFS/GVFS/CommandLine/UpgradeVerb.cs diff --git a/GVFS.sln b/GVFS.sln index e5378e7502..e0c6ba22de 100644 --- a/GVFS.sln +++ b/GVFS.sln @@ -125,6 +125,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GVFS.Mac", "GVFS\GVFS\GVFS. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Windows", "GVFS\GVFS\GVFS.Windows.csproj", "{32220664-594C-4425-B9A0-88E0BE2F3D2A}" ProjectSection(ProjectDependencies) = postProject + {AECEC217-2499-403D-B0BB-2962B9BE5970} = {AECEC217-2499-403D-B0BB-2962B9BE5970} {2D23AB54-541F-4ABC-8DCA-08C199E97ABB} = {2D23AB54-541F-4ABC-8DCA-08C199E97ABB} {798DE293-6EDA-4DC4-9395-BE7A71C563E3} = {798DE293-6EDA-4DC4-9395-BE7A71C563E3} {5A6656D5-81C7-472C-9DC8-32D071CB2258} = {5A6656D5-81C7-472C-9DC8-32D071CB2258} @@ -167,6 +168,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.FunctionalTests.LockHo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GVFS.Hooks.Mac", "GVFS\GVFS.Hooks\GVFS.Hooks.Mac.csproj", "{4CC2A90D-D240-4382-B4BF-5E175515E492}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Upgrader", "GVFS\GVFS.Upgrader\GVFS.Upgrader.csproj", "{AECEC217-2499-403D-B0BB-2962B9BE5970}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug.Mac|x64 = Debug.Mac|x64 @@ -369,6 +372,12 @@ Global {4CC2A90D-D240-4382-B4BF-5E175515E492}.Release.Mac|x64.Build.0 = Release|x64 {4CC2A90D-D240-4382-B4BF-5E175515E492}.Release.Windows|x64.ActiveCfg = Release|x64 {4CC2A90D-D240-4382-B4BF-5E175515E492}.Release.Windows|x64.Build.0 = Release|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Debug.Mac|x64.ActiveCfg = Debug|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Debug.Windows|x64.ActiveCfg = Debug|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Debug.Windows|x64.Build.0 = Debug|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Release.Mac|x64.ActiveCfg = Release|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Release.Windows|x64.ActiveCfg = Release|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Release.Windows|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -403,6 +412,7 @@ Global {BD7C5776-82F2-40C6-AF01-B3CC1E2D83AF} = {C41F10F9-1163-4CFA-A465-EA728F75E9FA} {FA273F69-5762-43D8-AEA1-B4F08090D624} = {C41F10F9-1163-4CFA-A465-EA728F75E9FA} {4CC2A90D-D240-4382-B4BF-5E175515E492} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C} + {AECEC217-2499-403D-B0BB-2962B9BE5970} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A025908B-DAB1-46CB-83A3-56F3B863D8FA} diff --git a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs index f34e9ad867..5ec4115fb3 100644 --- a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs +++ b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs @@ -327,7 +327,7 @@ private static void StartLogFile(string enlistmentRoot, JsonTracer tracer) tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName( Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.LogPath), - GVFSConstants.LogFileTypes.Upgrade), + GVFSConstants.LogFileTypes.UpgradeDiskLayout), EventLevel.Informational, Keywords.Any); diff --git a/GVFS/GVFS.Common/FileBasedLock.cs b/GVFS/GVFS.Common/FileBasedLock.cs index 560ad091fa..9f709f2cf4 100644 --- a/GVFS/GVFS.Common/FileBasedLock.cs +++ b/GVFS/GVFS.Common/FileBasedLock.cs @@ -1,4 +1,4 @@ -using GVFS.Common.FileSystem; +using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using System; diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index 35fbdd1c11..f223caf460 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -11,5 +11,6 @@ public interface IKernelDriver string FlushDriverLogs(); bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception); bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error); + bool IsGVFSUpgradeSupported(); } } diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index ece894ef80..a88200be62 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -1,4 +1,4 @@ -using System.IO; +using System.IO; namespace GVFS.Common { @@ -33,6 +33,7 @@ public static class GitConfig public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; public const string HooksExtension = ".hooks"; + public const string UpgradeRing = GVFSPrefix + "upgradering"; } public static class GitStatusCache @@ -72,6 +73,7 @@ public static class SpecialGitFiles public static class LogFileTypes { public const string MountPrefix = "mount"; + public const string UpgradePrefix = "upgrade"; public const string Clone = "clone"; public const string Dehydrate = "dehydrate"; @@ -80,7 +82,9 @@ public static class LogFileTypes public const string Prefetch = "prefetch"; public const string Repair = "repair"; public const string Service = "service"; - public const string Upgrade = MountPrefix + "_upgrade"; + public const string UpgradeVerb = UpgradePrefix + "_verb"; + public const string UpgradeProcess = UpgradePrefix + "_process"; + public const string UpgradeDiskLayout = MountPrefix + "_upgrade"; } public static class DotGVFS @@ -215,5 +219,13 @@ public static class Unmount public const string SkipLock = "skip-wait-for-lock"; } } + + public static class UpgradeVerbMessages + { + public const string NoneRingConsoleAlert = "Upgrade ring set to None. No upgrade check was performed."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"] from an elevated command prompt."; + public const string UpgradeAvailable = "A newer version of GVFS is available."; + public const string UpgradeInstallAdvice = "Run `gvfs upgrade --confirm` from an elevated command prompt to install."; + } } } diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index 2297a75d93..db124104bb 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -37,7 +37,7 @@ public static void Register(GVFSPlatform platform) public abstract void StartBackgroundProcess(string programName, string[] args); public abstract bool IsProcessActive(int processId); - + public abstract void IsServiceInstalledAndRunning(string name, out bool installed, out bool running); public abstract string GetNamedPipeName(string enlistmentRoot); public abstract NamedPipeServerStream CreatePipeByName(string pipeName); @@ -111,6 +111,11 @@ public string MountExecutableName { get { return "GVFS.Mount" + this.ExecutableExtension; } } + + public string GVFSUpgraderExecutableName + { + get { return "GVFS.Upgrader" + this.ExecutableExtension; } + } } } } diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 5c3bba3cde..1ce9236e16 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -92,6 +92,32 @@ public static Result GetFromSystemConfig(string gitBinPath, string settingName) return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --system " + settingName); } + public static bool TryGetVersion(out GitVersion gitVersion, out string error) + { + try + { + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + GitProcess gitProcess = new GitProcess(gitPath, null, null); + Result result = gitProcess.InvokeGitOutsideEnlistment("--version"); + string version = result.Output; + + if (!GitVersion.TryParseGitVersionCommandResult(version, out gitVersion)) + { + error = "Unable to determine installed git version. " + version; + return false; + } + + error = null; + return true; + } + catch (NotSupportedException exception) + { + error = exception.ToString(); + gitVersion = null; + return false; + } + } + public virtual void RevokeCredential(string repoUrl) { this.InvokeGitOutsideEnlistment( diff --git a/GVFS/GVFS.Common/Git/GitVersion.cs b/GVFS/GVFS.Common/Git/GitVersion.cs index a9f1e4115a..97a1bdebc2 100644 --- a/GVFS/GVFS.Common/Git/GitVersion.cs +++ b/GVFS/GVFS.Common/Git/GitVersion.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace GVFS.Common.Git { @@ -21,6 +21,21 @@ public GitVersion(int major, int minor, int build, string platform, int revision public int Revision { get; private set; } public int MinorRevision { get; private set; } + public static bool TryParseGitVersionCommandResult(string input, out GitVersion version) + { + // git version output is of the form + // git version 2.17.0.gvfs.1.preview.3 + + const string GitVersionExpectedPrefix = "git version "; + + if (input.StartsWith(GitVersionExpectedPrefix)) + { + input = input.Substring(GitVersionExpectedPrefix.Length); + } + + return TryParseVersion(input, out version); + } + public static bool TryParseInstallerName(string input, out GitVersion version) { // Installer name is of the form diff --git a/GVFS/GVFS.Common/InstallerPreRunChecker.cs b/GVFS/GVFS.Common/InstallerPreRunChecker.cs new file mode 100644 index 0000000000..e92c12742f --- /dev/null +++ b/GVFS/GVFS.Common/InstallerPreRunChecker.cs @@ -0,0 +1,212 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; + +namespace GVFS.Upgrader +{ + public class InstallerPreRunChecker + { + private static readonly HashSet BlockingProcessSet = new HashSet { "GVFS", "GVFS.Mount", "git", "ssh-agent", "bash", "wish", "git-bash" }; + + private ITracer tracer; + + public InstallerPreRunChecker(ITracer tracer) + { + this.tracer = tracer; + this.CommandToRerun = string.Empty; + } + + public string CommandToRerun { get; set; } + + public bool TryRunPreUpgradeChecks(out string consoleError) + { + this.tracer.RelatedInfo("Checking if GVFS upgrade can be run on this machine."); + + if (this.IsUnattended()) + { + consoleError = "`gvfs upgrade` is not supported in unattended mode"; + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } + + if (this.IsDevelopmentVersion()) + { + consoleError = "Cannot run upgrade when development version of GVFS is installed."; + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } + + if (!this.IsGVFSUpgradeAllowed(out consoleError)) + { + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully finished pre upgrade checks. Okay to run GVFS upgrade."); + + consoleError = null; + return true; + } + + // TODO: Move repo mount calls to GVFS.Upgrader project. + public bool TryMountAllGVFSRepos(out string consoleError) + { + return this.TryRunGVFSWithArgs("service --mount-all", out consoleError); + } + + public bool TryUnmountAllGVFSRepos(out string consoleError) + { + consoleError = null; + + this.tracer.RelatedInfo("Unmounting any mounted GVFS repositories."); + + if (!this.TryRunGVFSWithArgs("service --unmount-all", out consoleError)) + { + this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); + return false; + } + + // While checking for blocking processes like GVFS.Mount immediately after un-mounting, + // then sometimes GVFS.Mount shows up as running. But if the check is done after waiting + // for some time, then eventually GVFS.Mount goes away. The retry loop below is to help + // account for this delay between the time un-mount call returns and when GVFS.Mount + // actually quits. + this.tracer.RelatedInfo("Checking if GVFS or dependent processes are running."); + int retryCount = 10; + List processList = null; + while (retryCount > 0) + { + if (!this.IsBlockingProcessRunning(out processList)) + { + break; + } + + Thread.Sleep(TimeSpan.FromMilliseconds(250)); + retryCount--; + } + + if (processList.Count > 0) + { + consoleError = string.Join( + Environment.NewLine, + "Blocking processes are running.", + $"Run `{this.CommandToRerun}` again after quitting these processes - " + string.Join(", ", processList.ToArray())); + this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully unmounted repositories."); + + return true; + } + + protected virtual bool IsElevated() + { + return GVFSPlatform.Instance.IsElevated(); + } + + protected virtual bool IsGVFSUpgradeSupported() + { + return GVFSPlatform.Instance.KernelDriver.IsGVFSUpgradeSupported(); + } + + protected virtual bool IsServiceInstalledAndNotRunning() + { + GVFSPlatform.Instance.IsServiceInstalledAndRunning(GVFSConstants.Service.ServiceName, out bool isInstalled, out bool isRunning); + + return isInstalled && !isRunning; + } + + protected virtual bool IsUnattended() + { + return GVFSEnlistment.IsUnattended(this.tracer); + } + + protected virtual bool IsDevelopmentVersion() + { + return ProcessHelper.IsDevelopmentVersion(); + } + + protected virtual bool IsBlockingProcessRunning(out List processes) + { + int currentProcessId = Process.GetCurrentProcess().Id; + Process[] allProcesses = Process.GetProcesses(); + HashSet matchingNames = new HashSet(); + + foreach (Process process in allProcesses) + { + if (process.Id == currentProcessId || !BlockingProcessSet.Contains(process.ProcessName)) + { + continue; + } + + matchingNames.Add(process.ProcessName); + } + + processes = matchingNames.ToList(); + return processes.Count > 0; + } + + protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) + { + consoleError = null; + + string gvfsPath = Path.Combine( + ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSExecutableName), + GVFSPlatform.Instance.Constants.GVFSExecutableName); + + ProcessResult processResult = ProcessHelper.Run(gvfsPath, args); + if (processResult.ExitCode == 0) + { + return true; + } + else + { + string output = string.IsNullOrEmpty(processResult.Output) ? string.Empty : processResult.Output; + string errorString = string.IsNullOrEmpty(processResult.Errors) ? "GVFS error" : processResult.Errors; + consoleError = string.Format("{0}. {1}", errorString, output); + return false; + } + } + + private bool IsGVFSUpgradeAllowed(out string consoleError) + { + consoleError = null; + + if (!this.IsElevated()) + { + consoleError = string.Join( + Environment.NewLine, + "The installer needs to be run from an elevated command prompt.", + $"Run `{this.CommandToRerun}` again from an elevated command prompt."); + return false; + } + + if (!this.IsGVFSUpgradeSupported()) + { + consoleError = string.Join( + Environment.NewLine, + "ProjFS configuration does not support `gvfs upgrade`.", + "Check your team's documentation for how to upgrade."); + return false; + } + + if (this.IsServiceInstalledAndNotRunning()) + { + consoleError = string.Join( + Environment.NewLine, + "GVFS Service is not running.", + $"Run `sc start GVFS.Service` and run `{this.CommandToRerun}` again."); + return false; + } + + return true; + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.Common/ProcessHelper.cs b/GVFS/GVFS.Common/ProcessHelper.cs index a949a97964..9250faaf5e 100644 --- a/GVFS/GVFS.Common/ProcessHelper.cs +++ b/GVFS/GVFS.Common/ProcessHelper.cs @@ -1,4 +1,4 @@ -using Microsoft.Win32.SafeHandles; +using Microsoft.Win32.SafeHandles; using System; using System.Diagnostics; using System.IO; @@ -49,6 +49,13 @@ public static string GetCurrentProcessVersion() return fileVersionInfo.ProductVersion; } + public static bool IsDevelopmentVersion() + { + Version currentVersion = new Version(ProcessHelper.GetCurrentProcessVersion()); + + return currentVersion.Major == 0; + } + public static string WhereDirectory(string processName) { ProcessResult result = ProcessHelper.Run("where", processName); diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs new file mode 100644 index 0000000000..e8a995efcc --- /dev/null +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -0,0 +1,69 @@ +using System; +using System.Diagnostics; +using System.IO; + +namespace GVFS.Common +{ + public partial class ProductUpgrader + { + public const string UpgradeDirectoryName = "GVFS.Upgrade"; + public const string LogDirectory = "Logs"; + + private const string RootDirectory = UpgradeDirectoryName; + private const string DownloadDirectory = "Downloads"; + private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; + + public static bool IsLocalUpgradeAvailable() + { + string downloadDirectory = GetAssetDownloadsPath(); + if (Directory.Exists(downloadDirectory)) + { + string[] installers = Directory.GetFiles( + GetAssetDownloadsPath(), + $"{GVFSInstallerFileNamePrefix}*.*", + SearchOption.TopDirectoryOnly); + return installers.Length > 0; + } + + return false; + } + + public static string GetUpgradesDirectoryPath() + { + return Paths.GetServiceDataRoot(RootDirectory); + } + + public static string GetLogDirectoryPath() + { + return Path.Combine(Paths.GetServiceDataRoot(RootDirectory), LogDirectory); + } + + private static bool TryCreateDirectory(string path, out Exception exception) + { + try + { + Directory.CreateDirectory(path); + } + catch (IOException e) + { + exception = e; + return false; + } + catch (UnauthorizedAccessException e) + { + exception = e; + return false; + } + + exception = null; + return true; + } + + private static string GetAssetDownloadsPath() + { + return Path.Combine( + Paths.GetServiceDataRoot(RootDirectory), + DownloadDirectory); + } + } +} diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs new file mode 100644 index 0000000000..96f8dc6839 --- /dev/null +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -0,0 +1,477 @@ +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Json; + +namespace GVFS.Common +{ + public partial class ProductUpgrader + { + private const string GitHubReleaseURL = @"https://api.github.com/repos/microsoft/vfsforgit/releases"; + private const string JSONMediaType = @"application/vnd.github.v3+json"; + private const string UserAgent = @"GVFS_Auto_Upgrader"; + private const string CommonInstallerArgs = "/VERYSILENT /CLOSEAPPLICATIONS /SUPPRESSMSGBOXES /NORESTART"; + private const string GVFSInstallerArgs = CommonInstallerArgs + " /MOUNTREPOS=false"; + private const string GitInstallerArgs = CommonInstallerArgs + " /ALLOWDOWNGRADE=1"; + private const string GitAssetNamePrefix = "Git"; + private const string GVFSAssetNamePrefix = "GVFS"; + private const string GitInstallerFileNamePrefix = "Git-"; + private const int RepoMountFailureExitCode = 17; + private const string ToolsDirectory = "Tools"; + private static readonly string UpgraderToolName = GVFSPlatform.Instance.Constants.GVFSUpgraderExecutableName; + private static readonly string UpgraderToolConfigFile = UpgraderToolName + ".config"; + private static readonly string[] UpgraderToolAndLibs = + { + UpgraderToolName, + UpgraderToolConfigFile, + "GVFS.Common.dll", + "GVFS.Platform.Windows.dll", + "Microsoft.Diagnostics.Tracing.EventSource.dll", + "netstandard.dll", + "System.Net.Http.dll", + "Newtonsoft.Json.dll" + }; + + private Version installedVersion; + private Release newestRelease; + private PhysicalFileSystem fileSystem; + private ITracer tracer; + + public ProductUpgrader( + string currentVersion, + ITracer tracer) + { + this.installedVersion = new Version(currentVersion); + this.fileSystem = new PhysicalFileSystem(); + this.Ring = RingType.Invalid; + this.tracer = tracer; + + string upgradesDirectoryPath = GetUpgradesDirectoryPath(); + this.fileSystem.CreateDirectory(upgradesDirectoryPath); + } + + public enum RingType + { + // The values here should be ascending. + // (Fast should be greater than Slow, + // Slow should be greater than None, None greater than Invalid.) + // This is required for the correct implementation of Ring based + // upgrade logic. + Invalid = 0, + None = 10, + Slow = 20, + Fast = 30, + } + + public RingType Ring { get; protected set; } + + public bool IsNoneRing() + { + return this.TryLoadRingConfig(out string _) && this.Ring == RingType.None; + } + + public bool TryGetNewerVersion( + out Version newVersion, + out string errorMessage) + { + List releases; + + newVersion = null; + if (this.Ring == RingType.Invalid && !this.TryLoadRingConfig(out errorMessage)) + { + return false; + } + + if (this.TryFetchReleases(out releases, out errorMessage)) + { + foreach (Release nextRelease in releases) + { + Version releaseVersion; + + if (nextRelease.Ring <= this.Ring && + nextRelease.TryParseVersion(out releaseVersion) && + releaseVersion > this.installedVersion) + { + newVersion = releaseVersion; + this.newestRelease = nextRelease; + break; + } + } + + return true; + } + + return false; + } + + public bool TryGetGitVersion(out GitVersion gitVersion, out string error) + { + gitVersion = null; + error = null; + + foreach (Asset asset in this.newestRelease.Assets) + { + if (asset.Name.StartsWith(GitInstallerFileNamePrefix) && + GitVersion.TryParseInstallerName(asset.Name, out gitVersion)) + { + return true; + } + } + + error = "Could not find Git version info in newest release"; + + return false; + } + + public bool TryDownloadNewestVersion(out string errorMessage) + { + errorMessage = null; + + foreach (Asset asset in this.newestRelease.Assets) + { + if (!this.TryDownloadAsset(asset, out errorMessage)) + { + return false; + } + } + + return true; + } + + public bool TryRunGitInstaller(out bool installationSucceeded, out string error) + { + error = null; + installationSucceeded = false; + + int exitCode = 0; + bool launched = this.TryRunInstallerForAsset(GitAssetNamePrefix, out exitCode, out error); + installationSucceeded = exitCode == 0; + + return launched; + } + + public bool TryRunGVFSInstaller(out bool installationSucceeded, out string error) + { + error = null; + installationSucceeded = false; + + int exitCode = 0; + bool launched = this.TryRunInstallerForAsset(GVFSAssetNamePrefix, out exitCode, out error); + installationSucceeded = exitCode == 0 || exitCode == RepoMountFailureExitCode; + + return launched; + } + + // TrySetupToolsDirectory - + // Copies GVFS Upgrader tool and its dependencies to a temporary location in ProgramData. + // Reason why this is needed - When GVFS.Upgrader.exe is run from C:\ProgramFiles\GVFS folder + // upgrade installer that is downloaded and run will fail. This is because it cannot overwrite + // C:\ProgramFiles\GVFS\GVFS.Upgrader.exe that is running. Moving GVFS.Upgrader.exe along with + // its dependencies to a temporary location inside ProgramData and running GVFS.Upgrader.exe + // from this temporary location helps avoid this problem. + public virtual bool TrySetupToolsDirectory(out string upgraderToolPath, out string error) + { + string rootDirectoryPath = ProductUpgrader.GetUpgradesDirectoryPath(); + string toolsDirectoryPath = Path.Combine(rootDirectoryPath, ToolsDirectory); + Exception exception; + if (TryCreateDirectory(toolsDirectoryPath, out exception)) + { + string currentPath = ProcessHelper.GetCurrentProcessLocation(); + error = null; + foreach (string name in UpgraderToolAndLibs) + { + string toolPath = Path.Combine(currentPath, name); + string destinationPath = Path.Combine(toolsDirectoryPath, name); + try + { + File.Copy(toolPath, destinationPath, overwrite: true); + } + catch (UnauthorizedAccessException e) + { + error = string.Join( + Environment.NewLine, + "File copy error - " + e.Message, + $"Make sure you have write permissions to directory {rootDirectoryPath} and run `gvfs upgrade --confirm` again."); + this.TraceException(e, nameof(this.TrySetupToolsDirectory), $"Error copying {toolPath} to {destinationPath}."); + break; + } + catch (IOException e) + { + error = "File copy error - " + e.Message; + this.TraceException(e, nameof(this.TrySetupToolsDirectory), $"Error copying {toolPath} to {destinationPath}."); + break; + } + } + + upgraderToolPath = string.IsNullOrEmpty(error) ? Path.Combine(toolsDirectoryPath, UpgraderToolName) : null; + return string.IsNullOrEmpty(error); + } + + upgraderToolPath = null; + error = exception.Message; + this.TraceException(exception, nameof(this.TrySetupToolsDirectory), $"Error creating upgrade tools directory {toolsDirectoryPath}."); + return false; + } + + public bool TryCleanup(out string error) + { + error = string.Empty; + if (this.newestRelease == null) + { + return true; + } + + foreach (Asset asset in this.newestRelease.Assets) + { + Exception exception; + if (!this.TryDeleteDownloadedAsset(asset, out exception)) + { + error += $"Could not delete {asset.LocalPath}. {exception.ToString()}." + Environment.NewLine; + } + } + + if (!string.IsNullOrEmpty(error)) + { + error.TrimEnd(Environment.NewLine.ToCharArray()); + return false; + } + + error = null; + return true; + } + + protected virtual bool TryDeleteDownloadedAsset(Asset asset, out Exception exception) + { + return this.fileSystem.TryDeleteFile(asset.LocalPath, out exception); + } + + protected virtual bool TryLoadRingConfig(out string error) + { + string errorAdvisory = "Run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"]` and run `gvfs upgrade [--confirm]` again."; + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + GitProcess.Result result = GitProcess.GetFromGlobalConfig(gitPath, GVFSConstants.GitConfig.UpgradeRing); + if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) + { + string ringConfig = result.Output.TrimEnd('\r', '\n'); + RingType ringType; + + if (Enum.TryParse(ringConfig, ignoreCase: true, result: out ringType) && + Enum.IsDefined(typeof(RingType), ringType) && + ringType != RingType.Invalid) + { + this.Ring = ringType; + error = null; + return true; + } + else + { + error = "Invalid upgrade ring `" + ringConfig + "` specified in Git config."; + } + } + else + { + error = string.IsNullOrEmpty(result.Errors) ? "Unable to determine upgrade ring." : result.Errors; + } + + error += Environment.NewLine + errorAdvisory; + this.Ring = RingType.Invalid; + return false; + } + + protected virtual bool TryDownloadAsset(Asset asset, out string errorMessage) + { + errorMessage = null; + + string downloadPath = GetAssetDownloadsPath(); + Exception exception; + if (!ProductUpgrader.TryCreateDirectory(downloadPath, out exception)) + { + errorMessage = exception.Message; + this.TraceException(exception, nameof(this.TryDownloadAsset), $"Error creating download directory {downloadPath}."); + return false; + } + + string localPath = Path.Combine(downloadPath, asset.Name); + WebClient webClient = new WebClient(); + + try + { + webClient.DownloadFile(asset.DownloadURL, localPath); + asset.LocalPath = localPath; + } + catch (WebException webException) + { + errorMessage = "Download error: " + exception.Message; + this.TraceException(webException, nameof(this.TryDownloadAsset), $"Error downloading asset {asset.Name}."); + return false; + } + + return true; + } + + protected virtual bool TryFetchReleases(out List releases, out string errorMessage) + { + HttpClient client = new HttpClient(); + + client.DefaultRequestHeaders.Accept.Clear(); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(JSONMediaType)); + client.DefaultRequestHeaders.Add("User-Agent", UserAgent); + + releases = null; + errorMessage = null; + + try + { + Stream result = client.GetStreamAsync(GitHubReleaseURL).GetAwaiter().GetResult(); + + DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(List)); + releases = serializer.ReadObject(result) as List; + return true; + } + catch (HttpRequestException exception) + { + errorMessage = string.Format("Network error: could not connect to GitHub. {0}", exception.Message); + this.TraceException(exception, nameof(this.TryFetchReleases), $"Error fetching release info."); + } + catch (SerializationException exception) + { + errorMessage = string.Format("Parse error: could not parse releases info from GitHub. {0}", exception.Message); + this.TraceException(exception, nameof(this.TryFetchReleases), $"Error parsing release info."); + } + + return false; + } + + protected virtual void RunInstaller(string path, string args, out int exitCode, out string error) + { + ProcessResult processResult = ProcessHelper.Run(path, args); + + exitCode = processResult.ExitCode; + error = processResult.Errors; + } + + private bool TryRunInstallerForAsset(string name, out int installerExitCode, out string error) + { + error = null; + installerExitCode = 0; + + bool installerIsRun = false; + string path; + string installerArgs; + if (this.TryGetLocalInstallerPath(name, out path, out installerArgs)) + { + string logFilePath = GVFSEnlistment.GetNewLogFileName(GetLogDirectoryPath(), Path.GetFileNameWithoutExtension(path)); + string args = installerArgs + " /Log=" + logFilePath; + this.RunInstaller(path, args, out installerExitCode, out error); + + if (installerExitCode != 0 && string.IsNullOrEmpty(error)) + { + error = name + " installer failed. Error log: " + logFilePath; + } + + installerIsRun = true; + } + else + { + error = "Could not find downloaded installer for " + name; + } + + return installerIsRun; + } + + private void TraceException(Exception exception, string method, string message) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Method", method); + metadata.Add("Exception", exception.ToString()); + this.tracer.RelatedError(metadata, message, Keywords.Telemetry); + } + + private bool TryGetLocalInstallerPath(string name, out string path, out string args) + { + foreach (Asset asset in this.newestRelease.Assets) + { + string extension = Path.GetExtension(asset.Name); + if (string.Equals(extension, ".exe", StringComparison.OrdinalIgnoreCase)) + { + path = asset.LocalPath; + if (name == GitAssetNamePrefix && asset.Name.StartsWith(GitInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + args = GitInstallerArgs; + return true; + } + + if (name == GVFSAssetNamePrefix && asset.Name.StartsWith(GVFSInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + args = GVFSInstallerArgs; + return true; + } + } + } + + path = null; + args = null; + return false; + } + + [DataContract(Name = "asset")] + protected class Asset + { + [DataMember(Name = "name")] + public string Name { get; set; } + + [DataMember(Name = "size")] + public long Size { get; set; } + + [DataMember(Name = "browser_download_url")] + public Uri DownloadURL { get; set; } + + [IgnoreDataMember] + public string LocalPath { get; set; } + } + + [DataContract(Name = "release")] + protected class Release + { + [DataMember(Name = "name")] + public string Name { get; set; } + + [DataMember(Name = "tag_name")] + public string Tag { get; set; } + + [DataMember(Name = "prerelease")] + public bool PreRelease { get; set; } + + [DataMember(Name = "assets")] + public List Assets { get; set; } + + [IgnoreDataMember] + public RingType Ring + { + get + { + return this.PreRelease == true ? RingType.Fast : RingType.Slow; + } + } + + public bool TryParseVersion(out Version version) + { + version = null; + + if (this.Tag.StartsWith("v", StringComparison.CurrentCultureIgnoreCase)) + { + return Version.TryParse(this.Tag.Substring(1), out version); + } + + return false; + } + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs new file mode 100644 index 0000000000..837f21d273 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs @@ -0,0 +1,88 @@ +using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.Tools; +using GVFS.Tests.Should; +using NUnit.Framework; +using System; +using System.IO; +using System.Linq; + +namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture +{ + [TestFixture] + [NonParallelizable] + [Category(Categories.FullSuiteOnly)] + [Category(Categories.WindowsOnly)] + public class UpgradeReminderTests : TestsWithEnlistmentPerFixture + { + private const string GVFSInstallerName = "SetupGVFS.1.0.18234.1.exe"; + private const string GitInstallerName = "Git-2.17.1.gvfs.2.5.g2962052-64-bit.exe"; + + private string upgradeDirectory; + private FileSystemRunner fileSystem; + + public UpgradeReminderTests() + { + this.fileSystem = new SystemIORunner(); + this.upgradeDirectory = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData, Environment.SpecialFolderOption.Create), + "GVFS", + "GVFS.Upgrade", + "Downloads"); + } + + [TestCase] + public void NoNagWhenUpgradeNotAvailable() + { + this.EmptyDownloadDirectory(); + + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status"); + + string.IsNullOrEmpty(result.Errors).ShouldBeTrue(); + } + + [TestCase] + public void NagWhenUpgradeAvailable() + { + this.CreateUpgradeInstallers(); + + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status"); + + result.Errors.ShouldContain(new string[] + { + "A newer version of GVFS is available.", + "Run `gvfs upgrade --confirm` from an elevated command prompt to install." + }); + + this.EmptyDownloadDirectory(); + } + + private void EmptyDownloadDirectory() + { + if (Directory.Exists(this.upgradeDirectory)) + { + Directory.Delete(this.upgradeDirectory, recursive: true); + } + + Directory.CreateDirectory(this.upgradeDirectory); + Directory.Exists(this.upgradeDirectory).ShouldBeTrue(); + Directory.EnumerateFiles(this.upgradeDirectory).Any().ShouldBeFalse(); + } + + private void CreateUpgradeInstallers() + { + string gvfsInstallerPath = Path.Combine(this.upgradeDirectory, GVFSInstallerName); + string gitInstallerPath = Path.Combine(this.upgradeDirectory, GitInstallerName); + + this.EmptyDownloadDirectory(); + + this.fileSystem.CreateEmptyFile(gvfsInstallerPath); + this.fileSystem.CreateEmptyFile(gitInstallerPath); + this.fileSystem.FileExists(gvfsInstallerPath).ShouldBeTrue(); + this.fileSystem.FileExists(gitInstallerPath).ShouldBeTrue(); + } + } +} diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj index dccf8bdd31..4cdc8852b3 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj @@ -77,6 +77,7 @@ Common\ProcessResult.cs + Common\Tracing\EventLevel.cs diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj index 05629c39e3..e7dee24e31 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj @@ -1,4 +1,4 @@ - + @@ -101,6 +101,9 @@ Common\ProcessResult.cs + + Common\ProductUpgrader.Shared.cs + Common\Tracing\EventLevel.cs diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 5ffaea8c0c..e535b34520 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -1,4 +1,4 @@ -using GVFS.Common; +using GVFS.Common; using GVFS.Common.Git; using GVFS.Common.NamedPipes; using GVFS.Hooks.HooksPlatform; @@ -83,6 +83,12 @@ public static void Main(string[] args) private static void RunPreCommands(string[] args) { + if (ProductUpgrader.IsLocalUpgradeAvailable()) + { + Console.WriteLine(GVFSConstants.UpgradeVerbMessages.UpgradeAvailable); + Console.WriteLine(GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); + } + string command = GetGitCommand(args); switch (command) { diff --git a/GVFS/GVFS.Installer/Setup.iss b/GVFS/GVFS.Installer/Setup.iss index f2875b699a..e01597ee2b 100644 --- a/GVFS/GVFS.Installer/Setup.iss +++ b/GVFS/GVFS.Installer/Setup.iss @@ -14,6 +14,7 @@ #define GVFSMountDir BuildOutputDir + "\GVFS.Mount.Windows\bin\" + PlatformAndConfiguration #define ReadObjectDir BuildOutputDir + "\GVFS.ReadObjectHook.Windows\bin\" + PlatformAndConfiguration #define VirtualFileSystemDir BuildOutputDir + "\GVFS.VirtualFileSystemHook.Windows\bin\" + PlatformAndConfiguration +#define GVFSUpgraderDir BuildOutputDir + "\GVFS.Upgrader\bin\" + PlatformAndConfiguration #define MyAppName "GVFS" #define MyAppInstallerVersion GetFileVersion(GVFSDir + "\GVFS.exe") @@ -34,7 +35,7 @@ AppPublisher={#MyAppPublisher} AppPublisherURL={#MyAppPublisherURL} AppSupportURL={#MyAppURL} AppUpdatesURL={#MyAppURL} -AppCopyright=Copyright © Microsoft 2017 +AppCopyright=Copyright � Microsoft 2018 BackColor=clWhite BackSolid=yes DefaultDirName={pf}\{#MyAppName} @@ -93,6 +94,11 @@ DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSMountDir}\GVFS.Mount.pdb" DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSMountDir}\GVFS.Mount.exe" DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSMountDir}\GVFS.Mount.exe.config" +; GVFS.Upgrader Files +DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSUpgraderDir}\GVFS.Upgrader.pdb" +DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSUpgraderDir}\GVFS.Upgrader.exe" +DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSUpgraderDir}\GVFS.Upgrader.exe.config" + ; GVFS.ReadObjectHook files DestDir: "{app}"; Flags: ignoreversion; Source:"{#ReadObjectDir}\GVFS.ReadObjectHook.pdb" DestDir: "{app}"; Flags: ignoreversion; Source:"{#ReadObjectDir}\GVFS.ReadObjectHook.exe" @@ -159,6 +165,7 @@ DestDir: "{app}"; Flags: ignoreversion; Source:"{#ServiceDir}\GVFS.Service.exe"; [UninstallDelete] ; Deletes the entire installation directory, including files and subdirectories Type: filesandordirs; Name: "{app}"; +Type: filesandordirs; Name: "{commonappdata}\GVFS\GVFS.Upgrade"; [Registry] Root: HKLM; Subkey: "{#EnvironmentKey}"; \ @@ -623,7 +630,10 @@ begin end; ssPostInstall: begin - MountRepos(); + if ExpandConstant('{param:REMOUNTREPOS|true}') = 'true' then + begin + MountRepos(); + end end; end; end; @@ -650,7 +660,10 @@ begin Result := ''; if ConfirmUnmountAll() then begin - UnmountRepos(); + if ExpandConstant('{param:REMOUNTREPOS|true}') = 'true' then + begin + UnmountRepos(); + end end; if not EnsureGvfsNotRunning() then begin diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index 9be9a5ed9b..6dade62c76 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -56,6 +56,11 @@ public override bool IsProcessActive(int processId) return MacPlatform.IsProcessActiveImplementation(processId); } + public override void IsServiceInstalledAndRunning(string name, out bool installed, out bool running) + { + throw new NotImplementedException(); + } + public override void StartBackgroundProcess(string programName, string[] args) { ProcessLauncher.StartBackgroundProcess(programName, args); diff --git a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs index ede5623fc4..01663e6064 100644 --- a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs +++ b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs @@ -1,4 +1,4 @@ -using GVFS.Common; +using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using PrjFSLib.Mac; @@ -14,6 +14,12 @@ public class ProjFSKext : IKernelDriver public string DriverLogFolderName => throw new NotImplementedException(); + public bool IsGVFSUpgradeSupported() + { + // TODO(Mac) + return false; + } + public bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error) { warning = null; diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 3d7b4ce926..a5a0c3208e 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -1,4 +1,4 @@ -using GVFS.Common; +using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using Microsoft.Win32; @@ -35,7 +35,16 @@ public class ProjFSFilter : IKernelDriver private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; + private enum ProjFSInboxStatus + { + Invalid, + NotInbox = 2, + Enabled = 3, + Disabled = 4, + } + public bool EnumerationExpandsDirectories { get; } = false; + public string DriverLogFolderName { get; } = ProjFSFilter.ServiceName; public static bool TryAttach(ITracer tracer, string enlistmentRoot, out string errorMessage) @@ -272,6 +281,11 @@ public static bool IsNativeLibInstalled(ITracer tracer, PhysicalFileSystem fileS return existsInSystem32 || existsInAppDirectory; } + public bool IsGVFSUpgradeSupported() + { + return IsInboxAndEnabled(); + } + public bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error) { warning = null; @@ -362,6 +376,12 @@ public bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error) TryAttach(tracer, enlistmentRoot, out error); } + private static bool IsInboxAndEnabled() + { + ProcessResult getOptionalFeatureResult = GetProjFSOptionalFeatureStatus(); + return getOptionalFeatureResult.ExitCode == (int)ProjFSInboxStatus.Enabled; + } + private static bool TryGetIsInboxProjFSFinalAPI(ITracer tracer, out uint windowsBuildNumber, out bool isProjFSInbox) { isProjFSInbox = false; @@ -495,20 +515,13 @@ private static void GetNativeLibPaths(string gvfsAppDirectory, out string instal private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileSystem fileSystem, out bool isProjFSFeatureAvailable) { EventMetadata metadata = CreateEventMetadata(); - - const int ProjFSNotAnOptionalFeature = 2; - const int ProjFSEnabled = 3; - const int ProjFSDisabled = 4; - - ProcessResult getOptionalFeatureResult = CallPowershellCommand( - "$var=(Get-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + "); if($var -eq $null){exit " + - ProjFSNotAnOptionalFeature + "}else{if($var.State -eq 'Enabled'){exit " + ProjFSEnabled + "}else{exit " + ProjFSDisabled + "}}"); + ProcessResult getOptionalFeatureResult = GetProjFSOptionalFeatureStatus(); isProjFSFeatureAvailable = true; bool projFSEnabled = false; switch (getOptionalFeatureResult.ExitCode) { - case ProjFSNotAnOptionalFeature: + case (int)ProjFSInboxStatus.NotInbox: metadata.Add("getOptionalFeatureResult.Output", getOptionalFeatureResult.Output); metadata.Add("getOptionalFeatureResult.Errors", getOptionalFeatureResult.Errors); tracer.RelatedWarning(metadata, $"{nameof(TryEnableProjFSOptionalFeature)}: {OptionalFeatureName} optional feature is missing"); @@ -516,7 +529,7 @@ private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileS isProjFSFeatureAvailable = false; break; - case ProjFSEnabled: + case (int)ProjFSInboxStatus.Enabled: tracer.RelatedEvent( EventLevel.Informational, $"{nameof(TryEnableProjFSOptionalFeature)}_ClientProjFSAlreadyEnabled", @@ -525,7 +538,7 @@ private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileS projFSEnabled = true; break; - case ProjFSDisabled: + case (int)ProjFSInboxStatus.Disabled: ProcessResult enableOptionalFeatureResult = CallPowershellCommand("try {Enable-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + " -NoRestart}catch{exit 1}"); metadata.Add("enableOptionalFeatureResult.Output", enableOptionalFeatureResult.Output.Trim().Replace("\r\n", ",")); metadata.Add("enableOptionalFeatureResult.Errors", enableOptionalFeatureResult.Errors); @@ -564,6 +577,18 @@ private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileS return false; } + private static ProcessResult GetProjFSOptionalFeatureStatus() + { + return CallPowershellCommand( + "$var=(Get-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + "); if($var -eq $null){exit " + + (int)ProjFSInboxStatus.NotInbox + "}else{if($var.State -eq 'Enabled'){exit " + (int)ProjFSInboxStatus.Enabled + "}else{exit " + (int)ProjFSInboxStatus.Disabled + "}}"); + } + + private static ProcessResult CallPowershellCommand(string command) + { + return ProcessHelper.Run("powershell.exe", "-NonInteractive -NoProfile -Command \"& { " + command + " }\""); + } + private static EventMetadata CreateEventMetadata(Exception e = null) { EventMetadata metadata = new EventMetadata(); diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs index aa8c3953b6..0880ce9a48 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs @@ -12,6 +12,7 @@ using System.Linq; using System.Security.AccessControl; using System.Security.Principal; +using System.ServiceProcess; using System.Text; namespace GVFS.Platform.Windows @@ -170,6 +171,14 @@ public override bool IsProcessActive(int processId) return WindowsPlatform.IsProcessActiveImplementation(processId); } + public override void IsServiceInstalledAndRunning(string name, out bool installed, out bool running) + { + ServiceController service = ServiceController.GetServices().FirstOrDefault(s => s.ServiceName.Equals(name, StringComparison.Ordinal)); + + installed = service != null; + running = service != null ? service.Status == ServiceControllerStatus.Running : false; + } + public override string GetNamedPipeName(string enlistmentRoot) { return WindowsPlatform.GetNamedPipeNameImplementation(enlistmentRoot); diff --git a/GVFS/GVFS.Service/GVFS.Service.csproj b/GVFS/GVFS.Service/GVFS.Service.csproj index 32c7a47533..19975fabe5 100644 --- a/GVFS/GVFS.Service/GVFS.Service.csproj +++ b/GVFS/GVFS.Service/GVFS.Service.csproj @@ -71,6 +71,7 @@ + diff --git a/GVFS/GVFS.Service/GvfsService.cs b/GVFS/GVFS.Service/GvfsService.cs index 0134cd244f..4a51e988d0 100644 --- a/GVFS/GVFS.Service/GvfsService.cs +++ b/GVFS/GVFS.Service/GvfsService.cs @@ -1,4 +1,4 @@ -using GVFS.Common; +using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.NamedPipes; using GVFS.Common.Tracing; @@ -6,6 +6,7 @@ using System; using System.IO; using System.Linq; +using System.Reflection; using System.Runtime.Serialization; using System.ServiceProcess; using System.Threading; @@ -23,12 +24,14 @@ public class GVFSService : ServiceBase private string serviceName; private string serviceDataLocation; private RepoRegistry repoRegistry; + private ProductUpgradeTimer productUpgradeTimer; public GVFSService(JsonTracer tracer) { this.tracer = tracer; this.serviceName = GVFSConstants.Service.ServiceName; this.CanHandleSessionChangeEvent = true; + this.productUpgradeTimer = new ProductUpgradeTimer(tracer); } public void Run() @@ -37,6 +40,7 @@ public void Run() { this.repoRegistry = new RepoRegistry(this.tracer, new PhysicalFileSystem(), this.serviceDataLocation); this.repoRegistry.Upgrade(); + this.productUpgradeTimer.Start(); string pipeName = this.serviceName + ".Pipe"; this.tracer.RelatedInfo("Starting pipe server with name: " + pipeName); @@ -68,6 +72,11 @@ public void StopRunning() try { + if (this.productUpgradeTimer != null) + { + this.productUpgradeTimer.Stop(); + } + if (this.tracer != null) { this.tracer.RelatedInfo("Stopping"); diff --git a/GVFS/GVFS.Service/ProductUpgradeTimer.cs b/GVFS/GVFS.Service/ProductUpgradeTimer.cs new file mode 100644 index 0000000000..48c37d4154 --- /dev/null +++ b/GVFS/GVFS.Service/ProductUpgradeTimer.cs @@ -0,0 +1,83 @@ +using GVFS.Common; +using GVFS.Common.Tracing; +using GVFS.Upgrader; +using System; +using System.Collections.Generic; +using System.Threading; + +namespace GVFS.Service +{ + public class ProductUpgradeTimer + { + private static readonly TimeSpan TimeInterval = TimeSpan.FromDays(1); + private JsonTracer tracer; + private Timer timer; + + public ProductUpgradeTimer(JsonTracer tracer) + { + this.tracer = tracer; + } + + public void Start() + { + Random random = new Random(); + TimeSpan startTime = TimeSpan.FromMinutes(random.Next(0, 60)); + + this.tracer.RelatedInfo($"Starting auto upgrade checks. Start time - {startTime.ToString()}"); + this.timer = new Timer( + this.TimerCallback, + null, + startTime, + TimeInterval); + } + + public void Stop() + { + this.tracer.RelatedInfo("Stopping auto upgrade checks"); + this.timer.Dispose(); + } + + private void TimerCallback(object unusedState) + { + string errorMessage = null; + + InstallerPreRunChecker prerunChecker = new InstallerPreRunChecker(this.tracer); + if (prerunChecker.TryRunPreUpgradeChecks(out string _) && !this.TryDownloadUpgrade(out errorMessage)) + { + this.tracer.RelatedError(errorMessage); + } + } + + private bool TryDownloadUpgrade(out string errorMessage) + { + this.tracer.RelatedInfo("Checking for product upgrades."); + + ProductUpgrader productUpgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + Version newerVersion = null; + string detailedError = null; + if (!productUpgrader.TryGetNewerVersion(out newerVersion, out detailedError)) + { + errorMessage = "Could not fetch new version info. " + detailedError; + return false; + } + + if (newerVersion == null) + { + // Already up-to-date + errorMessage = null; + return true; + } + + if (productUpgrader.TryDownloadNewestVersion(out detailedError)) + { + errorMessage = null; + return true; + } + else + { + errorMessage = "Could not download product upgrade. " + detailedError; + return false; + } + } + } +} diff --git a/GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj b/GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj index 7f14680f1c..d3dacff460 100644 --- a/GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj +++ b/GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj @@ -47,6 +47,7 @@ $(BuildOutputDir)\GVFS.Service.UI\bin\$(Platform)\$(Configuration)\GVFS.Service.UI.exe; $(BuildOutputDir)\GVFS.Virtualization\bin\$(Platform)\$(Configuration)\netstandard2.0\GVFS.Virtualization.dll; $(BuildOutputDir)\GVFS.VirtualFileSystemHook.Windows\bin\$(Platform)\$(Configuration)\GVFS.VirtualFileSystemHook.exe; + $(BuildOutputDir)\GVFS.Upgrader\bin\$(Platform)\$(Configuration)\GVFS.Upgrader.exe; $(BuildOutputDir)\GVFS.Windows\bin\$(Platform)\$(Configuration)\GVFS.exe;"> Microsoft false diff --git a/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj b/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj index 6d9597a17c..aaf571282c 100644 --- a/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj +++ b/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj @@ -77,7 +77,15 @@ + + + + + + + + @@ -120,6 +128,10 @@ {72701bc3-5da9-4c7a-bf10-9e98c9fc8eac} GVFS.Tests + + {aecec217-2499-403d-b0bb-2962b9be5970} + GVFS.Upgrader + {F468B05A-95E5-46BC-8C67-B80A78527B7D} GVFS.Virtualization diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs new file mode 100644 index 0000000000..6b51f9a155 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs @@ -0,0 +1,129 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using GVFS.UnitTests.Windows.Upgrader; +using GVFS.Upgrader; +using System; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.UnitTests.Windows.Mock.Upgrader +{ + public class MockInstallerPrerunChecker : InstallerPreRunChecker + { + public const string GitUpgradeCheckError = "Unable to upgrade Git"; + + private FailOnCheckType failOnCheck; + + public MockInstallerPrerunChecker(ITracer tracer) : base(tracer) + { + } + + [Flags] + public enum FailOnCheckType + { + Invalid = 0, + ProjFSEnabled = 0x1, + IsElevated = 0x2, + BlockingProcessesRunning = 0x4, + UnattendedMode = 0x8, + IsDevelopmentVersion = 0x10, + IsGitUpgradeAllowed = 0x20, + UnMountRepos = 0x40, + RemountRepos = 0x80, + IsServiceInstalledAndNotRunning = 0x100 + } + + public List GVFSArgs { get; private set; } = new List(); + + public void SetReturnFalseOnCheck(FailOnCheckType prerunCheck) + { + this.failOnCheck |= prerunCheck; + } + + public void SetReturnTrueOnCheck(FailOnCheckType prerunCheck) + { + this.failOnCheck &= ~prerunCheck; + } + + public void Reset() + { + this.failOnCheck = FailOnCheckType.Invalid; + + this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsDevelopmentVersion); + this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.UnattendedMode); + this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.BlockingProcessesRunning); + this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsServiceInstalledAndNotRunning); + + this.GVFSArgs.Clear(); + } + + protected override bool IsServiceInstalledAndNotRunning() + { + return this.FakedResultOfCheck(FailOnCheckType.IsServiceInstalledAndNotRunning); + } + + protected override bool IsElevated() + { + return this.FakedResultOfCheck(FailOnCheckType.IsElevated); + } + + protected override bool IsGVFSUpgradeSupported() + { + return this.FakedResultOfCheck(FailOnCheckType.ProjFSEnabled); + } + + protected override bool IsUnattended() + { + return this.FakedResultOfCheck(FailOnCheckType.UnattendedMode); + } + + protected override bool IsDevelopmentVersion() + { + return this.FakedResultOfCheck(FailOnCheckType.IsDevelopmentVersion); + } + + protected override bool IsBlockingProcessRunning(out List processes) + { + processes = new List(); + + bool isRunning = this.FakedResultOfCheck(FailOnCheckType.BlockingProcessesRunning); + if (isRunning) + { + processes.Add("GVFS.Mount"); + processes.Add("git"); + } + + return isRunning; + } + + protected override bool TryRunGVFSWithArgs(string args, out string error) + { + this.GVFSArgs.Add(args); + + if (string.CompareOrdinal(args, "service --unmount-all") == 0) + { + bool result = this.FakedResultOfCheck(FailOnCheckType.UnMountRepos); + error = result == false ? "Unmount of some of the repositories failed." : null; + return result; + } + + if (string.CompareOrdinal(args, "service --mount-all") == 0) + { + bool result = this.FakedResultOfCheck(FailOnCheckType.RemountRepos); + error = result == false ? "Auto remount failed." : null; + return result; + } + + error = "Unknown GVFS command"; + return false; + } + + private bool FakedResultOfCheck(FailOnCheckType checkType) + { + bool result = this.failOnCheck.HasFlag(checkType) ? false : true; + + return result; + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs new file mode 100644 index 0000000000..dc3ddde556 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs @@ -0,0 +1,45 @@ +using System; +using static GVFS.CommandLine.UpgradeVerb; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + public class MockProcessLauncher : ProcessLauncher + { + private int exitCode; + private bool hasExited; + private bool startResult; + + public MockProcessLauncher( + int exitCode, + bool hasExited, + bool startResult) : base() + { + this.exitCode = exitCode; + this.hasExited = hasExited; + this.startResult = startResult; + } + + public bool IsLaunched { get; private set; } + + public string LaunchPath { get; private set; } + + public override bool HasExited + { + get { return this.hasExited; } + } + + public override int ExitCode + { + get { return this.exitCode; } + } + + public override bool TryStart(string path, out Exception exception) + { + this.LaunchPath = path; + this.IsLaunched = true; + + exception = null; + return this.startResult; + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs new file mode 100644 index 0000000000..2299943935 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -0,0 +1,241 @@ +using GVFS.Common; +using GVFS.Common.Tracing; +using GVFS.UnitTests.Windows.Upgrader; +using System; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.UnitTests.Windows.Mock.Upgrader +{ + public class MockProductUpgrader : ProductUpgrader + { + private string expectedGVFSAssetName; + private string expectedGitAssetName; + private ActionType failActionTypes; + + public MockProductUpgrader( + string currentVersion, + ITracer tracer) : base(currentVersion, tracer) + { + this.DownloadedFiles = new List(); + this.InstallerArgs = new Dictionary>(); + } + + [Flags] + public enum ActionType + { + Invalid = 0, + FetchReleaseInfo = 0x1, + CopyTools = 0x2, + GitDownload = 0x4, + GVFSDownload = 0x8, + GitInstall = 0x10, + GVFSInstall = 0x20, + GVFSCleanup = 0x40, + GitCleanup = 0x80, + } + + public RingType LocalRingConfig { get; set; } + public List DownloadedFiles { get; private set; } + public Dictionary> InstallerArgs { get; private set; } + + private Release FakeUpgradeRelease { get; set; } + + public void SetFailOnAction(ActionType failureType) + { + this.failActionTypes |= failureType; + } + + public void SetSucceedOnAction(ActionType failureType) + { + this.failActionTypes &= ~failureType; + } + + public void ResetFailedAction() + { + this.failActionTypes = ActionType.Invalid; + } + + public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType remoteRing) + { + string assetDownloadURLPrefix = "https://github.com/Microsoft/VFSForGit/releases/download/v" + upgradeVersion; + + Release release = new Release(); + + release.Name = "GVFS " + upgradeVersion; + release.Tag = "v" + upgradeVersion; + release.PreRelease = remoteRing == RingType.Fast; + release.Assets = new List(); + + Random random = new Random(); + Asset gvfsAsset = new Asset(); + gvfsAsset.Name = "SetupGVFS." + upgradeVersion + ".exe"; + + // This is not cross-checked anywhere, random value is good. + gvfsAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); + gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/SetupGVFS." + upgradeVersion + ".exe"); + release.Assets.Add(gvfsAsset); + + Asset gitAsset = new Asset(); + gitAsset.Name = "Git-2.17.1.gvfs.2.1.4.g4385455-64-bit.exe"; + gitAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); + gitAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/Git-2.17.1.gvfs.2.1.4.g4385455-64-bit.exe"); + release.Assets.Add(gitAsset); + + this.expectedGVFSAssetName = gvfsAsset.Name; + this.expectedGitAssetName = gitAsset.Name; + this.FakeUpgradeRelease = release; + } + + public override bool TrySetupToolsDirectory(out string upgraderToolPath, out string error) + { + if (this.failActionTypes.HasFlag(ActionType.CopyTools)) + { + upgraderToolPath = null; + error = "Unable to copy upgrader tools"; + return false; + } + + upgraderToolPath = @"C:\ProgramData\GVFS\GVFS.Upgrade\Tools\GVFS.Upgrader.exe"; + error = null; + return true; + } + + protected override bool TryLoadRingConfig(out string error) + { + this.Ring = this.LocalRingConfig; + + if (this.LocalRingConfig == RingType.Invalid) + { + error = "Invalid upgrade ring `Invalid` specified in Git config."; + return false; + } + + error = null; + return true; + } + + protected override bool TryDownloadAsset(Asset asset, out string errorMessage) + { + bool validAsset = true; + if (this.expectedGVFSAssetName.Equals(asset.Name, StringComparison.OrdinalIgnoreCase)) + { + if (this.failActionTypes.HasFlag(ActionType.GVFSDownload)) + { + errorMessage = "Error downloading GVFS from GitHub"; + return false; + } + } + else if (this.expectedGitAssetName.Equals(asset.Name, StringComparison.OrdinalIgnoreCase)) + { + if (this.failActionTypes.HasFlag(ActionType.GitDownload)) + { + errorMessage = "Error downloading Git from GitHub"; + return false; + } + } + else + { + validAsset = false; + } + + if (validAsset) + { + string fakeDownloadDirectory = @"C:\ProgramData\GVFS\GVFS.Upgrade\Downloads"; + asset.LocalPath = Path.Combine(fakeDownloadDirectory, asset.Name); + this.DownloadedFiles.Add(asset.LocalPath); + + errorMessage = null; + return true; + } + + errorMessage = "Cannot download unknown asset."; + return false; + } + + protected override bool TryDeleteDownloadedAsset(Asset asset, out Exception exception) + { + if (this.expectedGVFSAssetName.Equals(asset.Name, StringComparison.OrdinalIgnoreCase)) + { + if (this.failActionTypes.HasFlag(ActionType.GVFSCleanup)) + { + exception = new Exception("Error deleting downloaded GVFS installer."); + return false; + } + + exception = null; + return true; + } + else if (this.expectedGitAssetName.Equals(asset.Name, StringComparison.OrdinalIgnoreCase)) + { + if (this.failActionTypes.HasFlag(ActionType.GitCleanup)) + { + exception = new Exception("Error deleting downloaded Git installer."); + return false; + } + + exception = null; + return true; + } + else + { + exception = new Exception("Unknown asset."); + return false; + } + } + + protected override bool TryFetchReleases(out List releases, out string errorMessage) + { + if (this.failActionTypes.HasFlag(ActionType.FetchReleaseInfo)) + { + releases = null; + errorMessage = "Error fetching upgrade release info."; + return false; + } + + releases = new List { this.FakeUpgradeRelease }; + errorMessage = null; + + return true; + } + + protected override void RunInstaller(string path, string args, out int exitCode, out string error) + { + string fileName = Path.GetFileName(path); + Dictionary installationInfo = new Dictionary(); + installationInfo.Add("Installer", fileName); + installationInfo.Add("Args", args); + + exitCode = 0; + error = null; + + if (fileName.Equals(this.expectedGitAssetName, StringComparison.OrdinalIgnoreCase)) + { + this.InstallerArgs.Add("Git", installationInfo); + if (this.failActionTypes.HasFlag(ActionType.GitInstall)) + { + exitCode = -1; + error = "Git installation failed"; + } + + return; + } + + if (fileName.Equals(this.expectedGVFSAssetName, StringComparison.OrdinalIgnoreCase)) + { + this.InstallerArgs.Add("GVFS", installationInfo); + if (this.failActionTypes.HasFlag(ActionType.GVFSInstall)) + { + exitCode = -1; + error = "GVFS installation failed"; + } + + return; + } + + exitCode = -1; + error = "Cannot launch unknown installer"; + return; + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockTextWriter.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockTextWriter.cs new file mode 100644 index 0000000000..e6e8c7fd93 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockTextWriter.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace GVFS.UnitTests.Windows.Mock.Upgrader +{ + public class MockTextWriter : TextWriter + { + private StringBuilder stringBuilder; + + public MockTextWriter() : base() + { + this.AllLines = new List(); + this.stringBuilder = new StringBuilder(); + } + + public List AllLines { get; private set; } + + public override Encoding Encoding + { + get { return Encoding.Default; } + } + + public override void Write(char value) + { + if (value.Equals('\r')) + { + return; + } + + if (value.Equals('\n')) + { + this.AllLines.Add(this.stringBuilder.ToString()); + this.stringBuilder.Clear(); + return; + } + + this.stringBuilder.Append(value); + } + + public bool ContainsLine(string line) + { + return this.AllLines.Exists(x => x.Equals(line, StringComparison.Ordinal)); + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/ProductUpgraderTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/ProductUpgraderTests.cs new file mode 100644 index 0000000000..dec15f652d --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/ProductUpgraderTests.cs @@ -0,0 +1,135 @@ +using GVFS.Common; +using GVFS.Tests.Should; +using NUnit.Framework; +using System; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + [TestFixture] + public class ProductUpgraderTests : UpgradeTests + { + [SetUp] + public override void Setup() + { + base.Setup(); + } + + [TestCase] + public void UpgradeAvailableOnFastWhileOnLocalNoneRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Fast, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.None, + expectedReturn: true, + expectedUpgradeVersion: null); + } + + [TestCase] + public void UpgradeAvailableOnSlowWhileOnLocalNoneRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Slow, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.None, + expectedReturn: true, + expectedUpgradeVersion: null); + } + + [TestCase] + public void UpgradeAvailableOnFastWhileOnLocalSlowRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Fast, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.Slow, + expectedReturn: true, + expectedUpgradeVersion: null); + } + + [TestCase] + public void UpgradeAvailableOnSlowWhileOnLocalSlowRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Slow, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.Slow, + expectedReturn: true, + expectedUpgradeVersion: UpgradeTests.NewerThanLocalVersion); + } + + [TestCase] + public void UpgradeAvailableOnFastWhileOnLocalFastRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Fast, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.Fast, + expectedReturn: true, + expectedUpgradeVersion: UpgradeTests.NewerThanLocalVersion); + } + + [TestCase] + public void UpgradeAvailableOnSlowWhileOnLocalFastRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Slow, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing:ProductUpgrader.RingType.Fast, + expectedReturn: true, + expectedUpgradeVersion:UpgradeTests.NewerThanLocalVersion); + } + + public override void NoneLocalRing() + { + throw new NotImplementedException(); + } + + public override void InvalidUpgradeRing() + { + throw new NotImplementedException(); + } + + public override void FetchReleaseInfo() + { + throw new NotImplementedException(); + } + + protected override void RunUpgrade() + { + throw new NotImplementedException(); + } + + protected override ReturnCode ExitCode() + { + return ReturnCode.Success; + } + + private void SimulateUpgradeAvailable( + ProductUpgrader.RingType remoteRing, + string remoteVersion, + ProductUpgrader.RingType localRing, + bool expectedReturn, + string expectedUpgradeVersion) + { + this.Upgrader.LocalRingConfig = localRing; + this.Upgrader.PretendNewReleaseAvailableAtRemote( + remoteVersion, + remoteRing); + + Version newVersion; + string errorMessage; + this.Upgrader.TryGetNewerVersion(out newVersion, out errorMessage).ShouldEqual(expectedReturn); + + if (string.IsNullOrEmpty(expectedUpgradeVersion)) + { + newVersion.ShouldBeNull(); + } + else + { + newVersion.ShouldNotBeNull(); + newVersion.ShouldEqual(new Version(expectedUpgradeVersion)); + } + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs new file mode 100644 index 0000000000..abced85c4a --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs @@ -0,0 +1,250 @@ +using GVFS.Common; +using GVFS.Tests.Should; +using GVFS.UnitTests.Windows.Mock.Upgrader; +using GVFS.Upgrader; +using NUnit.Framework; +using System.Collections.Generic; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + [TestFixture] + public class UpgradeOrchestratorTests : UpgradeTests + { + private UpgradeOrchestrator Orchestrator { get; set; } + + [SetUp] + public override void Setup() + { + base.Setup(); + + this.Orchestrator = new UpgradeOrchestrator( + this.Upgrader, + this.Tracer, + this.PrerunChecker, + input: null, + output: this.Output, + shouldExitOnError: false); + this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + } + + [TestCase] + public void UpgradeNoError() + { + this.Orchestrator.Execute(); + this.Orchestrator.ExitCode.ShouldEqual(ReturnCode.Success); + this.Tracer.RelatedErrorEvents.ShouldBeEmpty(); + } + + [TestCase] + public void AutoUnmountError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.PrerunChecker.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.UnMountRepos); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Unmount of some of the repositories failed." + }, + expectedErrors: new List + { + "Unmount of some of the repositories failed." + }); + } + + [TestCase] + public void AbortOnBlockingProcess() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.BlockingProcessesRunning); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "ERROR: Blocking processes are running.", + $"Run `gvfs upgrade --confirm` again after quitting these processes - GVFS.Mount, git" + }, + expectedErrors: new List + { + $"Run `gvfs upgrade --confirm` again after quitting these processes - GVFS.Mount, git" + }); + } + + [TestCase] + public void GVFSDownloadError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GVFSDownload); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Error downloading GVFS from GitHub" + }, + expectedErrors: new List + { + "Error downloading GVFS from GitHub" + }); + } + + [TestCase] + public void GitDownloadError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GitDownload); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Error downloading Git from GitHub" + }, + expectedErrors: new List + { + "Error downloading Git from GitHub" + }); + } + + [TestCase] + public void GitInstallationArgs() + { + this.Orchestrator.Execute(); + + this.Orchestrator.ExitCode.ShouldEqual(ReturnCode.Success); + + Dictionary gitInstallerInfo; + this.Upgrader.InstallerArgs.ShouldBeNonEmpty(); + this.Upgrader.InstallerArgs.TryGetValue("Git", out gitInstallerInfo).ShouldBeTrue(); + + string args; + gitInstallerInfo.TryGetValue("Args", out args).ShouldBeTrue(); + args.ShouldContain(new string[] { "/VERYSILENT", "/CLOSEAPPLICATIONS", "/SUPPRESSMSGBOXES", "/NORESTART", "/Log" }); + } + + [TestCase] + public void GitInstallError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GitInstall); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Git installation failed" + }, + expectedErrors: new List + { + "Git installation failed" + }); + } + + [TestCase] + public void GVFSInstallationArgs() + { + this.Orchestrator.Execute(); + + this.Orchestrator.ExitCode.ShouldEqual(ReturnCode.Success); + + Dictionary gitInstallerInfo; + this.Upgrader.InstallerArgs.ShouldBeNonEmpty(); + this.Upgrader.InstallerArgs.TryGetValue("GVFS", out gitInstallerInfo).ShouldBeTrue(); + + string args; + gitInstallerInfo.TryGetValue("Args", out args).ShouldBeTrue(); + args.ShouldContain(new string[] { "/VERYSILENT", "/CLOSEAPPLICATIONS", "/SUPPRESSMSGBOXES", "/NORESTART", "/Log", "/MOUNTREPOS=false" }); + } + + [TestCase] + public void GVFSInstallError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GVFSInstall); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "GVFS installation failed" + }, + expectedErrors: new List + { + "GVFS installation failed" + }); + } + + [TestCase] + public void GVFSCleanupError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GVFSCleanup); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + }, + expectedErrors: new List + { + "Error deleting downloaded GVFS installer." + }); + } + + [TestCase] + public void GitCleanupError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GitCleanup); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + }, + expectedErrors: new List + { + "Error deleting downloaded Git installer." + }); + } + + [TestCase] + public void RemountReposError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.PrerunChecker.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.RemountRepos); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + "Auto remount failed." + }, + expectedErrors: new List + { + "Auto remount failed." + }); + } + + protected override void RunUpgrade() + { + this.Orchestrator.Execute(); + } + + protected override ReturnCode ExitCode() + { + return this.Orchestrator.ExitCode; + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs new file mode 100644 index 0000000000..9f9e1b3d37 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs @@ -0,0 +1,128 @@ +using GVFS.Common; +using GVFS.Tests.Should; +using GVFS.UnitTests.Category; +using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Windows.Mock.Upgrader; +using NUnit.Framework; +using System; +using System.Collections.Generic; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + public abstract class UpgradeTests + { + protected const string OlderThanLocalVersion = "1.0.17000.1"; + protected const string LocalGVFSVersion = "1.0.18115.1"; + protected const string NewerThanLocalVersion = "1.1.18115.1"; + + protected MockTracer Tracer { get; private set; } + protected MockTextWriter Output { get; private set; } + protected MockInstallerPrerunChecker PrerunChecker { get; private set; } + protected MockProductUpgrader Upgrader { get; private set; } + + public virtual void Setup() + { + this.Tracer = new MockTracer(); + this.Output = new MockTextWriter(); + this.PrerunChecker = new MockInstallerPrerunChecker(this.Tracer); + this.Upgrader = new MockProductUpgrader(LocalGVFSVersion, this.Tracer); + + this.PrerunChecker.Reset(); + this.Upgrader.PretendNewReleaseAvailableAtRemote( + upgradeVersion: NewerThanLocalVersion, + remoteRing: ProductUpgrader.RingType.Slow); + this.Upgrader.LocalRingConfig = ProductUpgrader.RingType.Slow; + } + + [TestCase] + public virtual void NoneLocalRing() + { + string message = "Upgrade ring set to None. No upgrade check was performed."; + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.LocalRingConfig = ProductUpgrader.RingType.None; + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + message + }, + expectedErrors: new List + { + }); + } + + [TestCase] + public virtual void InvalidUpgradeRing() + { + string errorString = "Invalid upgrade ring `Invalid` specified in Git config."; + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.LocalRingConfig = GVFS.Common.ProductUpgrader.RingType.Invalid; + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + errorString + }, + expectedErrors: new List + { + errorString + }); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public virtual void FetchReleaseInfo() + { + string errorString = "Error fetching upgrade release info."; + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.FetchReleaseInfo); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + errorString + }, + expectedErrors: new List + { + errorString + }); + } + + protected abstract void RunUpgrade(); + + protected abstract ReturnCode ExitCode(); + + protected void ConfigureRunAndVerify( + Action configure, + ReturnCode expectedReturn, + List expectedOutput, + List expectedErrors) + { + configure(); + + this.RunUpgrade(); + + this.ExitCode().ShouldEqual(expectedReturn); + + if (expectedOutput != null) + { + this.Output.AllLines.ShouldContain( + expectedOutput, + (line, expectedLine) => { return line.Contains(expectedLine); }); + } + + if (expectedErrors != null) + { + this.Tracer.RelatedErrorEvents.ShouldContain( + expectedErrors, + (error, expectedError) => { return error.Contains(expectedError); }); + } + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs new file mode 100644 index 0000000000..bdade42071 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -0,0 +1,247 @@ +using GVFS.CommandLine; +using GVFS.Common; +using GVFS.Tests.Should; +using GVFS.UnitTests.Category; +using GVFS.UnitTests.Windows.Mock.Upgrader; +using NUnit.Framework; +using System.Collections.Generic; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + [TestFixture] + public class UpgradeVerbTests : UpgradeTests + { + private MockProcessLauncher ProcessWrapper { get; set; } + private UpgradeVerb UpgradeVerb { get; set; } + + [SetUp] + public override void Setup() + { + base.Setup(); + + this.ProcessWrapper = new MockProcessLauncher(exitCode: 0, hasExited: true, startResult: true); + this.UpgradeVerb = new UpgradeVerb( + this.Upgrader, + this.Tracer, + this.PrerunChecker, + this.ProcessWrapper, + this.Output); + this.UpgradeVerb.Confirmed = false; + this.PrerunChecker.CommandToRerun = "gvfs upgrade"; + } + + [TestCase] + public void UpgradeAvailabilityReporting() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.PretendNewReleaseAvailableAtRemote( + upgradeVersion: NewerThanLocalVersion, + remoteRing: ProductUpgrader.RingType.Slow); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + "New GVFS version available: " + NewerThanLocalVersion, + "Run `gvfs upgrade --confirm` to install it" + }, + expectedErrors: null); + } + + [TestCase] + public void DowngradePrevention() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.PretendNewReleaseAvailableAtRemote( + upgradeVersion: OlderThanLocalVersion, + remoteRing: ProductUpgrader.RingType.Slow); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + "Checking for GVFS upgrades...Succeeded", + "Great news, you're all caught up on upgrades in the Slow ring!" + }, + expectedErrors: null); + } + + [TestCase] + public void LaunchInstaller() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + "New GVFS version available: " + NewerThanLocalVersion, + "Launching upgrade tool...Succeeded" + }, + expectedErrors:null); + + this.ProcessWrapper.IsLaunched.ShouldBeTrue(); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public override void NoneLocalRing() + { + base.NoneLocalRing(); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public override void InvalidUpgradeRing() + { + base.InvalidUpgradeRing(); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void CopyTools() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.CopyTools); + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Could not launch upgrade tool. Unable to copy upgrader tools" + }, + expectedErrors: new List + { + "Could not launch upgrade tool. Unable to copy upgrader tools" + }); + } + + [TestCase] + public void ProjFSPreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.ProjFSEnabled); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "ERROR: ProjFS configuration does not support `gvfs upgrade`.", + "Check your team's documentation for how to upgrade." + }, + expectedErrors: new List + { + "ProjFS configuration does not support `gvfs upgrade`." + }); + } + + [TestCase] + public void IsGVFSServiceRunningPreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsServiceInstalledAndNotRunning); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "GVFS Service is not running.", + "Run `sc start GVFS.Service` and run `gvfs upgrade --confirm` again." + }, + expectedErrors: new List + { + "GVFS Service is not running." + }); + } + + [TestCase] + public void ElevatedRunPreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsElevated); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "The installer needs to be run from an elevated command prompt.", + "Run `gvfs upgrade --confirm` again from an elevated command prompt." + }, + expectedErrors: new List + { + "The installer needs to be run from an elevated command prompt." + }); + } + + [TestCase] + public void UnAttendedModePreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.UnattendedMode); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "`gvfs upgrade` is not supported in unattended mode" + }, + expectedErrors: new List + { + "`gvfs upgrade` is not supported in unattended mode" + }); + } + + [TestCase] + public void DeveloperMachinePreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsDevelopmentVersion); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Cannot run upgrade when development version of GVFS is installed." + }, + expectedErrors: new List + { + "Cannot run upgrade when development version of GVFS is installed." + }); + } + + protected override void RunUpgrade() + { + try + { + this.UpgradeVerb.Execute(); + } + catch (GVFSVerb.VerbAbortedException) + { + // ignore. exceptions are expected while simulating some failures. + } + } + + protected override ReturnCode ExitCode() + { + return this.UpgradeVerb.ReturnCode; + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index b0cd6d43f5..4fb290f507 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -90,6 +90,11 @@ public override bool IsProcessActive(int processId) throw new NotSupportedException(); } + public override void IsServiceInstalledAndRunning(string name, out bool installed, out bool running) + { + throw new NotSupportedException(); + } + public override bool TryGetGVFSEnlistmentRoot(string directory, out string enlistmentRoot, out string errorMessage) { throw new NotSupportedException(); diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs index 17ecb7a93b..d3bb8a5d9f 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs @@ -1,4 +1,4 @@ -using GVFS.Common.Tracing; +using GVFS.Common.Tracing; using Newtonsoft.Json; using System; using System.Collections.Generic; @@ -15,12 +15,14 @@ public MockTracer() this.waitEvent = new AutoResetEvent(false); this.RelatedInfoEvents = new List(); this.RelatedWarningEvents = new List(); + this.RelatedErrorEvents = new List(); } public string WaitRelatedEventName { get; set; } public List RelatedInfoEvents { get; } public List RelatedWarningEvents { get; } + public List RelatedErrorEvents { get; } public void WaitForRelatedEvent() { @@ -71,18 +73,23 @@ public void RelatedWarning(string format, params object[] args) public void RelatedError(EventMetadata metadata, string message) { + metadata[TracingConstants.MessageKey.ErrorMessage] = message; + this.RelatedErrorEvents.Add(JsonConvert.SerializeObject(metadata)); } public void RelatedError(EventMetadata metadata, string message, Keywords keyword) { + this.RelatedError(metadata, message); } public void RelatedError(string message) { + this.RelatedErrorEvents.Add(message); } public void RelatedError(string format, params object[] args) { + this.RelatedErrorEvents.Add(string.Format(format, args)); } public ITracer StartActivity(string activityName, EventLevel level) diff --git a/GVFS/GVFS.Upgrader/App.config b/GVFS/GVFS.Upgrader/App.config new file mode 100644 index 0000000000..00bfd114af --- /dev/null +++ b/GVFS/GVFS.Upgrader/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj b/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj new file mode 100644 index 0000000000..8fb2f91ac7 --- /dev/null +++ b/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj @@ -0,0 +1,122 @@ + + + + + Debug + AnyCPU + {AECEC217-2499-403D-B0BB-2962B9BE5970} + Exe + GVFS.Upgrader + GVFS.Upgrader + v4.6.1 + 512 + true + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true + + + + + true + ..\..\..\BuildOutput\GVFS.Upgrader\bin\x64\Debug\ + DEBUG;TRACE + true + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + ..\..\..\BuildOutput\GVFS.Upgrader\bin\x64\Release\ + TRACE + true + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + + ..\..\..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll + + + + + + + + + + + + + + CommonAssemblyVersion.cs + + + PlatformLoader.Windows.cs + + + + + + + + + Designer + + + + + False + Microsoft .NET Framework 4.6.1 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 + false + + + + + + {374bf1e5-0b2d-4d4a-bd5e-4212299def09} + GVFS.Common + + + {4ce404e7-d3fc-471c-993c-64615861ea63} + GVFS.Platform.Windows + + + {f468b05a-95e5-46bc-8c67-b80a78527b7d} + GVFS.Virtualization + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + \ No newline at end of file diff --git a/GVFS/GVFS.Upgrader/Program.cs b/GVFS/GVFS.Upgrader/Program.cs new file mode 100644 index 0000000000..9f61701b21 --- /dev/null +++ b/GVFS/GVFS.Upgrader/Program.cs @@ -0,0 +1,16 @@ +using GVFS.PlatformLoader; + +namespace GVFS.Upgrader +{ + public class Program + { + public static void Main(string[] args) + { + GVFSPlatformLoader.Initialize(); + + UpgradeOrchestrator upgrader = new UpgradeOrchestrator(); + + upgrader.Execute(); + } + } +} diff --git a/GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs b/GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..4fddbd3e3a --- /dev/null +++ b/GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs @@ -0,0 +1,22 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("GVFS.Upgrader")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("GVFS.Upgrader")] +[assembly: AssemblyCopyright("Copyright © Microsoft 2018")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("aecec217-2499-403d-b0bb-2962b9be5970")] diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs new file mode 100644 index 0000000000..18780cf3dd --- /dev/null +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -0,0 +1,389 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using System; +using System.IO; + +namespace GVFS.Upgrader +{ + public class UpgradeOrchestrator + { + private const EventLevel DefaultEventLevel = EventLevel.Informational; + + private ProductUpgrader upgrader; + private ITracer tracer; + private InstallerPreRunChecker preRunChecker; + private TextWriter output; + private TextReader input; + private bool remount; + private bool shouldExitOnError; + + public UpgradeOrchestrator( + ProductUpgrader upgrader, + ITracer tracer, + InstallerPreRunChecker preRunChecker, + TextReader input, + TextWriter output, + bool shouldExitOnError) + { + this.upgrader = upgrader; + this.tracer = tracer; + this.preRunChecker = preRunChecker; + this.output = output; + this.input = input; + this.remount = false; + this.shouldExitOnError = shouldExitOnError; + this.ExitCode = ReturnCode.Success; + } + + public UpgradeOrchestrator() + { + string logFilePath = GVFSEnlistment.GetNewGVFSLogFileName( + ProductUpgrader.GetLogDirectoryPath(), + GVFSConstants.LogFileTypes.UpgradeProcess); + JsonTracer jsonTracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "UpgradeProcess"); + jsonTracer.AddLogFileEventListener( + logFilePath, + DefaultEventLevel, + Keywords.Any); + + this.tracer = jsonTracer; + this.preRunChecker = new InstallerPreRunChecker(this.tracer); + this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + this.output = Console.Out; + this.input = Console.In; + this.remount = false; + this.shouldExitOnError = false; + this.ExitCode = ReturnCode.Success; + } + + public ReturnCode ExitCode { get; private set; } + + public void Execute() + { + string error = null; + + if (this.upgrader.IsNoneRing()) + { + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); + } + else + { + try + { + Version newVersion = null; + if (!this.TryRunUpgradeInstall(out newVersion, out error)) + { + this.ExitCode = ReturnCode.GenericError; + } + } + finally + { + string remountError = null; + if (!this.TryRemountRepositories(out remountError)) + { + remountError = Environment.NewLine + "WARNING: " + remountError; + this.output.WriteLine(remountError); + this.ExitCode = ReturnCode.Success; + } + + this.DeletedDownloadedAssets(); + } + } + + if (this.ExitCode == ReturnCode.GenericError) + { + error = Environment.NewLine + "ERROR: " + error; + this.output.WriteLine(error); + } + + if (this.input == Console.In) + { + this.output.WriteLine("Press Enter to exit."); + this.input.ReadLine(); + } + + if (this.shouldExitOnError) + { + Environment.Exit((int)this.ExitCode); + } + } + + private bool LaunchInsideSpinner(Func method, string message) + { + return ConsoleHelper.ShowStatusWhileRunning( + method, + message, + this.output, + this.output == Console.Out && !GVFSPlatform.Instance.IsConsoleOutputRedirectedToFile(), + null); + } + + private bool TryRunUpgradeInstall(out Version newVersion, out string consoleError) + { + newVersion = null; + + Version newGVFSVersion = null; + GitVersion newGitVersion = null; + string errorMessage = null; + if (!this.LaunchInsideSpinner( + () => + { + if (!this.TryCheckIfUpgradeAvailable(out newGVFSVersion, out errorMessage) || + !this.TryGetNewGitVersion(out newGitVersion, out errorMessage)) + { + return false; + } + + this.LogInstalledVersionInfo(); + this.LogVersionInfo(newGVFSVersion, newGitVersion, "Available Version"); + + this.preRunChecker.CommandToRerun = "gvfs upgrade --confirm"; + if (!this.preRunChecker.TryRunPreUpgradeChecks(out errorMessage)) + { + return false; + } + + if (!this.TryDownloadUpgrade(newGVFSVersion, out errorMessage)) + { + return false; + } + + return true; + }, + "Downloading")) + { + consoleError = errorMessage; + return false; + } + + if (!this.LaunchInsideSpinner( + () => + { + if (!this.preRunChecker.TryUnmountAllGVFSRepos(out errorMessage)) + { + return false; + } + + this.remount = true; + + return true; + }, + "Unmounting repositories")) + { + consoleError = errorMessage; + return false; + } + + if (!this.LaunchInsideSpinner( + () => + { + if (!this.TryInstallGitUpgrade(newGitVersion, out errorMessage)) + { + return false; + } + + return true; + }, + $"Installing Git version: {newGitVersion}")) + { + consoleError = errorMessage; + return false; + } + + if (!this.LaunchInsideSpinner( + () => + { + if (!this.TryInstallGVFSUpgrade(newGVFSVersion, out errorMessage)) + { + return false; + } + + return true; + }, + $"Installing VFSForGit version: {newGVFSVersion}")) + { + consoleError = errorMessage; + return false; + } + + this.LogVersionInfo(newGVFSVersion, newGitVersion, "Newly Installed Version"); + + newVersion = newGVFSVersion; + consoleError = null; + return true; + } + + private bool TryRemountRepositories(out string consoleError) + { + string errorMessage = string.Empty; + if (this.remount && !this.LaunchInsideSpinner( + () => + { + string remountError; + if (!this.preRunChecker.TryMountAllGVFSRepos(out remountError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryRemountRepositories)); + metadata.Add("Remount Error", remountError); + this.tracer.RelatedError(metadata, $"{nameof(this.preRunChecker.TryMountAllGVFSRepos)} failed."); + errorMessage += remountError; + return false; + } + + return true; + }, + "Mounting repositories")) + { + consoleError = errorMessage; + return false; + } + + consoleError = null; + return true; + } + + private void DeletedDownloadedAssets() + { + string downloadsCleanupError; + if (!this.upgrader.TryCleanup(out downloadsCleanupError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.DeletedDownloadedAssets)); + metadata.Add("Download cleanup error", downloadsCleanupError); + this.tracer.RelatedError(metadata, $"{nameof(this.DeletedDownloadedAssets)} failed."); + } + } + + private bool TryGetNewGitVersion(out GitVersion gitVersion, out string consoleError) + { + gitVersion = null; + + this.tracer.RelatedInfo("Reading Git version from release info"); + + if (!this.upgrader.TryGetGitVersion(out gitVersion, out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryGetNewGitVersion)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetGitVersion)} failed. {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully read Git version {0}", gitVersion); + + return true; + } + + private bool TryCheckIfUpgradeAvailable(out Version newestVersion, out string consoleError) + { + newestVersion = null; + + this.tracer.RelatedInfo("Checking upgrade server for new releases"); + + if (!this.upgrader.TryGetNewerVersion(out newestVersion, out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryCheckIfUpgradeAvailable)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetNewerVersion)} failed. {consoleError}"); + return false; + } + + if (newestVersion == null) + { + consoleError = "No upgrades available in ring: " + this.upgrader.Ring; + this.tracer.RelatedInfo("No new upgrade releases available"); + return false; + } + + this.tracer.RelatedInfo("Successfully checked for new release. {0}", newestVersion); + + return true; + } + + private bool TryDownloadUpgrade(Version version, out string consoleError) + { + this.tracer.RelatedInfo("Downloading version: " + version.ToString()); + + if (!this.upgrader.TryDownloadNewestVersion(out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryDownloadUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryDownloadNewestVersion)} failed. {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully downloaded version: " + version.ToString()); + + return true; + } + + private bool TryInstallGitUpgrade(GitVersion version, out string consoleError) + { + this.tracer.RelatedInfo("Installing Git version: " + version.ToString()); + + bool installSuccess = false; + if (!this.upgrader.TryRunGitInstaller(out installSuccess, out consoleError) || + !installSuccess) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryInstallGitUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGitInstaller)} failed. {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully installed Git version: " + version.ToString()); + + return installSuccess; + } + + private bool TryInstallGVFSUpgrade(Version version, out string consoleError) + { + this.tracer.RelatedInfo("Installing GVFS version: " + version.ToString()); + + bool installSuccess = false; + if (!this.upgrader.TryRunGVFSInstaller(out installSuccess, out consoleError) || + !installSuccess) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryInstallGVFSUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGVFSInstaller)} failed. {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully installed GVFS version: " + version.ToString()); + + return installSuccess; + } + + private void LogVersionInfo( + Version gvfsVersion, + GitVersion gitVersion, + string message) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(gvfsVersion), gvfsVersion.ToString()); + metadata.Add(nameof(gitVersion), gitVersion.ToString()); + + this.tracer.RelatedEvent(EventLevel.Informational, message, metadata); + } + + private void LogInstalledVersionInfo() + { + EventMetadata metadata = new EventMetadata(); + string installedGVFSVersion = ProcessHelper.GetCurrentProcessVersion(); + metadata.Add(nameof(installedGVFSVersion), installedGVFSVersion); + + GitVersion installedGitVersion = null; + string error = null; + if (GitProcess.TryGetVersion( + out installedGitVersion, + out error)) + { + metadata.Add(nameof(installedGitVersion), installedGitVersion.ToString()); + } + + this.tracer.RelatedEvent(EventLevel.Informational, "Installed Version", metadata); + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.Upgrader/packages.config b/GVFS/GVFS.Upgrader/packages.config new file mode 100644 index 0000000000..88e2388195 --- /dev/null +++ b/GVFS/GVFS.Upgrader/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index 138259e8c8..b2e932f96c 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.Git; @@ -113,6 +113,14 @@ protected override void Execute(GVFSEnlistment enlistment) this.ServiceName, copySubFolders: true); + // upgrader + this.CopyAllFiles( + ProductUpgrader.GetUpgradesDirectoryPath(), + archiveFolderPath, + ProductUpgrader.LogDirectory, + copySubFolders: true, + targetFolderName: ProductUpgrader.UpgradeDirectoryName); + return true; }, "Copying logs"); @@ -160,10 +168,16 @@ private void RecordVersionInformation() this.diagnosticLogFileWriter.WriteLine(information); } - private void CopyAllFiles(string sourceRoot, string targetRoot, string folderName, bool copySubFolders, bool hideErrorsFromStdout = false) + private void CopyAllFiles( + string sourceRoot, + string targetRoot, + string folderName, + bool copySubFolders, + bool hideErrorsFromStdout = false, + string targetFolderName = null) { string sourceFolder = Path.Combine(sourceRoot, folderName); - string targetFolder = Path.Combine(targetRoot, folderName); + string targetFolder = Path.Combine(targetRoot, targetFolderName ?? folderName); try { diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 54f3fe204d..eb4370621f 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.Git; @@ -642,8 +642,7 @@ private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out stri this.ReportErrorAndExit(tracer, "Error: Invalid version of git {0}. Must use gvfs version.", version); } - Version gvfsVersion = new Version(ProcessHelper.GetCurrentProcessVersion()); - if (gvfsVersion.Major == 0) + if (ProcessHelper.IsDevelopmentVersion()) { if (gitVersion.IsLessThan(GVFSConstants.SupportedGitVersion)) { diff --git a/GVFS/GVFS/CommandLine/LogVerb.cs b/GVFS/GVFS/CommandLine/LogVerb.cs index da93d76144..d1eb5c9c42 100644 --- a/GVFS/GVFS/CommandLine/LogVerb.cs +++ b/GVFS/GVFS/CommandLine/LogVerb.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using GVFS.Common; using System.IO; using System.Linq; @@ -60,6 +60,9 @@ public override void Execute() string serviceLogsRoot = Paths.GetServiceLogsPath(this.ServiceName); this.DisplayMostRecent(serviceLogsRoot, GVFSConstants.LogFileTypes.Service); + + string autoUpgradeLogsRoot = Paths.GetServiceLogsPath(ProductUpgrader.UpgradeDirectoryName); + this.DisplayMostRecent(autoUpgradeLogsRoot, GVFSConstants.LogFileTypes.UpgradePrefix); } else { diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs new file mode 100644 index 0000000000..e914842a8c --- /dev/null +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -0,0 +1,325 @@ +using CommandLine; +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using GVFS.Upgrader; +using System; +using System.Diagnostics; +using System.IO; + +namespace GVFS.CommandLine +{ + [Verb(UpgradeVerbName, HelpText = "Checks if a new GVFS release is available.")] + public class UpgradeVerb : GVFSVerb + { + private const string UpgradeVerbName = "upgrade"; + private ITracer tracer; + private ProductUpgrader upgrader; + private InstallerPreRunChecker prerunChecker; + private ProcessLauncher processWrapper; + + public UpgradeVerb( + ProductUpgrader upgrader, + ITracer tracer, + InstallerPreRunChecker prerunChecker, + ProcessLauncher processWrapper, + TextWriter output) + { + this.upgrader = upgrader; + this.tracer = tracer; + this.prerunChecker = prerunChecker; + this.processWrapper = processWrapper; + this.Output = output; + } + + public UpgradeVerb() + { + JsonTracer jsonTracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "UpgradeVerb"); + string logFilePath = GVFSEnlistment.GetNewGVFSLogFileName( + ProductUpgrader.GetLogDirectoryPath(), + GVFSConstants.LogFileTypes.UpgradeVerb); + jsonTracer.AddLogFileEventListener(logFilePath, EventLevel.Informational, Keywords.Any); + + this.tracer = jsonTracer; + this.prerunChecker = new InstallerPreRunChecker(this.tracer); + this.processWrapper = new ProcessLauncher(); + this.Output = Console.Out; + this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + } + + [Option( + "confirm", + Default = false, + Required = false, + HelpText = "Pass in this flag to actually install the newest release")] + public bool Confirmed { get; set; } + + public override string EnlistmentRootPathParameter { get; set; } + + protected override string VerbName + { + get { return UpgradeVerbName; } + } + + public override void Execute() + { + ReturnCode exitCode = ReturnCode.Success; + if (!this.TryRunProductUpgrade()) + { + exitCode = ReturnCode.GenericError; + this.ReportErrorAndExit(this.tracer, exitCode, string.Empty); + } + } + + private bool TryRunProductUpgrade() + { + string errorOutputFormat = Environment.NewLine + "ERROR: {0}"; + string error = null; + Version newestVersion = null; + bool isInstallable = false; + + if (this.upgrader.IsNoneRing()) + { + this.tracer.RelatedInfo($"{nameof(this.TryRunProductUpgrade)}: {GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert}"); + this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); + this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); + return true; + } + + if (!this.TryRunUpgradeChecks(out newestVersion, out isInstallable, out error)) + { + this.Output.WriteLine(errorOutputFormat, error); + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade checks failed. {error}"); + return false; + } + + if (newestVersion == null) + { + this.ReportInfoToConsole($"Great news, you're all caught up on upgrades in the {this.upgrader.Ring} ring!"); + return true; + } + + this.ReportInfoToConsole("New GVFS version available: {0}", newestVersion.ToString()); + + if (!this.Confirmed && isInstallable) + { + this.ReportInfoToConsole("Run `gvfs upgrade --confirm` to install it"); + return true; + } + + if (!isInstallable) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: {error}"); + this.Output.WriteLine(errorOutputFormat, error); + return false; + } + + if (!this.TryRunInstaller(out error)) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Could not launch upgrade tool. {error}"); + this.Output.WriteLine(errorOutputFormat, "Could not launch upgrade tool. " + error); + return false; + } + + return true; + } + + private bool TryRunUpgradeChecks( + out Version latestVersion, + out bool isUpgradeInstallable, + out string consoleError) + { + bool upgradeCheckSuccess = false; + bool upgradeIsInstallable = false; + string errorMessage = null; + Version version = null; + + this.ShowStatusWhileRunning( + () => + { + upgradeCheckSuccess = this.TryCheckUpgradeAvailable(out version, out errorMessage); + if (upgradeCheckSuccess && version != null) + { + upgradeIsInstallable = true; + + if (!this.TryCheckUpgradeInstallable(out errorMessage)) + { + upgradeIsInstallable = false; + } + } + + return upgradeCheckSuccess; + }, + "Checking for GVFS upgrades", + suppressGvfsLogMessage: true); + + latestVersion = version; + isUpgradeInstallable = upgradeIsInstallable; + consoleError = errorMessage; + + return upgradeCheckSuccess; + } + + private bool TryRunInstaller(out string consoleError) + { + string upgraderPath = null; + string errorMessage = null; + + bool preUpgradeSuccess = this.ShowStatusWhileRunning( + () => + { + if (this.TryCopyUpgradeTool(out upgraderPath, out errorMessage) && + this.TryLaunchUpgradeTool(upgraderPath, out errorMessage)) + { + return true; + } + + return false; + }, + "Launching upgrade tool", + suppressGvfsLogMessage: true); + + if (!preUpgradeSuccess) + { + consoleError = errorMessage; + return false; + } + + consoleError = null; + return true; + } + + private bool TryCopyUpgradeTool(out string upgraderExePath, out string consoleError) + { + upgraderExePath = null; + + this.tracer.RelatedInfo("Copying upgrade tool"); + + if (!this.upgrader.TrySetupToolsDirectory(out upgraderExePath, out consoleError)) + { + return false; + } + + this.tracer.RelatedInfo($"Successfully Copied upgrade tool to {upgraderExePath}"); + + return true; + } + + private bool TryLaunchUpgradeTool(string path, out string consoleError) + { + this.tracer.RelatedInfo("Launching upgrade tool"); + + Exception exception; + if (!this.processWrapper.TryStart(path, out exception)) + { + if (exception != null) + { + consoleError = exception.Message; + this.tracer.RelatedError($"Error launching upgrade tool. {exception.ToString()}"); + } + else + { + consoleError = $"Error launching upgrade tool"; + } + + return false; + } + + this.tracer.RelatedInfo("Successfully launched upgrade tool."); + + consoleError = null; + return true; + } + + private bool TryCheckUpgradeAvailable( + out Version latestVersion, + out string consoleError) + { + latestVersion = null; + consoleError = null; + + this.tracer.RelatedInfo("Checking server for available upgrades."); + + bool checkSucceeded = false; + Version version = null; + + checkSucceeded = this.upgrader.TryGetNewerVersion(out version, out consoleError); + if (!checkSucceeded) + { + return false; + } + + latestVersion = version; + + this.tracer.RelatedInfo("Successfully checked server for GVFS upgrades."); + + return true; + } + + private bool TryCheckUpgradeInstallable(out string consoleError) + { + consoleError = null; + + this.tracer.RelatedInfo("Checking if upgrade is installable on this machine."); + + this.prerunChecker.CommandToRerun = this.Confirmed ? "gvfs upgrade --confirm" : "gvfs upgrade"; + + if (!this.prerunChecker.TryRunPreUpgradeChecks( + out consoleError)) + { + return false; + } + + this.tracer.RelatedInfo("Upgrade is installable."); + + return true; + } + + private void ReportInfoToConsole(string message, params object[] args) + { + this.Output.WriteLine(message, args); + } + + public class ProcessLauncher + { + public ProcessLauncher() + { + this.Process = new Process(); + } + + public Process Process { get; private set; } + + public virtual bool HasExited + { + get { return this.Process.HasExited; } + } + + public virtual int ExitCode + { + get { return this.Process.ExitCode; } + } + + public virtual bool TryStart(string path, out Exception exception) + { + this.Process.StartInfo = new ProcessStartInfo(path) + { + UseShellExecute = true, + WindowStyle = ProcessWindowStyle.Normal + }; + + exception = null; + + try + { + return this.Process.Start(); + } + catch (Exception ex) + { + exception = ex; + } + + return false; + } + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS/GVFS.Windows.csproj b/GVFS/GVFS/GVFS.Windows.csproj index 306a8073c7..d8de69db99 100644 --- a/GVFS/GVFS/GVFS.Windows.csproj +++ b/GVFS/GVFS/GVFS.Windows.csproj @@ -1,4 +1,4 @@ - + @@ -80,6 +80,7 @@ + diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index f906114dc5..43061a95ac 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using GVFS.CommandLine; using GVFS.Common; using GVFS.PlatformLoader; @@ -27,6 +27,7 @@ public static void Main(string[] args) typeof(ServiceVerb), typeof(StatusVerb), typeof(UnmountVerb), + typeof(UpgradeVerb) }; int consoleWidth = 80; @@ -52,7 +53,7 @@ public static void Main(string[] args) settings.CaseSensitive = false; settings.EnableDashDash = true; settings.IgnoreUnknownArguments = false; - settings.HelpWriter = Console.Error; + settings.HelpWriter = Console.Error; settings.MaximumDisplayWidth = consoleWidth; }) .ParseArguments(args, verbTypes) @@ -80,6 +81,14 @@ public static void Main(string[] args) service.Execute(); Environment.Exit((int)ReturnCode.Success); }) + .WithParsed( + upgrade => + { + // The upgrade verb doesn't operate on a repo, so it doesn't use the enlistment + // path at all. + upgrade.Execute(); + Environment.Exit((int)ReturnCode.Success); + }) .WithParsed( verb => { diff --git a/Scripts/UninstallGVFS.bat b/Scripts/UninstallGVFS.bat index c22ccab385..27ecc7a9e8 100644 --- a/Scripts/UninstallGVFS.bat +++ b/Scripts/UninstallGVFS.bat @@ -17,4 +17,6 @@ for /F "delims=" %%f in ('dir "c:\Program Files\GVFS\unins*.exe" /B /S /O:-D') d :deleteGVFS rmdir /q/s "c:\Program Files\GVFS" +rmdir /q/s "C:\ProgramData\GVFS\GVFS.Upgrade" + :end From d0668d481dd4074e55204ae6b4761c87df8d415a Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 19 Sep 2018 15:33:34 -0400 Subject: [PATCH 063/244] Incorporating suggestions from test feedback. - Display nag only 10% of the times a git command is run - Removed "Run gvfs upgrade..." from the reminder notification message. - Use ITracer.StartActivity for method tracing. - Service Verb redirects list of repos that failed to mount to stderr. - Upgrader tool reads stderr and displays the list on Console. - New console message displayed after launching installer advising user to not run gvfs, git commands until installation has finished. - Added a new Console message to let user know upgrade completed successfully - inserted blank line before final instruction to run `gvfs upgrade --confirm` command - Early exit when no upgrade or invalid upgade ring is set - Upgrade verb crash fix on Mac. - Early exit when upgrade is not installable and `gvfs upgrade --confirm` is run --- GVFS/GVFS.Common/GVFSConstants.cs | 10 +- GVFS/GVFS.Common/InstallerPreRunChecker.cs | 106 ++++---- GVFS/GVFS.Common/ProductUpgrader.cs | 21 +- .../GVFSUpgradeReminderTests.cs | 27 +- GVFS/GVFS.Hooks/Program.cs | 28 ++- .../Mock/MockInstallerPreRunChecker.cs | 4 +- .../Windows/Mock/MockProductUpgrader.cs | 2 +- .../Windows/Upgrader/UpgradeTests.cs | 2 +- .../Windows/Upgrader/UpgradeVerbTests.cs | 2 +- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 163 +++++++----- GVFS/GVFS/CommandLine/ServiceVerb.cs | 23 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 234 +++++++++++------- start | 1 + 13 files changed, 386 insertions(+), 237 deletions(-) create mode 100644 start diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index a88200be62..b9f6c56d27 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -222,10 +222,12 @@ public static class Unmount public static class UpgradeVerbMessages { - public const string NoneRingConsoleAlert = "Upgrade ring set to None. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"] from an elevated command prompt."; - public const string UpgradeAvailable = "A newer version of GVFS is available."; - public const string UpgradeInstallAdvice = "Run `gvfs upgrade --confirm` from an elevated command prompt to install."; + public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; + public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; + public const string ReminderNotification = "A new version of GVFS is available. Run `gvfs upgrade` to start the upgrade."; + public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; + public const string UpgradeInstallAdvice = "When you are ready, run `gvfs upgrade --confirm` from an elevated command prompt."; } } } diff --git a/GVFS/GVFS.Common/InstallerPreRunChecker.cs b/GVFS/GVFS.Common/InstallerPreRunChecker.cs index e92c12742f..aa5f919e51 100644 --- a/GVFS/GVFS.Common/InstallerPreRunChecker.cs +++ b/GVFS/GVFS.Common/InstallerPreRunChecker.cs @@ -26,29 +26,30 @@ public InstallerPreRunChecker(ITracer tracer) public bool TryRunPreUpgradeChecks(out string consoleError) { - this.tracer.RelatedInfo("Checking if GVFS upgrade can be run on this machine."); - - if (this.IsUnattended()) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryRunPreUpgradeChecks), EventLevel.Informational)) { - consoleError = "`gvfs upgrade` is not supported in unattended mode"; - this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); - return false; - } + if (this.IsUnattended()) + { + consoleError = "`gvfs upgrade` is not supported in unattended mode"; + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } - if (this.IsDevelopmentVersion()) - { - consoleError = "Cannot run upgrade when development version of GVFS is installed."; - this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); - return false; - } + if (this.IsDevelopmentVersion()) + { + consoleError = "Cannot run upgrade when development version of GVFS is installed."; + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } - if (!this.IsGVFSUpgradeAllowed(out consoleError)) - { - this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); - return false; - } + if (!this.IsGVFSUpgradeAllowed(out consoleError)) + { + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } - this.tracer.RelatedInfo("Successfully finished pre upgrade checks. Okay to run GVFS upgrade."); + activity.RelatedInfo("Successfully finished pre upgrade checks. Okay to run GVFS upgrade."); + } consoleError = null; return true; @@ -57,7 +58,7 @@ public bool TryRunPreUpgradeChecks(out string consoleError) // TODO: Move repo mount calls to GVFS.Upgrader project. public bool TryMountAllGVFSRepos(out string consoleError) { - return this.TryRunGVFSWithArgs("service --mount-all", out consoleError); + return this.TryRunGVFSWithArgs("service --mount-all --log-mount-failure-in-stderr", out consoleError); } public bool TryUnmountAllGVFSRepos(out string consoleError) @@ -66,43 +67,46 @@ public bool TryUnmountAllGVFSRepos(out string consoleError) this.tracer.RelatedInfo("Unmounting any mounted GVFS repositories."); - if (!this.TryRunGVFSWithArgs("service --unmount-all", out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryUnmountAllGVFSRepos), EventLevel.Informational)) { - this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); - return false; - } + if (!this.TryRunGVFSWithArgs("service --unmount-all --log-mount-failure-in-stderr", out consoleError)) + { + this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); + return false; + } - // While checking for blocking processes like GVFS.Mount immediately after un-mounting, - // then sometimes GVFS.Mount shows up as running. But if the check is done after waiting - // for some time, then eventually GVFS.Mount goes away. The retry loop below is to help - // account for this delay between the time un-mount call returns and when GVFS.Mount - // actually quits. - this.tracer.RelatedInfo("Checking if GVFS or dependent processes are running."); - int retryCount = 10; - List processList = null; - while (retryCount > 0) - { - if (!this.IsBlockingProcessRunning(out processList)) + // While checking for blocking processes like GVFS.Mount immediately after un-mounting, + // then sometimes GVFS.Mount shows up as running. But if the check is done after waiting + // for some time, then eventually GVFS.Mount goes away. The retry loop below is to help + // account for this delay between the time un-mount call returns and when GVFS.Mount + // actually quits. + this.tracer.RelatedInfo("Checking if GVFS or dependent processes are running."); + int retryCount = 10; + List processList = null; + while (retryCount > 0) { - break; + if (!this.IsBlockingProcessRunning(out processList)) + { + break; + } + + Thread.Sleep(TimeSpan.FromMilliseconds(250)); + retryCount--; } - Thread.Sleep(TimeSpan.FromMilliseconds(250)); - retryCount--; - } + if (processList.Count > 0) + { + consoleError = string.Join( + Environment.NewLine, + "Blocking processes are running.", + $"Run `{this.CommandToRerun}` again after quitting these processes - " + string.Join(", ", processList.ToArray())); + this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); + return false; + } - if (processList.Count > 0) - { - consoleError = string.Join( - Environment.NewLine, - "Blocking processes are running.", - $"Run `{this.CommandToRerun}` again after quitting these processes - " + string.Join(", ", processList.ToArray())); - this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); - return false; + activity.RelatedInfo("Successfully unmounted repositories."); } - this.tracer.RelatedInfo("Successfully unmounted repositories."); - return true; } @@ -168,9 +172,7 @@ protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) } else { - string output = string.IsNullOrEmpty(processResult.Output) ? string.Empty : processResult.Output; - string errorString = string.IsNullOrEmpty(processResult.Errors) ? "GVFS error" : processResult.Errors; - consoleError = string.Format("{0}. {1}", errorString, output); + consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"GVFS {args} failed." : processResult.Errors; return false; } } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index 96f8dc6839..b8b411a0c1 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -71,12 +71,7 @@ public enum RingType } public RingType Ring { get; protected set; } - - public bool IsNoneRing() - { - return this.TryLoadRingConfig(out string _) && this.Ring == RingType.None; - } - + public bool TryGetNewerVersion( out Version newVersion, out string errorMessage) @@ -246,13 +241,8 @@ public bool TryCleanup(out string error) error = null; return true; } - - protected virtual bool TryDeleteDownloadedAsset(Asset asset, out Exception exception) - { - return this.fileSystem.TryDeleteFile(asset.LocalPath, out exception); - } - - protected virtual bool TryLoadRingConfig(out string error) + + public virtual bool TryLoadRingConfig(out string error) { string errorAdvisory = "Run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"]` and run `gvfs upgrade [--confirm]` again."; string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); @@ -285,6 +275,11 @@ protected virtual bool TryLoadRingConfig(out string error) return false; } + protected virtual bool TryDeleteDownloadedAsset(Asset asset, out Exception exception) + { + return this.fileSystem.TryDeleteFile(asset.LocalPath, out exception); + } + protected virtual bool TryDownloadAsset(Asset asset, out string errorMessage) { errorMessage = null; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs index 837f21d273..376e35ea54 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs @@ -31,30 +31,41 @@ public UpgradeReminderTests() } [TestCase] - public void NoNagWhenUpgradeNotAvailable() + public void NoReminderWhenUpgradeNotAvailable() { this.EmptyDownloadDirectory(); - ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( + for (int count = 0; count < 50; count++) + { + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( this.Enlistment.RepoRoot, "status"); - string.IsNullOrEmpty(result.Errors).ShouldBeTrue(); + string.IsNullOrEmpty(result.Errors).ShouldBeTrue(); + } } [TestCase] - public void NagWhenUpgradeAvailable() + public void RemindWhenUpgradeAvailable() { this.CreateUpgradeInstallers(); - ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( + string errors = string.Empty; + for (int count = 0; count < 50; count++) + { + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( this.Enlistment.RepoRoot, "status"); - result.Errors.ShouldContain(new string[] + if (!string.IsNullOrEmpty(result.Errors)) + { + errors += result.Errors; + } + } + + errors.ShouldContain(new string[] { - "A newer version of GVFS is available.", - "Run `gvfs upgrade --confirm` from an elevated command prompt to install." + "A new version of GVFS is available." }); this.EmptyDownloadDirectory(); diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index e535b34520..94df57679e 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -22,6 +22,7 @@ public class Program private static Dictionary specialArgValues = new Dictionary(); private static string enlistmentRoot; private static string enlistmentPipename; + private static Random random = new Random(); private delegate void LockRequestDelegate(bool unattended, string[] args, int pid, NamedPipeClient pipeClient); @@ -68,6 +69,7 @@ public static void Main(string[] args) RunLockRequest(args, unattended, ReleaseGVFSLock); } + RunPostCommands(args); break; default: @@ -83,12 +85,6 @@ public static void Main(string[] args) private static void RunPreCommands(string[] args) { - if (ProductUpgrader.IsLocalUpgradeAvailable()) - { - Console.WriteLine(GVFSConstants.UpgradeVerbMessages.UpgradeAvailable); - Console.WriteLine(GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); - } - string command = GetGitCommand(args); switch (command) { @@ -99,6 +95,26 @@ private static void RunPreCommands(string[] args) } } + private static void RunPostCommands(string[] args) + { + RemindUpgradeAvailable(); + } + + private static void RemindUpgradeAvailable() + { + // The idea is to generate a random number between 0 and 100. To make + // sure that the reminder is displayed only 10% of the times a git + // command is run, check that the random number is between 0 and 10, + // which will have a probability of 10/100 == 10%. + int reminderFrequency = 10; + int randomValue = random.Next(0, 100); + + if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable()) + { + Console.WriteLine(Environment.NewLine + GVFSConstants.UpgradeVerbMessages.ReminderNotification); + } + } + private static void ExitWithError(params string[] messages) { foreach (string message in messages) diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs index 6b51f9a155..6dc432e4a4 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs @@ -101,14 +101,14 @@ protected override bool TryRunGVFSWithArgs(string args, out string error) { this.GVFSArgs.Add(args); - if (string.CompareOrdinal(args, "service --unmount-all") == 0) + if (string.CompareOrdinal(args, "service --unmount-all --log-mount-failure-in-stderr") == 0) { bool result = this.FakedResultOfCheck(FailOnCheckType.UnMountRepos); error = result == false ? "Unmount of some of the repositories failed." : null; return result; } - if (string.CompareOrdinal(args, "service --mount-all") == 0) + if (string.CompareOrdinal(args, "service --mount-all --log-mount-failure-in-stderr") == 0) { bool result = this.FakedResultOfCheck(FailOnCheckType.RemountRepos); error = result == false ? "Auto remount failed." : null; diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index 2299943935..3b8c30633e 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -101,7 +101,7 @@ public override bool TrySetupToolsDirectory(out string upgraderToolPath, out str return true; } - protected override bool TryLoadRingConfig(out string error) + public override bool TryLoadRingConfig(out string error) { this.Ring = this.LocalRingConfig; diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs index 9f9e1b3d37..4ac28563b8 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs @@ -37,7 +37,7 @@ public virtual void Setup() [TestCase] public virtual void NoneLocalRing() { - string message = "Upgrade ring set to None. No upgrade check was performed."; + string message = "Upgrade ring set to \"None\". No upgrade check was performed."; this.ConfigureRunAndVerify( configure: () => { diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs index bdade42071..d76ef303ce 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -44,7 +44,7 @@ public void UpgradeAvailabilityReporting() expectedOutput: new List { "New GVFS version available: " + NewerThanLocalVersion, - "Run `gvfs upgrade --confirm` to install it" + "When you are ready, run `gvfs upgrade --confirm` from an elevated command prompt." }, expectedErrors: null); } diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index 18780cf3dd..a5e51dae00 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -63,10 +63,19 @@ public void Execute() { string error = null; - if (this.upgrader.IsNoneRing()) + ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; + + if (!this.TryLoadUpgradeRing(out ring, out error) || ring == ProductUpgrader.RingType.None) { - this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); - this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); + string message = ring == ProductUpgrader.RingType.None ? + GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert : + GVFSConstants.UpgradeVerbMessages.InvalidRingConsoleAlert; + this.output.WriteLine(message); + + if (ring == ProductUpgrader.RingType.None) + { + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); + } } else { @@ -91,12 +100,16 @@ public void Execute() this.DeletedDownloadedAssets(); } } - + if (this.ExitCode == ReturnCode.GenericError) { error = Environment.NewLine + "ERROR: " + error; this.output.WriteLine(error); } + else + { + this.output.WriteLine(Environment.NewLine + "Upgrade completed successfully!"); + } if (this.input == Console.In) { @@ -120,6 +133,27 @@ private bool LaunchInsideSpinner(Func method, string message) null); } + private bool TryLoadUpgradeRing(out ProductUpgrader.RingType ring, out string consoleError) + { + bool loaded = false; + if (!this.upgrader.TryLoadRingConfig(out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryLoadUpgradeRing)); + metadata.Add("Load Error", consoleError); + this.tracer.RelatedError(metadata, $"{nameof(this.TryLoadUpgradeRing)} failed."); + this.ExitCode = ReturnCode.GenericError; + } + else + { + consoleError = null; + loaded = true; + } + + ring = this.upgrader.Ring; + return loaded; + } + private bool TryRunUpgradeInstall(out Version newVersion, out string consoleError) { newVersion = null; @@ -202,7 +236,7 @@ private bool TryRunUpgradeInstall(out Version newVersion, out string consoleErro return true; }, - $"Installing VFSForGit version: {newGVFSVersion}")) + $"Installing GVFS version: {newGVFSVersion}")) { consoleError = errorMessage; return false; @@ -260,17 +294,18 @@ private bool TryGetNewGitVersion(out GitVersion gitVersion, out string consoleEr { gitVersion = null; - this.tracer.RelatedInfo("Reading Git version from release info"); - - if (!this.upgrader.TryGetGitVersion(out gitVersion, out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryGetNewGitVersion), EventLevel.Informational)) { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryGetNewGitVersion)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetGitVersion)} failed. {consoleError}"); - return false; + if (!this.upgrader.TryGetGitVersion(out gitVersion, out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryGetNewGitVersion)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetGitVersion)} failed. {consoleError}"); + return false; + } + + activity.RelatedInfo("Successfully read Git version {0}", gitVersion); } - - this.tracer.RelatedInfo("Successfully read Git version {0}", gitVersion); return true; } @@ -279,79 +314,89 @@ private bool TryCheckIfUpgradeAvailable(out Version newestVersion, out string co { newestVersion = null; - this.tracer.RelatedInfo("Checking upgrade server for new releases"); - - if (!this.upgrader.TryGetNewerVersion(out newestVersion, out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCheckIfUpgradeAvailable), EventLevel.Informational)) { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryCheckIfUpgradeAvailable)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetNewerVersion)} failed. {consoleError}"); - return false; - } + if (!this.upgrader.TryGetNewerVersion(out newestVersion, out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryCheckIfUpgradeAvailable)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetNewerVersion)} failed. {consoleError}"); + return false; + } - if (newestVersion == null) - { - consoleError = "No upgrades available in ring: " + this.upgrader.Ring; - this.tracer.RelatedInfo("No new upgrade releases available"); - return false; - } + if (newestVersion == null) + { + consoleError = "No upgrades available in ring: " + this.upgrader.Ring; + this.tracer.RelatedInfo("No new upgrade releases available"); + return false; + } - this.tracer.RelatedInfo("Successfully checked for new release. {0}", newestVersion); + activity.RelatedInfo("Successfully checked for new release. {0}", newestVersion); + } return true; } private bool TryDownloadUpgrade(Version version, out string consoleError) { - this.tracer.RelatedInfo("Downloading version: " + version.ToString()); - - if (!this.upgrader.TryDownloadNewestVersion(out consoleError)) + using (ITracer activity = this.tracer.StartActivity( + $"{nameof(this.TryDownloadUpgrade)}({version.ToString()})", + EventLevel.Informational)) { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryDownloadUpgrade)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryDownloadNewestVersion)} failed. {consoleError}"); - return false; - } + if (!this.upgrader.TryDownloadNewestVersion(out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryDownloadUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryDownloadNewestVersion)} failed. {consoleError}"); + return false; + } - this.tracer.RelatedInfo("Successfully downloaded version: " + version.ToString()); + activity.RelatedInfo("Successfully downloaded version: " + version.ToString()); + } return true; } private bool TryInstallGitUpgrade(GitVersion version, out string consoleError) { - this.tracer.RelatedInfo("Installing Git version: " + version.ToString()); - bool installSuccess = false; - if (!this.upgrader.TryRunGitInstaller(out installSuccess, out consoleError) || - !installSuccess) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryInstallGitUpgrade)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGitInstaller)} failed. {consoleError}"); - return false; - } + using (ITracer activity = this.tracer.StartActivity( + $"{nameof(this.TryInstallGitUpgrade)}({version.ToString()})", + EventLevel.Informational)) + { + if (!this.upgrader.TryRunGitInstaller(out installSuccess, out consoleError) || + !installSuccess) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryInstallGitUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGitInstaller)} failed. {consoleError}"); + return false; + } - this.tracer.RelatedInfo("Successfully installed Git version: " + version.ToString()); + activity.RelatedInfo("Successfully installed Git version: " + version.ToString()); + } return installSuccess; } private bool TryInstallGVFSUpgrade(Version version, out string consoleError) { - this.tracer.RelatedInfo("Installing GVFS version: " + version.ToString()); - bool installSuccess = false; - if (!this.upgrader.TryRunGVFSInstaller(out installSuccess, out consoleError) || - !installSuccess) + using (ITracer activity = this.tracer.StartActivity( + $"{nameof(this.TryInstallGVFSUpgrade)}({version.ToString()})", + EventLevel.Informational)) { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryInstallGVFSUpgrade)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGVFSInstaller)} failed. {consoleError}"); - return false; - } + if (!this.upgrader.TryRunGVFSInstaller(out installSuccess, out consoleError) || + !installSuccess) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryInstallGVFSUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGVFSInstaller)} failed. {consoleError}"); + return false; + } - this.tracer.RelatedInfo("Successfully installed GVFS version: " + version.ToString()); + activity.RelatedInfo("Successfully installed GVFS version: " + version.ToString()); + } return installSuccess; } diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index 13b4a56892..e6ef09f66c 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -35,6 +35,13 @@ public class ServiceVerb : GVFSVerb HelpText = "Prints a list of all mounted repos")] public bool List { get; set; } + [Option( + "log-mount-failure-in-stderr", + Default = false, + Required = false, + HelpText = "This parameter is reserved for internal use.")] + public bool RedirectMountFailuresToStderr { get; set; } + public override string EnlistmentRootPathParameter { get { throw new InvalidOperationException(); } @@ -103,7 +110,13 @@ public override void Execute() if (failedRepoRoots.Count() > 0) { - this.ReportErrorAndExit("\r\nThe following repos failed to mount:\r\n" + string.Join("\r\n", failedRepoRoots.ToArray())); + string errorString = $"The following repos failed to mount:{Environment.NewLine}{string.Join("\r\n", failedRepoRoots.ToArray())}"; + if (this.RedirectMountFailuresToStderr) + { + Console.Error.WriteLine(errorString); + } + + this.ReportErrorAndExit(Environment.NewLine + errorString); } } else if (this.UnmountAll) @@ -132,7 +145,13 @@ public override void Execute() if (failedRepoRoots.Count() > 0) { - this.ReportErrorAndExit("\r\nThe following repos failed to unmount:\r\n" + string.Join("\r\n", failedRepoRoots.ToArray())); + string errorString = $"The following repos failed to unmount:{Environment.NewLine}{string.Join(Environment.NewLine, failedRepoRoots.ToArray())}"; + if (this.RedirectMountFailuresToStderr) + { + Console.Error.WriteLine(errorString); + } + + this.ReportErrorAndExit(Environment.NewLine + errorString); } } } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index e914842a8c..289d0ac07d 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -1,6 +1,5 @@ using CommandLine; using GVFS.Common; -using GVFS.Common.Git; using GVFS.Common.Tracing; using GVFS.Upgrader; using System; @@ -34,17 +33,8 @@ public UpgradeVerb( public UpgradeVerb() { - JsonTracer jsonTracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "UpgradeVerb"); - string logFilePath = GVFSEnlistment.GetNewGVFSLogFileName( - ProductUpgrader.GetLogDirectoryPath(), - GVFSConstants.LogFileTypes.UpgradeVerb); - jsonTracer.AddLogFileEventListener(logFilePath, EventLevel.Informational, Keywords.Any); - - this.tracer = jsonTracer; - this.prerunChecker = new InstallerPreRunChecker(this.tracer); this.processWrapper = new ProcessLauncher(); this.Output = Console.Out; - this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); } [Option( @@ -64,29 +54,74 @@ protected override string VerbName public override void Execute() { ReturnCode exitCode = ReturnCode.Success; - if (!this.TryRunProductUpgrade()) + if (!this.TryInitializeUpgrader() || !this.TryRunProductUpgrade()) { exitCode = ReturnCode.GenericError; this.ReportErrorAndExit(this.tracer, exitCode, string.Empty); } } - + + private bool TryInitializeUpgrader() + { + OperatingSystem os_info = Environment.OSVersion; + + if (os_info.Platform == PlatformID.Win32NT) + { + if (this.upgrader == null) + { + JsonTracer jsonTracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "UpgradeVerb"); + string logFilePath = GVFSEnlistment.GetNewGVFSLogFileName( + ProductUpgrader.GetLogDirectoryPath(), + GVFSConstants.LogFileTypes.UpgradeVerb); + jsonTracer.AddLogFileEventListener(logFilePath, EventLevel.Informational, Keywords.Any); + + this.tracer = jsonTracer; + this.prerunChecker = new InstallerPreRunChecker(this.tracer); + this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + } + + return true; + } + else + { + this.ReportInfoToConsole($"ERROR: `gvfs upgrade` in only supported on Microsoft Windows Operating System."); + return false; + } + } + private bool TryRunProductUpgrade() { string errorOutputFormat = Environment.NewLine + "ERROR: {0}"; string error = null; Version newestVersion = null; - bool isInstallable = false; + ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; + + bool isInstallable = this.TryCheckUpgradeInstallable(out error); + if (this.Confirmed && !isInstallable) + { + this.ReportInfoToConsole($"Cannot install upgrade on this machine."); + this.Output.WriteLine(errorOutputFormat, error); + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade is not installable. {error}"); + return false; + } - if (this.upgrader.IsNoneRing()) + if (!this.TryLoadUpgradeRing(out ring, out error)) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Could not load upgrade ring. {error}"); + this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.InvalidRingConsoleAlert); + this.Output.WriteLine(errorOutputFormat, error); + return false; + } + + if (ring == ProductUpgrader.RingType.None) { this.tracer.RelatedInfo($"{nameof(this.TryRunProductUpgrade)}: {GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert}"); this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); return true; } - - if (!this.TryRunUpgradeChecks(out newestVersion, out isInstallable, out error)) + + if (!this.TryRunUpgradeChecks(out newestVersion, out error)) { this.Output.WriteLine(errorOutputFormat, error); this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade checks failed. {error}"); @@ -98,39 +133,68 @@ private bool TryRunProductUpgrade() this.ReportInfoToConsole($"Great news, you're all caught up on upgrades in the {this.upgrader.Ring} ring!"); return true; } - - this.ReportInfoToConsole("New GVFS version available: {0}", newestVersion.ToString()); - - if (!this.Confirmed && isInstallable) + + string upgradeAvailableMessage = $"New GVFS version available: {newestVersion.ToString()}"; + if (this.Confirmed) { - this.ReportInfoToConsole("Run `gvfs upgrade --confirm` to install it"); - return true; - } + this.ReportInfoToConsole(upgradeAvailableMessage); - if (!isInstallable) - { - this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: {error}"); - this.Output.WriteLine(errorOutputFormat, error); - return false; + if (!isInstallable) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: {error}"); + this.Output.WriteLine(errorOutputFormat, error); + return false; + } + + if (!this.TryRunInstaller(out error)) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Could not launch upgrade tool. {error}"); + this.Output.WriteLine(errorOutputFormat, "Could not launch upgrade tool. " + error); + return false; + } } + else + { + if (isInstallable) + { + string message = string.Join( + Environment.NewLine + Environment.NewLine, + upgradeAvailableMessage, + GVFSConstants.UpgradeVerbMessages.UnmountRepoWarning, + GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); + this.ReportInfoToConsole(message); + } + else + { + this.ReportInfoToConsole($"{Environment.NewLine}{upgradeAvailableMessage}"); + } + } - if (!this.TryRunInstaller(out error)) + return true; + } + + private bool TryLoadUpgradeRing(out ProductUpgrader.RingType ring, out string consoleError) + { + bool loaded = false; + if (!this.upgrader.TryLoadRingConfig(out consoleError)) + { + this.tracer.RelatedError($"{nameof(this.TryLoadUpgradeRing)} failed. {consoleError}"); + } + else { - this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Could not launch upgrade tool. {error}"); - this.Output.WriteLine(errorOutputFormat, "Could not launch upgrade tool. " + error); - return false; + consoleError = null; + loaded = true; } - return true; + ring = this.upgrader.Ring; + return loaded; } private bool TryRunUpgradeChecks( out Version latestVersion, - out bool isUpgradeInstallable, out string consoleError) { bool upgradeCheckSuccess = false; - bool upgradeIsInstallable = false; string errorMessage = null; Version version = null; @@ -138,23 +202,12 @@ private bool TryRunUpgradeChecks( () => { upgradeCheckSuccess = this.TryCheckUpgradeAvailable(out version, out errorMessage); - if (upgradeCheckSuccess && version != null) - { - upgradeIsInstallable = true; - - if (!this.TryCheckUpgradeInstallable(out errorMessage)) - { - upgradeIsInstallable = false; - } - } - return upgradeCheckSuccess; }, "Checking for GVFS upgrades", suppressGvfsLogMessage: true); latestVersion = version; - isUpgradeInstallable = upgradeIsInstallable; consoleError = errorMessage; return upgradeCheckSuccess; @@ -185,6 +238,7 @@ private bool TryRunInstaller(out string consoleError) return false; } + this.ReportInfoToConsole($"{Environment.NewLine}Installer launched in a new window. Do not run any git or gvfs commands until the installer has completed."); consoleError = null; return true; } @@ -193,40 +247,42 @@ private bool TryCopyUpgradeTool(out string upgraderExePath, out string consoleEr { upgraderExePath = null; - this.tracer.RelatedInfo("Copying upgrade tool"); - - if (!this.upgrader.TrySetupToolsDirectory(out upgraderExePath, out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCopyUpgradeTool), EventLevel.Informational)) { - return false; - } + if (!this.upgrader.TrySetupToolsDirectory(out upgraderExePath, out consoleError)) + { + return false; + } - this.tracer.RelatedInfo($"Successfully Copied upgrade tool to {upgraderExePath}"); + activity.RelatedInfo($"Successfully Copied upgrade tool to {upgraderExePath}"); + } return true; } private bool TryLaunchUpgradeTool(string path, out string consoleError) { - this.tracer.RelatedInfo("Launching upgrade tool"); - - Exception exception; - if (!this.processWrapper.TryStart(path, out exception)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryLaunchUpgradeTool), EventLevel.Informational)) { - if (exception != null) + Exception exception; + if (!this.processWrapper.TryStart(path, out exception)) { - consoleError = exception.Message; - this.tracer.RelatedError($"Error launching upgrade tool. {exception.ToString()}"); - } - else - { - consoleError = $"Error launching upgrade tool"; + if (exception != null) + { + consoleError = exception.Message; + this.tracer.RelatedError($"Error launching upgrade tool. {exception.ToString()}"); + } + else + { + consoleError = "Error launching upgrade tool"; + } + + return false; } - - return false; - } - - this.tracer.RelatedInfo("Successfully launched upgrade tool."); + activity.RelatedInfo("Successfully launched upgrade tool."); + } + consoleError = null; return true; } @@ -238,20 +294,21 @@ private bool TryCheckUpgradeAvailable( latestVersion = null; consoleError = null; - this.tracer.RelatedInfo("Checking server for available upgrades."); - - bool checkSucceeded = false; - Version version = null; - - checkSucceeded = this.upgrader.TryGetNewerVersion(out version, out consoleError); - if (!checkSucceeded) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCheckUpgradeAvailable), EventLevel.Informational)) { - return false; - } + bool checkSucceeded = false; + Version version = null; - latestVersion = version; + checkSucceeded = this.upgrader.TryGetNewerVersion(out version, out consoleError); + if (!checkSucceeded) + { + return false; + } + + latestVersion = version; - this.tracer.RelatedInfo("Successfully checked server for GVFS upgrades."); + activity.RelatedInfo("Successfully checked server for GVFS upgrades."); + } return true; } @@ -260,18 +317,19 @@ private bool TryCheckUpgradeInstallable(out string consoleError) { consoleError = null; - this.tracer.RelatedInfo("Checking if upgrade is installable on this machine."); - - this.prerunChecker.CommandToRerun = this.Confirmed ? "gvfs upgrade --confirm" : "gvfs upgrade"; - - if (!this.prerunChecker.TryRunPreUpgradeChecks( - out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCheckUpgradeInstallable), EventLevel.Informational)) { - return false; - } + this.prerunChecker.CommandToRerun = this.Confirmed ? "gvfs upgrade --confirm" : "gvfs upgrade"; - this.tracer.RelatedInfo("Upgrade is installable."); + if (!this.prerunChecker.TryRunPreUpgradeChecks( + out consoleError)) + { + return false; + } + activity.RelatedInfo("Upgrade is installable."); + } + return true; } diff --git a/start b/start new file mode 100644 index 0000000000..c342dc54f5 --- /dev/null +++ b/start @@ -0,0 +1 @@ +prjflt From 5681da5273567803a9dc191933d5513a7ef1c626 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Fri, 21 Sep 2018 15:17:21 -0400 Subject: [PATCH 064/244] - renamed upgrade ring config to "gvfs.upgrade-ring" - Cleanup renamed UpgradeDiskLayout constant to MountUpgrade. - Cleanup - replaced usage of the string "gvfs.upgrade-ring" with const. - Added TODO to remove NotSupportedException from GitProcess::TryGitVersion. - Made InstallerTracer.CommandToRerun protected property. Updated UT. - Added issue reference url for InstallerPreRunChecker refactoring TODO. - Added new constants for "gvfs upgrade", "gvfs upgrade --confirm" strings. - Removed pre-check for development version. Upgrade can now be run on Developer machines. - Optimization - InstallerPreRunChecker::IsBlockingProcessRunning now returns HashSet. Removed unnecessary conversion to list. - Better error handling in InstallerPreRunChecker while launching GVFS CLI - Updated messaging in InstallerPreRunChecker. The text "elevated command prompt" was missing in some of the messaging. - Cleanup. Renamed processWrapper to ProcessLauncher. - Added URL info while logging network errors during fetch release. - Removed Randomization of upgrade timer in the service. - Removed GVFSArgs from MockInstallerPreRunChecker UT class. - Removed ShouldExitOnError property in UpgradOrchestrator class. Using Environment.ExitCode instead. - Cleanup. Replaced all occurances of "remount" with "mount". - removed log-mount-failure-in-stderr flag from Service verb. - renamed log files to productupgrade_and mount_repoupgrade --- GVFS/GVFS.Common/DiskLayoutUpgrade.cs | 2 +- GVFS/GVFS.Common/GVFSConstants.cs | 15 ++-- GVFS/GVFS.Common/Git/GitProcess.cs | 2 + GVFS/GVFS.Common/InstallerPreRunChecker.cs | 75 +++++++++---------- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 23 +----- GVFS/GVFS.Common/ProductUpgrader.cs | 36 +++++++-- GVFS/GVFS.Service/ProductUpgradeTimer.cs | 61 +++++++-------- .../Mock/MockInstallerPreRunChecker.cs | 37 ++++----- .../Windows/Mock/MockProcessLauncher.cs | 5 +- .../Upgrader/UpgradeOrchestratorTests.cs | 5 +- .../Windows/Upgrader/UpgradeVerbTests.cs | 30 ++------ GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 50 ++++++------- GVFS/GVFS/CommandLine/ServiceVerb.cs | 19 +---- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 16 ++-- 14 files changed, 161 insertions(+), 215 deletions(-) diff --git a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs index 5ec4115fb3..6655905950 100644 --- a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs +++ b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs @@ -327,7 +327,7 @@ private static void StartLogFile(string enlistmentRoot, JsonTracer tracer) tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName( Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.LogPath), - GVFSConstants.LogFileTypes.UpgradeDiskLayout), + GVFSConstants.LogFileTypes.MountUpgrade), EventLevel.Informational, Keywords.Any); diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index b9f6c56d27..39fc289305 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -33,7 +33,7 @@ public static class GitConfig public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; public const string HooksExtension = ".hooks"; - public const string UpgradeRing = GVFSPrefix + "upgradering"; + public const string UpgradeRing = GVFSPrefix + "upgrade-ring"; } public static class GitStatusCache @@ -73,7 +73,7 @@ public static class SpecialGitFiles public static class LogFileTypes { public const string MountPrefix = "mount"; - public const string UpgradePrefix = "upgrade"; + public const string UpgradePrefix = "productupgrade"; public const string Clone = "clone"; public const string Dehydrate = "dehydrate"; @@ -84,7 +84,7 @@ public static class LogFileTypes public const string Service = "service"; public const string UpgradeVerb = UpgradePrefix + "_verb"; public const string UpgradeProcess = UpgradePrefix + "_process"; - public const string UpgradeDiskLayout = MountPrefix + "_upgrade"; + public const string MountUpgrade = MountPrefix + "_repoupgrade"; } public static class DotGVFS @@ -222,12 +222,15 @@ public static class Unmount public static class UpgradeVerbMessages { + public const string GVFSUpgrade = "`gvfs upgrade`"; + public const string GVFSUpgradeConfirm = "`gvfs upgrade --confirm`"; + public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; - public const string ReminderNotification = "A new version of GVFS is available. Run `gvfs upgrade` to start the upgrade."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global " + GitConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; + public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; - public const string UpgradeInstallAdvice = "When you are ready, run `gvfs upgrade --confirm` from an elevated command prompt."; + public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; } } } diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 1ce9236e16..2ed4e1bf47 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -94,6 +94,8 @@ public static Result GetFromSystemConfig(string gitBinPath, string settingName) public static bool TryGetVersion(out GitVersion gitVersion, out string error) { + // TODO Implement IGitInstallation in MockPlatform.cs & remove + // NotSupportedException handler. try { string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); diff --git a/GVFS/GVFS.Common/InstallerPreRunChecker.cs b/GVFS/GVFS.Common/InstallerPreRunChecker.cs index aa5f919e51..ba46681a62 100644 --- a/GVFS/GVFS.Common/InstallerPreRunChecker.cs +++ b/GVFS/GVFS.Common/InstallerPreRunChecker.cs @@ -15,14 +15,14 @@ public class InstallerPreRunChecker private static readonly HashSet BlockingProcessSet = new HashSet { "GVFS", "GVFS.Mount", "git", "ssh-agent", "bash", "wish", "git-bash" }; private ITracer tracer; - - public InstallerPreRunChecker(ITracer tracer) + + public InstallerPreRunChecker(ITracer tracer, string commandToRerun) { this.tracer = tracer; - this.CommandToRerun = string.Empty; + this.CommandToRerun = commandToRerun; } - public string CommandToRerun { get; set; } + protected string CommandToRerun { private get; set; } public bool TryRunPreUpgradeChecks(out string consoleError) { @@ -30,35 +30,29 @@ public bool TryRunPreUpgradeChecks(out string consoleError) { if (this.IsUnattended()) { - consoleError = "`gvfs upgrade` is not supported in unattended mode"; - this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); - return false; - } - - if (this.IsDevelopmentVersion()) - { - consoleError = "Cannot run upgrade when development version of GVFS is installed."; + consoleError = $"{GVFSConstants.UpgradeVerbMessages.GVFSUpgrade} is not supported in unattended mode"; this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); return false; } - + if (!this.IsGVFSUpgradeAllowed(out consoleError)) { this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); return false; } - activity.RelatedInfo("Successfully finished pre upgrade checks. Okay to run GVFS upgrade."); + activity.RelatedInfo($"Successfully finished pre upgrade checks. Okay to run {GVFSConstants.UpgradeVerbMessages.GVFSUpgrade}."); } consoleError = null; return true; } - + // TODO: Move repo mount calls to GVFS.Upgrader project. + // https://github.com/Microsoft/VFSForGit/issues/293 public bool TryMountAllGVFSRepos(out string consoleError) { - return this.TryRunGVFSWithArgs("service --mount-all --log-mount-failure-in-stderr", out consoleError); + return this.TryRunGVFSWithArgs("service --mount-all", out consoleError); } public bool TryUnmountAllGVFSRepos(out string consoleError) @@ -69,7 +63,7 @@ public bool TryUnmountAllGVFSRepos(out string consoleError) using (ITracer activity = this.tracer.StartActivity(nameof(this.TryUnmountAllGVFSRepos), EventLevel.Informational)) { - if (!this.TryRunGVFSWithArgs("service --unmount-all --log-mount-failure-in-stderr", out consoleError)) + if (!this.TryRunGVFSWithArgs("service --unmount-all", out consoleError)) { this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); return false; @@ -82,7 +76,7 @@ public bool TryUnmountAllGVFSRepos(out string consoleError) // actually quits. this.tracer.RelatedInfo("Checking if GVFS or dependent processes are running."); int retryCount = 10; - List processList = null; + HashSet processList = null; while (retryCount > 0) { if (!this.IsBlockingProcessRunning(out processList)) @@ -99,7 +93,7 @@ public bool TryUnmountAllGVFSRepos(out string consoleError) consoleError = string.Join( Environment.NewLine, "Blocking processes are running.", - $"Run `{this.CommandToRerun}` again after quitting these processes - " + string.Join(", ", processList.ToArray())); + $"Run {this.CommandToRerun} again after quitting these processes - " + string.Join(", ", processList.ToArray())); this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); return false; } @@ -131,13 +125,8 @@ protected virtual bool IsUnattended() { return GVFSEnlistment.IsUnattended(this.tracer); } - - protected virtual bool IsDevelopmentVersion() - { - return ProcessHelper.IsDevelopmentVersion(); - } - - protected virtual bool IsBlockingProcessRunning(out List processes) + + protected virtual bool IsBlockingProcessRunning(out HashSet processes) { int currentProcessId = Process.GetCurrentProcess().Id; Process[] allProcesses = Process.GetProcesses(); @@ -153,26 +142,32 @@ protected virtual bool IsBlockingProcessRunning(out List processes) matchingNames.Add(process.ProcessName); } - processes = matchingNames.ToList(); + processes = matchingNames; return processes.Count > 0; } protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) { - consoleError = null; - - string gvfsPath = Path.Combine( - ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSExecutableName), - GVFSPlatform.Instance.Constants.GVFSExecutableName); - - ProcessResult processResult = ProcessHelper.Run(gvfsPath, args); - if (processResult.ExitCode == 0) + string gvfsDirectory = ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSExecutableName); + if (!string.IsNullOrEmpty(gvfsDirectory)) { - return true; + string gvfsPath = Path.Combine(gvfsDirectory, GVFSPlatform.Instance.Constants.GVFSExecutableName); + + ProcessResult processResult = ProcessHelper.Run(gvfsPath, args); + if (processResult.ExitCode == 0) + { + consoleError = null; + return true; + } + else + { + consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"GVFS {args} failed." : processResult.Errors; + return false; + } } else { - consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"GVFS {args} failed." : processResult.Errors; + consoleError = $"Could not locate {GVFSPlatform.Instance.Constants.GVFSExecutableName}"; return false; } } @@ -186,7 +181,7 @@ private bool IsGVFSUpgradeAllowed(out string consoleError) consoleError = string.Join( Environment.NewLine, "The installer needs to be run from an elevated command prompt.", - $"Run `{this.CommandToRerun}` again from an elevated command prompt."); + $"Run {this.CommandToRerun} again from an elevated command prompt."); return false; } @@ -194,7 +189,7 @@ private bool IsGVFSUpgradeAllowed(out string consoleError) { consoleError = string.Join( Environment.NewLine, - "ProjFS configuration does not support `gvfs upgrade`.", + $"ProjFS configuration does not support {GVFSConstants.UpgradeVerbMessages.GVFSUpgrade}.", "Check your team's documentation for how to upgrade."); return false; } @@ -204,7 +199,7 @@ private bool IsGVFSUpgradeAllowed(out string consoleError) consoleError = string.Join( Environment.NewLine, "GVFS Service is not running.", - $"Run `sc start GVFS.Service` and run `{this.CommandToRerun}` again."); + $"Run `sc start GVFS.Service` and run {this.CommandToRerun} again from an elevated command prompt."); return false; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index e8a995efcc..30d8aed521 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -19,7 +19,7 @@ public static bool IsLocalUpgradeAvailable() if (Directory.Exists(downloadDirectory)) { string[] installers = Directory.GetFiles( - GetAssetDownloadsPath(), + downloadDirectory, $"{GVFSInstallerFileNamePrefix}*.*", SearchOption.TopDirectoryOnly); return installers.Length > 0; @@ -38,27 +38,6 @@ public static string GetLogDirectoryPath() return Path.Combine(Paths.GetServiceDataRoot(RootDirectory), LogDirectory); } - private static bool TryCreateDirectory(string path, out Exception exception) - { - try - { - Directory.CreateDirectory(path); - } - catch (IOException e) - { - exception = e; - return false; - } - catch (UnauthorizedAccessException e) - { - exception = e; - return false; - } - - exception = null; - return true; - } - private static string GetAssetDownloadsPath() { return Path.Combine( diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index b8b411a0c1..af0a7e813c 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -66,8 +66,8 @@ public enum RingType // upgrade logic. Invalid = 0, None = 10, - Slow = 20, - Fast = 30, + Slow = None + 1, + Fast = Slow + 1, } public RingType Ring { get; protected set; } @@ -193,7 +193,7 @@ public virtual bool TrySetupToolsDirectory(out string upgraderToolPath, out stri error = string.Join( Environment.NewLine, "File copy error - " + e.Message, - $"Make sure you have write permissions to directory {rootDirectoryPath} and run `gvfs upgrade --confirm` again."); + $"Make sure you have write permissions to directory {rootDirectoryPath} and run {GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm} again."); this.TraceException(e, nameof(this.TrySetupToolsDirectory), $"Error copying {toolPath} to {destinationPath}."); break; } @@ -244,7 +244,6 @@ public bool TryCleanup(out string error) public virtual bool TryLoadRingConfig(out string error) { - string errorAdvisory = "Run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"]` and run `gvfs upgrade [--confirm]` again."; string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); GitProcess.Result result = GitProcess.GetFromGlobalConfig(gitPath, GVFSConstants.GitConfig.UpgradeRing); if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) @@ -270,7 +269,7 @@ public virtual bool TryLoadRingConfig(out string error) error = string.IsNullOrEmpty(result.Errors) ? "Unable to determine upgrade ring." : result.Errors; } - error += Environment.NewLine + errorAdvisory; + error += Environment.NewLine + GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand; this.Ring = RingType.Invalid; return false; } @@ -332,12 +331,12 @@ protected virtual bool TryFetchReleases(out List releases, out string e } catch (HttpRequestException exception) { - errorMessage = string.Format("Network error: could not connect to GitHub. {0}", exception.Message); + errorMessage = string.Format("Network error: could not connect to GitHub({0}). {1}", GitHubReleaseURL, exception.Message); this.TraceException(exception, nameof(this.TryFetchReleases), $"Error fetching release info."); } catch (SerializationException exception) { - errorMessage = string.Format("Parse error: could not parse releases info from GitHub. {0}", exception.Message); + errorMessage = string.Format("Parse error: could not parse releases info from GitHub({0}). {1}", GitHubReleaseURL, exception.Message); this.TraceException(exception, nameof(this.TryFetchReleases), $"Error parsing release info."); } @@ -352,6 +351,27 @@ protected virtual void RunInstaller(string path, string args, out int exitCode, error = processResult.Errors; } + private static bool TryCreateDirectory(string path, out Exception exception) + { + try + { + Directory.CreateDirectory(path); + } + catch (IOException e) + { + exception = e; + return false; + } + catch (UnauthorizedAccessException e) + { + exception = e; + return false; + } + + exception = null; + return true; + } + private bool TryRunInstallerForAsset(string name, out int installerExitCode, out string error) { error = null; @@ -415,7 +435,7 @@ private bool TryGetLocalInstallerPath(string name, out string path, out string a args = null; return false; } - + [DataContract(Name = "asset")] protected class Asset { diff --git a/GVFS/GVFS.Service/ProductUpgradeTimer.cs b/GVFS/GVFS.Service/ProductUpgradeTimer.cs index 48c37d4154..b276b28ebc 100644 --- a/GVFS/GVFS.Service/ProductUpgradeTimer.cs +++ b/GVFS/GVFS.Service/ProductUpgradeTimer.cs @@ -21,14 +21,14 @@ public ProductUpgradeTimer(JsonTracer tracer) public void Start() { Random random = new Random(); - TimeSpan startTime = TimeSpan.FromMinutes(random.Next(0, 60)); + TimeSpan startTime = TimeSpan.Zero; - this.tracer.RelatedInfo($"Starting auto upgrade checks. Start time - {startTime.ToString()}"); + this.tracer.RelatedInfo("Starting auto upgrade checks."); this.timer = new Timer( this.TimerCallback, - null, - startTime, - TimeInterval); + state: null, + dueTime: startTime, + period: TimeInterval); } public void Stop() @@ -41,7 +41,7 @@ private void TimerCallback(object unusedState) { string errorMessage = null; - InstallerPreRunChecker prerunChecker = new InstallerPreRunChecker(this.tracer); + InstallerPreRunChecker prerunChecker = new InstallerPreRunChecker(this.tracer, string.Empty); if (prerunChecker.TryRunPreUpgradeChecks(out string _) && !this.TryDownloadUpgrade(out errorMessage)) { this.tracer.RelatedError(errorMessage); @@ -50,33 +50,34 @@ private void TimerCallback(object unusedState) private bool TryDownloadUpgrade(out string errorMessage) { - this.tracer.RelatedInfo("Checking for product upgrades."); - - ProductUpgrader productUpgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); - Version newerVersion = null; - string detailedError = null; - if (!productUpgrader.TryGetNewerVersion(out newerVersion, out detailedError)) + using (ITracer activity = this.tracer.StartActivity("Checking for product upgrades.", EventLevel.Informational)) { - errorMessage = "Could not fetch new version info. " + detailedError; - return false; - } + ProductUpgrader productUpgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + Version newerVersion = null; + string detailedError = null; + if (!productUpgrader.TryGetNewerVersion(out newerVersion, out detailedError)) + { + errorMessage = "Could not fetch new version info. " + detailedError; + return false; + } - if (newerVersion == null) - { - // Already up-to-date - errorMessage = null; - return true; - } + if (newerVersion == null) + { + // Already up-to-date + errorMessage = null; + return true; + } - if (productUpgrader.TryDownloadNewestVersion(out detailedError)) - { - errorMessage = null; - return true; - } - else - { - errorMessage = "Could not download product upgrade. " + detailedError; - return false; + if (productUpgrader.TryDownloadNewestVersion(out detailedError)) + { + errorMessage = null; + return true; + } + else + { + errorMessage = "Could not download product upgrade. " + detailedError; + return false; + } } } } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs index 6dc432e4a4..55255631e4 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs @@ -15,7 +15,7 @@ public class MockInstallerPrerunChecker : InstallerPreRunChecker private FailOnCheckType failOnCheck; - public MockInstallerPrerunChecker(ITracer tracer) : base(tracer) + public MockInstallerPrerunChecker(ITracer tracer) : base(tracer, string.Empty) { } @@ -27,15 +27,11 @@ public enum FailOnCheckType IsElevated = 0x2, BlockingProcessesRunning = 0x4, UnattendedMode = 0x8, - IsDevelopmentVersion = 0x10, - IsGitUpgradeAllowed = 0x20, - UnMountRepos = 0x40, - RemountRepos = 0x80, - IsServiceInstalledAndNotRunning = 0x100 + UnMountRepos = 0x10, + RemountRepos = 0x20, + IsServiceInstalledAndNotRunning = 0x40, } - public List GVFSArgs { get; private set; } = new List(); - public void SetReturnFalseOnCheck(FailOnCheckType prerunCheck) { this.failOnCheck |= prerunCheck; @@ -50,12 +46,14 @@ public void Reset() { this.failOnCheck = FailOnCheckType.Invalid; - this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsDevelopmentVersion); this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.UnattendedMode); this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.BlockingProcessesRunning); this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsServiceInstalledAndNotRunning); + } - this.GVFSArgs.Clear(); + public void SetCommandToRerun(string command) + { + this.CommandToRerun = command; } protected override bool IsServiceInstalledAndNotRunning() @@ -77,15 +75,10 @@ protected override bool IsUnattended() { return this.FakedResultOfCheck(FailOnCheckType.UnattendedMode); } - - protected override bool IsDevelopmentVersion() - { - return this.FakedResultOfCheck(FailOnCheckType.IsDevelopmentVersion); - } - - protected override bool IsBlockingProcessRunning(out List processes) + + protected override bool IsBlockingProcessRunning(out HashSet processes) { - processes = new List(); + processes = new HashSet(); bool isRunning = this.FakedResultOfCheck(FailOnCheckType.BlockingProcessesRunning); if (isRunning) @@ -98,17 +91,15 @@ protected override bool IsBlockingProcessRunning(out List processes) } protected override bool TryRunGVFSWithArgs(string args, out string error) - { - this.GVFSArgs.Add(args); - - if (string.CompareOrdinal(args, "service --unmount-all --log-mount-failure-in-stderr") == 0) + { + if (string.CompareOrdinal(args, "service --unmount-all") == 0) { bool result = this.FakedResultOfCheck(FailOnCheckType.UnMountRepos); error = result == false ? "Unmount of some of the repositories failed." : null; return result; } - if (string.CompareOrdinal(args, "service --mount-all --log-mount-failure-in-stderr") == 0) + if (string.CompareOrdinal(args, "service --mount-all") == 0) { bool result = this.FakedResultOfCheck(FailOnCheckType.RemountRepos); error = result == false ? "Auto remount failed." : null; diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs index dc3ddde556..3eb47e4bfa 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs @@ -1,9 +1,8 @@ using System; -using static GVFS.CommandLine.UpgradeVerb; namespace GVFS.UnitTests.Windows.Upgrader { - public class MockProcessLauncher : ProcessLauncher + public class MockProcessLauncher : GVFS.CommandLine.UpgradeVerb.ProcessLauncher { private int exitCode; private bool hasExited; @@ -12,7 +11,7 @@ public class MockProcessLauncher : ProcessLauncher public MockProcessLauncher( int exitCode, bool hasExited, - bool startResult) : base() + bool startResult) { this.exitCode = exitCode; this.hasExited = hasExited; diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs index abced85c4a..356d806280 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs @@ -22,9 +22,8 @@ public override void Setup() this.Tracer, this.PrerunChecker, input: null, - output: this.Output, - shouldExitOnError: false); - this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + output: this.Output); + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); } [TestCase] diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs index d76ef303ce..cc4b43065e 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -27,7 +27,7 @@ public override void Setup() this.ProcessWrapper, this.Output); this.UpgradeVerb.Confirmed = false; - this.PrerunChecker.CommandToRerun = "gvfs upgrade"; + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade`"); } [TestCase] @@ -75,7 +75,7 @@ public void LaunchInstaller() configure: () => { this.UpgradeVerb.Confirmed = true; - this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); }, expectedReturn: ReturnCode.Success, expectedOutput: new List @@ -111,7 +111,7 @@ public void CopyTools() { this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.CopyTools); this.UpgradeVerb.Confirmed = true; - this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); }, expectedReturn: ReturnCode.GenericError, expectedOutput: new List @@ -148,6 +148,7 @@ public void ProjFSPreCheck() [TestCase] public void IsGVFSServiceRunningPreCheck() { + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); this.ConfigureRunAndVerify( configure: () => { @@ -158,7 +159,7 @@ public void IsGVFSServiceRunningPreCheck() expectedOutput: new List { "GVFS Service is not running.", - "Run `sc start GVFS.Service` and run `gvfs upgrade --confirm` again." + "Run `sc start GVFS.Service` and run `gvfs upgrade --confirm` again from an elevated command prompt." }, expectedErrors: new List { @@ -169,6 +170,7 @@ public void IsGVFSServiceRunningPreCheck() [TestCase] public void ElevatedRunPreCheck() { + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); this.ConfigureRunAndVerify( configure: () => { @@ -207,26 +209,6 @@ public void UnAttendedModePreCheck() }); } - [TestCase] - public void DeveloperMachinePreCheck() - { - this.ConfigureRunAndVerify( - configure: () => - { - this.UpgradeVerb.Confirmed = true; - this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsDevelopmentVersion); - }, - expectedReturn: ReturnCode.GenericError, - expectedOutput: new List - { - "Cannot run upgrade when development version of GVFS is installed." - }, - expectedErrors: new List - { - "Cannot run upgrade when development version of GVFS is installed." - }); - } - protected override void RunUpgrade() { try diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index a5e51dae00..155adf9f37 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -15,24 +15,21 @@ public class UpgradeOrchestrator private InstallerPreRunChecker preRunChecker; private TextWriter output; private TextReader input; - private bool remount; - private bool shouldExitOnError; + private bool mount; public UpgradeOrchestrator( ProductUpgrader upgrader, ITracer tracer, InstallerPreRunChecker preRunChecker, TextReader input, - TextWriter output, - bool shouldExitOnError) + TextWriter output) { this.upgrader = upgrader; this.tracer = tracer; this.preRunChecker = preRunChecker; this.output = output; this.input = input; - this.remount = false; - this.shouldExitOnError = shouldExitOnError; + this.mount = false; this.ExitCode = ReturnCode.Success; } @@ -48,12 +45,11 @@ public UpgradeOrchestrator() Keywords.Any); this.tracer = jsonTracer; - this.preRunChecker = new InstallerPreRunChecker(this.tracer); + this.preRunChecker = new InstallerPreRunChecker(this.tracer, GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm); this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); this.output = Console.Out; this.input = Console.In; - this.remount = false; - this.shouldExitOnError = false; + this.mount = false; this.ExitCode = ReturnCode.Success; } @@ -82,18 +78,18 @@ public void Execute() try { Version newVersion = null; - if (!this.TryRunUpgradeInstall(out newVersion, out error)) + if (!this.TryRunUpgrade(out newVersion, out error)) { this.ExitCode = ReturnCode.GenericError; } } finally { - string remountError = null; - if (!this.TryRemountRepositories(out remountError)) + string mountError = null; + if (!this.TryMountRepositories(out mountError)) { - remountError = Environment.NewLine + "WARNING: " + remountError; - this.output.WriteLine(remountError); + mountError = Environment.NewLine + "WARNING: " + mountError; + this.output.WriteLine(mountError); this.ExitCode = ReturnCode.Success; } @@ -116,11 +112,8 @@ public void Execute() this.output.WriteLine("Press Enter to exit."); this.input.ReadLine(); } - - if (this.shouldExitOnError) - { - Environment.Exit((int)this.ExitCode); - } + + Environment.ExitCode = (int)this.ExitCode; } private bool LaunchInsideSpinner(Func method, string message) @@ -154,7 +147,7 @@ private bool TryLoadUpgradeRing(out ProductUpgrader.RingType ring, out string co return loaded; } - private bool TryRunUpgradeInstall(out Version newVersion, out string consoleError) + private bool TryRunUpgrade(out Version newVersion, out string consoleError) { newVersion = null; @@ -173,7 +166,6 @@ private bool TryRunUpgradeInstall(out Version newVersion, out string consoleErro this.LogInstalledVersionInfo(); this.LogVersionInfo(newGVFSVersion, newGitVersion, "Available Version"); - this.preRunChecker.CommandToRerun = "gvfs upgrade --confirm"; if (!this.preRunChecker.TryRunPreUpgradeChecks(out errorMessage)) { return false; @@ -200,7 +192,7 @@ private bool TryRunUpgradeInstall(out Version newVersion, out string consoleErro return false; } - this.remount = true; + this.mount = true; return true; }, @@ -249,20 +241,20 @@ private bool TryRunUpgradeInstall(out Version newVersion, out string consoleErro return true; } - private bool TryRemountRepositories(out string consoleError) + private bool TryMountRepositories(out string consoleError) { string errorMessage = string.Empty; - if (this.remount && !this.LaunchInsideSpinner( + if (this.mount && !this.LaunchInsideSpinner( () => { - string remountError; - if (!this.preRunChecker.TryMountAllGVFSRepos(out remountError)) + string mountError; + if (!this.preRunChecker.TryMountAllGVFSRepos(out mountError)) { EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryRemountRepositories)); - metadata.Add("Remount Error", remountError); + metadata.Add("Upgrade Step", nameof(this.TryMountRepositories)); + metadata.Add("Mount Error", mountError); this.tracer.RelatedError(metadata, $"{nameof(this.preRunChecker.TryMountAllGVFSRepos)} failed."); - errorMessage += remountError; + errorMessage += mountError; return false; } diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index e6ef09f66c..640c99a08b 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -35,13 +35,6 @@ public class ServiceVerb : GVFSVerb HelpText = "Prints a list of all mounted repos")] public bool List { get; set; } - [Option( - "log-mount-failure-in-stderr", - Default = false, - Required = false, - HelpText = "This parameter is reserved for internal use.")] - public bool RedirectMountFailuresToStderr { get; set; } - public override string EnlistmentRootPathParameter { get { throw new InvalidOperationException(); } @@ -111,11 +104,7 @@ public override void Execute() if (failedRepoRoots.Count() > 0) { string errorString = $"The following repos failed to mount:{Environment.NewLine}{string.Join("\r\n", failedRepoRoots.ToArray())}"; - if (this.RedirectMountFailuresToStderr) - { - Console.Error.WriteLine(errorString); - } - + Console.Error.WriteLine(errorString); this.ReportErrorAndExit(Environment.NewLine + errorString); } } @@ -146,11 +135,7 @@ public override void Execute() if (failedRepoRoots.Count() > 0) { string errorString = $"The following repos failed to unmount:{Environment.NewLine}{string.Join(Environment.NewLine, failedRepoRoots.ToArray())}"; - if (this.RedirectMountFailuresToStderr) - { - Console.Error.WriteLine(errorString); - } - + Console.Error.WriteLine(errorString); this.ReportErrorAndExit(Environment.NewLine + errorString); } } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index 289d0ac07d..020a16fdc0 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -8,14 +8,14 @@ namespace GVFS.CommandLine { - [Verb(UpgradeVerbName, HelpText = "Checks if a new GVFS release is available.")] + [Verb(UpgradeVerbName, HelpText = "Checks for new GVFS release, downloads and installs it when available.")] public class UpgradeVerb : GVFSVerb { private const string UpgradeVerbName = "upgrade"; private ITracer tracer; private ProductUpgrader upgrader; private InstallerPreRunChecker prerunChecker; - private ProcessLauncher processWrapper; + private ProcessLauncher processLauncher; public UpgradeVerb( ProductUpgrader upgrader, @@ -27,13 +27,13 @@ public UpgradeVerb( this.upgrader = upgrader; this.tracer = tracer; this.prerunChecker = prerunChecker; - this.processWrapper = processWrapper; + this.processLauncher = processWrapper; this.Output = output; } public UpgradeVerb() { - this.processWrapper = new ProcessLauncher(); + this.processLauncher = new ProcessLauncher(); this.Output = Console.Out; } @@ -76,7 +76,7 @@ private bool TryInitializeUpgrader() jsonTracer.AddLogFileEventListener(logFilePath, EventLevel.Informational, Keywords.Any); this.tracer = jsonTracer; - this.prerunChecker = new InstallerPreRunChecker(this.tracer); + this.prerunChecker = new InstallerPreRunChecker(this.tracer, this.Confirmed ? GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm : GVFSConstants.UpgradeVerbMessages.GVFSUpgrade); this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); } @@ -84,7 +84,7 @@ private bool TryInitializeUpgrader() } else { - this.ReportInfoToConsole($"ERROR: `gvfs upgrade` in only supported on Microsoft Windows Operating System."); + this.ReportInfoToConsole($"ERROR: {GVFSConstants.UpgradeVerbMessages.GVFSUpgrade} in only supported on Microsoft Windows Operating System."); return false; } } @@ -265,7 +265,7 @@ private bool TryLaunchUpgradeTool(string path, out string consoleError) using (ITracer activity = this.tracer.StartActivity(nameof(this.TryLaunchUpgradeTool), EventLevel.Informational)) { Exception exception; - if (!this.processWrapper.TryStart(path, out exception)) + if (!this.processLauncher.TryStart(path, out exception)) { if (exception != null) { @@ -319,8 +319,6 @@ private bool TryCheckUpgradeInstallable(out string consoleError) using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCheckUpgradeInstallable), EventLevel.Informational)) { - this.prerunChecker.CommandToRerun = this.Confirmed ? "gvfs upgrade --confirm" : "gvfs upgrade"; - if (!this.prerunChecker.TryRunPreUpgradeChecks( out consoleError)) { From 4206f2670ce27134696b82dda053100805f90176 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 24 Sep 2018 17:15:12 -0400 Subject: [PATCH 065/244] - resolving merge issues --- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index a5a0c3208e..d82693c424 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -584,11 +584,6 @@ private static ProcessResult GetProjFSOptionalFeatureStatus() (int)ProjFSInboxStatus.NotInbox + "}else{if($var.State -eq 'Enabled'){exit " + (int)ProjFSInboxStatus.Enabled + "}else{exit " + (int)ProjFSInboxStatus.Disabled + "}}"); } - private static ProcessResult CallPowershellCommand(string command) - { - return ProcessHelper.Run("powershell.exe", "-NonInteractive -NoProfile -Command \"& { " + command + " }\""); - } - private static EventMetadata CreateEventMetadata(Exception e = null) { EventMetadata metadata = new EventMetadata(); From 9e225a8c6ae01b91988fcda42f4167a54514c278 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Thu, 27 Sep 2018 11:30:55 -0400 Subject: [PATCH 066/244] - Removed GitProcess.Version() - Using GitProcess.TryGetVersion() in GVFSVerb and Diagnose Verb. - Save directory listing of GVFS.Upgrade\Downloads directory during diagnose. - Created MockGitInstallation.cs - Removed NotSupportedException handler from GitProcess.cs --- GVFS/GVFS.Common/Git/GitProcess.cs | 15 ++++--------- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 2 +- .../Mock/Common/MockPlatform.cs | 3 ++- .../Mock/Git/MockGitInstallation.cs | 22 +++++++++++++++++++ GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 18 ++++++++++++++- GVFS/GVFS/CommandLine/GVFSVerb.cs | 16 +++----------- 6 files changed, 49 insertions(+), 27 deletions(-) create mode 100644 GVFS/GVFS.UnitTests/Mock/Git/MockGitInstallation.cs diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 2ed4e1bf47..2d3a2206e7 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -77,11 +77,6 @@ public static Result Init(Enlistment enlistment) return new GitProcess(enlistment).InvokeGitOutsideEnlistment("init \"" + enlistment.WorkingDirectoryRoot + "\""); } - public static Result Version(Enlistment enlistment) - { - return new GitProcess(enlistment).InvokeGitOutsideEnlistment("--version"); - } - public static Result GetFromGlobalConfig(string gitBinPath, string settingName) { return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --global " + settingName); @@ -94,11 +89,9 @@ public static Result GetFromSystemConfig(string gitBinPath, string settingName) public static bool TryGetVersion(out GitVersion gitVersion, out string error) { - // TODO Implement IGitInstallation in MockPlatform.cs & remove - // NotSupportedException handler. - try + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + if (gitPath != null) { - string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); GitProcess gitProcess = new GitProcess(gitPath, null, null); Result result = gitProcess.InvokeGitOutsideEnlistment("--version"); string version = result.Output; @@ -112,10 +105,10 @@ public static bool TryGetVersion(out GitVersion gitVersion, out string error) error = null; return true; } - catch (NotSupportedException exception) + else { - error = exception.ToString(); gitVersion = null; + error = "Unable to determine installed git path."; return false; } } diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index 30d8aed521..87d5438b13 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -8,9 +8,9 @@ public partial class ProductUpgrader { public const string UpgradeDirectoryName = "GVFS.Upgrade"; public const string LogDirectory = "Logs"; + public const string DownloadDirectory = "Downloads"; private const string RootDirectory = UpgradeDirectoryName; - private const string DownloadDirectory = "Downloads"; private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; public static bool IsLocalUpgradeAvailable() diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index 4fb290f507..f4ba15d35a 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -4,6 +4,7 @@ using GVFS.Common.Tracing; using GVFS.UnitTests.Mock.Common.Tracing; using GVFS.UnitTests.Mock.FileSystem; +using GVFS.UnitTests.Mock.Git; using System; using System.Collections.Generic; using System.IO.Pipes; @@ -19,7 +20,7 @@ public MockPlatform() public override IKernelDriver KernelDriver => throw new NotSupportedException(); - public override IGitInstallation GitInstallation => throw new NotSupportedException(); + public override IGitInstallation GitInstallation { get; } = new MockGitInstallation(); public override IDiskLayoutUpgradeData DiskLayoutUpgrade => throw new NotSupportedException(); diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGitInstallation.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGitInstallation.cs new file mode 100644 index 0000000000..792b0291f6 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGitInstallation.cs @@ -0,0 +1,22 @@ +using GVFS.Common.Git; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace GVFS.UnitTests.Mock.Git +{ + public class MockGitInstallation : IGitInstallation + { + public bool GitExists(string gitBinPath) + { + return false; + } + + public string GetInstalledGitBinPath() + { + return null; + } + } +} diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index b2e932f96c..6f59d5901e 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -49,7 +49,18 @@ protected override void Execute(GVFSEnlistment enlistment) this.WriteMessage(string.Empty); this.WriteMessage("gvfs version " + ProcessHelper.GetCurrentProcessVersion()); - this.WriteMessage(GitProcess.Version(enlistment).Output); + + GitVersion gitVersion; + string error; + if (GitProcess.TryGetVersion(out gitVersion, out error)) + { + this.WriteMessage("git version " + gitVersion.ToString()); + } + else + { + this.WriteMessage("Could not determine git version. " + error); + } + this.WriteMessage(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); this.WriteMessage(string.Empty); this.WriteMessage("Enlistment root: " + enlistment.EnlistmentRoot); @@ -120,6 +131,11 @@ protected override void Execute(GVFSEnlistment enlistment) ProductUpgrader.LogDirectory, copySubFolders: true, targetFolderName: ProductUpgrader.UpgradeDirectoryName); + this.LogDirectoryEnumeration( + ProductUpgrader.GetUpgradesDirectoryPath(), + Path.Combine(archiveFolderPath, ProductUpgrader.UpgradeDirectoryName), + ProductUpgrader.DownloadDirectory, + "downloaded-assets.txt"); return true; }, diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index eb4370621f..8b9f1a8b63 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -619,23 +619,13 @@ private void CheckFileSystemSupportsRequiredFeatures(ITracer tracer, Enlistment private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out string version) { - GitProcess.Result versionResult = GitProcess.Version(enlistment); - if (versionResult.HasErrors) - { - this.ReportErrorAndExit(tracer, "Error: Unable to retrieve the git version"); - } - GitVersion gitVersion; - version = versionResult.Output; - if (version.StartsWith("git version ")) + if (!GitProcess.TryGetVersion(out gitVersion, out string _)) { - version = version.Substring(12); + this.ReportErrorAndExit(tracer, "Error: Unable to retrieve the git version"); } - if (!GitVersion.TryParseVersion(version, out gitVersion)) - { - this.ReportErrorAndExit(tracer, "Error: Unable to parse the git version. {0}", version); - } + version = gitVersion.ToString(); if (gitVersion.Platform != GVFSConstants.SupportedGitVersion.Platform) { From f4536802cc224216eb327cfaa1ba124122f611cf Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 10:40:23 -0400 Subject: [PATCH 067/244] - Renamed GVFSConfig.cs to ServerGVFSConfig.cs. - Renamed all variables with name gvfsConfig to serverGVFSConfig --- GVFS/GVFS.Common/Http/CacheServerResolver.cs | 10 +- GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs | 20 ++-- GVFS/GVFS.Common/LocalCacheResolver.cs | 14 +-- GVFS/GVFS.Common/LocalGVFSConfig.cs | 99 +++++++++++++++++++ .../{GVFSConfig.cs => ServerGVFSConfig.cs} | 4 +- .../Common/CacheServerResolverTests.cs | 8 +- GVFS/GVFS/CommandLine/CacheServerVerb.cs | 8 +- GVFS/GVFS/CommandLine/CloneVerb.cs | 22 ++--- GVFS/GVFS/CommandLine/DehydrateVerb.cs | 2 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 38 +++---- GVFS/GVFS/CommandLine/MountVerb.cs | 14 +-- GVFS/GVFS/CommandLine/PrefetchVerb.cs | 14 +-- 12 files changed, 176 insertions(+), 77 deletions(-) create mode 100644 GVFS/GVFS.Common/LocalGVFSConfig.cs rename GVFS/GVFS.Common/{GVFSConfig.cs => ServerGVFSConfig.cs} (89%) diff --git a/GVFS/GVFS.Common/Http/CacheServerResolver.cs b/GVFS/GVFS.Common/Http/CacheServerResolver.cs index ea39119d03..86a693ea6a 100644 --- a/GVFS/GVFS.Common/Http/CacheServerResolver.cs +++ b/GVFS/GVFS.Common/Http/CacheServerResolver.cs @@ -39,7 +39,7 @@ public static string GetUrlFromConfig(Enlistment enlistment) public bool TryResolveUrlFromRemote( string cacheServerName, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, out CacheServerInfo cacheServer, out string error) { @@ -54,12 +54,12 @@ public bool TryResolveUrlFromRemote( if (cacheServerName.Equals(CacheServerInfo.ReservedNames.Default, StringComparison.OrdinalIgnoreCase)) { cacheServer = - gvfsConfig.CacheServers.FirstOrDefault(cache => cache.GlobalDefault) + serverGVFSConfig.CacheServers.FirstOrDefault(cache => cache.GlobalDefault) ?? this.CreateNone(); } else { - cacheServer = gvfsConfig.CacheServers.FirstOrDefault(cache => + cacheServer = serverGVFSConfig.CacheServers.FirstOrDefault(cache => cache.Name.Equals(cacheServerName, StringComparison.OrdinalIgnoreCase)); if (cacheServer == null) @@ -74,7 +74,7 @@ public bool TryResolveUrlFromRemote( public CacheServerInfo ResolveNameFromRemote( string cacheServerUrl, - GVFSConfig gvfsConfig) + ServerGVFSConfig serverGVFSConfig) { if (string.IsNullOrWhiteSpace(cacheServerUrl)) { @@ -87,7 +87,7 @@ public CacheServerInfo ResolveNameFromRemote( } return - gvfsConfig.CacheServers.FirstOrDefault(cache => cache.Url.Equals(cacheServerUrl, StringComparison.OrdinalIgnoreCase)) + serverGVFSConfig.CacheServers.FirstOrDefault(cache => cache.Url.Equals(cacheServerUrl, StringComparison.OrdinalIgnoreCase)) ?? new CacheServerInfo(cacheServerUrl, CacheServerInfo.ReservedNames.UserDefined); } diff --git a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs index a44f079fd7..8c6e450c12 100644 --- a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs @@ -16,9 +16,9 @@ public ConfigHttpRequestor(ITracer tracer, Enlistment enlistment, RetryConfig re this.repoUrl = enlistment.RepoUrl; } - public bool TryQueryGVFSConfig(out GVFSConfig gvfsConfig) + public bool TryQueryGVFSConfig(out ServerGVFSConfig serverGVFSConfig) { - gvfsConfig = null; + serverGVFSConfig = null; Uri gvfsConfigEndpoint; string gvfsConfigEndpointString = this.repoUrl + GVFSConstants.Endpoints.GVFSConfig; @@ -38,10 +38,10 @@ public bool TryQueryGVFSConfig(out GVFSConfig gvfsConfig) } long requestId = HttpRequestor.GetNewRequestId(); - RetryWrapper retrier = new RetryWrapper(this.RetryConfig.MaxAttempts, CancellationToken.None); - retrier.OnFailure += RetryWrapper.StandardErrorHandler(this.Tracer, requestId, "QueryGvfsConfig"); + RetryWrapper retrier = new RetryWrapper(this.RetryConfig.MaxAttempts, CancellationToken.None); + retrier.OnFailure += RetryWrapper.StandardErrorHandler(this.Tracer, requestId, "QueryGvfsConfig"); - RetryWrapper.InvocationResult output = retrier.Invoke( + RetryWrapper.InvocationResult output = retrier.Invoke( tryCount => { using (GitEndPointResponseData response = this.SendRequest( @@ -53,25 +53,25 @@ public bool TryQueryGVFSConfig(out GVFSConfig gvfsConfig) { if (response.HasErrors) { - return new RetryWrapper.CallbackResult(response.Error, response.ShouldRetry); + return new RetryWrapper.CallbackResult(response.Error, response.ShouldRetry); } try { string configString = response.RetryableReadToEnd(); - GVFSConfig config = JsonConvert.DeserializeObject(configString); - return new RetryWrapper.CallbackResult(config); + ServerGVFSConfig config = JsonConvert.DeserializeObject(configString); + return new RetryWrapper.CallbackResult(config); } catch (JsonReaderException e) { - return new RetryWrapper.CallbackResult(e, shouldRetry: false); + return new RetryWrapper.CallbackResult(e, shouldRetry: false); } } }); if (output.Succeeded) { - gvfsConfig = output.Result; + serverGVFSConfig = output.Result; return true; } diff --git a/GVFS/GVFS.Common/LocalCacheResolver.cs b/GVFS/GVFS.Common/LocalCacheResolver.cs index 12a5a7c23b..9e4cba525f 100644 --- a/GVFS/GVFS.Common/LocalCacheResolver.cs +++ b/GVFS/GVFS.Common/LocalCacheResolver.cs @@ -44,15 +44,15 @@ public static string GetDefaultLocalCacheRoot(GVFSEnlistment enlistment) public bool TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( ITracer tracer, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo currentCacheServer, string localCacheRoot, out string localCacheKey, out string errorMessage) { - if (gvfsConfig == null) + if (serverGVFSConfig == null) { - throw new ArgumentNullException(nameof(gvfsConfig)); + throw new ArgumentNullException(nameof(serverGVFSConfig)); } localCacheKey = null; @@ -115,7 +115,7 @@ public bool TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( metadata.Add("currentCacheServer", currentCacheServer.ToString()); string getLocalCacheKeyError; - if (this.TryGetLocalCacheKeyFromRemoteCacheServers(tracer, gvfsConfig, currentCacheServer, mappingFile, out localCacheKey, out getLocalCacheKeyError)) + if (this.TryGetLocalCacheKeyFromRemoteCacheServers(tracer, serverGVFSConfig, currentCacheServer, mappingFile, out localCacheKey, out getLocalCacheKeyError)) { metadata.Add("localCacheKey", localCacheKey); metadata.Add(TracingConstants.MessageKey.InfoMessage, nameof(this.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers) + ": Generated new local cache key"); @@ -192,7 +192,7 @@ private bool TryOpenMappingFile(ITracer tracer, string localCacheRoot, out FileB private bool TryGetLocalCacheKeyFromRemoteCacheServers( ITracer tracer, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo currentCacheServer, FileBasedDictionary mappingFile, out string localCacheKey, @@ -203,7 +203,7 @@ private bool TryGetLocalCacheKeyFromRemoteCacheServers( try { - if (this.TryFindExistingLocalCacheKey(mappingFile, gvfsConfig.CacheServers, out localCacheKey)) + if (this.TryFindExistingLocalCacheKey(mappingFile, serverGVFSConfig.CacheServers, out localCacheKey)) { EventMetadata metadata = CreateEventMetadata(); metadata.Add("currentCacheServer", currentCacheServer.ToString()); @@ -233,7 +233,7 @@ private bool TryGetLocalCacheKeyFromRemoteCacheServers( mappingFileUpdates.Add(new KeyValuePair(this.ToMappingKey(currentCacheServer.Url), localCacheKey)); } - foreach (CacheServerInfo cacheServer in gvfsConfig.CacheServers) + foreach (CacheServerInfo cacheServer in serverGVFSConfig.CacheServers) { string persistedLocalCacheKey; if (mappingFile.TryGetValue(this.ToMappingKey(cacheServer.Url), out persistedLocalCacheKey)) diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs new file mode 100644 index 0000000000..62d4a40645 --- /dev/null +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -0,0 +1,99 @@ +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using System; +using System.IO; + +namespace GVFS.Common +{ + public class LocalGVFSConfig + { + private const string FileName = "local_config.dat"; + private string configFile; + private string gitPath; + private PhysicalFileSystem fileSystem; + + public LocalGVFSConfig(string gitPath) + { + string servicePath = Paths.GetServiceDataRoot(GVFSConstants.Service.ServiceName); + string gvfsDirectory = Path.GetDirectoryName(servicePath); + + this.configFile = Path.Combine(gvfsDirectory, FileName); + this.gitPath = gitPath; + this.fileSystem = new PhysicalFileSystem(); + } + + public bool TryGetValueForKey(string key, out string value, out string error) + { + return this.TryGetConfig(this.KeyWithGVFSSectionPrefix(key), out value, out error); + } + + public bool TrySetValueForKey(string key, string value, out string error) + { + return this.TrySetConfig(this.KeyWithGVFSSectionPrefix(key), value, out error); + } + + private bool TryGetConfig(string key, out string value, out string error) + { + if (!this.fileSystem.FileExists(this.configFile)) + { + error = $"Error reading {key}. Config file({this.configFile}) does not exist."; + value = null; + return false; + } + + GitProcess.Result result = GitProcess.GetFromFileConfig(this.gitPath, this.configFile, key); + if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) + { + error = null; + value = result.Output.TrimEnd('\r', '\n'); + return true; + } + else + { + error = string.IsNullOrEmpty(result.Errors) ? $"Error reading \"{key}\" from file {this.configFile}." : result.Errors; + value = null; + return false; + } + } + + private bool TrySetConfig(string key, string value, out string error) + { + if (!this.fileSystem.FileExists(this.configFile) && !this.TryCreateConfigFile(out error)) + { + error = $"Error setting config value {key}: {value}. {error}"; + return false; + } + + GitProcess git = new GitProcess(this.gitPath, workingDirectoryRoot: null, gvfsHooksRoot: null); + GitProcess.Result result = git.SetInFileConfig(this.configFile, key, value, replaceAll: true); + if (result.HasErrors) + { + error = string.IsNullOrEmpty(result.Errors) ? $"Error setting config value {key}: {value}. Config file {this.configFile}." : result.Errors; + return false; + } + + error = null; + return true; + } + + private bool TryCreateConfigFile(out string error) + { + Exception exception = null; + if (!this.fileSystem.TryWriteTempFileAndRename(this.configFile, string.Empty, out exception)) + { + error = $"Could not create config file {this.configFile}. {exception.Message}"; + return false; + } + + error = null; + return true; + } + + private string KeyWithGVFSSectionPrefix(string key) + { + const string GVFSSectionName = "gvfs"; + + return string.Join(".", GVFSSectionName, key); + } + } +} diff --git a/GVFS/GVFS.Common/GVFSConfig.cs b/GVFS/GVFS.Common/ServerGVFSConfig.cs similarity index 89% rename from GVFS/GVFS.Common/GVFSConfig.cs rename to GVFS/GVFS.Common/ServerGVFSConfig.cs index 17d805c391..1bc9bef363 100644 --- a/GVFS/GVFS.Common/GVFSConfig.cs +++ b/GVFS/GVFS.Common/ServerGVFSConfig.cs @@ -5,7 +5,7 @@ namespace GVFS.Common { - public class GVFSConfig + public class ServerGVFSConfig { public IEnumerable AllowedGVFSClientVersions { get; set; } @@ -17,4 +17,4 @@ public class VersionRange public Version Max { get; set; } } } -} +} \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs index c46d4b9669..3a5026a09a 100644 --- a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs +++ b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs @@ -204,9 +204,9 @@ private MockGVFSEnlistment CreateEnlistment(string newConfigValue = null, string return new MockGVFSEnlistment(gitProcess); } - private GVFSConfig CreateGVFSConfig() + private ServerGVFSConfig CreateGVFSConfig() { - return new GVFSConfig + return new ServerGVFSConfig { CacheServers = new[] { @@ -215,9 +215,9 @@ private GVFSConfig CreateGVFSConfig() }; } - private GVFSConfig CreateDefaultDeserializedGVFSConfig() + private ServerGVFSConfig CreateDefaultDeserializedGVFSConfig() { - return JsonConvert.DeserializeObject("{}"); + return JsonConvert.DeserializeObject("{}"); } private CacheServerResolver CreateResolver(MockGVFSEnlistment enlistment = null) diff --git a/GVFS/GVFS/CommandLine/CacheServerVerb.cs b/GVFS/GVFS/CommandLine/CacheServerVerb.cs index d922129f9e..0209bf86ee 100644 --- a/GVFS/GVFS/CommandLine/CacheServerVerb.cs +++ b/GVFS/GVFS/CommandLine/CacheServerVerb.cs @@ -42,7 +42,7 @@ protected override void Execute(GVFSEnlistment enlistment) using (ITracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "CacheVerb")) { - GVFSConfig gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + ServerGVFSConfig serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); string error = null; @@ -50,7 +50,7 @@ protected override void Execute(GVFSEnlistment enlistment) if (this.CacheToSet != null) { CacheServerInfo cacheServer = cacheServerResolver.ParseUrlOrFriendlyName(this.CacheToSet); - cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, gvfsConfig); + cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, serverGVFSConfig); if (!cacheServerResolver.TrySaveUrlToLocalConfig(cacheServer, out error)) { @@ -61,7 +61,7 @@ protected override void Execute(GVFSEnlistment enlistment) } else if (this.ListCacheServers) { - List cacheServers = gvfsConfig.CacheServers.ToList(); + List cacheServers = serverGVFSConfig.CacheServers.ToList(); if (cacheServers != null && cacheServers.Any()) { @@ -80,7 +80,7 @@ protected override void Execute(GVFSEnlistment enlistment) else { string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment); - CacheServerInfo cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig); + CacheServerInfo cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, serverGVFSConfig); this.Output.WriteLine("Using cache server: " + cacheServer); } diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index ef59eea865..d3861f9293 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -119,7 +119,7 @@ public override void Execute() Result cloneResult = new Result(false); CacheServerInfo cacheServer = null; - GVFSConfig gvfsConfig = null; + ServerGVFSConfig serverGVFSConfig = null; using (JsonTracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "GVFSClone")) { @@ -178,19 +178,19 @@ public override void Execute() } RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); - gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); - cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, gvfsConfig); + cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, serverGVFSConfig); if (!GVFSPlatform.Instance.IsUnderConstruction) { - this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: true); + this.ValidateClientVersions(tracer, enlistment, serverGVFSConfig, showWarnings: true); } this.ShowStatusWhileRunning( () => { - cloneResult = this.TryClone(tracer, enlistment, cacheServer, retryConfig, gvfsConfig, resolvedLocalCacheRoot); + cloneResult = this.TryClone(tracer, enlistment, cacheServer, retryConfig, serverGVFSConfig, resolvedLocalCacheRoot); return cloneResult.Success; }, "Cloning", @@ -214,7 +214,7 @@ public override void Execute() verb.Commits = true; verb.SkipVersionCheck = true; verb.ResolvedCacheServer = cacheServer; - verb.GVFSConfig = gvfsConfig; + verb.ServerGVFSConfig = serverGVFSConfig; }); if (result != ReturnCode.Success) @@ -238,7 +238,7 @@ public override void Execute() verb.SkipMountedCheck = true; verb.SkipVersionCheck = true; verb.ResolvedCacheServer = cacheServer; - verb.DownloadedGVFSConfig = gvfsConfig; + verb.DownloadedGVFSConfig = serverGVFSConfig; }); } } @@ -323,7 +323,7 @@ private Result TryClone( GVFSEnlistment enlistment, CacheServerInfo cacheServer, RetryConfig retryConfig, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, string resolvedLocalCacheRoot) { Result pipeResult; @@ -372,7 +372,7 @@ private Result TryClone( } string localCacheError; - if (!this.TryDetermineLocalCacheAndInitializePaths(tracer, enlistment, gvfsConfig, cacheServer, resolvedLocalCacheRoot, out localCacheError)) + if (!this.TryDetermineLocalCacheAndInitializePaths(tracer, enlistment, serverGVFSConfig, cacheServer, resolvedLocalCacheRoot, out localCacheError)) { tracer.RelatedError(localCacheError); return new Result(localCacheError); @@ -460,7 +460,7 @@ private void CheckNotInsideExistingRepo(string normalizedEnlistmentRootPath) private bool TryDetermineLocalCacheAndInitializePaths( ITracer tracer, GVFSEnlistment enlistment, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo currentCacheServer, string localCacheRoot, out string errorMessage) @@ -472,7 +472,7 @@ private bool TryDetermineLocalCacheAndInitializePaths( string localCacheKey; if (!localCacheResolver.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( tracer, - gvfsConfig, + serverGVFSConfig, currentCacheServer, localCacheRoot, localCacheKey: out localCacheKey, diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index 906745f3c9..742e8c2716 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -104,7 +104,7 @@ of your enlistment's src folder. } // Local cache and objects paths are required for TryDownloadGitObjects - this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, gvfsConfig: null, cacheServer: null); + this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, serverGVFSConfig: null, cacheServer: null); if (this.TryBackupFiles(tracer, enlistment, backupRoot)) { diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 8b9f1a8b63..a01fe866d1 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -244,15 +244,15 @@ protected RetryConfig GetRetryConfig(ITracer tracer, GVFSEnlistment enlistment, return retryConfig; } - protected GVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig) + protected ServerGVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig) { - GVFSConfig gvfsConfig = null; + ServerGVFSConfig serverGVFSConfig = null; if (!this.ShowStatusWhileRunning( () => { using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, retryConfig)) { - return configRequestor.TryQueryGVFSConfig(out gvfsConfig); + return configRequestor.TryQueryGVFSConfig(out serverGVFSConfig); } }, "Querying remote for config", @@ -260,11 +260,11 @@ protected GVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlistment, { this.ReportErrorAndExit(tracer, "Unable to query /gvfs/config"); } + + return serverGVFSConfig; + } - return gvfsConfig; - } - - protected void ValidateClientVersions(ITracer tracer, GVFSEnlistment enlistment, GVFSConfig gvfsConfig, bool showWarnings) + protected void ValidateClientVersions(ITracer tracer, GVFSEnlistment enlistment, ServerGVFSConfig gvfsConfig, bool showWarnings) { if (!GVFSPlatform.Instance.IsUnderConstruction) { @@ -354,7 +354,7 @@ protected CacheServerInfo ResolveCacheServer( ITracer tracer, CacheServerInfo cacheServer, CacheServerResolver cacheServerResolver, - GVFSConfig gvfsConfig) + ServerGVFSConfig serverGVFSConfig) { CacheServerInfo resolvedCacheServer = cacheServer; @@ -365,7 +365,7 @@ protected CacheServerInfo ResolveCacheServer( if (!cacheServerResolver.TryResolveUrlFromRemote( cacheServerName, - gvfsConfig, + serverGVFSConfig, out resolvedCacheServer, out error)) { @@ -374,7 +374,7 @@ protected CacheServerInfo ResolveCacheServer( } else if (cacheServer.Name.Equals(CacheServerInfo.ReservedNames.UserDefined)) { - resolvedCacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, gvfsConfig); + resolvedCacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, serverGVFSConfig); } this.Output.WriteLine("Using cache server: " + resolvedCacheServer); @@ -660,7 +660,7 @@ private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out stri } } - private bool TryValidateGVFSVersion(GVFSEnlistment enlistment, ITracer tracer, GVFSConfig config, out string errorMessage, out bool errorIsFatal) + private bool TryValidateGVFSVersion(GVFSEnlistment enlistment, ITracer tracer, ServerGVFSConfig config, out string errorMessage, out bool errorIsFatal) { errorMessage = null; errorIsFatal = false; @@ -669,7 +669,7 @@ private bool TryValidateGVFSVersion(GVFSEnlistment enlistment, ITracer tracer, G { Version currentVersion = new Version(ProcessHelper.GetCurrentProcessVersion()); - IEnumerable allowedGvfsClientVersions = + IEnumerable allowedGvfsClientVersions = config != null ? config.AllowedGVFSClientVersions : null; @@ -692,7 +692,7 @@ private bool TryValidateGVFSVersion(GVFSEnlistment enlistment, ITracer tracer, G return false; } - foreach (GVFSConfig.VersionRange versionRange in config.AllowedGVFSClientVersions) + foreach (ServerGVFSConfig.VersionRange versionRange in config.AllowedGVFSClientVersions) { if (currentVersion >= versionRange.Min && (versionRange.Max == null || currentVersion <= versionRange.Max)) @@ -751,7 +751,7 @@ protected void InitializeLocalCacheAndObjectsPaths( ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo cacheServer) { string error; @@ -765,7 +765,7 @@ protected void InitializeLocalCacheAndObjectsPaths( // Note: Repos cloned with a version of GVFS that predates the local cache will not have a local cache configured if (!string.IsNullOrWhiteSpace(enlistment.LocalCacheRoot)) { - this.EnsureLocalCacheIsHealthy(tracer, enlistment, retryConfig, gvfsConfig, cacheServer); + this.EnsureLocalCacheIsHealthy(tracer, enlistment, retryConfig, serverGVFSConfig, cacheServer); } RepoMetadata.Shutdown(); @@ -813,7 +813,7 @@ private void EnsureLocalCacheIsHealthy( ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo cacheServer) { if (!Directory.Exists(enlistment.LocalCacheRoot)) @@ -901,7 +901,7 @@ private void EnsureLocalCacheIsHealthy( } string error; - if (gvfsConfig == null) + if (serverGVFSConfig == null) { if (retryConfig == null) { @@ -911,14 +911,14 @@ private void EnsureLocalCacheIsHealthy( } } - gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); } string localCacheKey; LocalCacheResolver localCacheResolver = new LocalCacheResolver(enlistment); if (!localCacheResolver.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( tracer, - gvfsConfig, + serverGVFSConfig, cacheServer, enlistment.LocalCacheRoot, localCacheKey: out localCacheKey, diff --git a/GVFS/GVFS/CommandLine/MountVerb.cs b/GVFS/GVFS/CommandLine/MountVerb.cs index 9487b2f1ee..ad8e0a2f4b 100644 --- a/GVFS/GVFS/CommandLine/MountVerb.cs +++ b/GVFS/GVFS/CommandLine/MountVerb.cs @@ -37,7 +37,7 @@ public class MountVerb : GVFSVerb.ForExistingEnlistment public bool SkipMountedCheck { get; set; } public bool SkipVersionCheck { get; set; } public CacheServerInfo ResolvedCacheServer { get; set; } - public GVFSConfig DownloadedGVFSConfig { get; set; } + public ServerGVFSConfig DownloadedGVFSConfig { get; set; } protected override string VerbName { @@ -128,7 +128,7 @@ protected override void Execute(GVFSEnlistment enlistment) } RetryConfig retryConfig = null; - GVFSConfig gvfsConfig = this.DownloadedGVFSConfig; + ServerGVFSConfig serverGVFSConfig = this.DownloadedGVFSConfig; if (!this.SkipVersionCheck) { string authErrorMessage = null; @@ -140,24 +140,24 @@ protected override void Execute(GVFSEnlistment enlistment) this.Output.WriteLine(" Mount will proceed, but new files cannot be accessed until GVFS can authenticate."); } - if (gvfsConfig == null) + if (serverGVFSConfig == null) { if (retryConfig == null) { retryConfig = this.GetRetryConfig(tracer, enlistment); } - gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); } - this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: true); + this.ValidateClientVersions(tracer, enlistment, serverGVFSConfig, showWarnings: true); CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); - cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, gvfsConfig); + cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, serverGVFSConfig); this.Output.WriteLine("Configured cache server: " + cacheServer); } - this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, gvfsConfig, cacheServer); + this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, serverGVFSConfig, cacheServer); if (!this.ShowStatusWhileRunning( () => { return this.PerformPreMountValidation(tracer, enlistment, out mountExecutableLocation, out errorMessage); }, diff --git a/GVFS/GVFS/CommandLine/PrefetchVerb.cs b/GVFS/GVFS/CommandLine/PrefetchVerb.cs index aa9d1c1be8..acea80b6b9 100644 --- a/GVFS/GVFS/CommandLine/PrefetchVerb.cs +++ b/GVFS/GVFS/CommandLine/PrefetchVerb.cs @@ -70,7 +70,7 @@ public class PrefetchVerb : GVFSVerb.ForExistingEnlistment public bool SkipVersionCheck { get; set; } public CacheServerInfo ResolvedCacheServer { get; set; } - public GVFSConfig GVFSConfig { get; set; } + public ServerGVFSConfig ServerGVFSConfig { get; set; } protected override string VerbName { @@ -100,7 +100,7 @@ protected override void Execute(GVFSEnlistment enlistment) RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); CacheServerInfo cacheServer = this.ResolvedCacheServer; - GVFSConfig gvfsConfig = this.GVFSConfig; + ServerGVFSConfig serverGVFSConfig = this.ServerGVFSConfig; if (!this.SkipVersionCheck) { string authErrorMessage; @@ -111,23 +111,23 @@ protected override void Execute(GVFSEnlistment enlistment) this.ReportErrorAndExit(tracer, "Unable to prefetch because authentication failed"); } - if (gvfsConfig == null) + if (serverGVFSConfig == null) { - gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); } if (cacheServer == null) { CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); - cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig); + cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, serverGVFSConfig); } - this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: false); + this.ValidateClientVersions(tracer, enlistment, serverGVFSConfig, showWarnings: false); this.Output.WriteLine("Configured cache server: " + cacheServer); } - this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, gvfsConfig, cacheServer); + this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, serverGVFSConfig, cacheServer); try { From 127435051345688dc7e1893124b387f6dd6b1ef8 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 10:41:19 -0400 Subject: [PATCH 068/244] - New ConfigVerb. - Usage gvfs config - New LocalGVFSConfig.cs file. It writes config key value pairs into a git config file. - Updated ProductUpgrader.cs to use the new LocalGVFSConfig class to read ring config. --- GVFS/GVFS.Common/GVFSConstants.cs | 4 +- GVFS/GVFS.Common/Git/GitProcess.cs | 15 +++++++ GVFS/GVFS.Common/ProductUpgrader.cs | 17 ++++---- GVFS/GVFS/CommandLine/ConfigVerb.cs | 61 +++++++++++++++++++++++++++++ GVFS/GVFS/GVFS.Windows.csproj | 3 +- GVFS/GVFS/Program.cs | 9 ++++- 6 files changed, 95 insertions(+), 14 deletions(-) create mode 100644 GVFS/GVFS/CommandLine/ConfigVerb.cs diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 39fc289305..27d7d4da8f 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -33,7 +33,7 @@ public static class GitConfig public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; public const string HooksExtension = ".hooks"; - public const string UpgradeRing = GVFSPrefix + "upgrade-ring"; + public const string UpgradeRing = "upgrade-ring"; } public static class GitStatusCache @@ -227,7 +227,7 @@ public static class UpgradeVerbMessages public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global " + GitConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + GitConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from an elevated command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 2d3a2206e7..8701605ab0 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -87,6 +87,11 @@ public static Result GetFromSystemConfig(string gitBinPath, string settingName) return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --system " + settingName); } + public static Result GetFromFileConfig(string gitBinPath, string configFile, string settingName) + { + return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --file " + configFile + " " + settingName); + } + public static bool TryGetVersion(out GitVersion gitVersion, out string error) { string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); @@ -203,6 +208,16 @@ public Result AddInLocalConfig(string settingName, string value) value)); } + public Result SetInFileConfig(string configFile, string settingName, string value, bool replaceAll = false) + { + return this.InvokeGitOutsideEnlistment(string.Format( + "config --file {0} {1} \"{2}\" \"{3}\"", + configFile, + replaceAll ? "--replace-all " : string.Empty, + settingName, + value)); + } + public bool TryGetAllConfig(bool localOnly, out Dictionary configSettings) { string localParameter = localOnly ? "--local" : string.Empty; diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index af0a7e813c..d65421064b 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -245,13 +245,14 @@ public bool TryCleanup(out string error) public virtual bool TryLoadRingConfig(out string error) { string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); - GitProcess.Result result = GitProcess.GetFromGlobalConfig(gitPath, GVFSConstants.GitConfig.UpgradeRing); - if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) + LocalGVFSConfig localConfig = new LocalGVFSConfig(gitPath); + + string ringConfig = null; + if (localConfig.TryGetValueForKey(GVFSConstants.GitConfig.UpgradeRing, out ringConfig, out error)) { - string ringConfig = result.Output.TrimEnd('\r', '\n'); RingType ringType; - if (Enum.TryParse(ringConfig, ignoreCase: true, result: out ringType) && + if (Enum.TryParse(ringConfig, ignoreCase: true, result: out ringType) && Enum.IsDefined(typeof(RingType), ringType) && ringType != RingType.Invalid) { @@ -261,14 +262,10 @@ public virtual bool TryLoadRingConfig(out string error) } else { - error = "Invalid upgrade ring `" + ringConfig + "` specified in Git config."; + error = "Invalid upgrade ring `" + ringConfig + "` specified in Git config."; } } - else - { - error = string.IsNullOrEmpty(result.Errors) ? "Unable to determine upgrade ring." : result.Errors; - } - + error += Environment.NewLine + GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand; this.Ring = RingType.Invalid; return false; diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs new file mode 100644 index 0000000000..4641f16c15 --- /dev/null +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -0,0 +1,61 @@ +using CommandLine; +using GVFS.Common; +using System; + +namespace GVFS.CommandLine +{ + [Verb(ConfigVerbName, HelpText = "Set and Get GVFS config settings.")] + public class ConfigVerb : GVFSVerb + { + private const string ConfigVerbName = "config"; + private LocalGVFSConfig localConfig; + + public override string EnlistmentRootPathParameter { get; set; } + + protected override string VerbName + { + get { return ConfigVerbName; } + } + + public override void Execute() + { + string[] args = Environment.GetCommandLineArgs(); + if (args.Length < 3 || args.Length > 4) + { + string usageString = string.Join( + Environment.NewLine, + "Error: wrong number of arguments.", + "Usage: gvfs config "); + this.ReportErrorAndExit(usageString); + } + + this.localConfig = new LocalGVFSConfig(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); + + string key = args[2]; + string value = null; + string error = null; + bool isRead = args.Length == 3; + + key = args[2]; + if (isRead) + { + if (this.localConfig.TryGetValueForKey(key, out value, out error)) + { + Console.WriteLine(value); + } + else + { + this.ReportErrorAndExit(error); + } + } + else + { + value = args[3]; + if (!this.localConfig.TrySetValueForKey(key, value, out error)) + { + this.ReportErrorAndExit(error); + } + } + } + } +} diff --git a/GVFS/GVFS/GVFS.Windows.csproj b/GVFS/GVFS/GVFS.Windows.csproj index d8de69db99..392c6cbb40 100644 --- a/GVFS/GVFS/GVFS.Windows.csproj +++ b/GVFS/GVFS/GVFS.Windows.csproj @@ -1,4 +1,4 @@ - + @@ -74,6 +74,7 @@ + diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index 43061a95ac..e97ad33342 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -27,7 +27,8 @@ public static void Main(string[] args) typeof(ServiceVerb), typeof(StatusVerb), typeof(UnmountVerb), - typeof(UpgradeVerb) + typeof(UpgradeVerb), + typeof(ConfigVerb) }; int consoleWidth = 80; @@ -89,6 +90,12 @@ public static void Main(string[] args) upgrade.Execute(); Environment.Exit((int)ReturnCode.Success); }) + .WithParsed( + config => + { + config.Execute(); + Environment.Exit((int)ReturnCode.Success); + }) .WithParsed( verb => { From 1827893fa5cb0c8e1fa6286d99e2e46473dc37c8 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 13:23:04 -0400 Subject: [PATCH 069/244] - Replaced git config format and writer with FileBasedDictionary - New base class for Verbs that don't need an enlistment. - Cleanups --- GVFS/GVFS.Common/GVFSConstants.cs | 10 +- GVFS/GVFS.Common/LocalGVFSConfig.cs | 105 ++++++++++-------- GVFS/GVFS.Common/ProductUpgrader.cs | 6 +- .../Windows/Mock/MockProductUpgrader.cs | 2 +- .../Windows/Upgrader/UpgradeTests.cs | 2 +- GVFS/GVFS/CommandLine/ConfigVerb.cs | 43 +++---- GVFS/GVFS/CommandLine/GVFSVerb.cs | 13 +++ GVFS/GVFS/CommandLine/ServiceVerb.cs | 8 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 4 +- GVFS/GVFS/Program.cs | 24 +--- 10 files changed, 112 insertions(+), 105 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 27d7d4da8f..813663a0ca 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -32,8 +32,12 @@ public static class GitConfig public const string DeprecatedCacheEndpointSuffix = ".cache-server-url"; public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; - public const string HooksExtension = ".hooks"; - public const string UpgradeRing = "upgrade-ring"; + public const string HooksExtension = ".hooks"; + } + + public static class LocalGVFSConfig + { + public const string UpgradeRing = "upgrade.ring"; } public static class GitStatusCache @@ -227,7 +231,7 @@ public static class UpgradeVerbMessages public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + GitConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from an elevated command prompt."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index 62d4a40645..a7c225873a 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -1,99 +1,112 @@ using GVFS.Common.FileSystem; -using GVFS.Common.Git; -using System; +using GVFS.Common.Tracing; using System.IO; namespace GVFS.Common { public class LocalGVFSConfig { - private const string FileName = "local_config.dat"; - private string configFile; - private string gitPath; - private PhysicalFileSystem fileSystem; + private const string FileName = "gvfs.config"; + private readonly string configFile; + private readonly PhysicalFileSystem fileSystem; - public LocalGVFSConfig(string gitPath) + public LocalGVFSConfig() { string servicePath = Paths.GetServiceDataRoot(GVFSConstants.Service.ServiceName); string gvfsDirectory = Path.GetDirectoryName(servicePath); this.configFile = Path.Combine(gvfsDirectory, FileName); - this.gitPath = gitPath; this.fileSystem = new PhysicalFileSystem(); } - public bool TryGetValueForKey(string key, out string value, out string error) - { - return this.TryGetConfig(this.KeyWithGVFSSectionPrefix(key), out value, out error); - } - - public bool TrySetValueForKey(string key, string value, out string error) - { - return this.TrySetConfig(this.KeyWithGVFSSectionPrefix(key), value, out error); - } + private FileBasedDictionary AllSettings { get; set; } - private bool TryGetConfig(string key, out string value, out string error) + public bool TryGetConfig( + string key, + out string value, + out string error, + ITracer tracer) { - if (!this.fileSystem.FileExists(this.configFile)) + if (!this.TryLoadSettings(tracer, out error)) { - error = $"Error reading {key}. Config file({this.configFile}) does not exist."; + error = $"Error getting config value {key}. {error}"; value = null; return false; } - - GitProcess.Result result = GitProcess.GetFromFileConfig(this.gitPath, this.configFile, key); - if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) + + try { + this.AllSettings.TryGetValue(key, out value); error = null; - value = result.Output.TrimEnd('\r', '\n'); return true; } - else + catch (FileBasedCollectionException exception) { - error = string.IsNullOrEmpty(result.Errors) ? $"Error reading \"{key}\" from file {this.configFile}." : result.Errors; + const string ErrorFormat = "Error getting config value for {0}. Config file {1}. {2}"; + if (tracer != null) + { + tracer.RelatedError(ErrorFormat, key, this.configFile, exception.ToString()); + } + + error = string.Format(ErrorFormat, key, this.configFile, exception.Message); value = null; return false; } } - private bool TrySetConfig(string key, string value, out string error) + public bool TrySetConfig( + string key, + string value, + out string error, + ITracer tracer) { - if (!this.fileSystem.FileExists(this.configFile) && !this.TryCreateConfigFile(out error)) + if (!this.TryLoadSettings(tracer, out error)) { error = $"Error setting config value {key}: {value}. {error}"; return false; } - GitProcess git = new GitProcess(this.gitPath, workingDirectoryRoot: null, gvfsHooksRoot: null); - GitProcess.Result result = git.SetInFileConfig(this.configFile, key, value, replaceAll: true); - if (result.HasErrors) + try { - error = string.IsNullOrEmpty(result.Errors) ? $"Error setting config value {key}: {value}. Config file {this.configFile}." : result.Errors; - return false; + this.AllSettings.SetValueAndFlush(key, value); + error = null; + return true; } + catch (FileBasedCollectionException exception) + { + const string ErrorFormat = "Error setting config value {0}: {1}. Config file {2}. {3}"; + if (tracer != null) + { + tracer.RelatedError(ErrorFormat, key, value, this.configFile, exception.ToString()); + } - error = null; - return true; + error = string.Format(ErrorFormat, key, value, this.configFile, exception.Message); + value = null; + return false; + } } - private bool TryCreateConfigFile(out string error) + private bool TryLoadSettings(ITracer tracer, out string error) { - Exception exception = null; - if (!this.fileSystem.TryWriteTempFileAndRename(this.configFile, string.Empty, out exception)) + if (this.AllSettings == null) { - error = $"Could not create config file {this.configFile}. {exception.Message}"; + FileBasedDictionary config = null; + if (FileBasedDictionary.TryCreate( + tracer: tracer, + dictionaryPath: this.configFile, + fileSystem: this.fileSystem, + output: out config, + error: out error)) + { + this.AllSettings = config; + return true; + } + return false; } error = null; return true; } - - private string KeyWithGVFSSectionPrefix(string key) - { - const string GVFSSectionName = "gvfs"; - - return string.Join(".", GVFSSectionName, key); - } } } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index d65421064b..eb023e1d89 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -245,10 +245,10 @@ public bool TryCleanup(out string error) public virtual bool TryLoadRingConfig(out string error) { string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); - LocalGVFSConfig localConfig = new LocalGVFSConfig(gitPath); + LocalGVFSConfig localConfig = new LocalGVFSConfig(); string ringConfig = null; - if (localConfig.TryGetValueForKey(GVFSConstants.GitConfig.UpgradeRing, out ringConfig, out error)) + if (localConfig.TryGetConfig(GVFSConstants.LocalGVFSConfig.UpgradeRing, out ringConfig, out error, this.tracer)) { RingType ringType; @@ -262,7 +262,7 @@ public virtual bool TryLoadRingConfig(out string error) } else { - error = "Invalid upgrade ring `" + ringConfig + "` specified in Git config."; + error = "Invalid upgrade ring `" + ringConfig + "` specified in gvfs config."; } } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index 3b8c30633e..9227d2cd84 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -107,7 +107,7 @@ public override bool TryLoadRingConfig(out string error) if (this.LocalRingConfig == RingType.Invalid) { - error = "Invalid upgrade ring `Invalid` specified in Git config."; + error = "Invalid upgrade ring `Invalid` specified in gvfs config."; return false; } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs index 4ac28563b8..da82211c36 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs @@ -56,7 +56,7 @@ public virtual void NoneLocalRing() [TestCase] public virtual void InvalidUpgradeRing() { - string errorString = "Invalid upgrade ring `Invalid` specified in Git config."; + string errorString = "Invalid upgrade ring `Invalid` specified in gvfs config."; this.ConfigureRunAndVerify( configure: () => { diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index 4641f16c15..24d0725508 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -4,13 +4,27 @@ namespace GVFS.CommandLine { - [Verb(ConfigVerbName, HelpText = "Set and Get GVFS config settings.")] - public class ConfigVerb : GVFSVerb + [Verb(ConfigVerbName, HelpText = "Get and set GVFS options.")] + public class ConfigVerb : GVFSVerb.NonRepoVerb { private const string ConfigVerbName = "config"; private LocalGVFSConfig localConfig; - public override string EnlistmentRootPathParameter { get; set; } + [Value( + 0, + Required = true, + Default = "", + MetaName = "GVFS config setting name", + HelpText = "Name of GVFS config setting that is to be set or read")] + public string Key { get; set; } + + [Value( + 1, + Required = false, + Default = "", + MetaName = "GVFS config setting value", + HelpText = "Value of GVFS config setting to be set")] + public string Value { get; set; } protected override string VerbName { @@ -19,27 +33,15 @@ protected override string VerbName public override void Execute() { - string[] args = Environment.GetCommandLineArgs(); - if (args.Length < 3 || args.Length > 4) - { - string usageString = string.Join( - Environment.NewLine, - "Error: wrong number of arguments.", - "Usage: gvfs config "); - this.ReportErrorAndExit(usageString); - } - - this.localConfig = new LocalGVFSConfig(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); + this.localConfig = new LocalGVFSConfig(); - string key = args[2]; - string value = null; string error = null; - bool isRead = args.Length == 3; + bool isRead = string.IsNullOrEmpty(this.Value); - key = args[2]; if (isRead) { - if (this.localConfig.TryGetValueForKey(key, out value, out error)) + string value = null; + if (this.localConfig.TryGetConfig(this.Key, out value, out error, tracer: null)) { Console.WriteLine(value); } @@ -50,8 +52,7 @@ public override void Execute() } else { - value = args[3]; - if (!this.localConfig.TrySetValueForKey(key, value, out error)) + if (!this.localConfig.TrySetConfig(this.Key, this.Value, out error, tracer: null)) { this.ReportErrorAndExit(error); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index a01fe866d1..bf5de3379f 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -1041,6 +1041,19 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) } } + public abstract class NonRepoVerb : GVFSVerb + { + public NonRepoVerb(bool validateOrigin = true) : base(validateOrigin) + { + } + + public override string EnlistmentRootPathParameter + { + get { throw new InvalidOperationException(); } + set { throw new InvalidOperationException(); } + } + } + public class VerbAbortedException : Exception { public VerbAbortedException(GVFSVerb verb) diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index 640c99a08b..b547b3bcc5 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -10,7 +10,7 @@ namespace GVFS.CommandLine { [Verb(ServiceVerbName, HelpText = "Runs commands for the GVFS service.")] - public class ServiceVerb : GVFSVerb + public class ServiceVerb : GVFSVerb.NonRepoVerb { private const string ServiceVerbName = "service"; @@ -35,12 +35,6 @@ public class ServiceVerb : GVFSVerb HelpText = "Prints a list of all mounted repos")] public bool List { get; set; } - public override string EnlistmentRootPathParameter - { - get { throw new InvalidOperationException(); } - set { throw new InvalidOperationException(); } - } - protected override string VerbName { get { return ServiceVerbName; } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index 020a16fdc0..e34f624026 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -9,7 +9,7 @@ namespace GVFS.CommandLine { [Verb(UpgradeVerbName, HelpText = "Checks for new GVFS release, downloads and installs it when available.")] - public class UpgradeVerb : GVFSVerb + public class UpgradeVerb : GVFSVerb.NonRepoVerb { private const string UpgradeVerbName = "upgrade"; private ITracer tracer; @@ -44,8 +44,6 @@ public UpgradeVerb() HelpText = "Pass in this flag to actually install the newest release")] public bool Confirmed { get; set; } - public override string EnlistmentRootPathParameter { get; set; } - protected override string VerbName { get { return UpgradeVerbName; } diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index e97ad33342..b72b8355d3 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -28,7 +28,7 @@ public static void Main(string[] args) typeof(StatusVerb), typeof(UnmountVerb), typeof(UpgradeVerb), - typeof(ConfigVerb) + typeof(ConfigVerb), }; int consoleWidth = 80; @@ -74,26 +74,10 @@ public static void Main(string[] args) clone.Execute(); Environment.Exit((int)ReturnCode.Success); }) - .WithParsed( - service => - { - // The service verb doesn't operate on a repo, so it doesn't use the enlistment - // path at all. - service.Execute(); - Environment.Exit((int)ReturnCode.Success); - }) - .WithParsed( - upgrade => - { - // The upgrade verb doesn't operate on a repo, so it doesn't use the enlistment - // path at all. - upgrade.Execute(); - Environment.Exit((int)ReturnCode.Success); - }) - .WithParsed( - config => + .WithParsed( + verb => { - config.Execute(); + verb.Execute(); Environment.Exit((int)ReturnCode.Success); }) .WithParsed( From bab403f406b54dbf9afa37ffe2f42a48e782ada5 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 16:56:37 -0400 Subject: [PATCH 070/244] - Allow gvfs config to succeed from a Non-elevated command prompt. GVFS Config is written to a temporary file and then renamed to its final destination. For this rename operation to succeed, user needs to have delete permission on the destination file, in case it is pre-existing. If the pre-existing file was created by a different user, then the delete will fail. Reference: https://stackoverflow.com/questions/22107812/privileges-owner-issue-when-writing-in-c-programdata. This work around allows safe write to succeed in C:\ProgramData directory. --- GVFS/GVFS.Service/GvfsService.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/GVFS/GVFS.Service/GvfsService.cs b/GVFS/GVFS.Service/GvfsService.cs index 4a51e988d0..9f47a6f7e9 100644 --- a/GVFS/GVFS.Service/GvfsService.cs +++ b/GVFS/GVFS.Service/GvfsService.cs @@ -8,6 +8,8 @@ using System.Linq; using System.Reflection; using System.Runtime.Serialization; +using System.Security.AccessControl; +using System.Security.Principal; using System.ServiceProcess; using System.Threading; @@ -150,6 +152,7 @@ protected override void OnStart(string[] args) this.serviceDataLocation = Paths.GetServiceDataRoot(this.serviceName); Directory.CreateDirectory(this.serviceDataLocation); + this.EnableAccessToAuthenticatedUsers(Path.GetDirectoryName(this.serviceDataLocation)); this.tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName(Paths.GetServiceLogsPath(this.serviceName), GVFSConstants.LogFileTypes.Service), @@ -344,5 +347,20 @@ private void LogExceptionAndExit(Exception e, string method) this.tracer.RelatedError(metadata, "Unhandled exception in " + method); Environment.Exit((int)ReturnCode.GenericError); } + + private void EnableAccessToAuthenticatedUsers(string rootDirectory) + { + // GVFS Config is written to a temporary file and then renamed to its final destination. + // For this rename operation to succeed, user needs to have delete permission on the + // destination file, in case it is pre-existing. If the pre-existing file was created + // by a different user, then the delete will fail. + // Reference: https://stackoverflow.com/questions/22107812/privileges-owner-issue-when-writing-in-c-programdata + // This work around allows safe write to succeed in C:\ProgramData directory. + + DirectorySecurity security = Directory.GetAccessControl(Path.GetDirectoryName(this.serviceDataLocation)); + SecurityIdentifier authenticatedUsers = new SecurityIdentifier(WellKnownSidType.AuthenticatedUserSid, null); + security.AddAccessRule(new FileSystemAccessRule(authenticatedUsers, FileSystemRights.FullControl, AccessControlType.Allow)); + Directory.SetAccessControl(Path.GetDirectoryName(this.serviceDataLocation), security); + } } } From 380fd992352313434221011e8f026fe7337b005c Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 17:14:33 -0400 Subject: [PATCH 071/244] - Renamed GVFSVerb.NonRepoVerb to GVFSVerb.ForNoEnlistment. --- GVFS/GVFS/CommandLine/ConfigVerb.cs | 2 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 4 ++-- GVFS/GVFS/CommandLine/ServiceVerb.cs | 2 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 2 +- GVFS/GVFS/Program.cs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index 24d0725508..c978888dab 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -5,7 +5,7 @@ namespace GVFS.CommandLine { [Verb(ConfigVerbName, HelpText = "Get and set GVFS options.")] - public class ConfigVerb : GVFSVerb.NonRepoVerb + public class ConfigVerb : GVFSVerb.ForNoEnlistment { private const string ConfigVerbName = "config"; private LocalGVFSConfig localConfig; diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index bf5de3379f..d38a28ed44 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -1041,9 +1041,9 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) } } - public abstract class NonRepoVerb : GVFSVerb + public abstract class ForNoEnlistment : GVFSVerb { - public NonRepoVerb(bool validateOrigin = true) : base(validateOrigin) + public ForNoEnlistment(bool validateOrigin = true) : base(validateOrigin) { } diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index b547b3bcc5..e701a74c83 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -10,7 +10,7 @@ namespace GVFS.CommandLine { [Verb(ServiceVerbName, HelpText = "Runs commands for the GVFS service.")] - public class ServiceVerb : GVFSVerb.NonRepoVerb + public class ServiceVerb : GVFSVerb.ForNoEnlistment { private const string ServiceVerbName = "service"; diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index e34f624026..282dacea23 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -9,7 +9,7 @@ namespace GVFS.CommandLine { [Verb(UpgradeVerbName, HelpText = "Checks for new GVFS release, downloads and installs it when available.")] - public class UpgradeVerb : GVFSVerb.NonRepoVerb + public class UpgradeVerb : GVFSVerb.ForNoEnlistment { private const string UpgradeVerbName = "upgrade"; private ITracer tracer; diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index b72b8355d3..4e52ee1d9f 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -74,7 +74,7 @@ public static void Main(string[] args) clone.Execute(); Environment.Exit((int)ReturnCode.Success); }) - .WithParsed( + .WithParsed( verb => { verb.Execute(); From d263eb165904c2dad2fad3a8f430eac1bc76d8d1 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Fri, 28 Sep 2018 17:36:55 -0400 Subject: [PATCH 072/244] - Support for New Asset installer name format. - New GVFSPlatform constant for installer file name extension. - Renamed installer file name from "SetupGVFS.version.exe" to "VFSGit.version.exe" - Updated UT & FT. --- GVFS/GVFS.Common/GVFSPlatform.cs | 9 ++++++--- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 2 +- GVFS/GVFS.Common/ProductUpgrader.cs | 6 +++--- .../EnlistmentPerFixture/GVFSUpgradeReminderTests.cs | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.cs | 2 +- GVFS/GVFS.Platform.Windows/WindowsPlatform.cs | 2 +- .../Windows/Mock/MockProductUpgrader.cs | 4 ++-- GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs | 2 +- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 1 - 9 files changed, 16 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index db124104bb..4341b589f8 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -10,9 +10,9 @@ namespace GVFS.Common { public abstract class GVFSPlatform { - public GVFSPlatform(string executableExtension) + public GVFSPlatform(string executableExtension, string installerExtension) { - this.Constants = new GVFSPlatformConstants(executableExtension); + this.Constants = new GVFSPlatformConstants(executableExtension, installerExtension); } public static GVFSPlatform Instance { get; private set; } @@ -81,12 +81,15 @@ public bool TryGetNormalizedPathRoot(string path, out string pathRoot, out strin public class GVFSPlatformConstants { - public GVFSPlatformConstants(string executableExtension) + public GVFSPlatformConstants(string executableExtension, string installerExtension) { this.ExecutableExtension = executableExtension; + this.InstallerExtension = installerExtension; } public string ExecutableExtension { get; } + public string InstallerExtension { get; } + public string GVFSExecutableName { get { return "GVFS" + this.ExecutableExtension; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index 87d5438b13..edd4db8746 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -11,7 +11,7 @@ public partial class ProductUpgrader public const string DownloadDirectory = "Downloads"; private const string RootDirectory = UpgradeDirectoryName; - private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; + private const string GVFSInstallerFileNamePrefix = "VFSGit"; public static bool IsLocalUpgradeAvailable() { diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index eb023e1d89..d95c30624d 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -131,7 +131,8 @@ public bool TryDownloadNewestVersion(out string errorMessage) foreach (Asset asset in this.newestRelease.Assets) { - if (!this.TryDownloadAsset(asset, out errorMessage)) + if (string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase) && + !this.TryDownloadAsset(asset, out errorMessage)) { return false; } @@ -410,8 +411,7 @@ private bool TryGetLocalInstallerPath(string name, out string path, out string a { foreach (Asset asset in this.newestRelease.Assets) { - string extension = Path.GetExtension(asset.Name); - if (string.Equals(extension, ".exe", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase)) { path = asset.LocalPath; if (name == GitAssetNamePrefix && asset.Name.StartsWith(GitInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs index 376e35ea54..54e4c8eff0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs @@ -14,7 +14,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture [Category(Categories.WindowsOnly)] public class UpgradeReminderTests : TestsWithEnlistmentPerFixture { - private const string GVFSInstallerName = "SetupGVFS.1.0.18234.1.exe"; + private const string GVFSInstallerName = "VFSGit.1.0.18234.1.exe"; private const string GitInstallerName = "Git-2.17.1.gvfs.2.5.g2962052-64-bit.exe"; private string upgradeDirectory; diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index 6dade62c76..06dcc527ac 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -12,7 +12,7 @@ namespace GVFS.Platform.Mac public partial class MacPlatform : GVFSPlatform { public MacPlatform() - : base(executableExtension: string.Empty) + : base(executableExtension: string.Empty, installerExtension: string.Empty) { } diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs index 0880ce9a48..763f2d61b5 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs @@ -24,7 +24,7 @@ public partial class WindowsPlatform : GVFSPlatform private const string BuildLabExRegistryValue = "BuildLabEx"; public WindowsPlatform() - : base(executableExtension: ".exe") + : base(executableExtension: ".exe", installerExtension: ".exe") { } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index 9227d2cd84..7312d0d495 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -69,11 +69,11 @@ public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType r Random random = new Random(); Asset gvfsAsset = new Asset(); - gvfsAsset.Name = "SetupGVFS." + upgradeVersion + ".exe"; + gvfsAsset.Name = "VFSGit." + upgradeVersion + ".exe"; // This is not cross-checked anywhere, random value is good. gvfsAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); - gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/SetupGVFS." + upgradeVersion + ".exe"); + gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSGit." + upgradeVersion + ".exe"); release.Assets.Add(gvfsAsset); Asset gitAsset = new Asset(); diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index f4ba15d35a..38381800c2 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -14,7 +14,7 @@ namespace GVFS.UnitTests.Mock.Common public class MockPlatform : GVFSPlatform { public MockPlatform() - : base(executableExtension: ".mockexe") + : base(executableExtension: ".mockexe", installerExtension: ".exe") { } diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index 155adf9f37..deb0cec57c 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -90,7 +90,6 @@ public void Execute() { mountError = Environment.NewLine + "WARNING: " + mountError; this.output.WriteLine(mountError); - this.ExitCode = ReturnCode.Success; } this.DeletedDownloadedAssets(); From 28f1633b51071a4b4d1623ff27994cbcb2f8f88c Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 1 Oct 2018 12:15:15 -0400 Subject: [PATCH 073/244] - Updated InstallerExtension of Mock & Mac platforms. New extenstions are .mockexe and .dmg - Update UT - Changed GVFS installer name to New Format "VFSGit.Version.Extension" --- GVFS/GVFS.Common/Git/GitVersion.cs | 4 ++-- GVFS/GVFS.Common/ProductUpgrader.cs | 2 +- GVFS/GVFS.Installer/Setup.iss | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.cs | 2 +- .../Windows/Mock/MockProductUpgrader.cs | 9 ++++----- GVFS/GVFS.UnitTests/Common/GitVersionTests.cs | 11 ++++++----- GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs | 2 +- Readme.md | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitVersion.cs b/GVFS/GVFS.Common/Git/GitVersion.cs index 97a1bdebc2..4ffcdd4773 100644 --- a/GVFS/GVFS.Common/Git/GitVersion.cs +++ b/GVFS/GVFS.Common/Git/GitVersion.cs @@ -36,7 +36,7 @@ public static bool TryParseGitVersionCommandResult(string input, out GitVersion return TryParseVersion(input, out version); } - public static bool TryParseInstallerName(string input, out GitVersion version) + public static bool TryParseInstallerName(string input, string installerExtension, out GitVersion version) { // Installer name is of the form // Git-2.14.1.gvfs.1.1.gb16030b-64-bit.exe @@ -48,7 +48,7 @@ public static bool TryParseInstallerName(string input, out GitVersion version) return false; } - if (!input.EndsWith("-64-bit.exe", StringComparison.InvariantCultureIgnoreCase)) + if (!input.EndsWith("-64-bit" + installerExtension, StringComparison.InvariantCultureIgnoreCase)) { return false; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index d95c30624d..05f3127c1e 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -114,7 +114,7 @@ public bool TryGetGitVersion(out GitVersion gitVersion, out string error) foreach (Asset asset in this.newestRelease.Assets) { if (asset.Name.StartsWith(GitInstallerFileNamePrefix) && - GitVersion.TryParseInstallerName(asset.Name, out gitVersion)) + GitVersion.TryParseInstallerName(asset.Name, GVFSPlatform.Instance.Constants.InstallerExtension, out gitVersion)) { return true; } diff --git a/GVFS/GVFS.Installer/Setup.iss b/GVFS/GVFS.Installer/Setup.iss index e01597ee2b..c6648f7ac0 100644 --- a/GVFS/GVFS.Installer/Setup.iss +++ b/GVFS/GVFS.Installer/Setup.iss @@ -39,7 +39,7 @@ AppCopyright=Copyright � Microsoft 2018 BackColor=clWhite BackSolid=yes DefaultDirName={pf}\{#MyAppName} -OutputBaseFilename=SetupGVFS.{#GVFSVersion} +OutputBaseFilename=VFSGit.{#GVFSVersion} OutputDir=Setup Compression=lzma2 InternalCompressLevel=ultra64 diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index 06dcc527ac..e31d892bbd 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -12,7 +12,7 @@ namespace GVFS.Platform.Mac public partial class MacPlatform : GVFSPlatform { public MacPlatform() - : base(executableExtension: string.Empty, installerExtension: string.Empty) + : base(executableExtension: string.Empty, installerExtension: ".dmg") { } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index 7312d0d495..f18ae29984 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -59,7 +59,6 @@ public void ResetFailedAction() public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType remoteRing) { string assetDownloadURLPrefix = "https://github.com/Microsoft/VFSForGit/releases/download/v" + upgradeVersion; - Release release = new Release(); release.Name = "GVFS " + upgradeVersion; @@ -69,17 +68,17 @@ public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType r Random random = new Random(); Asset gvfsAsset = new Asset(); - gvfsAsset.Name = "VFSGit." + upgradeVersion + ".exe"; + gvfsAsset.Name = "VFSGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension; // This is not cross-checked anywhere, random value is good. gvfsAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); - gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSGit." + upgradeVersion + ".exe"); + gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension); release.Assets.Add(gvfsAsset); Asset gitAsset = new Asset(); - gitAsset.Name = "Git-2.17.1.gvfs.2.1.4.g4385455-64-bit.exe"; + gitAsset.Name = "Git-2.17.1.gvfs.2.1.4.g4385455-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension; gitAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); - gitAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/Git-2.17.1.gvfs.2.1.4.g4385455-64-bit.exe"); + gitAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/Git-2.17.1.gvfs.2.1.4.g4385455-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension); release.Assets.Add(gitAsset); this.expectedGVFSAssetName = gvfsAsset.Name; diff --git a/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs b/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs index e33443ab93..1f196b3e00 100644 --- a/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs +++ b/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs @@ -1,4 +1,5 @@ -using GVFS.Common.Git; +using GVFS.Common; +using GVFS.Common.Git; using GVFS.Tests.Should; using NUnit.Framework; @@ -10,9 +11,9 @@ public class GitVersionTests [TestCase] public void TryParseInstallerName() { - this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit.exe"); - this.ParseAndValidateInstallerVersion("git-1.2.3.gvfs.4.5.gb16030b-64-bit.exe"); - this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit.EXE"); + this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension); + this.ParseAndValidateInstallerVersion("git-1.2.3.gvfs.4.5.gb16030b-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension); + this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension); } [TestCase] @@ -206,7 +207,7 @@ public void Allow_Invalid_Minor_Revision() private void ParseAndValidateInstallerVersion(string installerName) { GitVersion version; - bool success = GitVersion.TryParseInstallerName(installerName, out version); + bool success = GitVersion.TryParseInstallerName(installerName, GVFSPlatform.Instance.Constants.InstallerExtension, out version); success.ShouldBeTrue(); version.Major.ShouldEqual(1); diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index 38381800c2..996cf2ff47 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -14,7 +14,7 @@ namespace GVFS.UnitTests.Mock.Common public class MockPlatform : GVFSPlatform { public MockPlatform() - : base(executableExtension: ".mockexe", installerExtension: ".exe") + : base(executableExtension: ".mockexe", installerExtension: ".mockexe") { } diff --git a/Readme.md b/Readme.md index 133b937547..5af238112b 100644 --- a/Readme.md +++ b/Readme.md @@ -50,7 +50,7 @@ If you'd like to build your own VFS for Git Windows installer: build will fail, and the second and subsequent builds will succeed. This is because the build requires a prebuild code generation step. For details, see the build script in the previous step. -The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\SetupGVFS..exe` +The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\VFSGit..exe` ## Trying out VFS for Git From 49802305692458783776e2f491afd5e5f8e71bf3 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 1 Oct 2018 12:43:09 -0400 Subject: [PATCH 074/244] - Fixing FunctionalTest failure. --- GVFS/GVFS.Build/GVFS.PreBuild.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Build/GVFS.PreBuild.csproj b/GVFS/GVFS.Build/GVFS.PreBuild.csproj index 00ff6598c4..1a64a9536f 100644 --- a/GVFS/GVFS.Build/GVFS.PreBuild.csproj +++ b/GVFS/GVFS.Build/GVFS.PreBuild.csproj @@ -140,7 +140,7 @@ - $(BuildOutputDir)\GVFS.Installer\bin\x64\$(Configuration)\SetupGVFS.$(GVFSVersion).exe + $(BuildOutputDir)\GVFS.Installer\bin\x64\$(Configuration)\VFSGit.$(GVFSVersion).exe $(BuildOutputDir)\GVFS.Build\ $(OutDir)GVFSConstants.GitVersion.cs $(OutDir)InstallG4W.bat From f215afc94eb00659d80fd8ac90bb5aea0b1a0b68 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 1 Oct 2018 14:58:28 -0400 Subject: [PATCH 075/244] Fixing a typo --- GVFS/GVFS.Common/GVFSConstants.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 813663a0ca..9f54961592 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -231,7 +231,7 @@ public static class UpgradeVerbMessages public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"]` from a command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; From 0322ab5bc507ee39e739349c599573d7facf117e Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 3 Oct 2018 11:04:40 -0400 Subject: [PATCH 076/244] - Revert back to old installer name "SetupGVFS..exe" - Upgrade supports both old and new gvfs installer name formats now. - Early exit upgrade when either Git or GVFS asset installers are missing in the Release. - New success message when repository mount fails. - updated messaging when gvfs mount fails - Don't re-mount when GVFS installation fails. - Shorter pre-upgrade warning text. - rephrased ProjFS error message - Rephrased Cannot install upgrade messaging. - Added advice on what to do next when an upgrade is available but not installable. - Include Ring information in upgrade-available message. Updated UT. - cleanup made LocalGVFSConfig.allSettings a private property --- GVFS/GVFS.Build/GVFS.PreBuild.csproj | 2 +- GVFS/GVFS.Common/GVFSConstants.cs | 4 +- GVFS/GVFS.Common/InstallerPreRunChecker.cs | 15 ++-- GVFS/GVFS.Common/LocalGVFSConfig.cs | 11 ++- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 8 +- GVFS/GVFS.Common/ProductUpgrader.cs | 73 +++++++++++++++---- .../GVFSUpgradeReminderTests.cs | 2 +- GVFS/GVFS.Installer/Setup.iss | 2 +- .../Windows/Mock/MockProductUpgrader.cs | 4 +- .../Windows/Upgrader/UpgradeVerbTests.cs | 10 +-- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 6 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 21 ++++-- Readme.md | 2 +- 13 files changed, 107 insertions(+), 53 deletions(-) diff --git a/GVFS/GVFS.Build/GVFS.PreBuild.csproj b/GVFS/GVFS.Build/GVFS.PreBuild.csproj index 1a64a9536f..00ff6598c4 100644 --- a/GVFS/GVFS.Build/GVFS.PreBuild.csproj +++ b/GVFS/GVFS.Build/GVFS.PreBuild.csproj @@ -140,7 +140,7 @@ - $(BuildOutputDir)\GVFS.Installer\bin\x64\$(Configuration)\VFSGit.$(GVFSVersion).exe + $(BuildOutputDir)\GVFS.Installer\bin\x64\$(Configuration)\SetupGVFS.$(GVFSVersion).exe $(BuildOutputDir)\GVFS.Build\ $(OutDir)GVFSConstants.GitVersion.cs $(OutDir)InstallG4W.bat diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 9f54961592..393004b5c1 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -233,8 +233,8 @@ public static class UpgradeVerbMessages public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"]` from a command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; - public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; - public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; + public const string UnmountRepoWarning = "Upgrade will unmount and remount gvfs repos, ensure you are at a stopping point."; + public const string UpgradeInstallAdvice = "When ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; } } } diff --git a/GVFS/GVFS.Common/InstallerPreRunChecker.cs b/GVFS/GVFS.Common/InstallerPreRunChecker.cs index ba46681a62..1970d08648 100644 --- a/GVFS/GVFS.Common/InstallerPreRunChecker.cs +++ b/GVFS/GVFS.Common/InstallerPreRunChecker.cs @@ -161,7 +161,7 @@ protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) } else { - consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"GVFS {args} failed." : processResult.Errors; + consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"`gvfs {args}` failed." : processResult.Errors; return false; } } @@ -174,14 +174,15 @@ protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) private bool IsGVFSUpgradeAllowed(out string consoleError) { - consoleError = null; - + bool isConfirmed = string.Equals(this.CommandToRerun, GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm, StringComparison.OrdinalIgnoreCase); + string adviceText = null; if (!this.IsElevated()) { + adviceText = isConfirmed ? $"Run {this.CommandToRerun} again from an elevated command prompt." : $"To install, run {GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm} from an elevated command prompt."; consoleError = string.Join( Environment.NewLine, "The installer needs to be run from an elevated command prompt.", - $"Run {this.CommandToRerun} again from an elevated command prompt."); + adviceText); return false; } @@ -189,20 +190,22 @@ private bool IsGVFSUpgradeAllowed(out string consoleError) { consoleError = string.Join( Environment.NewLine, - $"ProjFS configuration does not support {GVFSConstants.UpgradeVerbMessages.GVFSUpgrade}.", + $"{GVFSConstants.UpgradeVerbMessages.GVFSUpgrade} is not supported because you have previously installed an out of band ProjFS driver.", "Check your team's documentation for how to upgrade."); return false; } if (this.IsServiceInstalledAndNotRunning()) { + adviceText = isConfirmed ? $"Run `sc start GVFS.Service` and run {this.CommandToRerun} again from an elevated command prompt." : $"To install, run `sc start GVFS.Service` and run {GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm} from an elevated command prompt."; consoleError = string.Join( Environment.NewLine, "GVFS Service is not running.", - $"Run `sc start GVFS.Service` and run {this.CommandToRerun} again from an elevated command prompt."); + adviceText); return false; } + consoleError = null; return true; } } diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index a7c225873a..722b280811 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -9,6 +9,7 @@ public class LocalGVFSConfig private const string FileName = "gvfs.config"; private readonly string configFile; private readonly PhysicalFileSystem fileSystem; + private FileBasedDictionary allSettings; public LocalGVFSConfig() { @@ -19,8 +20,6 @@ public LocalGVFSConfig() this.fileSystem = new PhysicalFileSystem(); } - private FileBasedDictionary AllSettings { get; set; } - public bool TryGetConfig( string key, out string value, @@ -36,7 +35,7 @@ public bool TryGetConfig( try { - this.AllSettings.TryGetValue(key, out value); + this.allSettings.TryGetValue(key, out value); error = null; return true; } @@ -68,7 +67,7 @@ public bool TrySetConfig( try { - this.AllSettings.SetValueAndFlush(key, value); + this.allSettings.SetValueAndFlush(key, value); error = null; return true; } @@ -88,7 +87,7 @@ public bool TrySetConfig( private bool TryLoadSettings(ITracer tracer, out string error) { - if (this.AllSettings == null) + if (this.allSettings == null) { FileBasedDictionary config = null; if (FileBasedDictionary.TryCreate( @@ -98,7 +97,7 @@ private bool TryLoadSettings(ITracer tracer, out string error) output: out config, error: out error)) { - this.AllSettings = config; + this.allSettings = config; return true; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index edd4db8746..178f58038d 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -11,16 +11,18 @@ public partial class ProductUpgrader public const string DownloadDirectory = "Downloads"; private const string RootDirectory = UpgradeDirectoryName; - private const string GVFSInstallerFileNamePrefix = "VFSGit"; + private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; + private const string VFSForGitInstallerFileNamePrefix = "VFSForGit"; public static bool IsLocalUpgradeAvailable() { string downloadDirectory = GetAssetDownloadsPath(); if (Directory.Exists(downloadDirectory)) { + const string PotentialInstallerName = "*VFS*.*"; string[] installers = Directory.GetFiles( - downloadDirectory, - $"{GVFSInstallerFileNamePrefix}*.*", + downloadDirectory, + PotentialInstallerName, SearchOption.TopDirectoryOnly); return installers.Length > 0; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index 05f3127c1e..33a584fe8b 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -20,8 +20,8 @@ public partial class ProductUpgrader private const string CommonInstallerArgs = "/VERYSILENT /CLOSEAPPLICATIONS /SUPPRESSMSGBOXES /NORESTART"; private const string GVFSInstallerArgs = CommonInstallerArgs + " /MOUNTREPOS=false"; private const string GitInstallerArgs = CommonInstallerArgs + " /ALLOWDOWNGRADE=1"; - private const string GitAssetNamePrefix = "Git"; - private const string GVFSAssetNamePrefix = "GVFS"; + private const string GitAssetId = "Git"; + private const string GVFSAssetId = "GVFS"; private const string GitInstallerFileNamePrefix = "Git-"; private const int RepoMountFailureExitCode = 17; private const string ToolsDirectory = "Tools"; @@ -127,17 +127,37 @@ public bool TryGetGitVersion(out GitVersion gitVersion, out string error) public bool TryDownloadNewestVersion(out string errorMessage) { - errorMessage = null; - + bool downloadedGit = false; + bool downloadedGVFS = false; foreach (Asset asset in this.newestRelease.Assets) { - if (string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase) && - !this.TryDownloadAsset(asset, out errorMessage)) + bool targetOSMatch = string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase); + bool isGitAsset = this.IsGitAsset(asset); + bool isGVFSAsset = isGitAsset ? false : this.IsGVFSAsset(asset); + if (!targetOSMatch || (!isGVFSAsset && !isGitAsset)) + { + continue; + } + + if (!this.TryDownloadAsset(asset, out errorMessage)) { + errorMessage = $"Could not download {(isGVFSAsset ? GVFSAssetId : GitAssetId)} installer. {errorMessage}"; return false; } + else + { + downloadedGit = isGitAsset ? true : downloadedGit; + downloadedGVFS = isGVFSAsset ? true : downloadedGVFS; + } } + if (!downloadedGit || !downloadedGVFS) + { + errorMessage = $"Could not find {(!downloadedGit ? GitAssetId : GVFSAssetId)} installer in the latest release."; + return false; + } + + errorMessage = null; return true; } @@ -147,7 +167,7 @@ public bool TryRunGitInstaller(out bool installationSucceeded, out string error) installationSucceeded = false; int exitCode = 0; - bool launched = this.TryRunInstallerForAsset(GitAssetNamePrefix, out exitCode, out error); + bool launched = this.TryRunInstallerForAsset(GitAssetId, out exitCode, out error); installationSucceeded = exitCode == 0; return launched; @@ -159,7 +179,7 @@ public bool TryRunGVFSInstaller(out bool installationSucceeded, out string error installationSucceeded = false; int exitCode = 0; - bool launched = this.TryRunInstallerForAsset(GVFSAssetNamePrefix, out exitCode, out error); + bool launched = this.TryRunInstallerForAsset(GVFSAssetId, out exitCode, out error); installationSucceeded = exitCode == 0 || exitCode == RepoMountFailureExitCode; return launched; @@ -370,7 +390,7 @@ private static bool TryCreateDirectory(string path, out Exception exception) return true; } - private bool TryRunInstallerForAsset(string name, out int installerExitCode, out string error) + private bool TryRunInstallerForAsset(string assetId, out int installerExitCode, out string error) { error = null; installerExitCode = 0; @@ -378,7 +398,7 @@ private bool TryRunInstallerForAsset(string name, out int installerExitCode, out bool installerIsRun = false; string path; string installerArgs; - if (this.TryGetLocalInstallerPath(name, out path, out installerArgs)) + if (this.TryGetLocalInstallerPath(assetId, out path, out installerArgs)) { string logFilePath = GVFSEnlistment.GetNewLogFileName(GetLogDirectoryPath(), Path.GetFileNameWithoutExtension(path)); string args = installerArgs + " /Log=" + logFilePath; @@ -386,14 +406,14 @@ private bool TryRunInstallerForAsset(string name, out int installerExitCode, out if (installerExitCode != 0 && string.IsNullOrEmpty(error)) { - error = name + " installer failed. Error log: " + logFilePath; + error = assetId + " installer failed. Error log: " + logFilePath; } installerIsRun = true; } else { - error = "Could not find downloaded installer for " + name; + error = "Could not find downloaded installer for " + assetId; } return installerIsRun; @@ -407,20 +427,20 @@ private void TraceException(Exception exception, string method, string message) this.tracer.RelatedError(metadata, message, Keywords.Telemetry); } - private bool TryGetLocalInstallerPath(string name, out string path, out string args) + private bool TryGetLocalInstallerPath(string assetId, out string path, out string args) { foreach (Asset asset in this.newestRelease.Assets) { if (string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase)) { path = asset.LocalPath; - if (name == GitAssetNamePrefix && asset.Name.StartsWith(GitInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) + if (assetId == GitAssetId && this.IsGitAsset(asset)) { args = GitInstallerArgs; return true; } - if (name == GVFSAssetNamePrefix && asset.Name.StartsWith(GVFSInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) + if (assetId == GVFSAssetId && this.IsGVFSAsset(asset)) { args = GVFSInstallerArgs; return true; @@ -433,6 +453,29 @@ private bool TryGetLocalInstallerPath(string name, out string path, out string a return false; } + private bool IsGVFSAsset(Asset asset) + { + return this.AssetInstallerNameCompare(asset, GVFSInstallerFileNamePrefix, VFSForGitInstallerFileNamePrefix); + } + + private bool IsGitAsset(Asset asset) + { + return this.AssetInstallerNameCompare(asset, GitInstallerFileNamePrefix); + } + + private bool AssetInstallerNameCompare(Asset asset, params string[] expectedFileNamePrefixes) + { + foreach (string fileNamePrefix in expectedFileNamePrefixes) + { + if (asset.Name.StartsWith(fileNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + [DataContract(Name = "asset")] protected class Asset { diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs index 54e4c8eff0..905f9e9392 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs @@ -14,7 +14,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture [Category(Categories.WindowsOnly)] public class UpgradeReminderTests : TestsWithEnlistmentPerFixture { - private const string GVFSInstallerName = "VFSGit.1.0.18234.1.exe"; + private const string GVFSInstallerName = "VFSForGit.1.0.18234.1.exe"; private const string GitInstallerName = "Git-2.17.1.gvfs.2.5.g2962052-64-bit.exe"; private string upgradeDirectory; diff --git a/GVFS/GVFS.Installer/Setup.iss b/GVFS/GVFS.Installer/Setup.iss index c6648f7ac0..e01597ee2b 100644 --- a/GVFS/GVFS.Installer/Setup.iss +++ b/GVFS/GVFS.Installer/Setup.iss @@ -39,7 +39,7 @@ AppCopyright=Copyright � Microsoft 2018 BackColor=clWhite BackSolid=yes DefaultDirName={pf}\{#MyAppName} -OutputBaseFilename=VFSGit.{#GVFSVersion} +OutputBaseFilename=SetupGVFS.{#GVFSVersion} OutputDir=Setup Compression=lzma2 InternalCompressLevel=ultra64 diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index f18ae29984..68281dfa38 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -68,11 +68,11 @@ public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType r Random random = new Random(); Asset gvfsAsset = new Asset(); - gvfsAsset.Name = "VFSGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension; + gvfsAsset.Name = "VFSForGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension; // This is not cross-checked anywhere, random value is good. gvfsAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); - gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension); + gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSForGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension); release.Assets.Add(gvfsAsset); Asset gitAsset = new Asset(); diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs index cc4b43065e..f5e5604659 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -43,8 +43,8 @@ public void UpgradeAvailabilityReporting() expectedReturn: ReturnCode.Success, expectedOutput: new List { - "New GVFS version available: " + NewerThanLocalVersion, - "When you are ready, run `gvfs upgrade --confirm` from an elevated command prompt." + "New GVFS version " + NewerThanLocalVersion + " available in ring Slow", + "When ready, run `gvfs upgrade --confirm` from an elevated command prompt." }, expectedErrors: null); } @@ -80,7 +80,7 @@ public void LaunchInstaller() expectedReturn: ReturnCode.Success, expectedOutput: new List { - "New GVFS version available: " + NewerThanLocalVersion, + "New GVFS version " + NewerThanLocalVersion + " available in ring Slow", "Launching upgrade tool...Succeeded" }, expectedErrors:null); @@ -136,12 +136,12 @@ public void ProjFSPreCheck() expectedReturn: ReturnCode.GenericError, expectedOutput: new List { - "ERROR: ProjFS configuration does not support `gvfs upgrade`.", + "ERROR: `gvfs upgrade` is not supported because you have previously installed an out of band ProjFS driver.", "Check your team's documentation for how to upgrade." }, expectedErrors: new List { - "ProjFS configuration does not support `gvfs upgrade`." + "`gvfs upgrade` is not supported because you have previously installed an out of band ProjFS driver." }); } diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index deb0cec57c..df18ceb2dc 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -60,6 +60,7 @@ public void Execute() string error = null; ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; + string mountError = null; if (!this.TryLoadUpgradeRing(out ring, out error) || ring == ProductUpgrader.RingType.None) { @@ -85,7 +86,6 @@ public void Execute() } finally { - string mountError = null; if (!this.TryMountRepositories(out mountError)) { mountError = Environment.NewLine + "WARNING: " + mountError; @@ -103,7 +103,7 @@ public void Execute() } else { - this.output.WriteLine(Environment.NewLine + "Upgrade completed successfully!"); + this.output.WriteLine($"{Environment.NewLine}{(string.IsNullOrEmpty(mountError) ? "U" : "Repository mount failed. But u")}pgrade completed successfully!"); } if (this.input == Console.In) @@ -229,6 +229,8 @@ private bool TryRunUpgrade(out Version newVersion, out string consoleError) }, $"Installing GVFS version: {newGVFSVersion}")) { + this.mount = false; + consoleError = errorMessage; return false; } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index 282dacea23..edd1af8c37 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -91,15 +91,16 @@ private bool TryRunProductUpgrade() { string errorOutputFormat = Environment.NewLine + "ERROR: {0}"; string error = null; + string cannotInstallReason = null; Version newestVersion = null; ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; - bool isInstallable = this.TryCheckUpgradeInstallable(out error); + bool isInstallable = this.TryCheckUpgradeInstallable(out cannotInstallReason); if (this.Confirmed && !isInstallable) { - this.ReportInfoToConsole($"Cannot install upgrade on this machine."); - this.Output.WriteLine(errorOutputFormat, error); - this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade is not installable. {error}"); + this.ReportInfoToConsole($"Cannot upgrade GVFS on this machine."); + this.Output.WriteLine(errorOutputFormat, cannotInstallReason); + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade is not installable. {cannotInstallReason}"); return false; } @@ -132,7 +133,7 @@ private bool TryRunProductUpgrade() return true; } - string upgradeAvailableMessage = $"New GVFS version available: {newestVersion.ToString()}"; + string upgradeAvailableMessage = $"New GVFS version {newestVersion.ToString()} available in ring {ring}"; if (this.Confirmed) { this.ReportInfoToConsole(upgradeAvailableMessage); @@ -156,15 +157,19 @@ private bool TryRunProductUpgrade() if (isInstallable) { string message = string.Join( - Environment.NewLine + Environment.NewLine, - upgradeAvailableMessage, + Environment.NewLine, GVFSConstants.UpgradeVerbMessages.UnmountRepoWarning, GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); - this.ReportInfoToConsole(message); + this.ReportInfoToConsole(upgradeAvailableMessage + Environment.NewLine + Environment.NewLine + message + Environment.NewLine); } else { this.ReportInfoToConsole($"{Environment.NewLine}{upgradeAvailableMessage}"); + + if (!string.IsNullOrEmpty(cannotInstallReason)) + { + this.ReportInfoToConsole($"{Environment.NewLine}{cannotInstallReason}"); + } } } diff --git a/Readme.md b/Readme.md index 5af238112b..133b937547 100644 --- a/Readme.md +++ b/Readme.md @@ -50,7 +50,7 @@ If you'd like to build your own VFS for Git Windows installer: build will fail, and the second and subsequent builds will succeed. This is because the build requires a prebuild code generation step. For details, see the build script in the previous step. -The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\VFSGit..exe` +The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\SetupGVFS..exe` ## Trying out VFS for Git From 46f9bc5b51cf31af648d2026fddf70e26f33ed4b Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Thu, 4 Oct 2018 11:30:15 -0400 Subject: [PATCH 077/244] Repair Functional Tests: test for confirm on and off Enrich existing tests to include confirm off --- .../EnlistmentPerTestCase/RepairTests.cs | 49 +++++++++++-------- .../MultiEnlistmentTests/SharedCacheTests.cs | 4 +- .../Tools/GVFSFunctionalTestEnlistment.cs | 4 +- .../GVFS.FunctionalTests/Tools/GVFSProcess.cs | 5 +- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs index b53d2caf85..7f44dcf809 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs @@ -17,8 +17,8 @@ public class RepairTests : TestsWithEnlistmentPerTestCase public void NoFixesNeeded() { this.Enlistment.UnmountGVFS(); - - this.Enlistment.Repair(); + this.Enlistment.Repair(confirm: false); + this.Enlistment.Repair(confirm: true); } [TestCase] @@ -28,12 +28,11 @@ public void FixesCorruptHeadSha() string headFilePath = Path.Combine(this.Enlistment.RepoRoot, ".git", "HEAD"); File.WriteAllText(headFilePath, "0000"); - this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when HEAD is corrupt"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -43,12 +42,11 @@ public void FixesCorruptHeadSymRef() string headFilePath = Path.Combine(this.Enlistment.RepoRoot, ".git", "HEAD"); File.WriteAllText(headFilePath, "ref: refs"); - this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when HEAD is corrupt"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -58,12 +56,11 @@ public void FixesMissingGitIndex() string gitIndexPath = Path.Combine(this.Enlistment.RepoRoot, ".git", "index"); File.Delete(gitIndexPath); - this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when git index is missing"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -84,9 +81,9 @@ public void FixesGitIndexCorruptedWithBadData() this.Enlistment.TryMountGVFS(out output).ShouldEqual(false, "GVFS shouldn't mount when index is corrupt"); output.ShouldContain("Index validation failed"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -108,9 +105,9 @@ public void FixesGitIndexContainingAllNulls() this.Enlistment.TryMountGVFS(out output).ShouldEqual(false, "GVFS shouldn't mount when index is corrupt"); output.ShouldContain("Index validation failed"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -135,9 +132,9 @@ public void FixesGitIndexCorruptedByTruncation() this.Enlistment.TryMountGVFS(out output).ShouldEqual(false, "GVFS shouldn't mount when index is corrupt"); output.ShouldContain("Index validation failed"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -150,11 +147,11 @@ public void FixesCorruptGitConfig() this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when git config is missing"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); + this.Enlistment.Repair(confirm: true); ProcessResult result = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "remote add origin " + this.Enlistment.RepoUrl); - result.ExitCode.ShouldEqual(0, result.Errors); - + result.ExitCode.ShouldEqual(0, result.Errors); this.Enlistment.MountGVFS(); } @@ -170,5 +167,17 @@ private void CreateCorruptIndexAndRename(string indexPath, Action Date: Thu, 4 Oct 2018 09:54:27 -0700 Subject: [PATCH 078/244] PR Feedback: Use fsid&inode instead of path as key in mutex map --- .../PrjFSKext.xcodeproj/project.pbxproj | 2 + .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 37 +++++++- .../PrjFSKext/PrjFSKext/Message_Kernel.cpp | 2 + .../PrjFSKext/VirtualizationRoots.cpp | 23 +++-- .../PrjFSKext/VirtualizationRoots.hpp | 5 +- .../PrjFSKext/PrjFSKext/VnodeUtilities.cpp | 2 +- .../PrjFSKext/PrjFSKext/VnodeUtilities.hpp | 9 +- ProjFS.Mac/PrjFSKext/public/FsidInode.h | 20 ++++ ProjFS.Mac/PrjFSKext/public/Message.h | 5 + ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 94 ++++++++++--------- 10 files changed, 130 insertions(+), 69 deletions(-) create mode 100644 ProjFS.Mac/PrjFSKext/public/FsidInode.h diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj index f55c074035..74700506e7 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj @@ -30,6 +30,7 @@ /* End PBXBuildFile section */ /* Begin PBXFileReference section */ + 263E066C21667C11005F756A /* FsidInode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FsidInode.h; sourceTree = ""; }; 4A08829C20D80B8300E17FEE /* PrjFSXattrs.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PrjFSXattrs.h; sourceTree = ""; }; 4A63CB0B20AB009000157B95 /* VnodeUtilities.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = VnodeUtilities.cpp; sourceTree = ""; }; 4A63CB0C20AB009000157B95 /* VnodeUtilities.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = VnodeUtilities.hpp; sourceTree = ""; }; @@ -90,6 +91,7 @@ 4ABB734C20C1A65B00DC0D17 /* PrjFSProviderClientShared.h */, 4A08829C20D80B8300E17FEE /* PrjFSXattrs.h */, C6BDD37A208C2FD700CB7E58 /* Message.h */, + 263E066C21667C11005F756A /* FsidInode.h */, ); path = public; sourceTree = ""; diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index dbf425d533..a0ed82e25c 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -7,6 +7,7 @@ #include "PrjFSCommon.h" #include "VirtualizationRoots.hpp" +#include "VnodeUtilities.hpp" #include "KauthHandler.hpp" #include "KextLog.hpp" #include "Message.h" @@ -48,6 +49,7 @@ static bool TrySendRequestAndWaitForResponse( const VirtualizationRoot* root, MessageType messageType, const vnode_t vnode, + const FsidInode& vnodeFsidInode, const char* vnodePath, int pid, const char* procname, @@ -66,6 +68,7 @@ static bool ShouldHandleVnodeOpEvent( VirtualizationRoot** root, vtype* vnodeType, uint32_t* vnodeFileFlags, + FsidInode* vnodeFsidInode, int* pid, char procname[MAXCOMLEN + 1], int* kauthResult); @@ -78,6 +81,7 @@ static bool ShouldHandleFileOpEvent( // Out params: VirtualizationRoot** root, + FsidInode* vnodeFsidInode, int* pid); // Structs @@ -264,6 +268,7 @@ static int HandleVnodeOperation( VirtualizationRoot* root = nullptr; vtype vnodeType; uint32_t currentVnodeFileFlags; + FsidInode vnodeFsidInode; int pid; char procname[MAXCOMLEN + 1]; bool isDeleteAction = false; @@ -284,6 +289,7 @@ static int HandleVnodeOperation( &root, &vnodeType, ¤tVnodeFileFlags, + &vnodeFsidInode, &pid, procname, &kauthResult)) @@ -302,6 +308,7 @@ static int HandleVnodeOperation( MessageType_KtoU_NotifyDirectoryPreDelete : MessageType_KtoU_NotifyFilePreDelete, currentVnode, + vnodeFsidInode, vnodePath, pid, procname, @@ -332,6 +339,7 @@ static int HandleVnodeOperation( MessageType_KtoU_RecursivelyEnumerateDirectory : MessageType_KtoU_EnumerateDirectory, currentVnode, + vnodeFsidInode, vnodePath, pid, procname, @@ -362,6 +370,7 @@ static int HandleVnodeOperation( root, MessageType_KtoU_HydrateFile, currentVnode, + vnodeFsidInode, vnodePath, pid, procname, @@ -410,12 +419,14 @@ static int HandleFileOpOperation( } VirtualizationRoot* root = nullptr; + FsidInode vnodeFsidInode; int pid; if (!ShouldHandleFileOpEvent( context, currentVnodeFromPath, action, &root, + &vnodeFsidInode, &pid)) { goto CleanupAndReturn; @@ -440,6 +451,7 @@ static int HandleFileOpOperation( root, messageType, currentVnodeFromPath, + vnodeFsidInode, newPath, pid, procname, @@ -467,12 +479,14 @@ static int HandleFileOpOperation( } VirtualizationRoot* root = nullptr; + FsidInode vnodeFsidInode; int pid; if (!ShouldHandleFileOpEvent( context, currentVnode, action, &root, + &vnodeFsidInode, &pid)) { goto CleanupAndReturn; @@ -489,6 +503,7 @@ static int HandleFileOpOperation( root, MessageType_KtoU_NotifyFileModified, currentVnode, + vnodeFsidInode, path, pid, procname, @@ -506,6 +521,7 @@ static int HandleFileOpOperation( root, MessageType_KtoU_NotifyFileCreated, currentVnode, + vnodeFsidInode, path, pid, procname, @@ -541,6 +557,7 @@ static bool ShouldHandleVnodeOpEvent( VirtualizationRoot** root, vtype* vnodeType, uint32_t* vnodeFileFlags, + FsidInode* vnodeFsidInode, int* pid, char procname[MAXCOMLEN + 1], int* kauthResult) @@ -591,7 +608,8 @@ static bool ShouldHandleVnodeOpEvent( } } - *root = VirtualizationRoots_FindForVnode(vnode); + *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); + *root = VirtualizationRoots_FindForVnode(vnode, *vnodeFsidInode); if (nullptr == *root) { @@ -629,6 +647,7 @@ static bool ShouldHandleFileOpEvent( // Out params: VirtualizationRoot** root, + FsidInode* vnodeFsidInode, int* pid) { vtype vnodeType = vnode_vtype(vnode); @@ -636,8 +655,9 @@ static bool ShouldHandleFileOpEvent( { return false; } - - *root = VirtualizationRoots_FindForVnode(vnode); + + *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); + *root = VirtualizationRoots_FindForVnode(vnode, *vnodeFsidInode); if (nullptr == *root) { return false; @@ -662,6 +682,7 @@ static bool TrySendRequestAndWaitForResponse( const VirtualizationRoot* root, MessageType messageType, const vnode_t vnode, + const FsidInode& vnodeFsidInode, const char* vnodePath, int pid, const char* procname, @@ -686,7 +707,15 @@ static bool TrySendRequestAndWaitForResponse( int nextMessageId = OSIncrementAtomic(&s_nextMessageId); Message messageSpec = {}; - Message_Init(&messageSpec, &(message.request), nextMessageId, messageType, pid, procname, relativePath); + Message_Init( + &messageSpec, + &(message.request), + nextMessageId, + messageType, + vnodeFsidInode, + pid, + procname, + relativePath); bool isShuttingDown = false; Mutex_Acquire(s_outstandingMessagesMutex); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp index 1c0d80ced2..aebb999647 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp @@ -6,12 +6,14 @@ void Message_Init( MessageHeader* header, uint64_t messageId, MessageType messageType, + const FsidInode& fsidInode, int32_t pid, const char* procname, const char* path) { header->messageId = messageId; header->messageType = messageType; + header->fsidInode = fsidInode; header->pid = pid; if (nullptr != procname) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index 443a8d30f3..46a5704d9f 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -20,9 +20,9 @@ static const size_t MaxVirtualizationRoots = 128; static VirtualizationRoot s_virtualizationRoots[MaxVirtualizationRoots] = {}; -static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, VnodeFsidInode fileId); +static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId); static int16_t FindUnusedIndex_Locked(); -static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, VnodeFsidInode persistentIds, const char* path); +static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); kern_return_t VirtualizationRoots_Init() { @@ -56,7 +56,7 @@ kern_return_t VirtualizationRoots_Cleanup() return KERN_FAILURE; } -VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode) +VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode) { VirtualizationRoot* root = nullptr; @@ -64,7 +64,7 @@ VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode) // Search up the tree until we hit a known virtualization root or THE root of the file system while (nullptr == root && NULLVP != vnode && !vnode_isvroot(vnode)) { - int16_t rootIndex = VirtualizationRoots_LookupVnode(vnode, nullptr); + int16_t rootIndex = VirtualizationRoots_LookupVnode(vnode, /*context*/ nullptr, vnodeFsidInode); if (rootIndex >= 0) { root = &s_virtualizationRoots[rootIndex]; @@ -84,16 +84,15 @@ VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode) return root; } -int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context) +int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode) { - VnodeFsidInode fsidInode = Vnode_GetFsidAndInode(vnode, context); uint32_t vid = vnode_vid(vnode); int16_t rootIndex; RWLock_AcquireShared(s_rwLock); { - rootIndex = FindRootForVnode_Locked(vnode, vid, fsidInode); + rootIndex = FindRootForVnode_Locked(vnode, vid, vnodeFsidInode); } RWLock_ReleaseShared(s_rwLock); @@ -112,12 +111,12 @@ int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context) RWLock_AcquireExclusive(s_rwLock); { // Vnode may already have been inserted as a root in the interim - rootIndex = FindRootForVnode_Locked(vnode, vid, fsidInode); + rootIndex = FindRootForVnode_Locked(vnode, vid, vnodeFsidInode); if (rootIndex < 0) { // Insert new offline root - rootIndex = InsertVirtualizationRoot_Locked(nullptr, 0, vnode, vid, fsidInode, path); + rootIndex = InsertVirtualizationRoot_Locked(nullptr, 0, vnode, vid, vnodeFsidInode, path); // TODO: error handling assert(rootIndex >= 0); @@ -149,7 +148,7 @@ static bool FsidsAreEqual(fsid_t a, fsid_t b) return a.val[0] == b.val[0] && a.val[1] == b.val[1]; } -static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, VnodeFsidInode fileId) +static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId) { for (int16_t i = 0; i < MaxVirtualizationRoots; ++i) { @@ -177,7 +176,7 @@ static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, VnodeFsidIno } // Returns negative value if it failed, or inserted index on success -static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, VnodeFsidInode persistentIds, const char* path) +static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path) { // New root int16_t rootIndex = FindUnusedIndex_Locked(); @@ -230,7 +229,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide } else { - VnodeFsidInode vnodeIds = Vnode_GetFsidAndInode(virtualizationRootVNode, vfsContext); + FsidInode vnodeIds = Vnode_GetFsidAndInode(virtualizationRootVNode, vfsContext); uint32_t rootVid = vnode_vid(virtualizationRootVNode); RWLock_AcquireExclusive(s_rwLock); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp index 49389c10cf..3d1c86aefa 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp @@ -1,5 +1,6 @@ #pragma once +#include "FsidInode.h" #include "PrjFSClasses.hpp" #include "kernel-header-wrappers/vnode.h" @@ -27,7 +28,7 @@ struct VirtualizationRoot kern_return_t VirtualizationRoots_Init(void); kern_return_t VirtualizationRoots_Cleanup(void); -VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode); +VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode); struct VirtualizationRootResult { @@ -41,4 +42,4 @@ struct Message; errno_t ActiveProvider_SendMessage(int32_t rootIndex, const Message message); bool VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode_t vnode); -int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context); +int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp index f129087515..c73aa5bfed 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp @@ -5,7 +5,7 @@ extern "C" int mac_vnop_getxattr(struct vnode *, const char *, char *, size_t, size_t *); -VnodeFsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context) +FsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context) { vnode_attr attrs; VATTR_INIT(&attrs); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp index 7ab01a0534..068c94efdf 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp @@ -2,6 +2,7 @@ #include #include +#include "FsidInode.h" struct SizeOrError { @@ -10,10 +11,4 @@ struct SizeOrError }; SizeOrError Vnode_ReadXattr(vnode_t vnode, const char* xattrName, void* buffer, size_t bufferSize, vfs_context_t context); - -struct VnodeFsidInode -{ - fsid_t fsid; - uint64_t inode; -}; -VnodeFsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context); +FsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context); diff --git a/ProjFS.Mac/PrjFSKext/public/FsidInode.h b/ProjFS.Mac/PrjFSKext/public/FsidInode.h new file mode 100644 index 0000000000..caf2f7715e --- /dev/null +++ b/ProjFS.Mac/PrjFSKext/public/FsidInode.h @@ -0,0 +1,20 @@ +// +// FsidInode.h +// PrjFSKext +// +// Created by William Baker on 10/4/18. +// Copyright © 2018 GVFS. All rights reserved. +// + +#ifndef FsidInode_h +#define FsidInode_h + +#include + +struct FsidInode +{ + fsid_t fsid; + uint64_t inode; +}; + +#endif /* FsidInode_h */ diff --git a/ProjFS.Mac/PrjFSKext/public/Message.h b/ProjFS.Mac/PrjFSKext/public/Message.h index 43a99ab066..7257cb385f 100644 --- a/ProjFS.Mac/PrjFSKext/public/Message.h +++ b/ProjFS.Mac/PrjFSKext/public/Message.h @@ -3,6 +3,7 @@ #include #include "PrjFSCommon.h" +#include "FsidInode.h" typedef enum { @@ -39,6 +40,9 @@ struct MessageHeader // The message type indicates the type of request or response uint32_t messageType; // values of type MessageType + // fsid and inode of the file + FsidInode fsidInode; + // For messages from kernel to user mode, indicates the PID of the process that initiated the I/O int32_t pid; char procname[MAXCOMLEN + 1]; @@ -59,6 +63,7 @@ void Message_Init( MessageHeader* header, uint64_t messageId, MessageType messageType, + const FsidInode& fsidInode, int32_t pid, const char* procname, const char* path); diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index a901cfed26..132ccd3830 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -55,6 +55,32 @@ struct _PrjFS_FileHandle FILE* file; }; +struct FsidInodeCompare +{ + bool operator() (const FsidInode& lhs, const FsidInode& rhs) const + { + if (lhs.fsid.val[0] != rhs.fsid.val[0]) + { + return lhs.fsid.val[0] < rhs.fsid.val[0]; + } + + if (lhs.fsid.val[1] != rhs.fsid.val[1]) + { + return lhs.fsid.val[1] < rhs.fsid.val[1]; + } + + return lhs.inode < rhs.inode; + } +}; + +struct MutexAndUseCount +{ + shared_ptr mutex; + int useCount; +}; + +typedef map FileMutexMap; + // Function prototypes static bool SetBitInFileFlags(const char* path, uint32_t bit, bool value); static bool IsBitSetInFileFlags(const char* path, uint32_t bit); @@ -90,6 +116,9 @@ static void ClearMachNotification(mach_port_t port); static const char* NotificationTypeToString(PrjFS_NotificationType notificationType); #endif +static FileMutexMap::iterator CheckoutFileMutexIterator(const FsidInode& fsidInode); +static void ReturnFileMutexIterator(FileMutexMap::iterator lockIterator); + // State static io_connect_t s_kernelServiceConnection = IO_OBJECT_NULL; static string s_virtualizationRootFullPath; @@ -97,27 +126,9 @@ static PrjFS_Callbacks s_callbacks; static dispatch_queue_t s_messageQueueDispatchQueue; static dispatch_queue_t s_kernelRequestHandlingConcurrentQueue; -struct CaseInsensitiveStringCompare -{ - bool operator() (const std::string& lhs, const std::string& rhs) const - { - return strcasecmp(lhs.c_str(), rhs.c_str()) < 0; - } -}; - -struct MutexAndUseCount -{ - shared_ptr mutex; - int useCount; -}; - -// Map of relative path -> MutexAndUseCount for that path, plus mutex to protect the map itself. -typedef map PathToMutexMap; -PathToMutexMap s_inProgressExpansions; -mutex s_inProgressExpansionsMutex; - -static shared_ptr CheckoutPathMutex(const string& fullPath); -static void ReturnPathMutex(const string& fullPath, const shared_ptr& mutex); +// Map of FsidInode -> MutexAndUseCount for that FsidInode, plus mutex to protect the map itself. +FileMutexMap s_fileLocks; +mutex s_fileLocksMutex; // The full API is defined in the header, but only the minimal set of functions needed // for the initial MirrorProvider implementation are listed here. Calling any other function @@ -643,9 +654,9 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request } PrjFS_Result result; - shared_ptr expansionMutex = CheckoutPathMutex(fullPath); + FileMutexMap::iterator mutexIterator = CheckoutFileMutexIterator(request->fsidInode); { - mutex_lock lock(*expansionMutex); + mutex_lock lock(*(mutexIterator->second.mutex)); if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) { result = PrjFS_Result_Success; @@ -670,7 +681,7 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request } CleanupAndReturn: - ReturnPathMutex(fullPath, expansionMutex); + ReturnFileMutexIterator(mutexIterator); return result; } @@ -755,10 +766,10 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const PrjFS_Result result; PrjFS_FileHandle fileHandle; - shared_ptr hydrationMutex = CheckoutPathMutex(fullPath); + FileMutexMap::iterator mutexIterator = CheckoutFileMutexIterator(request->fsidInode); { - mutex_lock lock(*hydrationMutex); + mutex_lock lock(*(mutexIterator->second.mutex)); if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) { result = PrjFS_Result_Success; @@ -820,7 +831,7 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const } CleanupAndReturn: - ReturnPathMutex(fullPath, hydrationMutex); + ReturnFileMutexIterator(mutexIterator); return result; } @@ -1057,33 +1068,30 @@ static const char* NotificationTypeToString(PrjFS_NotificationType notificationT } #endif -static shared_ptr CheckoutPathMutex(const string& fullPath) +static FileMutexMap::iterator CheckoutFileMutexIterator(const FsidInode& fsidInode) { - mutex_lock lock(s_inProgressExpansionsMutex); - PathToMutexMap::iterator iter = s_inProgressExpansions.find(fullPath); - if (iter == s_inProgressExpansions.end()) + mutex_lock lock(s_fileLocksMutex); + FileMutexMap::iterator iter = s_fileLocks.find(fsidInode); + if (iter == s_fileLocks.end()) { - pair newEntry = s_inProgressExpansions.insert( - PathToMutexMap::value_type(fullPath, { make_shared(), 1 })); + pair newEntry = s_fileLocks.insert( + FileMutexMap::value_type(fsidInode, { make_shared(), 1 })); assert(newEntry.second); - return newEntry.first->second.mutex; + return newEntry.first; } else { iter->second.useCount++; - return iter->second.mutex; + return iter; } } -static void ReturnPathMutex(const string& fullPath, const shared_ptr& mutex) +static void ReturnFileMutexIterator(FileMutexMap::iterator lockIterator) { - mutex_lock lock(s_inProgressExpansionsMutex); - PathToMutexMap::iterator iter = s_inProgressExpansions.find(fullPath); - assert(iter != s_inProgressExpansions.end()); - assert(iter->second.mutex.get() == mutex.get()); - iter->second.useCount--; - if (iter->second.useCount == 0) + mutex_lock lock(s_fileLocksMutex); + lockIterator->second.useCount--; + if (lockIterator->second.useCount == 0) { - s_inProgressExpansions.erase(iter); + s_fileLocks.erase(lockIterator); } } From 5ab446331bbb33ae61477855d58cae0efcdd664d Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 30 Aug 2018 15:31:08 -0700 Subject: [PATCH 079/244] Add Mac Setup instructions needed for building on Mac --- Readme.md | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index 133b937547..a41a9c1e7a 100644 --- a/Readme.md +++ b/Readme.md @@ -30,10 +30,10 @@ built executables, and releases may still refer to the old GVFS name. See https: * VFS for Git requires Windows 10 Anniversary Update (Windows 10 version 1607) or later * Run the latest GVFS and Git for Windows installers from https://github.com/Microsoft/VFSForGit/releases -## Building VFS for Git +## Building VFS for Git on Windows If you'd like to build your own VFS for Git Windows installer: -* Install Visual Studio 2017 Community Edition or higher (https://www.visualstudio.com/downloads/). +* Install Visual Studio 2017 Community Edition or higher (https://www.visualstudio.com/downloads/). * Include the following workloads: * .NET desktop development * Desktop development with C++ @@ -46,15 +46,47 @@ If you'd like to build your own VFS for Git Windows installer: * Create a folder to clone into, e.g. `C:\Repos\VFSForGit` * Clone this repo into the `src` subfolder, e.g. `C:\Repos\VFSForGit\src` * Run `\src\Scripts\BuildGVFSForWindows.bat` -* You can also build in Visual Studio by opening `src\GVFS.sln` (do not upgrade any projects) and building. However, the very first +* You can also build in Visual Studio by opening `src\GVFS.sln` (do not upgrade any projects) and building. However, the very first build will fail, and the second and subsequent builds will succeed. This is because the build requires a prebuild code generation step. For details, see the build script in the previous step. The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\SetupGVFS..exe` +## Building VFS for Git on Mac + +### First setup your Mac + +* Ensure you have Xcode installed and have accepted the terms of use (Launch Xcode at least once). +* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac) +* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) +* [Install Homebrew](https://brew.sh/) +* Install Java with + ``` + brew cask install java + ``` + +* Install the Git credential manager for Mac [by following their Homebrew instructions](https://github.com/Microsoft/Git-Credential-Manager-for-Mac-and-Linux/blob/master/Install.md#installing-on-mac-using-homebrew-or-on-linux-using-linuxbrew-recommended) + +* If you are not currently an Apple Developer you will need to become one so that you can either use Microsoft certs or create your own certs for signing purposes. (If you are a Microsoft employee you must use your alias@microsoft.com when creating your Apple account.) + +* Create a `VFS` directory and Clone the VFS repo into a directory called `src` inside it: + ``` + mkdir VFS && cd VFS && git clone https://github.com/Microsoft/VFSForGit.git src && cd src + ``` + +* Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). + Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: + ``` + csrutil disable + ``` + to disable SIP. + +* Now `cd` into + + ## Trying out VFS for Git -* VFS for Git will work with any git service that supports the GVFS [protocol](Protocol.md). For example, you can create a repo in +* VFS for Git will work with any git service that supports the GVFS [protocol](Protocol.md). For example, you can create a repo in Visual Studio Team Services (https://www.visualstudio.com/team-services/), and push some contents to it. There are two constraints: * Your repo must not enable any clean/smudge filters * Your repo must have a `.gitattributes` file in the root that includes the line `* -text` @@ -63,6 +95,7 @@ Visual Studio Team Services (https://www.visualstudio.com/team-services/), and p * Run git commands as you normally would * `gvfs unmount` when done + # Licenses The VFS for Git source code in this repo is available under the MIT license. See [License.md](License.md). From 84fdbf89392259ca00608d5ed8bdcb8c2faf291b Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Wed, 19 Sep 2018 20:21:50 -0700 Subject: [PATCH 080/244] Add initial set of build steps for Mac OS --- ProjFS.Mac/Scripts/Build.sh | 4 +-- Readme.md | 54 ++++++++++++++++++++++++++++++++-- Scripts/Mac/BuildGVFSForMac.sh | 8 ++--- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/ProjFS.Mac/Scripts/Build.sh b/ProjFS.Mac/Scripts/Build.sh index 2f298f876d..333883c37b 100755 --- a/ProjFS.Mac/Scripts/Build.sh +++ b/ProjFS.Mac/Scripts/Build.sh @@ -8,11 +8,11 @@ fi SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) SRCDIR=$SCRIPTDIR/../.. ROOTDIR=$SRCDIR/.. -PACKAGES=$ROOTDIR/packages +PACKAGES=$ROOTDIR/packages PROJFS=$SRCDIR/ProjFS.Mac -xcodebuild -sdk macosx10.13 -configuration $CONFIGURATION -workspace $PROJFS/PrjFS.xcworkspace build -scheme PrjFS -derivedDataPath $ROOTDIR/BuildOutput/ProjFS.Mac/Native || exit 1 +xcodebuild -configuration $CONFIGURATION -workspace $PROJFS/PrjFS.xcworkspace build -scheme PrjFS -derivedDataPath $ROOTDIR/BuildOutput/ProjFS.Mac/Native || exit 1 dotnet restore $PROJFS/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj /p:Configuration=$CONFIGURATION /p:Platform=x64 --packages $PACKAGES || exit 1 dotnet build $PROJFS/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj /p:Configuration=$CONFIGURATION /p:Platform=x64 || exit 1 diff --git a/Readme.md b/Readme.md index a41a9c1e7a..8f0dc078eb 100644 --- a/Readme.md +++ b/Readme.md @@ -56,8 +56,8 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ### First setup your Mac -* Ensure you have Xcode installed and have accepted the terms of use (Launch Xcode at least once). -* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac) +* Ensure you have Xcode installed and have accepted the terms of use and launched Xcode at least once. +* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). * If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) * [Install Homebrew](https://brew.sh/) * Install Java with @@ -81,7 +81,55 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ``` to disable SIP. -* Now `cd` into +* Now `cd` back into your VFS src directory and run + + ``` + Scripts/Mac/BuildGVFSForMac.sh [Release] + ``` + + **Troubleshooting** + + If you get + ``` + xcodebuild: error: SDK "macosx10.13" cannot be located. + ``` + You may have the "XCode Command Line Tools" installed (helpfully by Mac OS) instead of full XCode. + Make sure + ``` + xcode-select -p + ``` + + shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install Xcode and then launch it (you can close it afterwards.) + +* Now setup the git credential manager by running + + ``` + Scripts/Mac/PrepFunctionalTests.sh + ``` + +* Now you have to load the ProjFS Kext. + + ``` + ProjFS.Mac/Scripts/LoadPrjFSKext.sh [Release] + ``` + +* Now you can put your built gvfs program on your path. A simple way to do that is by adding `Path/to/VFS/Publish` to your path. + + Check you have it by running + + ``` + command -v gvfs + ``` + + You should see a path to the gvfs executable. + +* Try cloning! Now that you have `gvfs` ready you can try cloning a VFS for Git enabled repository! + + ``` + gvfs clone URL_TO_REPOSITORY [--cache-url] --local-cache-path ~/.gvfsCache + ``` + + Note you may have a cache server URL to use. Note the current use of `--local-cache-path`. That argument prevent gvfs from running into a permissions error trying to put the cache at the root of your Mac's hard-drive. ## Trying out VFS for Git diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index 2eb875c777..678bb89aaf 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -14,20 +14,20 @@ popd SRCDIR=$SCRIPTDIR/../.. ROOTDIR=$SRCDIR/.. -BUILDOUTPUT=$ROOTDIR/BuildOutput +BUILDOUTPUT=$ROOTDIR/BuildOutput PUBLISHDIR=$ROOTDIR/Publish if [ ! -d $BUILDOUTPUT ]; then mkdir $BUILDOUTPUT fi -PACKAGES=$ROOTDIR/packages +PACKAGES=$ROOTDIR/packages # Build the ProjFS kext and libraries $SRCDIR/ProjFS.Mac/Scripts/Build.sh $CONFIGURATION || exit 1 # Create the directory where we'll do pre build tasks -BUILDDIR=$BUILDOUTPUT/GVFS.Build +BUILDDIR=$BUILDOUTPUT/GVFS.Build if [ ! -d $BUILDDIR ]; then mkdir $BUILDDIR || exit 1 fi @@ -44,7 +44,7 @@ dotnet build $SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --conf dotnet publish $SRCDIR/GVFS.sln /p:Configuration=$DOTNETCONFIGURATION /p:Platform=x64 --runtime osx-x64 --framework netcoreapp2.1 --self-contained --output $PUBLISHDIR /maxcpucount:1 || exit 1 NATIVEDIR=$SRCDIR/GVFS/GVFS.Native.Mac -xcodebuild -sdk macosx10.13 -configuration $CONFIGURATION -workspace $NATIVEDIR/GVFS.Native.Mac.xcworkspace build -scheme GVFS.Native.Mac -derivedDataPath $ROOTDIR/BuildOutput/GVFS.Native.Mac || exit 1 +xcodebuild -configuration $CONFIGURATION -workspace $NATIVEDIR/GVFS.Native.Mac.xcworkspace build -scheme GVFS.Native.Mac -derivedDataPath $ROOTDIR/BuildOutput/GVFS.Native.Mac || exit 1 echo 'Copying native binaries to Publish directory' cp $BUILDOUTPUT/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.ReadObjectHook $PUBLISHDIR || exit 1 From 871f87c59a56e8566e10424a0f4c44a5e09da459 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 20 Sep 2018 14:54:48 -0700 Subject: [PATCH 081/244] Update mac os build subtitle --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 8f0dc078eb..aada7ffbd6 100644 --- a/Readme.md +++ b/Readme.md @@ -54,7 +54,7 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ## Building VFS for Git on Mac -### First setup your Mac +VFS for macOS is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. * Ensure you have Xcode installed and have accepted the terms of use and launched Xcode at least once. * Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). From 88b1c55254a966586c974f0455c4639b2f1080d4 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 20 Sep 2018 15:18:57 -0700 Subject: [PATCH 082/244] Remove steps covered by the prep script --- Readme.md | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/Readme.md b/Readme.md index aada7ffbd6..ee8c0562b2 100644 --- a/Readme.md +++ b/Readme.md @@ -57,15 +57,17 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer VFS for macOS is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. * Ensure you have Xcode installed and have accepted the terms of use and launched Xcode at least once. -* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). -* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) -* [Install Homebrew](https://brew.sh/) -* Install Java with + +* Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). + Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: ``` - brew cask install java + csrutil disable ``` + to disable SIP. -* Install the Git credential manager for Mac [by following their Homebrew instructions](https://github.com/Microsoft/Git-Credential-Manager-for-Mac-and-Linux/blob/master/Install.md#installing-on-mac-using-homebrew-or-on-linux-using-linuxbrew-recommended) +* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). + +* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) * If you are not currently an Apple Developer you will need to become one so that you can either use Microsoft certs or create your own certs for signing purposes. (If you are a Microsoft employee you must use your alias@microsoft.com when creating your Apple account.) @@ -74,20 +76,13 @@ VFS for macOS is still in progress. You can build it, but this will not create a mkdir VFS && cd VFS && git clone https://github.com/Microsoft/VFSForGit.git src && cd src ``` -* Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). - Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: - ``` - csrutil disable - ``` - to disable SIP. - * Now `cd` back into your VFS src directory and run ``` Scripts/Mac/BuildGVFSForMac.sh [Release] ``` - **Troubleshooting** + _Troubleshooting if this fails_ If you get ``` @@ -126,10 +121,10 @@ VFS for macOS is still in progress. You can build it, but this will not create a * Try cloning! Now that you have `gvfs` ready you can try cloning a VFS for Git enabled repository! ``` - gvfs clone URL_TO_REPOSITORY [--cache-url] --local-cache-path ~/.gvfsCache + gvfs clone URL_TO_REPOSITORY --local-cache-path ~/.gvfsCache ``` - Note you may have a cache server URL to use. Note the current use of `--local-cache-path`. That argument prevent gvfs from running into a permissions error trying to put the cache at the root of your Mac's hard-drive. + Note the current use of `--local-cache-path`. That argument prevent gvfs from running into a permissions error trying to put the cache at the root of your Mac's hard-drive. ## Trying out VFS for Git From fbc892a2df053d4322a40cc78420794dd84a6b56 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 20 Sep 2018 15:22:09 -0700 Subject: [PATCH 083/244] Remove deprecated java hack for git-credential-manager --- Scripts/Mac/PrepFunctionalTests.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index f1a0c4260d..fb35bde658 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -36,13 +36,6 @@ fi git-credential-manager install -# If our Java version is 9+ (the formatting of 'java -version' changed in Java 9), work around -# https://github.com/Microsoft/Git-Credential-Manager-for-Mac-and-Linux/issues/71 -JAVAVERSION="$(java -version 2>&1 | egrep -o '"[[:digit:]]+.[[:digit:]]+.[[:digit:]]+"' | xargs)" -if [[ ! -z $JAVAVERSION ]]; then - git config --global credential.helper "!/usr/bin/java -Ddebug=false --add-modules java.xml.bind -Djava.net.useSystemProxies=true -jar /usr/local/Cellar/git-credential-manager/2.0.3/libexec/git-credential-manager-2.0.3.jar" || exit 1 -fi - # If we're running on an agent where the PAT environment variable is set and a URL is passed into the script, add it to the keychain. PATURL=$1 PAT=$2 From 96218db5912168e5bd64406c724983de2696a70a Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 21 Sep 2018 11:23:34 -0700 Subject: [PATCH 084/244] Grammar --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index ee8c0562b2..9338be9133 100644 --- a/Readme.md +++ b/Readme.md @@ -56,7 +56,7 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer VFS for macOS is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. -* Ensure you have Xcode installed and have accepted the terms of use and launched Xcode at least once. +* Ensure you have Xcode installed, have accepted the terms of use, and launched Xcode at least once. * Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: From 54b73152e6282149a02dc9740b2b3786ecf3e09d Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 21 Sep 2018 11:58:56 -0700 Subject: [PATCH 085/244] Style updates --- Readme.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Readme.md b/Readme.md index 9338be9133..3544184330 100644 --- a/Readme.md +++ b/Readme.md @@ -54,29 +54,29 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ## Building VFS for Git on Mac -VFS for macOS is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. +VFS for Git on Mac is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. * Ensure you have Xcode installed, have accepted the terms of use, and launched Xcode at least once. -* Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). +* Disable the "System Integrity Protection" (for loading unsigned Kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: ``` csrutil disable ``` - to disable SIP. + to disable SIP. Then click the Apple logo in the top left and restart. * Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). -* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) +* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it]. You can check what version you have with `dotnet --version`.(https://www.microsoft.com/net/download/dotnet-core/2.1) -* If you are not currently an Apple Developer you will need to become one so that you can either use Microsoft certs or create your own certs for signing purposes. (If you are a Microsoft employee you must use your alias@microsoft.com when creating your Apple account.) +* You will need to manage and sign your own certificate. [Apple Developer Cert Docs](https://developer.apple.com/documentation/security/certificate_key_and_trust_services/certificates). -* Create a `VFS` directory and Clone the VFS repo into a directory called `src` inside it: +* Create a `VFS` directory and Clone VFSForGit into a directory called `src` inside it: ``` mkdir VFS && cd VFS && git clone https://github.com/Microsoft/VFSForGit.git src && cd src ``` -* Now `cd` back into your VFS src directory and run +* From the src directory run ``` Scripts/Mac/BuildGVFSForMac.sh [Release] @@ -96,7 +96,11 @@ VFS for macOS is still in progress. You can build it, but this will not create a shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install Xcode and then launch it (you can close it afterwards.) -* Now setup the git credential manager by running +* Prep your machine to use VFS for Git. The following are all done by the script below.wwwww + * install Homebrew + * install and setup the git credential manager (with `brew`) + * install/update Java (with `brew`) + * install a VFS for Git aware version of Git ``` Scripts/Mac/PrepFunctionalTests.sh @@ -124,7 +128,7 @@ VFS for macOS is still in progress. You can build it, but this will not create a gvfs clone URL_TO_REPOSITORY --local-cache-path ~/.gvfsCache ``` - Note the current use of `--local-cache-path`. That argument prevent gvfs from running into a permissions error trying to put the cache at the root of your Mac's hard-drive. + Note the current use of `--local-cache-path`. That argument prevents VFS for Git from running into a permissions error trying to put the cache at the root of your Mac's hard-drive because porting of this feature is still in progress. ## Trying out VFS for Git From 4fe765d05f4e78b71a96737cf93437e5b6686663 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 21 Sep 2018 15:27:40 -0700 Subject: [PATCH 086/244] Simplify steps language --- Readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Readme.md b/Readme.md index 3544184330..498a30433d 100644 --- a/Readme.md +++ b/Readme.md @@ -112,9 +112,9 @@ VFS for Git on Mac is still in progress. You can build it, but this will not cre ProjFS.Mac/Scripts/LoadPrjFSKext.sh [Release] ``` -* Now you can put your built gvfs program on your path. A simple way to do that is by adding `Path/to/VFS/Publish` to your path. +* Add your built gvfs program to your path. A simple way to do that is by adding `Path/to/VFS/Publish` to your path. - Check you have it by running + Confirm you have it by running ``` command -v gvfs @@ -122,7 +122,7 @@ VFS for Git on Mac is still in progress. You can build it, but this will not cre You should see a path to the gvfs executable. -* Try cloning! Now that you have `gvfs` ready you can try cloning a VFS for Git enabled repository! +* Try cloning a VFS for Git enabled repository! ``` gvfs clone URL_TO_REPOSITORY --local-cache-path ~/.gvfsCache From b824382829fb9424e55bfba40015ec320db9b464 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Wed, 26 Sep 2018 13:04:50 -0700 Subject: [PATCH 087/244] Update grammar issues --- Readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index 498a30433d..43dce75f2b 100644 --- a/Readme.md +++ b/Readme.md @@ -54,9 +54,9 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ## Building VFS for Git on Mac -VFS for Git on Mac is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. +Note that VFS for Git on Mac is under active development. -* Ensure you have Xcode installed, have accepted the terms of use, and launched Xcode at least once. +* Ensure you have Xcode installed, have accepted the terms of use, and have launched Xcode at least once. * Disable the "System Integrity Protection" (for loading unsigned Kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: @@ -71,9 +71,9 @@ VFS for Git on Mac is still in progress. You can build it, but this will not cre * You will need to manage and sign your own certificate. [Apple Developer Cert Docs](https://developer.apple.com/documentation/security/certificate_key_and_trust_services/certificates). -* Create a `VFS` directory and Clone VFSForGit into a directory called `src` inside it: +* Create a `VFSForGit` directory and Clone VFSForGit into a directory called `src` inside it: ``` - mkdir VFS && cd VFS && git clone https://github.com/Microsoft/VFSForGit.git src && cd src + mkdir VFSForGit && cd VFSForGit && git clone https://github.com/Microsoft/VFSForGit.git src && cd src ``` * From the src directory run From 01b68ac525c741e31d3ac41e12e871a5b704efbb Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Wed, 26 Sep 2018 13:07:19 -0700 Subject: [PATCH 088/244] Fix Spelling --- Readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 43dce75f2b..1b87e3bbb2 100644 --- a/Readme.md +++ b/Readme.md @@ -59,7 +59,8 @@ Note that VFS for Git on Mac is under active development. * Ensure you have Xcode installed, have accepted the terms of use, and have launched Xcode at least once. * Disable the "System Integrity Protection" (for loading unsigned Kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). - Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: + Once booted into recovery mode open Utilities -> Terminal to launch a terminal. Enter: + ``` csrutil disable ``` From 53f1defea4b3b9fb2df385a97e3b637dd3a8b1d6 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 4 Oct 2018 12:32:06 -0700 Subject: [PATCH 089/244] More style fixes --- Readme.md | 56 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/Readme.md b/Readme.md index 1b87e3bbb2..d6ddc27fe2 100644 --- a/Readme.md +++ b/Readme.md @@ -58,29 +58,36 @@ Note that VFS for Git on Mac is under active development. * Ensure you have Xcode installed, have accepted the terms of use, and have launched Xcode at least once. -* Disable the "System Integrity Protection" (for loading unsigned Kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). - Once booted into recovery mode open Utilities -> Terminal to launch a terminal. Enter: - - ``` - csrutil disable - ``` - to disable SIP. Then click the Apple logo in the top left and restart. - * Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). * If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it]. You can check what version you have with `dotnet --version`.(https://www.microsoft.com/net/download/dotnet-core/2.1) -* You will need to manage and sign your own certificate. [Apple Developer Cert Docs](https://developer.apple.com/documentation/security/certificate_key_and_trust_services/certificates). +* You will need to manage and sign your own certificate. + + If you're using Xcode for the first time, you may have to login to Xcode with your Apple ID to generate a codesigning certificate. You can do this by launching Xcode.app, opening the PrjFS.xcworkspace and trying to build. You can find the signing options in the General tab of the project's settings. * Create a `VFSForGit` directory and Clone VFSForGit into a directory called `src` inside it: ``` - mkdir VFSForGit && cd VFSForGit && git clone https://github.com/Microsoft/VFSForGit.git src && cd src + mkdir VFSForGit + cd VFSForGit + git clone https://github.com/Microsoft/VFSForGit.git src + cd src + ``` + +* Prep your machine to use VFS for Git. The following are all done by the script below. + * install Homebrew + * install and setup the Git Credential Manager (with `brew`) + * install/update Java (with `brew`) + * install a VFS for Git aware version of Git + + ``` + Scripts/Mac/PrepFunctionalTests.sh ``` * From the src directory run ``` - Scripts/Mac/BuildGVFSForMac.sh [Release] + Scripts/Mac/BuildGVFSForMac.sh [Debug|Release] ``` _Troubleshooting if this fails_ @@ -97,23 +104,32 @@ Note that VFS for Git on Mac is under active development. shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install Xcode and then launch it (you can close it afterwards.) -* Prep your machine to use VFS for Git. The following are all done by the script below.wwwww - * install Homebrew - * install and setup the git credential manager (with `brew`) - * install/update Java (with `brew`) - * install a VFS for Git aware version of Git +* For the time being, only for active development, you will have to disable the SIP (System Integrity Protection) in order to load the kext). + + **This is dangerous and very bad for the security of your machine. Do not do this on any production machine! If you no longer need to developer VFS for Git on Mac we recommend re-enabling SIP ASAP.** + + To disable SIP boot into recovery mode (`[Win/⌘] + R` while booting your Mac). + Once booted into recovery mode open Utilities -> Terminal to launch a terminal. Enter: ``` - Scripts/Mac/PrepFunctionalTests.sh + csrutil disable + # use "csrutil enable" to re-enable when you no longer need to build VFS for Git on Mac ``` + Then click the Apple logo in the top left and restart. * Now you have to load the ProjFS Kext. ``` - ProjFS.Mac/Scripts/LoadPrjFSKext.sh [Release] + ProjFS.Mac/Scripts/LoadPrjFSKext.sh [Debug|Release] + ``` + +* Add your built VFS for Git executable (`gvfs`) program to your path. A simple way to do that is by adding + + ``` + /VFSForGit/Publish ``` -* Add your built gvfs program to your path. A simple way to do that is by adding `Path/to/VFS/Publish` to your path. + to your `PATH`. Confirm you have it by running @@ -129,7 +145,7 @@ Note that VFS for Git on Mac is under active development. gvfs clone URL_TO_REPOSITORY --local-cache-path ~/.gvfsCache ``` - Note the current use of `--local-cache-path`. That argument prevents VFS for Git from running into a permissions error trying to put the cache at the root of your Mac's hard-drive because porting of this feature is still in progress. + Note the current use of `--local-cache-path`. Without this argument VFS for Git will encounter a permissions error when it attempts to create its cache at the root of your hard-drive. Automatic picking of the cache path has not yet been ported to VFS for Git on Mac. ## Trying out VFS for Git From 04b92037a4697d98e29e81d9aedd75a006590285 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 4 Oct 2018 13:11:00 -0700 Subject: [PATCH 090/244] More grammar resolutions --- Readme.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Readme.md b/Readme.md index d6ddc27fe2..ca80a3f571 100644 --- a/Readme.md +++ b/Readme.md @@ -62,9 +62,7 @@ Note that VFS for Git on Mac is under active development. * If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it]. You can check what version you have with `dotnet --version`.(https://www.microsoft.com/net/download/dotnet-core/2.1) -* You will need to manage and sign your own certificate. - - If you're using Xcode for the first time, you may have to login to Xcode with your Apple ID to generate a codesigning certificate. You can do this by launching Xcode.app, opening the PrjFS.xcworkspace and trying to build. You can find the signing options in the General tab of the project's settings. +* If you're using Xcode for the first time, you may have to login to Xcode with your Apple ID to generate a codesigning certificate. You can do this by launching Xcode.app, opening the PrjFS.xcworkspace and trying to build. You can find the signing options in the General tab of the project's settings. * Create a `VFSForGit` directory and Clone VFSForGit into a directory called `src` inside it: ``` @@ -104,9 +102,9 @@ Note that VFS for Git on Mac is under active development. shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install Xcode and then launch it (you can close it afterwards.) -* For the time being, only for active development, you will have to disable the SIP (System Integrity Protection) in order to load the kext). +* In order to build VFS for Git on Mac (and PrjFSKext) you will have to disable the SIP (System Integrity Protection) in order to load the kext). - **This is dangerous and very bad for the security of your machine. Do not do this on any production machine! If you no longer need to developer VFS for Git on Mac we recommend re-enabling SIP ASAP.** + **This is dangerous and very bad for the security of your machine. Do not do this on any production machine! If you no longer need to develop VFS for Git on Mac we recommend re-enabling SIP ASAP.** To disable SIP boot into recovery mode (`[Win/⌘] + R` while booting your Mac). Once booted into recovery mode open Utilities -> Terminal to launch a terminal. Enter: From 718b5d0464d47df28e267b52c472f699c2682f1f Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Thu, 4 Oct 2018 14:25:01 -0400 Subject: [PATCH 091/244] Add original description on NamedPipe protocol --- GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs index 83ac690e3d..07c2bff4a9 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs @@ -6,6 +6,22 @@ namespace GVFS.Common.NamedPipes { + /// + /// The server side of a Named Pipe used for interprocess communication. + /// + /// Named Pipe protocol: + /// The client / server process sends a "message" (or line) of data as a + /// sequence of bytes terminated by a 0x3 byte (ASCII control code for + /// End of text). The sender of a message must wait for a response from + /// the other side before sending another message. Text is encoded as + /// UTF-8 to be sent as bytes across the wire. + /// + /// This format was chosen so that: + /// 1) A reasonable range of values can be transmitted across the pipe, + /// including null and bytes that represent newline characters. + /// 2) It would be easy to implement in multiple places, as we + /// have managed and native implementations. + /// public class NamedPipeServer : IDisposable { // TODO(Mac) the limit is much shorter on macOS From 6f00c207c062c70a1d1fae45104beb6bb5429ad1 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 4 Oct 2018 11:10:26 -0400 Subject: [PATCH 092/244] SecondCloneSucceedsWithMissingTrees: Attempt 2 The previous approach to this test relied on deleting packfiles from the shared object cache. That failed on Windows because the .idx files have restrictive permissions that cause the delete to fail. Instead, use the loose object downloads to hydrate the commit and root tree for the WellKnownBranch and then do a clone pointing at that branch. This should trigger the failure case. --- .../MultiEnlistmentTests/SharedCacheTests.cs | 19 +++++++++++++++++++ .../TestsWithMultiEnlistment.cs | 11 +++++++++-- .../Tools/GVFSFunctionalTestEnlistment.cs | 8 ++++++-- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index 7ec39205a1..f4026dccc1 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -268,6 +268,25 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() this.AlternatesFileShouldHaveGitObjectsRoot(enlistment); } + [TestCase] + public void SecondCloneSucceedsWithMissingTrees() + { + string newCachePath = Path.Combine(this.localCacheParentPath, ".customGvfsCache2"); + GVFSFunctionalTestEnlistment enlistment1 = this.CreateNewEnlistment(localCacheRoot: newCachePath, skipPrefetch: true); + File.ReadAllText(Path.Combine(enlistment1.RepoRoot, WellKnownFile)); + this.AlternatesFileShouldHaveGitObjectsRoot(enlistment1); + + // This Git command loads the commit and root tree for WellKnownCommitSha, + // but does not download any more reachable objects. + string command = "cat-file -p origin/" + WellKnownBranch + "^{tree}"; + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo(enlistment1.RepoRoot, command); + result.ExitCode.ShouldEqual(0, $"git {command} failed with error: " + result.Errors); + + // If we did not properly check the failed checkout at this step, then clone will fail during checkout. + GVFSFunctionalTestEnlistment enlistment2 = this.CreateNewEnlistment(localCacheRoot: newCachePath, branch: WellKnownBranch, skipPrefetch: true); + File.ReadAllText(Path.Combine(enlistment2.RepoRoot, WellKnownFile)); + } + // Override OnTearDownEnlistmentsDeleted rathern than using [TearDown] as the enlistments need to be unmounted before // localCacheParentPath can be deleted (as the SQLite blob sizes database cannot be deleted while GVFS is mounted) protected override void OnTearDownEnlistmentsDeleted() diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs index f0afd93995..3c9a353272 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs @@ -28,9 +28,16 @@ protected virtual void OnTearDownEnlistmentsDeleted() { } - protected GVFSFunctionalTestEnlistment CreateNewEnlistment(string localCacheRoot = null, string branch = null) + protected GVFSFunctionalTestEnlistment CreateNewEnlistment( + string localCacheRoot = null, + string branch = null, + bool skipPrefetch = false) { - GVFSFunctionalTestEnlistment output = GVFSFunctionalTestEnlistment.CloneAndMount(GVFSTestConfig.PathToGVFS, branch, localCacheRoot); + GVFSFunctionalTestEnlistment output = GVFSFunctionalTestEnlistment.CloneAndMount( + GVFSTestConfig.PathToGVFS, + branch, + localCacheRoot, + skipPrefetch); this.enlistmentsToDelete.Add(output); return output; } diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index 427fef7b28..eb6ea827cb 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -89,10 +89,14 @@ public static GVFSFunctionalTestEnlistment CloneAndMountWithPerRepoCache(string return CloneAndMount(pathToGvfs, enlistmentRoot, null, localCache, skipPrefetch); } - public static GVFSFunctionalTestEnlistment CloneAndMount(string pathToGvfs, string commitish = null, string localCacheRoot = null) + public static GVFSFunctionalTestEnlistment CloneAndMount( + string pathToGvfs, + string commitish = null, + string localCacheRoot = null, + bool skipPrefetch = false) { string enlistmentRoot = GVFSFunctionalTestEnlistment.GetUniqueEnlistmentRoot(); - return CloneAndMount(pathToGvfs, enlistmentRoot, commitish, localCacheRoot); + return CloneAndMount(pathToGvfs, enlistmentRoot, commitish, localCacheRoot, skipPrefetch); } public static GVFSFunctionalTestEnlistment CloneAndMountEnlistmentWithSpacesInPath(string pathToGvfs, string commitish = null) From 43287ecf8d7f821a2e286fc6c67a2e1eaed2c0c2 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 4 Oct 2018 13:41:36 -0400 Subject: [PATCH 093/244] SharedCacheTests: Update WellKnownCommitSha This doesn't match, as we expect it to! --- .../Tests/MultiEnlistmentTests/SharedCacheTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index f4026dccc1..c7b3072f2a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -21,7 +21,7 @@ public class SharedCacheTests : TestsWithMultiEnlistment // This branch and commit sha should point to the same place. private const string WellKnownBranch = "FunctionalTests/20170602"; - private const string WellKnownCommitSha = "79dc4233df4d9a7e053662bff95df498f640022e"; + private const string WellKnownCommitSha = "42eb6632beffae26893a3d6e1a9f48d652327c6f"; private string localCachePath; private string localCacheParentPath; From 02d13b7ea0fb456c27573e9949df9879f349280b Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Thu, 4 Oct 2018 11:22:36 -0400 Subject: [PATCH 094/244] - Fixing merge issues with master - Added NoConfig RingType. NoConfig & None rings would be similar. During upgrade no error messaging will be displayed. RingType Invalid would continue to be treated as error. - Rephrased mount error messaging. - Making gvfs config keys case-insensitive - Removed spinner for "Launching upgrade tool" - Show pre-upgrade warning about repos being unmounted & mounted when upgrade is run from non-elevated command prompt as well - Log version info from UpgradeVerb after successfull upgrade check. --- GVFS/GVFS.Common/GVFSConstants.cs | 6 ++- GVFS/GVFS.Common/LocalGVFSConfig.cs | 4 +- GVFS/GVFS.Common/ProductUpgrader.cs | 17 +++++++-- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 16 ++++---- GVFS/GVFS/CommandLine/ConfigVerb.cs | 10 ++--- GVFS/GVFS/CommandLine/GVFSVerb.cs | 2 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 46 +++++++---------------- start | 1 - 8 files changed, 45 insertions(+), 57 deletions(-) delete mode 100644 start diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 393004b5c1..a387f89376 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -229,8 +229,10 @@ public static class UpgradeVerbMessages public const string GVFSUpgrade = "`gvfs upgrade`"; public const string GVFSUpgradeConfirm = "`gvfs upgrade --confirm`"; public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; - public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; - public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; + public const string NoUpgradeCheckPerformed = "No upgrade check was performed."; + public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". " + NoUpgradeCheckPerformed; + public const string NoRingConfigConsoleAlert = "Upgrade ring is not set. " + NoUpgradeCheckPerformed; + public const string InvalidRingConsoleAlert = "Upgrade ring set to unknown value. " + NoUpgradeCheckPerformed; public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"]` from a command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "Upgrade will unmount and remount gvfs repos, ensure you are at a stopping point."; diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index 722b280811..980d5956ac 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -35,7 +35,7 @@ public bool TryGetConfig( try { - this.allSettings.TryGetValue(key, out value); + this.allSettings.TryGetValue(key.ToUpper(), out value); error = null; return true; } @@ -67,7 +67,7 @@ public bool TrySetConfig( try { - this.allSettings.SetValueAndFlush(key, value); + this.allSettings.SetValueAndFlush(key.ToUpper(), value); error = null; return true; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index 33a584fe8b..636278ea8c 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -59,12 +59,16 @@ public ProductUpgrader( public enum RingType { - // The values here should be ascending. + // The values here should be ascending. + // Invalid - User has set an incorrect ring + // NoConfig - User has Not set any ring yet + // None - User has set a valid "None" ring // (Fast should be greater than Slow, // Slow should be greater than None, None greater than Invalid.) // This is required for the correct implementation of Ring based // upgrade logic. Invalid = 0, + NoConfig = None - 1, None = 10, Slow = None + 1, Fast = Slow + 1, @@ -281,13 +285,18 @@ public virtual bool TryLoadRingConfig(out string error) error = null; return true; } - else + + if (string.IsNullOrEmpty(ringConfig)) { - error = "Invalid upgrade ring `" + ringConfig + "` specified in gvfs config."; + this.Ring = RingType.NoConfig; + error = null; + return true; } + + error = "Invalid upgrade ring `" + ringConfig + "` specified in gvfs config." + Environment.NewLine; } - error += Environment.NewLine + GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand; + error += GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand; this.Ring = RingType.Invalid; return false; } diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index df18ceb2dc..66a729269f 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -62,17 +62,17 @@ public void Execute() ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; string mountError = null; - if (!this.TryLoadUpgradeRing(out ring, out error) || ring == ProductUpgrader.RingType.None) + if (!this.TryLoadUpgradeRing(out ring, out error)) + { + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.InvalidRingConsoleAlert); + } + else if (ring == ProductUpgrader.RingType.None || ring == ProductUpgrader.RingType.NoConfig) { string message = ring == ProductUpgrader.RingType.None ? GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert : - GVFSConstants.UpgradeVerbMessages.InvalidRingConsoleAlert; + GVFSConstants.UpgradeVerbMessages.NoRingConfigConsoleAlert; this.output.WriteLine(message); - - if (ring == ProductUpgrader.RingType.None) - { - this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); - } + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); } else { @@ -103,7 +103,7 @@ public void Execute() } else { - this.output.WriteLine($"{Environment.NewLine}{(string.IsNullOrEmpty(mountError) ? "U" : "Repository mount failed. But u")}pgrade completed successfully!"); + this.output.WriteLine($"{Environment.NewLine}Upgrade completed successfully{(string.IsNullOrEmpty(mountError) ? "." : ", but one or more repositories will need to be mounted manually.")}"); } if (this.input == Console.In) diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index c978888dab..6c64eb3d44 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -13,17 +13,15 @@ public class ConfigVerb : GVFSVerb.ForNoEnlistment [Value( 0, Required = true, - Default = "", - MetaName = "GVFS config setting name", - HelpText = "Name of GVFS config setting that is to be set or read")] + MetaName = "Setting name", + HelpText = "Name of setting that is to be set or read")] public string Key { get; set; } [Value( 1, Required = false, - Default = "", - MetaName = "GVFS config setting value", - HelpText = "Value of GVFS config setting to be set")] + MetaName = "Setting value", + HelpText = "Value of setting to be set")] public string Value { get; set; } protected override string VerbName diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index d38a28ed44..56b2eca612 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -260,7 +260,7 @@ protected ServerGVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlist { this.ReportErrorAndExit(tracer, "Unable to query /gvfs/config"); } - + return serverGVFSConfig; } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index edd1af8c37..b26cd1e661 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -112,10 +112,10 @@ private bool TryRunProductUpgrade() return false; } - if (ring == ProductUpgrader.RingType.None) + if (ring == ProductUpgrader.RingType.None || ring == ProductUpgrader.RingType.NoConfig) { this.tracer.RelatedInfo($"{nameof(this.TryRunProductUpgrade)}: {GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert}"); - this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); + this.ReportInfoToConsole(ring == ProductUpgrader.RingType.None ? GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert : GVFSConstants.UpgradeVerbMessages.NoRingConfigConsoleAlert); this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); return true; } @@ -154,23 +154,11 @@ private bool TryRunProductUpgrade() } else { - if (isInstallable) - { - string message = string.Join( - Environment.NewLine, + string message = string.Join( + Environment.NewLine, GVFSConstants.UpgradeVerbMessages.UnmountRepoWarning, GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); - this.ReportInfoToConsole(upgradeAvailableMessage + Environment.NewLine + Environment.NewLine + message + Environment.NewLine); - } - else - { - this.ReportInfoToConsole($"{Environment.NewLine}{upgradeAvailableMessage}"); - - if (!string.IsNullOrEmpty(cannotInstallReason)) - { - this.ReportInfoToConsole($"{Environment.NewLine}{cannotInstallReason}"); - } - } + this.ReportInfoToConsole(upgradeAvailableMessage + Environment.NewLine + Environment.NewLine + message + Environment.NewLine); } return true; @@ -221,23 +209,15 @@ private bool TryRunInstaller(out string consoleError) string upgraderPath = null; string errorMessage = null; - bool preUpgradeSuccess = this.ShowStatusWhileRunning( - () => - { - if (this.TryCopyUpgradeTool(out upgraderPath, out errorMessage) && - this.TryLaunchUpgradeTool(upgraderPath, out errorMessage)) - { - return true; - } + this.ReportInfoToConsole("Launching upgrade tool..."); - return false; - }, - "Launching upgrade tool", - suppressGvfsLogMessage: true); - - if (!preUpgradeSuccess) + if (!this.TryCopyUpgradeTool(out upgraderPath, out consoleError)) + { + return false; + } + + if (!this.TryLaunchUpgradeTool(upgraderPath, out errorMessage)) { - consoleError = errorMessage; return false; } @@ -310,7 +290,7 @@ private bool TryCheckUpgradeAvailable( latestVersion = version; - activity.RelatedInfo("Successfully checked server for GVFS upgrades."); + activity.RelatedInfo($"Successfully checked server for GVFS upgrades. New version available {latestVersion}"); } return true; diff --git a/start b/start deleted file mode 100644 index c342dc54f5..0000000000 --- a/start +++ /dev/null @@ -1 +0,0 @@ -prjflt From 5dc0e3517446bcf9e2bf624ece6cf5378740919d Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 4 Oct 2018 14:27:43 -0700 Subject: [PATCH 095/244] Remove autogenerated Xcode comment --- ProjFS.Mac/PrjFSKext/public/FsidInode.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/public/FsidInode.h b/ProjFS.Mac/PrjFSKext/public/FsidInode.h index caf2f7715e..d05bb26469 100644 --- a/ProjFS.Mac/PrjFSKext/public/FsidInode.h +++ b/ProjFS.Mac/PrjFSKext/public/FsidInode.h @@ -1,11 +1,3 @@ -// -// FsidInode.h -// PrjFSKext -// -// Created by William Baker on 10/4/18. -// Copyright © 2018 GVFS. All rights reserved. -// - #ifndef FsidInode_h #define FsidInode_h From 253c9f5c7b58cf606ca66aead2ac55fdafdad348 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 4 Oct 2018 14:45:51 -0700 Subject: [PATCH 096/244] Mac: Update MoveAndOverwriteFile to use native API --- GVFS/GVFS.Common/NativeMethods.Shared.cs | 10 +++++----- GVFS/GVFS.Platform.Mac/MacFileSystem.cs | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.Common/NativeMethods.Shared.cs b/GVFS/GVFS.Common/NativeMethods.Shared.cs index e7ed7f2864..7a796b15e6 100644 --- a/GVFS/GVFS.Common/NativeMethods.Shared.cs +++ b/GVFS/GVFS.Common/NativeMethods.Shared.cs @@ -143,6 +143,11 @@ public static string GetFinalPathName(string path) } } + public static void ThrowLastWin32Exception(string message) + { + throw new Win32Exception(Marshal.GetLastWin32Error(), message); + } + [DllImport("kernel32.dll", SetLastError = true)] public static extern SafeFileHandle OpenProcess( ProcessAccessFlags processAccess, @@ -153,11 +158,6 @@ public static extern SafeFileHandle OpenProcess( [return: MarshalAs(UnmanagedType.Bool)] public static extern bool GetExitCodeProcess(SafeFileHandle hProcess, out uint lpExitCode); - private static void ThrowLastWin32Exception(string message) - { - throw new Win32Exception(Marshal.GetLastWin32Error(), message); - } - [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] private static extern SafeFileHandle CreateFile( [In] string lpFileName, diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs index 4e72868a9a..96dec6a59b 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs @@ -1,4 +1,5 @@ -using GVFS.Common.FileSystem; +using GVFS.Common; +using GVFS.Common.FileSystem; using System.IO; using System.Runtime.InteropServices; @@ -15,13 +16,10 @@ public void FlushFileBuffers(string path) public void MoveAndOverwriteFile(string sourceFileName, string destinationFilename) { - // TODO(Mac): Use native API - if (File.Exists(destinationFilename)) + if (Rename(sourceFileName, destinationFilename) != 0) { - File.Delete(destinationFilename); + NativeMethods.ThrowLastWin32Exception($"Failed to renname {sourceFileName} to {destinationFilename}"); } - - File.Move(sourceFileName, destinationFilename); } public void CreateHardLink(string newFileName, string existingFileName) @@ -42,5 +40,8 @@ public bool TryGetNormalizedPath(string path, out string normalizedPath, out str [DllImport("libc", EntryPoint = "chmod", SetLastError = true)] private static extern int Chmod(string pathname, int mode); + + [DllImport("libc", EntryPoint = "rename", SetLastError = true)] + private static extern int Rename(string oldPath, string newPath); } } From 7aebab2206759401ab72b6adefeeb91ffb491db5 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Thu, 4 Oct 2018 18:02:14 -0400 Subject: [PATCH 097/244] - Fixing UT failure. Launching upgrade tool was moved out of the spinner, UT was still expecting the Succeeded message. --- .../GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs index f5e5604659..6edb15419d 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -81,7 +81,7 @@ public void LaunchInstaller() expectedOutput: new List { "New GVFS version " + NewerThanLocalVersion + " available in ring Slow", - "Launching upgrade tool...Succeeded" + "Launching upgrade tool..." }, expectedErrors:null); From df5ea192dbafe0c2f27f9b4eddc8908570229760 Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Thu, 4 Oct 2018 14:00:40 -0400 Subject: [PATCH 098/244] Simplify NamedPipeStreamReader logic The existing NamedPipe protocol expected that the sender of a message would wait for a response from the receiver before sending another message. It did not allow for multiple messages to be sent without the client responding. However, this is not a correct assumption, as there is at least 1 place where a component will send multiple messages without waiting for a response from the receiver (#333). This change updates the NamedPipeStreamReader to handle multiple messages sent sequentially, and simplifies the overall logic in the process. Tests are also included which exhibit this behavior. --- .../GVFS.Common/NamedPipes/NamedPipeServer.cs | 4 +- .../NamedPipes/NamedPipeStreamReader.cs | 63 +++++++------------ .../NamedPipeStreamReaderWriterTests.cs | 61 +++++++++--------- 3 files changed, 56 insertions(+), 72 deletions(-) diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs index 07c2bff4a9..5f93c602d9 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs @@ -12,9 +12,7 @@ namespace GVFS.Common.NamedPipes /// Named Pipe protocol: /// The client / server process sends a "message" (or line) of data as a /// sequence of bytes terminated by a 0x3 byte (ASCII control code for - /// End of text). The sender of a message must wait for a response from - /// the other side before sending another message. Text is encoded as - /// UTF-8 to be sent as bytes across the wire. + /// End of text). Text is encoded as UTF-8 to be sent as bytes across the wire. /// /// This format was chosen so that: /// 1) A reasonable range of values can be transmitted across the pipe, diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs index 17ced2e205..63e8f5f69e 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs @@ -11,23 +11,16 @@ namespace GVFS.Common.NamedPipes /// public class NamedPipeStreamReader { - private const int DefaultBufferSize = 1024; + private const int InitialListSize = 1024; private const byte TerminatorByte = 0x3; + private readonly byte[] buffer; - private int bufferSize; - private byte[] buffer; private Stream stream; - public NamedPipeStreamReader(Stream stream, int bufferSize) - { - this.stream = stream; - this.bufferSize = bufferSize; - this.buffer = new byte[this.bufferSize]; - } - public NamedPipeStreamReader(Stream stream) - : this(stream, DefaultBufferSize) { + this.stream = stream; + this.buffer = new byte[1]; } /// @@ -36,38 +29,23 @@ public NamedPipeStreamReader(Stream stream) /// The message read from the stream, or null if the end of the input stream has been reached. public string ReadMessage() { - int bytesRead = this.Read(); - if (bytesRead == 0) + byte currentByte; + + bool streamOpen = this.TryReadByte(out currentByte); + if (!streamOpen) { // The end of the stream has been reached - return null to indicate this. return null; } - // If we have read in the entire message in the first read (mainline scenario), - // then just process the data directly from the buffer. - if (this.buffer[bytesRead - 1] == TerminatorByte) - { - return Encoding.UTF8.GetString(this.buffer, 0, bytesRead - 1); - } + List bytes = new List(InitialListSize); - // We need to process multiple chunks - collect data from multiple chunks - // into a single list - List bytes = new List(this.bufferSize * 2); - - while (true) + do { - bool encounteredTerminatorByte = this.buffer[bytesRead - 1] == TerminatorByte; - int lengthToCopy = encounteredTerminatorByte ? bytesRead - 1 : bytesRead; + bytes.Add(currentByte); + streamOpen = this.TryReadByte(out currentByte); - bytes.AddRange(new ArraySegment(this.buffer, 0, lengthToCopy)); - if (encounteredTerminatorByte) - { - break; - } - - bytesRead = this.Read(); - - if (bytesRead == 0) + if (!streamOpen) { // We have read a partial message (the last byte received does not indicate that // this was the end of the message), but the stream has been closed. Throw an exception @@ -76,17 +54,24 @@ public string ReadMessage() throw new IOException("Incomplete message read from stream. The end of the stream was reached without the expected terminating byte."); } } + while (currentByte != TerminatorByte); return Encoding.UTF8.GetString(bytes.ToArray()); } /// - /// Read the next chunk of bytes from the stream. + /// Read a byte from the stream. /// - /// The number of bytes read. - private int Read() + /// The byte read from the stream + /// True if byte read, false if end of stream has been reached + private bool TryReadByte(out byte readByte) { - return this.stream.Read(this.buffer, 0, this.buffer.Length); + this.buffer[0] = 0; + + int numBytesRead = this.stream.Read(this.buffer, 0, 1); + readByte = this.buffer[0]; + + return numBytesRead == 1; } } } diff --git a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs index 7e630fea6a..36acb1d373 100644 --- a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs +++ b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs @@ -9,8 +9,6 @@ namespace GVFS.UnitTests.Common [TestFixture] public class NamedPipeStreamReaderWriterTests { - private const int BufferSize = 256; - private MemoryStream stream; private NamedPipeStreamWriter streamWriter; private NamedPipeStreamReader streamReader; @@ -20,11 +18,10 @@ public void Setup() { this.stream = new MemoryStream(); this.streamWriter = new NamedPipeStreamWriter(this.stream); - this.streamReader = new NamedPipeStreamReader(this.stream, BufferSize); + this.streamReader = new NamedPipeStreamReader(this.stream); } [Test] - [Description("Verify that we can transmit multiple messages")] public void CanWriteAndReadMessages() { string firstMessage = @"This is a new message"; @@ -41,24 +38,6 @@ public void CanWriteAndReadMessages() } [Test] - [Description("Verify that we can transmit a message that contains content that is the size of a NamedPipeStreamReader's buffer")] - public void CanSendBufferSizedContent() - { - string longMessage = new string('T', BufferSize); - this.TestTransmitMessage(longMessage); - } - - [Test] - [Description("Verify that we can transmit message that is the same size a NamedPipeStreamReader's buffer")] - public void CanSendBufferSizedMessage() - { - int numBytesInMessageTerminator = 1; - string longMessage = new string('T', BufferSize - numBytesInMessageTerminator); - this.TestTransmitMessage(longMessage); - } - - [Test] - [Description("Verify that the expected exception is thrown if message is not terminated with expected byte.")] [Category(CategoryConstants.ExceptionExpected)] public void ReadingPartialMessgeThrows() { @@ -71,19 +50,23 @@ public void ReadingPartialMessgeThrows() } [Test] - [Description("Verify that we can transmit message that is larger than the buffer")] - public void CanSendMultiBufferSizedMessage() + public void CanSendMessagesWithNewLines() { - string longMessage = new string('T', BufferSize * 3); - this.TestTransmitMessage(longMessage); + string messageWithNewLines = "This is a \nstringwith\nnewlines"; + this.TestTransmitMessage(messageWithNewLines); } [Test] - [Description("Verify that we can transmit message that newline characters")] - public void CanSendNewLines() + public void CanSendMultipleMessagesSequentially() { - string messageWithNewLines = "This is a \nstringwith\nnewlines"; - this.TestTransmitMessage(messageWithNewLines); + string[] messages = new string[] + { + "This is a new message", + "This is another message", + "This is the third message in a series of messages" + }; + + this.TestTransmitMessages(messages); } private void TestTransmitMessage(string message) @@ -96,6 +79,24 @@ private void TestTransmitMessage(string message) string readMessage = this.streamReader.ReadMessage(); readMessage.ShouldEqual(message, "The message read from the stream reader is not the same as the message that was sent."); } + + private void TestTransmitMessages(string[] messages) + { + long pos = this.ReadStreamPosition(); + + foreach (string message in messages) + { + this.streamWriter.WriteMessage(message); + } + + this.SetStreamPosition(pos); + + foreach (string message in messages) + { + string readMessage = this.streamReader.ReadMessage(); + readMessage.ShouldEqual(message, "The message read from the stream reader is not the same as the message that was sent."); + } + } private long ReadStreamPosition() { From 755dda6ff536e3a3471f0e8d1781edf40a40df80 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Mon, 1 Oct 2018 16:51:55 +0200 Subject: [PATCH 099/244] Mac kext: Handle named streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Occasionally, the vnode auth handler is called on a vnode representing a named stream/named fork. (Typically, the resource fork.) Forks don’t have their own xattrs, etc. so some of the logic we apply fails. We don’t care specifically about named forks, so treat any access to them as an access to the main file instead. Note: we need to balance the vnode_getparent() call with vnode_put() --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index a0ed82e25c..b0729babd9 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -260,6 +260,18 @@ static int HandleVnodeOperation( // arg2 is the (vnode_t) parent vnode int* kauthError = reinterpret_cast(arg3); int kauthResult = KAUTH_RESULT_DEFER; + bool putVnodeWhenDone = false; + + // A lot of our file checks such as attribute tests behave oddly if the vnode + // refers to a named fork/stream; apply the logic as if the vnode operation was + // occurring on the file itself. (/path/to/file/..namedfork/rsrc) + if (vnode_isnamedstream(currentVnode)) + { + vnode_t mainFileFork = vnode_getparent(currentVnode); + assert(NULLVP != mainFileFork); + currentVnode = mainFileFork; + putVnodeWhenDone = true; + } const char* vnodePath = nullptr; char vnodePathBuffer[PrjFSMaxPath]; @@ -384,6 +396,11 @@ static int HandleVnodeOperation( } CleanupAndReturn: + if (putVnodeWhenDone) + { + vnode_put(currentVnode); + } + atomic_fetch_sub(&s_numActiveKauthEvents, 1); return kauthResult; } From f147f8c02634c9d73299c4cac461783dfae1745a Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Fri, 28 Sep 2018 13:04:06 +0200 Subject: [PATCH 100/244] Mac kext: Replaces some C-style casts with C++ style. --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index b0729babd9..c782672781 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -425,7 +425,7 @@ static int HandleFileOpOperation( KAUTH_FILEOP_LINK == action) { // arg0 is the (const char *) fromPath (or the file being linked to) - const char* newPath = (const char*)arg1; + const char* newPath = reinterpret_cast(arg1); // TODO(Mac): We need to handle failures to lookup the vnode. If we fail to lookup the vnode // it's possible that we'll miss notifications @@ -481,7 +481,7 @@ static int HandleFileOpOperation( else if (KAUTH_FILEOP_CLOSE == action) { vnode_t currentVnode = reinterpret_cast(arg0); - const char* path = (const char*)arg1; + const char* path = reinterpret_cast(arg1); int closeFlags = static_cast(arg2); if (vnode_isdir(currentVnode)) From d21d165d15db312ee4b83a2fb3c8cf4d3f6707e4 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Mon, 1 Oct 2018 16:42:08 +0200 Subject: [PATCH 101/244] Mac kext: Enables warning for shadowed variables. Shadowing variables frequently is a source of bugs, so this enables the corresponding compiler warning. --- ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj index 74700506e7..dbd7e2453f 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj @@ -408,6 +408,7 @@ "$(inherited)", "MACH_ASSERT=1", ); + GCC_WARN_SHADOW = YES; INFOPLIST_FILE = PrjFSKext/Info.plist; MODULE_NAME = io.gvfs.PrjFSKext; MODULE_START = PrjFSKext_Start; @@ -430,7 +431,11 @@ buildSettings = { CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = UBF8T346G9; - GCC_PREPROCESSOR_DEFINITIONS = "MACH_ASSERT=1"; + GCC_PREPROCESSOR_DEFINITIONS = ( + "MACH_ASSERT=1", + "$(inherited)", + ); + GCC_WARN_SHADOW = YES; INFOPLIST_FILE = PrjFSKext/Info.plist; MODULE_NAME = io.gvfs.PrjFSKext; MODULE_START = PrjFSKext_Start; From e50e488ede2723162593b9a316a83915d29b1dae Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Mon, 1 Oct 2018 13:52:41 +0200 Subject: [PATCH 102/244] Mac kext: Adds typed array memory allocation function. --- ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp index eddfdf0fc5..0749a65533 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp @@ -7,4 +7,17 @@ kern_return_t Memory_Cleanup(); void* Memory_Alloc(uint32_t size); void Memory_Free(void* buffer, uint32_t size); +template +T* Memory_AllocArray(uint32_t arrayLength) +{ + size_t allocBytes = arrayLength * sizeof(T); + if (allocBytes > UINT32_MAX) + { + return nullptr; + } + + return static_cast(Memory_Alloc(static_cast(allocBytes))); +} + + #endif /* Memory_h */ From 3211ed1a87322fdf3173fc7a2c02527853f42674 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Fri, 28 Sep 2018 13:03:57 +0200 Subject: [PATCH 103/244] Mac kext: Use only VirtualizationRoot handles, not direct pointers This change moves the VirtualizationRoot structure definition out of the header file, and unifies the API to only use integer handles, which internally are array indices for non-negative values, and negative values from an enum of special cases. Where indices were previously used outside the VirtualizationRoot implementation functions, the terminology has been changed to "handles." An immediate advantage of the move away from pointers is an improvement in the ability to control thread safety of the root structures; additionally, unlike pointers, handles remain valid outside of held locks, so in a follow-on change, we can reallocate the array for resizing. --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 70 ++++---- .../PrjFSKext/PrjFSProviderUserClient.cpp | 20 ++- .../PrjFSKext/PrjFSProviderUserClient.hpp | 5 +- .../PrjFSKext/VirtualizationRoots.cpp | 155 ++++++++++++++---- .../PrjFSKext/VirtualizationRoots.hpp | 43 +++-- 5 files changed, 186 insertions(+), 107 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index c782672781..111c9355d8 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -42,11 +42,9 @@ static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); static bool IsFileSystemCrawler(char* procname); -static const char* GetRelativePath(const char* path, const char* root); - static void Sleep(int seconds, void* channel); static bool TrySendRequestAndWaitForResponse( - const VirtualizationRoot* root, + VirtualizationRootHandle root, MessageType messageType, const vnode_t vnode, const FsidInode& vnodeFsidInode, @@ -65,7 +63,7 @@ static bool ShouldHandleVnodeOpEvent( kauth_action_t action, // Out params: - VirtualizationRoot** root, + VirtualizationRootHandle* root, vtype* vnodeType, uint32_t* vnodeFileFlags, FsidInode* vnodeFsidInode, @@ -80,7 +78,7 @@ static bool ShouldHandleFileOpEvent( kauth_action_t action, // Out params: - VirtualizationRoot** root, + VirtualizationRootHandle* root, FsidInode* vnodeFsidInode, int* pid); @@ -277,7 +275,7 @@ static int HandleVnodeOperation( char vnodePathBuffer[PrjFSMaxPath]; int vnodePathLength = PrjFSMaxPath; - VirtualizationRoot* root = nullptr; + VirtualizationRootHandle root = RootHandle_None; vtype vnodeType; uint32_t currentVnodeFileFlags; FsidInode vnodeFsidInode; @@ -435,7 +433,7 @@ static int HandleFileOpOperation( goto CleanupAndReturn; } - VirtualizationRoot* root = nullptr; + VirtualizationRootHandle root = RootHandle_None; FsidInode vnodeFsidInode; int pid; if (!ShouldHandleFileOpEvent( @@ -495,7 +493,7 @@ static int HandleFileOpOperation( goto CleanupAndReturn; } - VirtualizationRoot* root = nullptr; + VirtualizationRootHandle root = RootHandle_None; FsidInode vnodeFsidInode; int pid; if (!ShouldHandleFileOpEvent( @@ -571,7 +569,7 @@ static bool ShouldHandleVnodeOpEvent( kauth_action_t action, // Out params: - VirtualizationRoot** root, + VirtualizationRootHandle* root, vtype* vnodeType, uint32_t* vnodeFileFlags, FsidInode* vnodeFsidInode, @@ -579,8 +577,8 @@ static bool ShouldHandleVnodeOpEvent( char procname[MAXCOMLEN + 1], int* kauthResult) { - *root = nullptr; *kauthResult = KAUTH_RESULT_DEFER; + *root = RootHandle_None; if (!VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode)) { @@ -626,19 +624,22 @@ static bool ShouldHandleVnodeOpEvent( } *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); - *root = VirtualizationRoots_FindForVnode(vnode, *vnodeFsidInode); + *root = VirtualizationRoot_FindForVnode(vnode, *vnodeFsidInode); - if (nullptr == *root) + if (RootHandle_ProviderTemporaryDirectory == *root) + { + *kauthResult = KAUTH_RESULT_DEFER; + return false; + } + else if (RootHandle_None == *root) { KextLog_FileNote(vnode, "No virtualization root found for file with set flag."); *kauthResult = KAUTH_RESULT_DEFER; return false; } - else if (nullptr == (*root)->providerUserClient) + else if (!VirtualizationRoot_IsOnline(*root)) { - // There is no registered provider for this root - // TODO(Mac): Protect files in the worktree from modification (and prevent // the creation of new files) when the provider is offline @@ -647,7 +648,7 @@ static bool ShouldHandleVnodeOpEvent( } // If the calling process is the provider, we must exit right away to avoid deadlocks - if (*pid == (*root)->providerPid) + if (VirtualizationRoot_PIDMatchesProvider(*root, *pid)) { *kauthResult = KAUTH_RESULT_DEFER; return false; @@ -663,10 +664,12 @@ static bool ShouldHandleFileOpEvent( kauth_action_t action, // Out params: - VirtualizationRoot** root, + VirtualizationRootHandle* root, FsidInode* vnodeFsidInode, int* pid) { + *root = RootHandle_None; + vtype vnodeType = vnode_vtype(vnode); if (ShouldIgnoreVnodeType(vnodeType, vnode)) { @@ -674,21 +677,17 @@ static bool ShouldHandleFileOpEvent( } *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); - *root = VirtualizationRoots_FindForVnode(vnode, *vnodeFsidInode); - if (nullptr == *root) - { - return false; - } - else if (nullptr == (*root)->providerUserClient) + *root = VirtualizationRoot_FindForVnode(vnode, *vnodeFsidInode); + if (!VirtualizationRoot_IsValidRootHandle(*root)) { - // There is no registered provider for this root + // This VNode is not part of a root return false; } - // If the calling process is the provider, we must exit right away to avoid deadlocks *pid = GetPid(context); - if (*pid == (*root)->providerPid) + if (VirtualizationRoot_PIDMatchesProvider(*root, *pid)) { + // If the calling process is the provider, we must exit right away to avoid deadlocks return false; } @@ -696,7 +695,7 @@ static bool ShouldHandleFileOpEvent( } static bool TrySendRequestAndWaitForResponse( - const VirtualizationRoot* root, + VirtualizationRootHandle root, MessageType messageType, const vnode_t vnode, const FsidInode& vnodeFsidInode, @@ -719,7 +718,7 @@ static bool TrySendRequestAndWaitForResponse( return false; } - const char* relativePath = GetRelativePath(vnodePath, root->path); + const char* relativePath = VirtualizationRoot_GetRootRelativePath(root, vnodePath); int nextMessageId = OSIncrementAtomic(&s_nextMessageId); @@ -748,7 +747,7 @@ static bool TrySendRequestAndWaitForResponse( // TODO(Mac): Should we pass in the root directly, rather than root->index? // The index seems more like a private implementation detail. - if (!isShuttingDown && 0 != ActiveProvider_SendMessage(root->index, messageSpec)) + if (!isShuttingDown && 0 != ActiveProvider_SendMessage(root, messageSpec)) { // TODO: appropriately handle unresponsive providers @@ -885,19 +884,6 @@ static bool IsFileSystemCrawler(char* procname) return false; } -static const char* GetRelativePath(const char* path, const char* root) -{ - assert(strlen(path) >= strlen(root)); - - const char* relativePath = path + strlen(root); - if (relativePath[0] == '/') - { - relativePath++; - } - - return relativePath; -} - static bool ShouldIgnoreVnodeType(vtype vnodeType, vnode_t vnode) { switch (vnodeType) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.cpp index 6e9b75427c..83b2fcb19b 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.cpp @@ -19,7 +19,11 @@ static const IOExternalMethodDispatch ProviderUserClientDispatch[] = { [ProviderSelector_RegisterVirtualizationRootPath] = { - &PrjFSProviderUserClient::registerVirtualizationRoot, 0, kIOUCVariableStructureSize, 1, 0 + .function = &PrjFSProviderUserClient::registerVirtualizationRoot, + .checkScalarInputCount = 0, + .checkStructureInputSize = kIOUCVariableStructureSize, // null-terminated string: virtualisation root path + .checkScalarOutputCount = 1, // returned errno + .checkStructureOutputSize = 0 }, [ProviderSelector_KernelMessageResponse] = { @@ -37,7 +41,7 @@ bool PrjFSProviderUserClient::initWithTask( UInt32 type, OSDictionary* properties) { - this->virtualizationRootIndex = -1; + this->virtualizationRootHandle = RootHandle_None; this->pid = proc_selfpid(); if (!this->super::initWithTask(owningTask, securityToken, type, properties)) @@ -92,9 +96,9 @@ void PrjFSProviderUserClient::free() // the connection. IOReturn PrjFSProviderUserClient::clientClose() { - int32_t root = this->virtualizationRootIndex; - this->virtualizationRootIndex = -1; - if (-1 != root) + VirtualizationRootHandle root = this->virtualizationRootHandle; + this->virtualizationRootHandle = RootHandle_None; + if (RootHandle_None != root) { ActiveProvider_Disconnect(root); } @@ -210,7 +214,7 @@ IOReturn PrjFSProviderUserClient::registerVirtualizationRoot(const char* rootPat *outError = EINVAL; return kIOReturnSuccess; } - else if (this->virtualizationRootIndex != -1) + else if (this->virtualizationRootHandle != RootHandle_None) { // Already set *outError = EBUSY; @@ -220,11 +224,11 @@ IOReturn PrjFSProviderUserClient::registerVirtualizationRoot(const char* rootPat VirtualizationRootResult result = VirtualizationRoot_RegisterProviderForPath(this, this->pid, rootPath); if (0 == result.error) { - this->virtualizationRootIndex = result.rootIndex; + this->virtualizationRootHandle = result.root; // Sets the root index in the IORegistry for diagnostic purposes char location[5] = ""; - snprintf(location, sizeof(location), "%d", result.rootIndex); + snprintf(location, sizeof(location), "%d", result.root); this->setLocation(location); } diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp index 12621ede4d..de487108c1 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp @@ -3,6 +3,7 @@ #include "PrjFSClasses.hpp" #include "Locks.hpp" #include "Message.h" +#include "VirtualizationRoots.hpp" #include struct MessageHeader; @@ -18,8 +19,8 @@ class PrjFSProviderUserClient : public IOUserClient Mutex dataQueueWriterMutex; public: pid_t pid; - // The root for which this is the provider; -1 prior to registration - int32_t virtualizationRootIndex; + // The root for which this is the provider; RootHandle_None prior to registration + VirtualizationRootHandle virtualizationRootHandle; // IOUserClient methods: virtual bool initWithTask( diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index 46a5704d9f..ecb7924481 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -12,6 +12,27 @@ #include "VnodeUtilities.hpp" +struct VirtualizationRoot +{ + bool inUse; + // If this is a nullptr, there is no active provider for this virtualization root (offline root) + PrjFSProviderUserClient* providerUserClient; + int providerPid; + // For an active root, this is retained (vnode_get), for an offline one, it is not, so it may be stale (check the vid) + vnode_t rootVNode; + uint32_t rootVNodeVid; + + // Mount point ID + persistent, on-disk ID for the root directory, so we can + // identify it if the vnode of an offline root gets recycled. + fsid_t rootFsid; + uint64_t rootInode; + + // TODO(Mac): this should eventually be entirely diagnostic and not used for decisions + char path[PrjFSMaxPath]; + + int32_t index; +}; + static RWLock s_rwLock = {}; // Arbitrary choice, but prevents user space attacker from causing @@ -20,9 +41,52 @@ static const size_t MaxVirtualizationRoots = 128; static VirtualizationRoot s_virtualizationRoots[MaxVirtualizationRoots] = {}; -static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId); -static int16_t FindUnusedIndex_Locked(); -static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); +// Looks up the vnode/vid and fsid/inode pairs among the known roots +static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId); + +// Looks up the vnode and fsid/inode pair among the known roots, and if not found, +// detects if there is a hitherto-unknown root at vnode by checking attributes. +static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode); + +static VirtualizationRootHandle FindUnusedIndex_Locked(); +static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); + +bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootIndex) +{ + if (rootIndex < 0 || rootIndex >= MaxVirtualizationRoots) + { + return false; + } + + bool result; + RWLock_AcquireShared(s_rwLock); + { + result = (nullptr != s_virtualizationRoots[rootIndex].providerUserClient); + } + RWLock_ReleaseShared(s_rwLock); + + return result; +} + +bool VirtualizationRoot_PIDMatchesProvider(VirtualizationRootHandle rootIndex, pid_t pid) +{ + bool result; + RWLock_AcquireShared(s_rwLock); + { + result = + (rootIndex >= 0 && rootIndex < MaxVirtualizationRoots) + && (nullptr != s_virtualizationRoots[rootIndex].providerUserClient) + && pid == s_virtualizationRoots[rootIndex].providerPid; + } + RWLock_ReleaseShared(s_rwLock); + + return result; +} + +bool VirtualizationRoot_IsValidRootHandle(VirtualizationRootHandle rootIndex) +{ + return (rootIndex > RootHandle_None); +} kern_return_t VirtualizationRoots_Init() { @@ -37,7 +101,7 @@ kern_return_t VirtualizationRoots_Init() return KERN_FAILURE; } - for (uint32_t i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) { s_virtualizationRoots[i].index = i; } @@ -56,18 +120,19 @@ kern_return_t VirtualizationRoots_Cleanup() return KERN_FAILURE; } -VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode) +VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode) { - VirtualizationRoot* root = nullptr; + VirtualizationRootHandle rootHandle = RootHandle_None; vnode_get(vnode); // Search up the tree until we hit a known virtualization root or THE root of the file system - while (nullptr == root && NULLVP != vnode && !vnode_isvroot(vnode)) + while (RootHandle_None == rootHandle && NULLVP != vnode && !vnode_isvroot(vnode)) { - int16_t rootIndex = VirtualizationRoots_LookupVnode(vnode, /*context*/ nullptr, vnodeFsidInode); - if (rootIndex >= 0) + rootHandle = FindOrDetectRootAtVnode(vnode, nullptr /* vfs context */, vnodeFsidInode); + // Note: if FindOrDetectRootAtVnode returns a "special" handle other + // than RootHandle_None, we want to stop the search and return that. + if (rootHandle != RootHandle_None) { - root = &s_virtualizationRoots[rootIndex]; break; } @@ -81,22 +146,22 @@ VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidIn vnode_put(vnode); } - return root; + return rootHandle; } -int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode) +static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode) { uint32_t vid = vnode_vid(vnode); - int16_t rootIndex; + VirtualizationRootHandle rootIndex; RWLock_AcquireShared(s_rwLock); { - rootIndex = FindRootForVnode_Locked(vnode, vid, vnodeFsidInode); + rootIndex = FindRootAtVnode_Locked(vnode, vid, vnodeFsidInode); } RWLock_ReleaseShared(s_rwLock); - if (rootIndex < 0) + if (rootIndex == RootHandle_None) { PrjFSVirtualizationRootXAttrData rootXattr = {}; SizeOrError xattrResult = Vnode_ReadXattr(vnode, PrjFSVirtualizationRootXAttrName, &rootXattr, sizeof(rootXattr), context); @@ -111,9 +176,9 @@ int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, co RWLock_AcquireExclusive(s_rwLock); { // Vnode may already have been inserted as a root in the interim - rootIndex = FindRootForVnode_Locked(vnode, vid, vnodeFsidInode); + rootIndex = FindRootAtVnode_Locked(vnode, vid, vnodeFsidInode); - if (rootIndex < 0) + if (RootHandle_None == rootIndex) { // Insert new offline root rootIndex = InsertVirtualizationRoot_Locked(nullptr, 0, vnode, vid, vnodeFsidInode, path); @@ -130,9 +195,9 @@ int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, co return rootIndex; } -static int16_t FindUnusedIndex_Locked() +static VirtualizationRootHandle FindUnusedIndex_Locked() { - for (int16_t i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) { if (!s_virtualizationRoots[i].inUse) { @@ -140,7 +205,7 @@ static int16_t FindUnusedIndex_Locked() } } - return -1; + return RootHandle_None; } static bool FsidsAreEqual(fsid_t a, fsid_t b) @@ -148,9 +213,9 @@ static bool FsidsAreEqual(fsid_t a, fsid_t b) return a.val[0] == b.val[0] && a.val[1] == b.val[1]; } -static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId) +static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId) { - for (int16_t i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) { VirtualizationRoot& rootEntry = s_virtualizationRoots[i]; if (!rootEntry.inUse) @@ -172,14 +237,13 @@ static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fi return i; } } - return -1; + return RootHandle_None; } // Returns negative value if it failed, or inserted index on success -static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path) +static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path) { - // New root - int16_t rootIndex = FindUnusedIndex_Locked(); + VirtualizationRootHandle rootIndex = FindUnusedIndex_Locked(); if (rootIndex >= 0) { @@ -215,7 +279,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide vnode_t virtualizationRootVNode = NULLVP; vfs_context_t vfsContext = vfs_context_create(nullptr); - int32_t rootIndex = -1; + VirtualizationRootHandle rootIndex = RootHandle_None; errno_t err = vnode_lookup(virtualizationRootPath, 0 /* flags */, &virtualizationRootVNode, vfsContext); if (0 == err) { @@ -234,7 +298,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide RWLock_AcquireExclusive(s_rwLock); { - rootIndex = FindRootForVnode_Locked(virtualizationRootVNode, rootVid, vnodeIds); + rootIndex = FindRootAtVnode_Locked(virtualizationRootVNode, rootVid, vnodeIds); if (rootIndex >= 0) { // Reattaching to existing root @@ -242,7 +306,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide { // Only one provider per root err = EBUSY; - rootIndex = -1; + rootIndex = RootHandle_None; } else { @@ -292,7 +356,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide return VirtualizationRootResult { err, rootIndex }; } -void ActiveProvider_Disconnect(int32_t rootIndex) +void ActiveProvider_Disconnect(VirtualizationRootHandle rootIndex) { assert(rootIndex >= 0); assert(rootIndex <= MaxVirtualizationRoots); @@ -311,7 +375,7 @@ void ActiveProvider_Disconnect(int32_t rootIndex) RWLock_ReleaseExclusive(s_rwLock); } -errno_t ActiveProvider_SendMessage(int32_t rootIndex, const Message message) +errno_t ActiveProvider_SendMessage(VirtualizationRootHandle rootIndex, const Message message) { assert(rootIndex >= 0); assert(rootIndex < MaxVirtualizationRoots); @@ -356,3 +420,32 @@ bool VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode_t vnode) || 0 == strncmp("apfs", vfsStat->f_fstypename, sizeof(vfsStat->f_fstypename)); } +static const char* GetRelativePath(const char* path, const char* root) +{ + assert(strlen(path) >= strlen(root)); + + const char* relativePath = path + strlen(root); + if (relativePath[0] == '/') + { + relativePath++; + } + + return relativePath; +} + +const char* VirtualizationRoot_GetRootRelativePath(VirtualizationRootHandle rootIndex, const char* path) +{ + assert(rootIndex >= 0); + assert(rootIndex <= MaxVirtualizationRoots); + + const char* relativePath; + + RWLock_AcquireShared(s_rwLock); + { + assert(s_virtualizationRoots[rootIndex].inUse); + relativePath = GetRelativePath(path, s_virtualizationRoots[rootIndex].path); + } + RWLock_ReleaseShared(s_rwLock); + + return relativePath; +} diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp index 3d1c86aefa..2b19b0681d 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp @@ -4,42 +4,37 @@ #include "PrjFSClasses.hpp" #include "kernel-header-wrappers/vnode.h" -struct VirtualizationRoot -{ - bool inUse; - // If this is a nullptr, there is no active provider for this virtualization root (offline root) - PrjFSProviderUserClient* providerUserClient; - int providerPid; - // For an active root, this is retained (vnode_get), for an offline one, it is not, so it may be stale (check the vid) - vnode_t rootVNode; - uint32_t rootVNodeVid; - - // Mount point ID + persistent, on-disk ID for the root directory, so we can - // identify it if the vnode of an offline root gets recycled. - fsid_t rootFsid; - uint64_t rootInode; - - // TODO(Mac): this should eventually be entirely diagnostic and not used for decisions - char path[PrjFSMaxPath]; +typedef int16_t VirtualizationRootHandle; - int32_t index; +// Zero and positive values indicate a handle for a valid virtualization +// root. Other values have special meanings: +enum VirtualizationRootSpecialHandle : VirtualizationRootHandle +{ + // Not in a virtualization root. + RootHandle_None = -1, + // Root/non-root state not known. Useful reset value for invalidating cached state. + RootHandle_Indeterminate = -2, + // Vnode is not in a virtualization root, but below a provider's registered temp directory + RootHandle_ProviderTemporaryDirectory = -3, }; kern_return_t VirtualizationRoots_Init(void); kern_return_t VirtualizationRoots_Cleanup(void); -VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode); +VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode); struct VirtualizationRootResult { errno_t error; - int32_t rootIndex; + VirtualizationRootHandle root; }; VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProviderUserClient* userClient, pid_t clientPID, const char* virtualizationRootPath); -void ActiveProvider_Disconnect(int32_t rootIndex); +void ActiveProvider_Disconnect(VirtualizationRootHandle rootHandle); struct Message; -errno_t ActiveProvider_SendMessage(int32_t rootIndex, const Message message); +errno_t ActiveProvider_SendMessage(VirtualizationRootHandle rootHandle, const Message message); bool VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode_t vnode); - -int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode); +bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootHandle); +bool VirtualizationRoot_PIDMatchesProvider(VirtualizationRootHandle rootHandle, pid_t pid); +bool VirtualizationRoot_IsValidRootHandle(VirtualizationRootHandle rootHandle); +const char* VirtualizationRoot_GetRootRelativePath(VirtualizationRootHandle rootHandle, const char* path); From 0a0ac8838f2a2a9daddaed88682e411e421bd9fc Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Mon, 1 Oct 2018 21:02:03 +0200 Subject: [PATCH 104/244] Mac kext: Dynamically alloc virtualisation root array & resize when full. --- .../PrjFSKext/VirtualizationRoots.cpp | 105 +++++++++++++----- 1 file changed, 76 insertions(+), 29 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index ecb7924481..88b0a97dba 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "PrjFSCommon.h" #include "PrjFSXattrs.h" @@ -29,17 +30,13 @@ struct VirtualizationRoot // TODO(Mac): this should eventually be entirely diagnostic and not used for decisions char path[PrjFSMaxPath]; - - int32_t index; }; static RWLock s_rwLock = {}; -// Arbitrary choice, but prevents user space attacker from causing -// allocation of too much wired kernel memory. -static const size_t MaxVirtualizationRoots = 128; - -static VirtualizationRoot s_virtualizationRoots[MaxVirtualizationRoots] = {}; +// Current length of the s_virtualizationRoots array +static uint16_t s_maxVirtualizationRoots = 0; +static VirtualizationRoot* s_virtualizationRoots = nullptr; // Looks up the vnode/vid and fsid/inode pairs among the known roots static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId); @@ -51,9 +48,9 @@ static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_conte static VirtualizationRootHandle FindUnusedIndex_Locked(); static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); -bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootIndex) +bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootHandle) { - if (rootIndex < 0 || rootIndex >= MaxVirtualizationRoots) + if (rootHandle < 0) { return false; } @@ -61,22 +58,27 @@ bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootIndex) bool result; RWLock_AcquireShared(s_rwLock); { - result = (nullptr != s_virtualizationRoots[rootIndex].providerUserClient); + result = + rootHandle < s_maxVirtualizationRoots + && s_virtualizationRoots[rootHandle].inUse + && nullptr != s_virtualizationRoots[rootHandle].providerUserClient; } RWLock_ReleaseShared(s_rwLock); return result; } -bool VirtualizationRoot_PIDMatchesProvider(VirtualizationRootHandle rootIndex, pid_t pid) +bool VirtualizationRoot_PIDMatchesProvider(VirtualizationRootHandle rootHandle, pid_t pid) { bool result; RWLock_AcquireShared(s_rwLock); { result = - (rootIndex >= 0 && rootIndex < MaxVirtualizationRoots) - && (nullptr != s_virtualizationRoots[rootIndex].providerUserClient) - && pid == s_virtualizationRoots[rootIndex].providerPid; + rootHandle >= 0 + && rootHandle < s_maxVirtualizationRoots + && s_virtualizationRoots[rootHandle].inUse + && nullptr != s_virtualizationRoots[rootHandle].providerUserClient + && pid == s_virtualizationRoots[rootHandle].providerPid; } RWLock_ReleaseShared(s_rwLock); @@ -101,11 +103,20 @@ kern_return_t VirtualizationRoots_Init() return KERN_FAILURE; } - for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) + s_maxVirtualizationRoots = 128; + s_virtualizationRoots = Memory_AllocArray(s_maxVirtualizationRoots); + if (nullptr == s_virtualizationRoots) { - s_virtualizationRoots[i].index = i; + return KERN_RESOURCE_SHORTAGE; } + for (VirtualizationRootHandle i = 0; i < s_maxVirtualizationRoots; ++i) + { + s_virtualizationRoots[i] = VirtualizationRoot{ }; + } + + atomic_thread_fence(memory_order_seq_cst); + return KERN_SUCCESS; } @@ -197,7 +208,7 @@ static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_conte static VirtualizationRootHandle FindUnusedIndex_Locked() { - for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < s_maxVirtualizationRoots; ++i) { if (!s_virtualizationRoots[i].inUse) { @@ -208,6 +219,42 @@ static VirtualizationRootHandle FindUnusedIndex_Locked() return RootHandle_None; } +static VirtualizationRootHandle FindUnusedIndexOrGrow_Locked() +{ + VirtualizationRootHandle rootIndex = FindUnusedIndex_Locked(); + + if (RootHandle_None == rootIndex) + { + // No space, resize array + uint16_t newLength = MIN(s_maxVirtualizationRoots * 2u, INT16_MAX + 1u); + if (newLength <= s_maxVirtualizationRoots) + { + return RootHandle_None; + } + + VirtualizationRoot* grownArray = Memory_AllocArray(newLength); + if (nullptr == grownArray) + { + return RootHandle_None; + } + + uint32_t oldSizeBytes = sizeof(s_virtualizationRoots[0]) * s_maxVirtualizationRoots; + memcpy(grownArray, s_virtualizationRoots, oldSizeBytes); + Memory_Free(s_virtualizationRoots, oldSizeBytes); + s_virtualizationRoots = grownArray; + + for (uint16_t i = s_maxVirtualizationRoots; i < newLength; ++i) + { + s_virtualizationRoots[i] = VirtualizationRoot{ }; + } + + rootIndex = s_maxVirtualizationRoots; + s_maxVirtualizationRoots = newLength; + } + + return rootIndex; +} + static bool FsidsAreEqual(fsid_t a, fsid_t b) { return a.val[0] == b.val[0] && a.val[1] == b.val[1]; @@ -215,7 +262,7 @@ static bool FsidsAreEqual(fsid_t a, fsid_t b) static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId) { - for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < s_maxVirtualizationRoots; ++i) { VirtualizationRoot& rootEntry = s_virtualizationRoots[i]; if (!rootEntry.inUse) @@ -243,17 +290,16 @@ static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t v // Returns negative value if it failed, or inserted index on success static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path) { - VirtualizationRootHandle rootIndex = FindUnusedIndex_Locked(); + VirtualizationRootHandle rootIndex = FindUnusedIndexOrGrow_Locked(); - if (rootIndex >= 0) + if (RootHandle_None != rootIndex) { - assert(rootIndex < MaxVirtualizationRoots); + assert(rootIndex < s_maxVirtualizationRoots); VirtualizationRoot* root = &s_virtualizationRoots[rootIndex]; root->providerUserClient = userClient; root->providerPid = clientPID; root->inUse = true; - root->index = rootIndex; root->rootVNode = vnode; root->rootVNodeVid = vid; @@ -321,7 +367,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide rootIndex = InsertVirtualizationRoot_Locked(userClient, clientPID, virtualizationRootVNode, rootVid, vnodeIds, virtualizationRootPath); if (rootIndex >= 0) { - assert(rootIndex < MaxVirtualizationRoots); + assert(rootIndex < s_maxVirtualizationRoots); VirtualizationRoot* root = &s_virtualizationRoots[rootIndex]; strlcpy(root->path, virtualizationRootPath, sizeof(root->path)); @@ -359,10 +405,10 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide void ActiveProvider_Disconnect(VirtualizationRootHandle rootIndex) { assert(rootIndex >= 0); - assert(rootIndex <= MaxVirtualizationRoots); - RWLock_AcquireExclusive(s_rwLock); { + assert(rootIndex <= s_maxVirtualizationRoots); + VirtualizationRoot* root = &s_virtualizationRoots[rootIndex]; assert(nullptr != root->providerUserClient); @@ -378,19 +424,20 @@ void ActiveProvider_Disconnect(VirtualizationRootHandle rootIndex) errno_t ActiveProvider_SendMessage(VirtualizationRootHandle rootIndex, const Message message) { assert(rootIndex >= 0); - assert(rootIndex < MaxVirtualizationRoots); PrjFSProviderUserClient* userClient = nullptr; - RWLock_AcquireExclusive(s_rwLock); + RWLock_AcquireShared(s_rwLock); { + assert(rootIndex < s_maxVirtualizationRoots); + userClient = s_virtualizationRoots[rootIndex].providerUserClient; if (nullptr != userClient) { userClient->retain(); } } - RWLock_ReleaseExclusive(s_rwLock); + RWLock_ReleaseShared(s_rwLock); if (nullptr != userClient) { @@ -436,12 +483,12 @@ static const char* GetRelativePath(const char* path, const char* root) const char* VirtualizationRoot_GetRootRelativePath(VirtualizationRootHandle rootIndex, const char* path) { assert(rootIndex >= 0); - assert(rootIndex <= MaxVirtualizationRoots); const char* relativePath; RWLock_AcquireShared(s_rwLock); { + assert(rootIndex < s_maxVirtualizationRoots); assert(s_virtualizationRoots[rootIndex].inUse); relativePath = GetRelativePath(path, s_virtualizationRoots[rootIndex].path); } From 25dab5d4afac154b2c5d9bc00b7422de268c729e Mon Sep 17 00:00:00 2001 From: John Briggs Date: Thu, 4 Oct 2018 13:20:11 -0400 Subject: [PATCH 105/244] Select installation of VS that can build the project on Windows. - Pass all required components to vswhere in BuildGVFSForWindows.bat. - Remove unnecessary component from Readme.md. --- Readme.md | 1 - Scripts/BuildGVFSForWindows.bat | 12 +++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index 133b937547..1a005324cf 100644 --- a/Readme.md +++ b/Readme.md @@ -40,7 +40,6 @@ If you'd like to build your own VFS for Git Windows installer: * .NET Core cross-platform development * Include the following additional components: * .NET Core runtime - * C++/CLI support * Windows 10 SDK (10.0.10240.0) * Install the .NET Core 2.1 SDK (https://www.microsoft.com/net/download/dotnet-core/2.1) * Create a folder to clone into, e.g. `C:\Repos\VFSForGit` diff --git a/Scripts/BuildGVFSForWindows.bat b/Scripts/BuildGVFSForWindows.bat index fd3adee6be..ac3ec7a1d3 100644 --- a/Scripts/BuildGVFSForWindows.bat +++ b/Scripts/BuildGVFSForWindows.bat @@ -19,14 +19,20 @@ SET vswhere=%~dp0..\..\packages\vswhere.%vswherever%\tools\vswhere.exe :: Use vswhere to find the latest VS installation (including prerelease installations) with the msbuild component. :: See https://github.com/Microsoft/vswhere/wiki/Find-MSBuild -for /f "usebackq tokens=*" %%i in (`%vswhere% -all -prerelease -latest -products * -requires Microsoft.Component.MSBuild -property installationPath`) do ( +for /f "usebackq tokens=*" %%i in (`%vswhere% -all -prerelease -latest -products * -requires Microsoft.Component.MSBuild Microsoft.VisualStudio.Workload.ManagedDesktop Microsoft.VisualStudio.Workload.NativeDesktop Microsoft.VisualStudio.Workload.NetCoreTools Microsoft.Component.NetFX.Core.Runtime Microsoft.VisualStudio.Component.Windows10SDK.10240 -property installationPath`) do ( set VsInstallDir=%%i ) +IF NOT DEFINED VsInstallDir ( + echo ERROR: Could not locate a Visual Studio installation with required components. + echo Refer to Readme.md for a list of the required Visual Studio components. + exit /b 10 +) + SET msbuild="%VsInstallDir%\MSBuild\15.0\Bin\amd64\msbuild.exe" IF NOT EXIST %msbuild% ( - echo Error: Could not find msbuild - exit /b 1 + echo ERROR: Could not find msbuild + exit /b 1 ) %msbuild% %~dp0\..\GVFS.sln /p:GVFSVersion=%GVFSVersion% /p:Configuration=%SolutionConfiguration% /p:Platform=x64 || exit /b 1 From c07d9eb6a7b5ae4df3bbf9a346a3c9697184690b Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Fri, 5 Oct 2018 17:42:23 -0400 Subject: [PATCH 106/244] - Removed wildcard matching while searching for downloaded installers - Added ToUpper() with ToUpperInvariant in LocalGVFSConfig - Fixed partial failure in build caused by UninstallGVFS.bat script. - Updated TryGetVersion in GitProcess to accept git path as input argument. --- GVFS/GVFS.Common/GVFSConstants.cs | 5 ++-- GVFS/GVFS.Common/Git/GitProcess.cs | 27 +++++++-------------- GVFS/GVFS.Common/LocalGVFSConfig.cs | 25 +++++++++++-------- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 28 +++++++++++++++------- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 5 ++-- GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 7 +++--- GVFS/GVFS/CommandLine/GVFSVerb.cs | 5 ++-- Scripts/UninstallGVFS.bat | 2 +- 8 files changed, 56 insertions(+), 48 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index a387f89376..8d2bd49891 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -83,12 +83,12 @@ public static class LogFileTypes public const string Dehydrate = "dehydrate"; public const string MountVerb = MountPrefix + "_verb"; public const string MountProcess = MountPrefix + "_process"; + public const string MountUpgrade = MountPrefix + "_repoupgrade"; public const string Prefetch = "prefetch"; public const string Repair = "repair"; public const string Service = "service"; public const string UpgradeVerb = UpgradePrefix + "_verb"; - public const string UpgradeProcess = UpgradePrefix + "_process"; - public const string MountUpgrade = MountPrefix + "_repoupgrade"; + public const string UpgradeProcess = UpgradePrefix + "_process"; } public static class DotGVFS @@ -228,7 +228,6 @@ public static class UpgradeVerbMessages { public const string GVFSUpgrade = "`gvfs upgrade`"; public const string GVFSUpgradeConfirm = "`gvfs upgrade --confirm`"; - public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoUpgradeCheckPerformed = "No upgrade check was performed."; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". " + NoUpgradeCheckPerformed; public const string NoRingConfigConsoleAlert = "Upgrade ring is not set. " + NoUpgradeCheckPerformed; diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 8701605ab0..41e63763d3 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -92,30 +92,21 @@ public static Result GetFromFileConfig(string gitBinPath, string configFile, str return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --file " + configFile + " " + settingName); } - public static bool TryGetVersion(out GitVersion gitVersion, out string error) + public static bool TryGetVersion(string gitBinPath, out GitVersion gitVersion, out string error) { - string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); - if (gitPath != null) - { - GitProcess gitProcess = new GitProcess(gitPath, null, null); - Result result = gitProcess.InvokeGitOutsideEnlistment("--version"); - string version = result.Output; - - if (!GitVersion.TryParseGitVersionCommandResult(version, out gitVersion)) - { - error = "Unable to determine installed git version. " + version; - return false; - } + GitProcess gitProcess = new GitProcess(gitBinPath, null, null); + Result result = gitProcess.InvokeGitOutsideEnlistment("--version"); + string version = result.Output; - error = null; - return true; - } - else + if (result.HasErrors || !GitVersion.TryParseGitVersionCommandResult(version, out gitVersion)) { gitVersion = null; - error = "Unable to determine installed git path."; + error = "Unable to determine installed git version. " + version; return false; } + + error = null; + return true; } public virtual void RevokeCredential(string repoUrl) diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index 980d5956ac..ac65b45893 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -21,21 +21,21 @@ public LocalGVFSConfig() } public bool TryGetConfig( - string key, + string name, out string value, out string error, ITracer tracer) { if (!this.TryLoadSettings(tracer, out error)) { - error = $"Error getting config value {key}. {error}"; + error = $"Error getting config value {name}. {error}"; value = null; return false; } try { - this.allSettings.TryGetValue(key.ToUpper(), out value); + this.allSettings.TryGetValue(this.KeyFromConfigName(name), out value); error = null; return true; } @@ -44,30 +44,30 @@ public bool TryGetConfig( const string ErrorFormat = "Error getting config value for {0}. Config file {1}. {2}"; if (tracer != null) { - tracer.RelatedError(ErrorFormat, key, this.configFile, exception.ToString()); + tracer.RelatedError(ErrorFormat, name, this.configFile, exception.ToString()); } - error = string.Format(ErrorFormat, key, this.configFile, exception.Message); + error = string.Format(ErrorFormat, name, this.configFile, exception.Message); value = null; return false; } } public bool TrySetConfig( - string key, + string name, string value, out string error, ITracer tracer) { if (!this.TryLoadSettings(tracer, out error)) { - error = $"Error setting config value {key}: {value}. {error}"; + error = $"Error setting config value {name}: {value}. {error}"; return false; } try { - this.allSettings.SetValueAndFlush(key.ToUpper(), value); + this.allSettings.SetValueAndFlush(this.KeyFromConfigName(name), value); error = null; return true; } @@ -76,15 +76,20 @@ public bool TrySetConfig( const string ErrorFormat = "Error setting config value {0}: {1}. Config file {2}. {3}"; if (tracer != null) { - tracer.RelatedError(ErrorFormat, key, value, this.configFile, exception.ToString()); + tracer.RelatedError(ErrorFormat, name, value, this.configFile, exception.ToString()); } - error = string.Format(ErrorFormat, key, value, this.configFile, exception.Message); + error = string.Format(ErrorFormat, name, value, this.configFile, exception.Message); value = null; return false; } } + private string KeyFromConfigName(string configName) + { + return configName.ToUpperInvariant(); + } + private bool TryLoadSettings(ITracer tracer, out string error) { if (this.allSettings == null) diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index 178f58038d..51125717e7 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -1,5 +1,4 @@ -using System; -using System.Diagnostics; +using System.Collections.Generic; using System.IO; namespace GVFS.Common @@ -19,12 +18,25 @@ public static bool IsLocalUpgradeAvailable() string downloadDirectory = GetAssetDownloadsPath(); if (Directory.Exists(downloadDirectory)) { - const string PotentialInstallerName = "*VFS*.*"; - string[] installers = Directory.GetFiles( - downloadDirectory, - PotentialInstallerName, - SearchOption.TopDirectoryOnly); - return installers.Length > 0; + // This method is used Only by Git hooks. Git hooks does not have access + // to GVFSPlatform to read platform specific file extensions. That is the + // reason possible installer file extensions are defined here. + HashSet extensions = new HashSet() { "EXE", "DMG", "RPM", "DEB" }; + HashSet installerNames = new HashSet() + { + GVFSInstallerFileNamePrefix.ToUpperInvariant(), + VFSForGitInstallerFileNamePrefix.ToUpperInvariant() + }; + + foreach (string file in Directory.EnumerateFiles(downloadDirectory, "*", SearchOption.TopDirectoryOnly)) + { + string[] components = Path.GetFileName(file).ToUpperInvariant().Split('.'); + int length = components.Length; + if (length >= 2 && installerNames.Contains(components[0]) && extensions.Contains(components[length - 1])) + { + return true; + } + } } return false; diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index 66a729269f..f3faf5f379 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -414,9 +414,8 @@ private void LogInstalledVersionInfo() GitVersion installedGitVersion = null; string error = null; - if (GitProcess.TryGetVersion( - out installedGitVersion, - out error)) + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + if (!string.IsNullOrEmpty(gitPath) && GitProcess.TryGetVersion(gitPath, out installedGitVersion, out error)) { metadata.Add(nameof(installedGitVersion), installedGitVersion.ToString()); } diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index a7979157d1..c8a0c0a52a 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -50,9 +50,10 @@ protected override void Execute(GVFSEnlistment enlistment) this.WriteMessage(string.Empty); this.WriteMessage("gvfs version " + ProcessHelper.GetCurrentProcessVersion()); - GitVersion gitVersion; - string error; - if (GitProcess.TryGetVersion(out gitVersion, out error)) + GitVersion gitVersion = null; + string error = null; + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + if (!string.IsNullOrEmpty(gitPath) && GitProcess.TryGetVersion(gitPath, out gitVersion, out error)) { this.WriteMessage("git version " + gitVersion.ToString()); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 56b2eca612..94bd33752f 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -619,8 +619,9 @@ private void CheckFileSystemSupportsRequiredFeatures(ITracer tracer, Enlistment private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out string version) { - GitVersion gitVersion; - if (!GitProcess.TryGetVersion(out gitVersion, out string _)) + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + GitVersion gitVersion = null; + if (string.IsNullOrEmpty(gitPath) || !GitProcess.TryGetVersion(gitPath, out gitVersion, out string _)) { this.ReportErrorAndExit(tracer, "Error: Unable to retrieve the git version"); } diff --git a/Scripts/UninstallGVFS.bat b/Scripts/UninstallGVFS.bat index 27ecc7a9e8..6a0451b500 100644 --- a/Scripts/UninstallGVFS.bat +++ b/Scripts/UninstallGVFS.bat @@ -17,6 +17,6 @@ for /F "delims=" %%f in ('dir "c:\Program Files\GVFS\unins*.exe" /B /S /O:-D') d :deleteGVFS rmdir /q/s "c:\Program Files\GVFS" -rmdir /q/s "C:\ProgramData\GVFS\GVFS.Upgrade" +if exist "C:\ProgramData\GVFS\GVFS.Upgrade" rmdir /q/s "C:\ProgramData\GVFS\GVFS.Upgrade" :end From 63a8405ccd02c8915ef0ee4ad963a12fd7e8a971 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 5 Oct 2018 14:47:08 -0700 Subject: [PATCH 107/244] Windows: Add delay and retry to attributes check in ExpandedFileAttributesAreUpdated --- .../BasicFileSystemTests.cs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index 3c05f81513..37061f8746 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -8,12 +8,17 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; namespace GVFS.FunctionalTests.Tests.LongRunningEnlistment { [TestFixture] public class BasicFileSystemTests : TestsWithEnlistmentPerFixture { + private const int FileAttributeSparseFile = 0x00000200; + private const int FileAttributeReparsePoint = 0x00000400; + private const int FileAttributeRecallOnOpen = 0x00040000; + [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] public void ShrinkFileContents(FileSystemRunner fileSystem, string parentFolder) { @@ -152,7 +157,24 @@ public void ExpandedFileAttributesAreUpdated() // Ignore the archive bit as it can be re-added to the file as part of its expansion to full FileAttributes attributes = info.Attributes & ~FileAttributes.Archive; - attributes.ShouldEqual(FileAttributes.Hidden, "Attributes do not match"); + + int retryCount = 0; + int maxRetries = 10; + while (attributes != FileAttributes.Hidden && retryCount < maxRetries) + { + // ProjFS attributes are remoted asynchronously when files are converted to full + FileAttributes attributesLessProjFS = attributes & (FileAttributes)~(FileAttributeSparseFile | FileAttributeRecallOnOpen | FileAttributeReparsePoint); + + attributesLessProjFS.ShouldEqual( + FileAttributes.Hidden, + $"Attributes (ignoring ProjFS attributes) do not match, expected: {FileAttributes.Hidden} actual: {attributesLessProjFS}"); + + ++retryCount; + Thread.Sleep(500); + attributes = new FileInfo(virtualFile).Attributes & ~FileAttributes.Archive; + } + + attributes.ShouldEqual(FileAttributes.Hidden, $"Attributes do not match, expected: {FileAttributes.Hidden} actual: {attributes}"); } [TestCase] From 320d04645f8a65ea597ff4cfe74a6f636684d2d3 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 5 Oct 2018 15:08:56 -0700 Subject: [PATCH 108/244] Use Refresh rather than creating a new FileInfo --- .../Tests/EnlistmentPerFixture/BasicFileSystemTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index 37061f8746..db86eb26ef 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -171,7 +171,9 @@ public void ExpandedFileAttributesAreUpdated() ++retryCount; Thread.Sleep(500); - attributes = new FileInfo(virtualFile).Attributes & ~FileAttributes.Archive; + + info.Refresh(); + attributes = info.Attributes & ~FileAttributes.Archive; } attributes.ShouldEqual(FileAttributes.Hidden, $"Attributes do not match, expected: {FileAttributes.Hidden} actual: {attributes}"); From c2910a10f4306e1985aa31b628899df21e76a0c4 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Wed, 20 Jun 2018 12:22:09 +0200 Subject: [PATCH 109/244] Mac kext: Scoped timing wrapper and user client for extracting profile data This adds a scoped time measurement class with splits. Probes are defined in an enum. When a ProfileSample instance is created, the Mach absolute time is taken, and when it is destroyed, another time stamp is taken, and the interval recorded with the sample aggregation probe ID specified during construction. Additionally, split time samples can be taken. The number of samples, total runtime, sum of squares, minimum, and maximum values are recorded. From these, mean and standard deviation can be derived. An external method for extracting the profiling data from the kernel has been added to the logging user client class. The simple log tool has been extended to fetch and dump this data to stdout every 15 seconds. --- .../PrjFSKext.xcodeproj/project.pbxproj | 6 + .../PrjFSKext/PerformanceTracing.cpp | 59 +++++++ .../PrjFSKext/PerformanceTracing.hpp | 155 ++++++++++++++++++ ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp | 3 + .../PrjFSKext/PrjFSLogUserClient.cpp | 43 +++++ .../PrjFSKext/PrjFSLogUserClient.hpp | 13 ++ ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h | 25 +++ .../PrjFSKext/public/PrjFSLogClientShared.h | 8 + .../PrjFSLib.xcodeproj/project.pbxproj | 6 + .../PrjFSLib/prjfs-log/kext-perf-tracing.cpp | 65 ++++++++ .../PrjFSLib/prjfs-log/kext-perf-tracing.hpp | 5 + ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp | 34 +++- 12 files changed, 419 insertions(+), 3 deletions(-) create mode 100644 ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp create mode 100644 ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp create mode 100644 ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp create mode 100644 ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.hpp diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj index dbd7e2453f..44b29d877d 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj @@ -15,6 +15,7 @@ 4AC1D7C12091FA0400786861 /* PrjFSProviderUserClient.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 4AC1D7BF2091FA0400786861 /* PrjFSProviderUserClient.hpp */; }; 4AC1D7C52091FBFC00786861 /* PrjFSLogUserClient.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 4AC1D7C32091FBFC00786861 /* PrjFSLogUserClient.cpp */; }; 4AC1D7C62091FBFC00786861 /* PrjFSLogUserClient.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 4AC1D7C42091FBFC00786861 /* PrjFSLogUserClient.hpp */; }; + 4AE187A821203A890026AC68 /* PerformanceTracing.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 4AE187A721203A890026AC68 /* PerformanceTracing.cpp */; }; C6BDD366208BC60400CB7E58 /* VirtualizationRoots.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C6BDD364208BC60400CB7E58 /* VirtualizationRoots.hpp */; }; C6BDD367208BC60400CB7E58 /* VirtualizationRoots.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C6BDD365208BC60400CB7E58 /* VirtualizationRoots.cpp */; }; C6BDD36A208BC99100CB7E58 /* Locks.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C6BDD368208BC99100CB7E58 /* Locks.hpp */; }; @@ -38,6 +39,7 @@ 4A9D139C208F675500376182 /* PrjFSService.hpp */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 4; lastKnownFileType = sourcecode.cpp.h; path = PrjFSService.hpp; sourceTree = ""; tabWidth = 4; usesTabs = 0; }; 4AA0BA9920A4500600F33D1C /* vnode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = vnode.h; sourceTree = ""; }; 4ABB733020B85DA500DC0D17 /* mount.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = mount.h; sourceTree = ""; }; + 4ABB734320B867E000DC0D17 /* PerformanceTracing.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = PerformanceTracing.hpp; sourceTree = ""; }; 4ABB734520BED11500DC0D17 /* PrjFSLogClientShared.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PrjFSLogClientShared.h; sourceTree = ""; }; 4ABB734C20C1A65B00DC0D17 /* PrjFSProviderClientShared.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PrjFSProviderClientShared.h; sourceTree = ""; }; 4AC1D7BE2091FA0400786861 /* PrjFSProviderUserClient.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PrjFSProviderUserClient.cpp; sourceTree = ""; }; @@ -45,6 +47,7 @@ 4AC1D7C22091FA2300786861 /* PrjFSClasses.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = PrjFSClasses.hpp; sourceTree = ""; }; 4AC1D7C32091FBFC00786861 /* PrjFSLogUserClient.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PrjFSLogUserClient.cpp; sourceTree = ""; }; 4AC1D7C42091FBFC00786861 /* PrjFSLogUserClient.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = PrjFSLogUserClient.hpp; sourceTree = ""; }; + 4AE187A721203A890026AC68 /* PerformanceTracing.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = PerformanceTracing.cpp; sourceTree = ""; }; C6BDD364208BC60400CB7E58 /* VirtualizationRoots.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = VirtualizationRoots.hpp; sourceTree = ""; }; C6BDD365208BC60400CB7E58 /* VirtualizationRoots.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = VirtualizationRoots.cpp; sourceTree = ""; }; C6BDD368208BC99100CB7E58 /* Locks.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = Locks.hpp; sourceTree = ""; }; @@ -124,6 +127,8 @@ C6C780CB207FD02400E7E054 /* KextLog.hpp */, C6C780CC207FD02400E7E054 /* KextLog.cpp */, C6C780B5207FC67200E7E054 /* Info.plist */, + 4ABB734320B867E000DC0D17 /* PerformanceTracing.hpp */, + 4AE187A721203A890026AC68 /* PerformanceTracing.cpp */, C6E9E116208BBB62004A5725 /* KauthHandler.hpp */, C6E9E117208BBB62004A5725 /* KauthHandler.cpp */, C6BDD364208BC60400CB7E58 /* VirtualizationRoots.hpp */, @@ -253,6 +258,7 @@ buildActionMask = 2147483647; files = ( C6BDD36B208BC99100CB7E58 /* Locks.cpp in Sources */, + 4AE187A821203A890026AC68 /* PerformanceTracing.cpp in Sources */, C6BDD367208BC60400CB7E58 /* VirtualizationRoots.cpp in Sources */, C6BDD374208C033200CB7E58 /* Memory.cpp in Sources */, 4AC1D7C52091FBFC00786861 /* PrjFSLogUserClient.cpp in Sources */, diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp new file mode 100644 index 0000000000..644cb574cb --- /dev/null +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp @@ -0,0 +1,59 @@ +#include "PerformanceTracing.hpp" +#include +#include +#include + +PerfTracingProbe profile_probes[Probe_Count]; + +void PerfTracing_Init() +{ + for (size_t i = 0; i < Probe_Count; ++i) + { + PerfTracing_ProbeInit(&profile_probes[i]); + } +} + +void PerfTracing_ProbeInit(PerfTracingProbe* probe) +{ + *probe = PerfTracingProbe{ .min = UINT64_MAX }; +} + +IOReturn PerfTracing_ExportDataUserClient(IOExternalMethodArguments* arguments) +{ + if (arguments->structureOutput == nullptr || arguments->structureOutputSize != sizeof(profile_probes)) + { + return kIOReturnBadArgument; + } + + memcpy(arguments->structureOutput, profile_probes, sizeof(profile_probes)); + return kIOReturnSuccess; +} + +void PerfTracing_RecordSample(PerfTracingProbe* probe, uint64_t startTime, uint64_t endTime) +{ + uint64_t interval = endTime - startTime; + + atomic_fetch_add(&probe->numSamples1, 1); + atomic_fetch_add(&probe->sum, interval); + + __uint128_t intervalSquared = interval; + intervalSquared *= intervalSquared; + atomic_fetch_add(&probe->sumSquares, intervalSquared); + + // Update minimum sample if necessary + { + uint64_t oldMin = atomic_load(&probe->min); + while (interval < oldMin && !atomic_compare_exchange_weak(&probe->min, &oldMin, interval)) + {} + } + + // Update maximum sample if necessary + { + uint64_t oldMax = atomic_load(&probe->max); + while (interval > oldMax && !atomic_compare_exchange_weak(&probe->max, &oldMax, interval)) + {} + } + + atomic_fetch_add(&probe->numSamples2, 1); +} + diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp new file mode 100644 index 0000000000..4f00998142 --- /dev/null +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp @@ -0,0 +1,155 @@ +#pragma once + +#include "PrjFSCommon.h" + +#include +#include + +void PerfTracing_Init(); +void PerfTracing_ProbeInit(PerfTracingProbe* probe); +void PerfTracing_RecordSample(PerfTracingProbe* probe, uint64_t startTime, uint64_t endTime); + +struct IOExternalMethodArguments; +IOReturn PerfTracing_ExportDataUserClient(IOExternalMethodArguments* arguments); + +extern PerfTracingProbe profile_probes[Probe_Count]; +// A scope-based manual instrumentation profiling mechanism. +// In the simplest case, the time between construction and destruction of the ProfileSample +// is measured and registered with the probe specified during construction: +// +// void MyFunction() +// { +// ProfileSample functionSample(Probe_MyFunction); +// // ... The code for which we're measuring the runtime ... +// } // <-- functionSample goes out of scope here, so that's when timing ends +// +// +// To allow runtimes different code paths in the same scope to be recorded separately, the +// probe identity can be modified using SetProbe(): +// +// void MyFunction() +// { +// ProfileSample functionSample(Probe_MyFunction); +// // ... The code for which we're measuring the runtime ... +// if (specialCase) +// { +// // We want to be able to distinguish between the runtimes of MyFunction for this +// // special case vs "normal" runs. +// functionSample.SetProbe(Probe_MyFunctionSpecialCase); +// // ... do something potentially expensive here ... +// } +// // ... more code ... +// } // <-- Runtime to here will be recorded either under Probe_MyFunction or Probe_MyFunctionSpecialCase +// +// +// For tracing sub-sections of code, such as the special case logic above, in isolation, +// we have 2 options: taking additional scoped samples, or split timings. Scoped samples are +// easier to understand: +// +// void MyFunction() +// { +// ProfileSample functionSample(Probe_MyFunction); +// // ... +// if (specialCase) +// { +// // Measure only the special case code on its own: +// ProfileSample specialCaseSample(Probe_MyFunctionSpecialCase); +// // ... do something potentially expensive here ... +// } // <-- scope of specialCaseSample ends here +// // ... more code ... +// } // <-- end of Probe_MyFunction in all cases +// +// In the above example, the runtimes of all MyFunction() calls are recorded under Probe_MyFunction, +// while the special case code on its own is recorded in Probe_MyFunctionSpecialCase. +// +// Taking split timings meanwhile allows us to carve up scoped samples into constituent sub-samples, +// useful for drilling down to find the source of performance issues: +// +// void MyFunction() +// { +// ProfileSample functionSample(Probe_MyFunction); +// // ... +// // The time from the creation of functionSample to this point is recorded as Probe_MyFunctionPart1. +// functionSample.TakeSplitSample(Probe_MyFunctionPart1, Probe_MyFunctionRemainder); +// if (specialCase) +// { +// // ... do something potentially expensive here ... +// functionSample.TakeSplitSample(Probe_MyFunctionSpecialCase); // This measures time since the Part1 split +// } // <-- scope of specialCaseSample ends here +// // ... more code ... +// } // <-- end of Probe_MyFunction in all cases; the time since the last split (Probe_MyFunctionPart1 +// // or Probe_MyFunctionSpecialCase depending on code path) is recorded as Probe_MyFunctionRemainder. +// +// The end time stamp for a split is taken as the start of the next split, and the overall start and end +// stamps of the scoped sample are alse the start and end of the first and last (remainder) split, +// respectively. +// So in this case, the sum total runtime of all samples of Probe_MyFunction is exactly equal to the sum of +// Probe_MyFunctionPart1 + Probe_MyFunctionSpecialCase + Probe_MyFunctionRemainder. +// Note that Probe_MyFunctionSpecialCase may have a lower sample count. +// Note also that the "remainder" split is optional - if only the 1-argument version of TakeSplitSample +// is used, the split time to the end of scope is not recorded. (And like the scoped sample, it can be changed, +// in this case using SetFinalSplitProbe()) +class ProfileSample +{ + ProfileSample(const ProfileSample&) = delete; + ProfileSample() = delete; + + const uint64_t startTimestamp; + PrjFS_PerfCounter wholeSampleProbe; + PrjFS_PerfCounter finalSplitProbe; + uint64_t splitTimestamp; + +public: + inline ProfileSample(PrjFS_PerfCounter defaultProbe); + inline void SetProbe(PrjFS_PerfCounter probe); + inline void TakeSplitSample(PrjFS_PerfCounter splitProbe); + inline void TakeSplitSample(PrjFS_PerfCounter splitProbe, PrjFS_PerfCounter newFinalSplitProbe); + inline void SetFinalSplitProbe(PrjFS_PerfCounter newFinalSplitProbe); + inline ~ProfileSample(); +}; + +ProfileSample::~ProfileSample() +{ + uint64_t endTimestamp = mach_absolute_time(); + if (this->wholeSampleProbe != Probe_None) + { + PerfTracing_RecordSample(&profile_probes[this->wholeSampleProbe], this->startTimestamp, endTimestamp); + } + + if (this->finalSplitProbe != Probe_None) + { + PerfTracing_RecordSample(&profile_probes[this->finalSplitProbe], this->splitTimestamp, endTimestamp); + } +}; + +void ProfileSample::TakeSplitSample(PrjFS_PerfCounter splitProbe) +{ + uint64_t newSplitTimestamp = mach_absolute_time(); + PerfTracing_RecordSample(&profile_probes[splitProbe], this->splitTimestamp, newSplitTimestamp); + this->splitTimestamp = newSplitTimestamp; +} + +void ProfileSample::TakeSplitSample(PrjFS_PerfCounter splitProbe, PrjFS_PerfCounter newFinalSplitProbe) +{ + this->TakeSplitSample(splitProbe); + this->finalSplitProbe = newFinalSplitProbe; +} + +void ProfileSample::SetFinalSplitProbe(PrjFS_PerfCounter newFinalSplitProbe) +{ + this->finalSplitProbe = newFinalSplitProbe; +} + +ProfileSample::ProfileSample(PrjFS_PerfCounter defaultProbe) : + startTimestamp(mach_absolute_time()), + wholeSampleProbe(defaultProbe), + finalSplitProbe(Probe_None), + splitTimestamp(this->startTimestamp) +{ +} + +void ProfileSample::SetProbe(PrjFS_PerfCounter probe) +{ + this->wholeSampleProbe = probe; +} + diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp index 508545bcd6..97a8ee6e54 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp @@ -5,12 +5,15 @@ #include "KauthHandler.hpp" #include "Locks.hpp" #include "Memory.hpp" +#include "PerformanceTracing.hpp" extern "C" kern_return_t PrjFSKext_Start(kmod_info_t* ki, void* d); extern "C" kern_return_t PrjFSKext_Stop(kmod_info_t* ki, void* d); kern_return_t PrjFSKext_Start(kmod_info_t* ki, void* d) { + PerfTracing_Init(); + if (Locks_Init()) { goto CleanupAndFail; diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.cpp index b39b73bf9f..69e59889d0 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.cpp @@ -2,6 +2,7 @@ #include "PrjFSLogClientShared.h" #include "KextLog.hpp" #include "PrjFSCommon.h" +#include "PerformanceTracing.hpp" #include @@ -9,6 +10,20 @@ OSDefineMetaClassAndStructors(PrjFSLogUserClient, IOUserClient); // Amount of memory to set aside for kernel -> userspace log messages. static const uint32_t LogMessageQueueCapacityBytes = 1024 * 1024; + +static const IOExternalMethodDispatch LogUserClientDispatch[] = +{ + [LogSelector_FetchProfilingData] = + { + .function = &PrjFSLogUserClient::fetchProfilingData, + .checkScalarInputCount = 0, + .checkStructureInputSize = 0, + .checkScalarOutputCount = 0, + .checkStructureOutputSize = Probe_Count * sizeof(PerfTracingProbe), // array of probes + }, +}; + + bool PrjFSLogUserClient::initWithTask( task_t owningTask, void* securityToken, @@ -129,3 +144,31 @@ void PrjFSLogUserClient::sendLogMessage(KextLog_MessageHeader* message, uint32_t Mutex_Release(this->dataQueueWriterMutex); } +IOReturn PrjFSLogUserClient::externalMethod( + uint32_t selector, + IOExternalMethodArguments* arguments, + IOExternalMethodDispatch* dispatch, + OSObject* target, + void* reference) +{ + IOExternalMethodDispatch local_dispatch = {}; + if (selector < sizeof(LogUserClientDispatch) / sizeof(LogUserClientDispatch[0])) + { + if (nullptr != LogUserClientDispatch[selector].function) + { + local_dispatch = LogUserClientDispatch[selector]; + dispatch = &local_dispatch; + target = this; + } + } + return this->super::externalMethod(selector, arguments, dispatch, target, reference); +} + +IOReturn PrjFSLogUserClient::fetchProfilingData( + OSObject* target, + void* reference, + IOExternalMethodArguments* arguments) +{ + return PerfTracing_ExportDataUserClient(arguments); +} + diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.hpp index a91d456a8c..28c3bb66ef 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.hpp @@ -26,5 +26,18 @@ class PrjFSLogUserClient : public IOUserClient virtual IOReturn clientMemoryForType(UInt32 type, IOOptionBits* options, IOMemoryDescriptor** memory) override; virtual IOReturn registerNotificationPort(mach_port_t port, UInt32 type, io_user_reference_t refCon) override; + virtual IOReturn externalMethod( + uint32_t selector, + IOExternalMethodArguments* arguments, + IOExternalMethodDispatch* dispatch = nullptr, + OSObject* target = nullptr, + void* reference = nullptr) override; + + + static IOReturn fetchProfilingData( + OSObject* target, + void* reference, + IOExternalMethodArguments* arguments); + void sendLogMessage(KextLog_MessageHeader* message, uint32_t size); }; diff --git a/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h b/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h index f7478ece94..35b386b442 100644 --- a/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h +++ b/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h @@ -33,4 +33,29 @@ enum PrjFSServiceUserClientType UserClientType_Log, }; +enum PrjFS_PerfCounter : int32_t +{ + // Note: ensure that any changes to this list are reflected in the PerfCounterNames array of strings + + + Probe_Count, + + Probe_None = -1 +}; + +struct PerfTracingProbe +{ + _Atomic uint64_t numSamples1; + _Atomic uint64_t numSamples2; + // Units: Mach absolute time (squared for sumSquares) + // Sum of measured sample intervals + _Atomic uint64_t sum; + // Smallest encountered interval + _Atomic uint64_t min; + // Largest encountered interval + _Atomic uint64_t max; + // Sum-of-squares of measured time intervals (for stddev) + _Atomic __uint128_t sumSquares; +}; + #endif /* PrjFSCommon_h */ diff --git a/ProjFS.Mac/PrjFSKext/public/PrjFSLogClientShared.h b/ProjFS.Mac/PrjFSKext/public/PrjFSLogClientShared.h index ca847b4e98..957a86a738 100644 --- a/ProjFS.Mac/PrjFSKext/public/PrjFSLogClientShared.h +++ b/ProjFS.Mac/PrjFSKext/public/PrjFSLogClientShared.h @@ -2,6 +2,14 @@ #include +// External method selectors for log user clients +enum PrjFSLogUserClientSelector +{ + LogSelector_Invalid = 0, + + LogSelector_FetchProfilingData, +}; + enum PrjFSLogUserClientMemoryType { LogMemoryType_Invalid = 0, diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.xcodeproj/project.pbxproj b/ProjFS.Mac/PrjFSLib/PrjFSLib.xcodeproj/project.pbxproj index 9bfd51e90c..f88b6cc2b1 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.xcodeproj/project.pbxproj +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.xcodeproj/project.pbxproj @@ -9,6 +9,7 @@ /* Begin PBXBuildFile section */ 4A440DDE2093AD3300AADA76 /* IOKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 4A440DDD2093AD3300AADA76 /* IOKit.framework */; }; 4A8A1BEE20A0D5940024BC10 /* CoreFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 4A8A1BED20A0D5940024BC10 /* CoreFoundation.framework */; }; + 4A91E086215E76A90079FE1B /* kext-perf-tracing.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 4A91E084215E76A90079FE1B /* kext-perf-tracing.cpp */; }; C6C780D120816BDC00E7E054 /* PrjFSLib.h in Headers */ = {isa = PBXBuildFile; fileRef = C6C780CF20816BDC00E7E054 /* PrjFSLib.h */; }; C6C780D220816BDC00E7E054 /* PrjFSLib.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C6C780D020816BDC00E7E054 /* PrjFSLib.cpp */; }; D308478120B4431200F69E92 /* prjfs-log.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D308478020B4431200F69E92 /* prjfs-log.cpp */; }; @@ -34,6 +35,8 @@ /* Begin PBXFileReference section */ 4A440DDD2093AD3300AADA76 /* IOKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = IOKit.framework; path = System/Library/Frameworks/IOKit.framework; sourceTree = SDKROOT; }; 4A8A1BED20A0D5940024BC10 /* CoreFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreFoundation.framework; path = System/Library/Frameworks/CoreFoundation.framework; sourceTree = SDKROOT; }; + 4A91E084215E76A90079FE1B /* kext-perf-tracing.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = "kext-perf-tracing.cpp"; sourceTree = ""; }; + 4A91E085215E76A90079FE1B /* kext-perf-tracing.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = "kext-perf-tracing.hpp"; sourceTree = ""; }; C6C780C4207FC6AB00E7E054 /* libPrjFSLib.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libPrjFSLib.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; C6C780CF20816BDC00E7E054 /* PrjFSLib.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PrjFSLib.h; sourceTree = ""; }; C6C780D020816BDC00E7E054 /* PrjFSLib.cpp */ = {isa = PBXFileReference; indentWidth = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PrjFSLib.cpp; sourceTree = ""; tabWidth = 4; usesTabs = 0; }; @@ -103,6 +106,8 @@ isa = PBXGroup; children = ( D308478020B4431200F69E92 /* prjfs-log.cpp */, + 4A91E085215E76A90079FE1B /* kext-perf-tracing.hpp */, + 4A91E084215E76A90079FE1B /* kext-perf-tracing.cpp */, ); path = "prjfs-log"; sourceTree = ""; @@ -207,6 +212,7 @@ buildActionMask = 2147483647; files = ( D308478920B4432500F69E92 /* PrjFSUser.cpp in Sources */, + 4A91E086215E76A90079FE1B /* kext-perf-tracing.cpp in Sources */, D308478120B4431200F69E92 /* prjfs-log.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp new file mode 100644 index 0000000000..30d46eda4c --- /dev/null +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp @@ -0,0 +1,65 @@ +#include "kext-perf-tracing.hpp" +#include "../../PrjFSKext/public/PrjFSCommon.h" +#include "../../PrjFSKext/public/PrjFSLogClientShared.h" +#include +#include +#include +#include +#include + +static mach_timebase_info_data_t s_machTimebase; + +static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime) +{ + return static_cast<__uint128_t>(machAbsoluteTime) * s_machTimebase.numer / s_machTimebase.denom; +} + +static const char* const PerfCounterNames[Probe_Count] = +{ +}; + +bool PrjFSLog_FetchAndPrintKextProfilingData(io_connect_t connection) +{ + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + mach_timebase_info(&s_machTimebase); + }); + + PerfTracingProbe probes[Probe_Count]; + size_t out_size = sizeof(probes); + IOReturn ret = IOConnectCallStructMethod(connection, LogSelector_FetchProfilingData, nullptr, 0, probes, &out_size); + if (ret == kIOReturnUnsupported) + { + return false; + } + else if (ret == kIOReturnSuccess) + { + for (unsigned i = 0; i < Probe_Count; ++i) + { + double samples = probes[i].numSamples1; + double sum_abs = probes[i].sum; + double stddev_abs = samples > 1 ? sqrt((samples * probes[i].sumSquares - sum_abs * sum_abs) / (samples * (samples - 1))) : 0.0; + + double sum_ns = nanosecondsFromAbsoluteTime(sum_abs); + double stddev_ns = nanosecondsFromAbsoluteTime(stddev_abs); + double mean_ns = samples > 0 ? sum_ns / samples : 0; + printf("%2u %40s %8llu [%8llu] samples, total time: %15.0f ns, mean: %10.2f ns +/- %11.2f", + i, PerfCounterNames[i], probes[i].numSamples1, probes[i].numSamples2, sum_ns, mean_ns, stddev_ns); + if (probes[i].min != UINT64_MAX) + { + printf(", min: %7llu ns, max: %10llu ns\n", nanosecondsFromAbsoluteTime(probes[i].min), nanosecondsFromAbsoluteTime(probes[i].max)); + } + else + { + printf("\n"); + } + } + } + else + { + fprintf(stderr, "fetching profiling data from kernel failed: 0x%x\n", ret); + return false; + } + fflush(stdout); + return true; +} diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.hpp b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.hpp new file mode 100644 index 0000000000..fedbf92dac --- /dev/null +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.hpp @@ -0,0 +1,5 @@ +#pragma once + +#include + +bool PrjFSLog_FetchAndPrintKextProfilingData(io_connect_t connection); diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp index 281fc2bd3b..aaa32b1c4e 100644 --- a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp @@ -1,4 +1,5 @@ #include "../PrjFSUser.hpp" +#include "kext-perf-tracing.hpp" #include "../../PrjFSKext/public/PrjFSLogClientShared.h" #include #include @@ -7,10 +8,12 @@ #include static const char* KextLogLevelAsString(KextLog_Level level); -static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime); +static uint64_t NanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime); +static dispatch_source_t StartKextProfilingDataPolling(io_connect_t connection); static mach_timebase_info_data_t s_machTimebase; + int main(int argc, const char * argv[]) { mach_timebase_info(&s_machTimebase); @@ -55,7 +58,7 @@ int main(int argc, const char * argv[]) const char* messageType = KextLogLevelAsString(message.level); int logStringLength = messageSize - sizeof(KextLog_MessageHeader) - 1; - uint64_t timeOffsetNS = nanosecondsFromAbsoluteTime(message.machAbsoluteTimestamp - machStartTime); + uint64_t timeOffsetNS = NanosecondsFromAbsoluteTime(message.machAbsoluteTimestamp - machStartTime); uint64_t timeOffsetMS = timeOffsetNS / NSEC_PER_MSEC; printf("(%d: %5llu.%03llu) %s: %.*s\n", lineCount, timeOffsetMS / 1000u, timeOffsetMS % 1000u, messageType, logStringLength, entry->data + sizeof(KextLog_MessageHeader)); @@ -66,8 +69,21 @@ int main(int argc, const char * argv[]) } }); dispatch_resume(dataQueue.dispatchSource); + + dispatch_source_t timer = nullptr; + if (PrjFSLog_FetchAndPrintKextProfilingData(connection)) + { + timer = StartKextProfilingDataPolling(connection); + } + CFRunLoopRun(); + if (nullptr != timer) + { + dispatch_cancel(timer); + dispatch_release(timer); + } + return 0; } @@ -86,7 +102,19 @@ static const char* KextLogLevelAsString(KextLog_Level level) } } -static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime) +static uint64_t NanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime) { return static_cast<__uint128_t>(machAbsoluteTime) * s_machTimebase.numer / s_machTimebase.denom; } + +static dispatch_source_t StartKextProfilingDataPolling(io_connect_t connection) +{ + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, dispatch_get_main_queue()); + dispatch_source_set_timer(timer, DISPATCH_TIME_NOW, 15 * NSEC_PER_SEC, 10 * NSEC_PER_SEC); + dispatch_source_set_event_handler(timer, ^{ + PrjFSLog_FetchAndPrintKextProfilingData(connection); + }); + dispatch_resume(timer); + return timer; +} + From 3c86170f8f4fab252af852d12aa5538964a5ec2f Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Fri, 5 Oct 2018 12:11:54 +0200 Subject: [PATCH 110/244] Mac kext: Incorporates code review style feedback In the review, it was pointed out we should be using nullptr instead of 0 for default function argument values which were copied from the declaration of the function being overridden in the log user client. The same is true for the provider user client, so this fixes it there too. --- ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp index de487108c1..650b5dc0fc 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp @@ -30,9 +30,9 @@ class PrjFSProviderUserClient : public IOUserClient virtual IOReturn externalMethod( uint32_t selector, IOExternalMethodArguments* arguments, - IOExternalMethodDispatch* dispatch = 0, - OSObject* target = 0, - void* reference = 0) override; + IOExternalMethodDispatch* dispatch = nullptr, + OSObject* target = nullptr, + void* reference = nullptr) override; virtual IOReturn clientMemoryForType( UInt32 type, IOOptionBits* options, From 5f7cc9721b8fff584c8da2991ddf6301bc86aa40 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Sat, 29 Sep 2018 11:41:51 +0200 Subject: [PATCH 111/244] Mac kext: Placement of profiling probes throughout the vnode listener Using the previously created profiling mechanism, this adds various measurement points to the vnode scope listener handler, and the relevant strings to the user space extraction tool. --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 29 ++++++++++++++++++- .../PrjFSKext/VirtualizationRoots.cpp | 5 ++++ ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h | 18 ++++++++++++ .../PrjFSLib/prjfs-log/kext-perf-tracing.cpp | 17 +++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 111c9355d8..50250b6318 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -13,6 +13,7 @@ #include "Message.h" #include "Locks.hpp" #include "PrjFSProviderUserClient.hpp" +#include "PerformanceTracing.hpp" // Function prototypes static int HandleVnodeOperation( @@ -61,6 +62,7 @@ static bool ShouldHandleVnodeOpEvent( vfs_context_t context, const vnode_t vnode, kauth_action_t action, + ProfileSample& operationSample, // Out params: VirtualizationRootHandle* root, @@ -252,6 +254,8 @@ static int HandleVnodeOperation( uintptr_t arg3) { atomic_fetch_add(&s_numActiveKauthEvents, 1); + + ProfileSample functionSample(Probe_VnodeOp); vfs_context_t context = reinterpret_cast(arg0); vnode_t currentVnode = reinterpret_cast(arg1); @@ -296,6 +300,7 @@ static int HandleVnodeOperation( context, currentVnode, action, + functionSample, &root, &vnodeType, ¤tVnodeFileFlags, @@ -343,6 +348,8 @@ static int HandleVnodeOperation( // Recursively expand directory on delete to ensure child placeholders are created before rename operations if (isDeleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { + functionSample.SetProbe(Probe_VnodeOp_PopulatePlaceholderDirectory); + if (!TrySendRequestAndWaitForResponse( root, isDeleteAction ? @@ -376,6 +383,8 @@ static int HandleVnodeOperation( { if (FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { + functionSample.SetProbe(Probe_VnodeOp_HydratePlaceholderFile); + if (!TrySendRequestAndWaitForResponse( root, MessageType_KtoU_HydrateFile, @@ -416,6 +425,8 @@ static int HandleFileOpOperation( { atomic_fetch_add(&s_numActiveKauthEvents, 1); + ProfileSample functionSample(Probe_FileOp); + vfs_context_t context = vfs_context_create(NULL); vnode_t currentVnodeFromPath = NULLVP; @@ -567,6 +578,7 @@ static bool ShouldHandleVnodeOpEvent( vfs_context_t context, const vnode_t vnode, kauth_action_t action, + ProfileSample& operationSample, // Out params: VirtualizationRootHandle* root, @@ -582,6 +594,7 @@ static bool ShouldHandleVnodeOpEvent( if (!VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode)) { + operationSample.SetProbe(Probe_Op_EarlyOut); *kauthResult = KAUTH_RESULT_DEFER; return false; } @@ -589,16 +602,22 @@ static bool ShouldHandleVnodeOpEvent( *vnodeType = vnode_vtype(vnode); if (ShouldIgnoreVnodeType(*vnodeType, vnode)) { + operationSample.SetProbe(Probe_Op_EarlyOut); *kauthResult = KAUTH_RESULT_DEFER; return false; } - *vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + { + ProfileSample readflags(Probe_ReadFileFlags); + *vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + } + if (!FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) { // This vnode is not part of ANY virtualization root, so exit now before doing any more work. // This gives us a cheap way to avoid adding overhead to IO outside of a virtualization root. + operationSample.SetProbe(Probe_Op_NoVirtualizationRootFlag); *kauthResult = KAUTH_RESULT_DEFER; return false; } @@ -618,14 +637,21 @@ static bool ShouldHandleVnodeOpEvent( // get called again, so we lose the opportunity to hydrate the file/directory and it will appear as though // it is missing its contents. + operationSample.SetProbe(Probe_Op_DenyCrawler); *kauthResult = KAUTH_RESULT_DENY; return false; } + + operationSample.SetProbe(Probe_Op_EmptyFlag); } + operationSample.TakeSplitSample(Probe_Op_IdentifySplit); + *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); *root = VirtualizationRoot_FindForVnode(vnode, *vnodeFsidInode); + operationSample.TakeSplitSample(Probe_Op_VirtualizationRootFindSplit); + if (RootHandle_ProviderTemporaryDirectory == *root) { *kauthResult = KAUTH_RESULT_DEFER; @@ -650,6 +676,7 @@ static bool ShouldHandleVnodeOpEvent( // If the calling process is the provider, we must exit right away to avoid deadlocks if (VirtualizationRoot_PIDMatchesProvider(*root, *pid)) { + operationSample.SetProbe(Probe_Op_Provider); *kauthResult = KAUTH_RESULT_DEFER; return false; } diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index 88b0a97dba..4949e357bf 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -11,6 +11,7 @@ #include "PrjFSProviderUserClient.hpp" #include "kernel-header-wrappers/mount.h" #include "VnodeUtilities.hpp" +#include "PerformanceTracing.hpp" struct VirtualizationRoot @@ -133,12 +134,16 @@ kern_return_t VirtualizationRoots_Cleanup() VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode) { + ProfileSample functionSample(Probe_VirtualizationRoot_Find); + VirtualizationRootHandle rootHandle = RootHandle_None; vnode_get(vnode); // Search up the tree until we hit a known virtualization root or THE root of the file system while (RootHandle_None == rootHandle && NULLVP != vnode && !vnode_isvroot(vnode)) { + ProfileSample iterationSample(Probe_VirtualizationRoot_FindIteration); + rootHandle = FindOrDetectRootAtVnode(vnode, nullptr /* vfs context */, vnodeFsidInode); // Note: if FindOrDetectRootAtVnode returns a "special" handle other // than RootHandle_None, we want to stop the search and return that. diff --git a/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h b/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h index 35b386b442..fe44364fbf 100644 --- a/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h +++ b/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h @@ -37,6 +37,24 @@ enum PrjFS_PerfCounter : int32_t { // Note: ensure that any changes to this list are reflected in the PerfCounterNames array of strings + Probe_VnodeOp = 0, + Probe_FileOp, + Probe_Op_EarlyOut, + Probe_Op_NoVirtualizationRootFlag, + Probe_Op_EmptyFlag, + Probe_Op_DenyCrawler, + Probe_Op_Offline, + Probe_Op_Provider, + Probe_VnodeOp_PopulatePlaceholderDirectory, + Probe_VnodeOp_HydratePlaceholderFile, + + Probe_Op_IdentifySplit, + Probe_Op_VirtualizationRootFindSplit, + + Probe_ReadFileFlags, + + Probe_VirtualizationRoot_Find, + Probe_VirtualizationRoot_FindIteration, Probe_Count, diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp index 30d46eda4c..45a519a196 100644 --- a/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp @@ -16,6 +16,23 @@ static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime) static const char* const PerfCounterNames[Probe_Count] = { + [Probe_VnodeOp] = "VnodeOp", + [Probe_FileOp] = "FileOp", + [Probe_Op_EarlyOut] = "Op_EarlyOut", + [Probe_Op_NoVirtualizationRootFlag] = "Op_NoVirtualizationRootFlag", + [Probe_Op_EmptyFlag] = "Op_EmptyFlag", + [Probe_Op_DenyCrawler] = "Op_DenyCrawler", + [Probe_Op_Offline] = "Op_Offline", + [Probe_Op_Provider] = "Op_Provider", + [Probe_VnodeOp_PopulatePlaceholderDirectory] = "VnodeOp_PopulatePlaceholderDirectory", + [Probe_VnodeOp_HydratePlaceholderFile] = "VnodeOp_HydratePlaceholderFile", + + [Probe_Op_IdentifySplit] = "Op_IdentifySplit", + [Probe_Op_VirtualizationRootFindSplit] = "Op_VirtualizationRootFindSplit", + + [Probe_ReadFileFlags] = "Probe_ReadFileFlags", + [Probe_VirtualizationRoot_Find] = "VirtualizationRoot_Find", + [Probe_VirtualizationRoot_FindIteration] = "VirtualizationRoot_FindIteration", }; bool PrjFSLog_FetchAndPrintKextProfilingData(io_connect_t connection) From b26859971b42772abfbe5a784d861ef8e0d3f121 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Fri, 28 Sep 2018 19:16:57 +0200 Subject: [PATCH 112/244] Mac kext: Conditionally enable performance tracing. --- .../PrjFSKext/PrjFSKext/PerformanceTracing.cpp | 4 ++++ .../PrjFSKext/PrjFSKext/PerformanceTracing.hpp | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp index 644cb574cb..738ee5c79c 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp @@ -20,6 +20,7 @@ void PerfTracing_ProbeInit(PerfTracingProbe* probe) IOReturn PerfTracing_ExportDataUserClient(IOExternalMethodArguments* arguments) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE if (arguments->structureOutput == nullptr || arguments->structureOutputSize != sizeof(profile_probes)) { return kIOReturnBadArgument; @@ -27,6 +28,9 @@ IOReturn PerfTracing_ExportDataUserClient(IOExternalMethodArguments* arguments) memcpy(arguments->structureOutput, profile_probes, sizeof(profile_probes)); return kIOReturnSuccess; +#else + return kIOReturnUnsupported; +#endif } void PerfTracing_RecordSample(PerfTracingProbe* probe, uint64_t startTime, uint64_t endTime) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp index 4f00998142..664f1d5e38 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp @@ -94,10 +94,12 @@ class ProfileSample ProfileSample(const ProfileSample&) = delete; ProfileSample() = delete; +#if PRJFS_PERFORMANCE_TRACING_ENABLE const uint64_t startTimestamp; PrjFS_PerfCounter wholeSampleProbe; PrjFS_PerfCounter finalSplitProbe; uint64_t splitTimestamp; +#endif public: inline ProfileSample(PrjFS_PerfCounter defaultProbe); @@ -110,6 +112,7 @@ class ProfileSample ProfileSample::~ProfileSample() { +#if PRJFS_PERFORMANCE_TRACING_ENABLE uint64_t endTimestamp = mach_absolute_time(); if (this->wholeSampleProbe != Probe_None) { @@ -120,36 +123,49 @@ ProfileSample::~ProfileSample() { PerfTracing_RecordSample(&profile_probes[this->finalSplitProbe], this->splitTimestamp, endTimestamp); } +#endif }; void ProfileSample::TakeSplitSample(PrjFS_PerfCounter splitProbe) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE uint64_t newSplitTimestamp = mach_absolute_time(); PerfTracing_RecordSample(&profile_probes[splitProbe], this->splitTimestamp, newSplitTimestamp); this->splitTimestamp = newSplitTimestamp; +#endif } void ProfileSample::TakeSplitSample(PrjFS_PerfCounter splitProbe, PrjFS_PerfCounter newFinalSplitProbe) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE this->TakeSplitSample(splitProbe); this->finalSplitProbe = newFinalSplitProbe; +#endif } void ProfileSample::SetFinalSplitProbe(PrjFS_PerfCounter newFinalSplitProbe) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE this->finalSplitProbe = newFinalSplitProbe; +#endif } -ProfileSample::ProfileSample(PrjFS_PerfCounter defaultProbe) : + +ProfileSample::ProfileSample(PrjFS_PerfCounter defaultProbe) +#if PRJFS_PERFORMANCE_TRACING_ENABLE + : startTimestamp(mach_absolute_time()), wholeSampleProbe(defaultProbe), finalSplitProbe(Probe_None), splitTimestamp(this->startTimestamp) +#endif { } void ProfileSample::SetProbe(PrjFS_PerfCounter probe) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE this->wholeSampleProbe = probe; +#endif } From 905258ae720f8069071b3dfa44b9498dcbb8aa3c Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Sat, 29 Sep 2018 14:24:42 +0200 Subject: [PATCH 113/244] Mac kext: New Profiling(Release) build configuration based on Release --- .../xcshareddata/xcschemes/PrjFS.xcscheme | 2 +- .../PrjFSKext.xcodeproj/project.pbxproj | 92 +++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme b/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme index 558130ad60..389d98c9a2 100644 --- a/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme +++ b/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme @@ -92,7 +92,7 @@ Date: Sat, 6 Oct 2018 10:25:24 +0200 Subject: [PATCH 114/244] Mac kext: Adds check to only handle fileops on allowed filesystems The vnode handler was already performing this check, but the fileop handler previously skipped it. This is part 1 of a fix for KP bug report #340. --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 111c9355d8..91e4e00410 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -670,6 +670,11 @@ static bool ShouldHandleFileOpEvent( { *root = RootHandle_None; + if (!VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode)) + { + return false; + } + vtype vnodeType = vnode_vtype(vnode); if (ShouldIgnoreVnodeType(vnodeType, vnode)) { From 5ee74b0b6c401d1eac84c619be043176623dbdb2 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Mon, 8 Oct 2018 09:02:00 -0400 Subject: [PATCH 115/244] Make GVFS.Upgrader rely on GVFS.cs.props to fix incremental build issue --- GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj b/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj index 8fb2f91ac7..008a3eb240 100644 --- a/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj +++ b/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj @@ -1,6 +1,7 @@  + Debug AnyCPU @@ -11,45 +12,26 @@ v4.6.1 512 true - publish\ - true - Disk - false - Foreground - 7 - Days - false - false - true - 0 - 1.0.0.%2a - false - false - true true - ..\..\..\BuildOutput\GVFS.Upgrader\bin\x64\Debug\ DEBUG;TRACE true full x64 prompt - MinimumRecommendedRules.ruleset - true + false - ..\..\..\BuildOutput\GVFS.Upgrader\bin\x64\Release\ TRACE true true pdbonly x64 prompt - MinimumRecommendedRules.ruleset - true + false From 95b9bf68e4f867a514746821529aa17426fbb352 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 3 Oct 2018 07:59:04 -0400 Subject: [PATCH 116/244] GitProcess: allow GIT_TRACE to point to full path --- GVFS/GVFS.Common/Git/GitProcess.cs | 26 ++++++++++++----- .../EnlistmentPerFixture/StatusVerbTests.cs | 28 +++++++++++++++++++ .../Tools/GVFSFunctionalTestEnlistment.cs | 4 +-- .../GVFS.FunctionalTests/Tools/GVFSProcess.cs | 15 ++++++---- 4 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/StatusVerbTests.cs diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 8701605ab0..89475752e0 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -1,6 +1,5 @@ using GVFS.Common.FileSystem; using GVFS.Common.Tracing; -using Microsoft.Win32; using System; using System.Collections.Generic; using System.ComponentModel; @@ -175,7 +174,7 @@ public bool IsValidRepo() Result result = this.InvokeGitAgainstDotGitFolder("rev-parse --show-toplevel"); return !result.HasErrors; } - + public Result RevParse(string gitRef) { return this.InvokeGitAgainstDotGitFolder("rev-parse " + gitRef); @@ -441,11 +440,24 @@ public Process GetGitProcess(string command, string workingDirectory, string dot // Removing trace variables that might change git output and break parsing // List of environment variables: https://git-scm.com/book/gr/v2/Git-Internals-Environment-Variables - foreach (string key in processInfo.EnvironmentVariables.Keys.Cast() - .Where(x => x.StartsWith("GIT_TRACE", StringComparison.OrdinalIgnoreCase)) - .ToList()) + foreach (string key in processInfo.EnvironmentVariables.Keys.Cast().ToList()) { - processInfo.EnvironmentVariables.Remove(key); + // If GIT_TRACE is set to a fully-rooted path, then Git sends the trace + // output to that path instead of stdout (GIT_TRACE=1) or stderr (GIT_TRACE=2). + if (key.StartsWith("GIT_TRACE", StringComparison.OrdinalIgnoreCase)) + { + try + { + if (!Path.IsPathRooted(processInfo.EnvironmentVariables[key])) + { + processInfo.EnvironmentVariables.Remove(key); + } + } + catch (ArgumentException) + { + processInfo.EnvironmentVariables.Remove(key); + } + } } processInfo.EnvironmentVariables["GIT_TERMINAL_PROMPT"] = "0"; @@ -637,7 +649,7 @@ private Result InvokeGitAgainstDotGitFolder( parseStdOutLine: parseStdOutLine, timeoutMs: -1); } - + public class Result { public const int SuccessCode = 0; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/StatusVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/StatusVerbTests.cs new file mode 100644 index 0000000000..f7def18b94 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/StatusVerbTests.cs @@ -0,0 +1,28 @@ +using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.Tests.Should; +using NUnit.Framework; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture +{ + [TestFixture] + public class StatusVerbTests : TestsWithEnlistmentPerFixture + { + [TestCase] + public void GitTrace() + { + Dictionary environmentVariables = new Dictionary(); + + this.Enlistment.Status(trace: "1"); + this.Enlistment.Status(trace: "2"); + + string logPath = Path.Combine(this.Enlistment.RepoRoot, "log-file.txt"); + this.Enlistment.Status(trace: logPath); + + FileSystemRunner fileSystem = new SystemIORunner(); + fileSystem.FileExists(logPath).ShouldBeTrue(); + string.IsNullOrWhiteSpace(fileSystem.ReadAllText(logPath)).ShouldBeFalse(); + } + } +} diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index 34ce2fd6dd..bc28530b85 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -211,9 +211,9 @@ public string Diagnose() return this.gvfsProcess.Diagnose(); } - public string Status() + public string Status(string trace = null) { - return this.gvfsProcess.Status(); + return this.gvfsProcess.Status(trace); } public bool WaitForBackgroundOperations(int maxWaitMilliseconds = DefaultMaxWaitMSForStatusCheck) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs index d9acce87d7..98939a8fb8 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs @@ -8,14 +8,14 @@ public class GVFSProcess private readonly string pathToGVFS; private readonly string enlistmentRoot; private readonly string localCacheRoot; - + public GVFSProcess(string pathToGVFS, string enlistmentRoot, string localCacheRoot) { this.pathToGVFS = pathToGVFS; this.enlistmentRoot = enlistmentRoot; this.localCacheRoot = localCacheRoot; } - + public void Clone(string repositorySource, string branchToCheckout, bool skipPrefetch) { string args = string.Format( @@ -60,9 +60,9 @@ public string Diagnose() return this.CallGVFS("diagnose \"" + this.enlistmentRoot + "\""); } - public string Status() + public string Status(string trace = null) { - return this.CallGVFS("status " + this.enlistmentRoot); + return this.CallGVFS("status " + this.enlistmentRoot, trace: trace); } public string CacheServer(string args) @@ -90,7 +90,7 @@ public string RunServiceVerb(string argument) return this.CallGVFS("service " + argument, failOnError: true); } - private string CallGVFS(string args, bool failOnError = false) + private string CallGVFS(string args, bool failOnError = false, string trace = null) { ProcessStartInfo processInfo = null; processInfo = new ProcessStartInfo(this.pathToGVFS); @@ -100,6 +100,11 @@ private string CallGVFS(string args, bool failOnError = false) processInfo.UseShellExecute = false; processInfo.RedirectStandardOutput = true; + if (trace != null) + { + processInfo.EnvironmentVariables["GIT_TRACE"] = trace; + } + using (Process process = Process.Start(processInfo)) { string result = process.StandardOutput.ReadToEnd(); From b1a31fc6abfc3032a30c26fbea20976980184d80 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 8 Oct 2018 08:55:23 -0700 Subject: [PATCH 117/244] Switch from FileAttributeRecallOnOpen to FileAttributeRecallOnDataAccess --- .../Tests/EnlistmentPerFixture/BasicFileSystemTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index db86eb26ef..982e02decf 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -17,7 +17,7 @@ public class BasicFileSystemTests : TestsWithEnlistmentPerFixture { private const int FileAttributeSparseFile = 0x00000200; private const int FileAttributeReparsePoint = 0x00000400; - private const int FileAttributeRecallOnOpen = 0x00040000; + private const int FileAttributeRecallOnDataAccess = 0x00400000; [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] public void ShrinkFileContents(FileSystemRunner fileSystem, string parentFolder) @@ -163,7 +163,7 @@ public void ExpandedFileAttributesAreUpdated() while (attributes != FileAttributes.Hidden && retryCount < maxRetries) { // ProjFS attributes are remoted asynchronously when files are converted to full - FileAttributes attributesLessProjFS = attributes & (FileAttributes)~(FileAttributeSparseFile | FileAttributeRecallOnOpen | FileAttributeReparsePoint); + FileAttributes attributesLessProjFS = attributes & (FileAttributes)~(FileAttributeSparseFile | FileAttributeReparsePoint | FileAttributeRecallOnDataAccess); attributesLessProjFS.ShouldEqual( FileAttributes.Hidden, From c3ea175df4d4aac0de2322ef71c02d278b2657b9 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 8 Oct 2018 11:19:30 -0400 Subject: [PATCH 118/244] - Added ability to set custom key comparers in FileBasedDictionary. Consumers can set case-insensitive comparers for case-insensitive dictionary lookups. - Added platform specific installer extensions. GVFS.Hooks.OSPlatform would read the extensions from GVFSHooksPlatform.OSPlatform classes. - Updated ProductUpgrader to do case-insensitive HashSet lookups while searching for downloaded installers. --- GVFS/GVFS.Common/FileBasedDictionary.cs | 20 ++++++++++++++++--- GVFS/GVFS.Common/LocalGVFSConfig.cs | 13 +++++------- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 16 +++++++-------- .../HooksPlatform/GVFSHooksPlatform.Mac.cs | 5 +++++ .../GVFSHooksPlatform.Windows.cs | 5 +++++ GVFS/GVFS.Hooks/Program.cs | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs | 2 ++ .../WindowsPlatform.Shared.cs | 2 ++ 8 files changed, 44 insertions(+), 21 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedDictionary.cs b/GVFS/GVFS.Common/FileBasedDictionary.cs index ace9e28040..e02086a377 100644 --- a/GVFS/GVFS.Common/FileBasedDictionary.cs +++ b/GVFS/GVFS.Common/FileBasedDictionary.cs @@ -11,14 +11,28 @@ public class FileBasedDictionary : FileBasedCollection { private ConcurrentDictionary data = new ConcurrentDictionary(); - private FileBasedDictionary(ITracer tracer, PhysicalFileSystem fileSystem, string dataFilePath) + private FileBasedDictionary( + ITracer tracer, + PhysicalFileSystem fileSystem, + string dataFilePath, + IEqualityComparer keyComparer = null) : base(tracer, fileSystem, dataFilePath, collectionAppendsDirectlyToFile: false) { + if (keyComparer != null) + { + this.data = new ConcurrentDictionary(keyComparer); + } } - public static bool TryCreate(ITracer tracer, string dictionaryPath, PhysicalFileSystem fileSystem, out FileBasedDictionary output, out string error) + public static bool TryCreate( + ITracer tracer, + string dictionaryPath, + PhysicalFileSystem fileSystem, + out FileBasedDictionary output, + out string error, + IEqualityComparer keyComparer = null) { - output = new FileBasedDictionary(tracer, fileSystem, dictionaryPath); + output = new FileBasedDictionary(tracer, fileSystem, dictionaryPath, keyComparer); if (!output.TryLoadFromDisk( output.TryParseAddLine, output.TryParseRemoveLine, diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index ac65b45893..6094122fc3 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -1,5 +1,6 @@ using GVFS.Common.FileSystem; using GVFS.Common.Tracing; +using System; using System.IO; namespace GVFS.Common @@ -35,7 +36,7 @@ public bool TryGetConfig( try { - this.allSettings.TryGetValue(this.KeyFromConfigName(name), out value); + this.allSettings.TryGetValue(name, out value); error = null; return true; } @@ -67,7 +68,7 @@ public bool TrySetConfig( try { - this.allSettings.SetValueAndFlush(this.KeyFromConfigName(name), value); + this.allSettings.SetValueAndFlush(name, value); error = null; return true; } @@ -85,11 +86,6 @@ public bool TrySetConfig( } } - private string KeyFromConfigName(string configName) - { - return configName.ToUpperInvariant(); - } - private bool TryLoadSettings(ITracer tracer, out string error) { if (this.allSettings == null) @@ -100,7 +96,8 @@ private bool TryLoadSettings(ITracer tracer, out string error) dictionaryPath: this.configFile, fileSystem: this.fileSystem, output: out config, - error: out error)) + error: out error, + keyComparer: StringComparer.OrdinalIgnoreCase)) { this.allSettings = config; return true; diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index 51125717e7..d26994bcc1 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; @@ -13,24 +14,21 @@ public partial class ProductUpgrader private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; private const string VFSForGitInstallerFileNamePrefix = "VFSForGit"; - public static bool IsLocalUpgradeAvailable() + public static bool IsLocalUpgradeAvailable(string[] installerExtensions) { string downloadDirectory = GetAssetDownloadsPath(); if (Directory.Exists(downloadDirectory)) { - // This method is used Only by Git hooks. Git hooks does not have access - // to GVFSPlatform to read platform specific file extensions. That is the - // reason possible installer file extensions are defined here. - HashSet extensions = new HashSet() { "EXE", "DMG", "RPM", "DEB" }; - HashSet installerNames = new HashSet() + HashSet extensions = new HashSet(installerExtensions, StringComparer.OrdinalIgnoreCase); + HashSet installerNames = new HashSet(StringComparer.OrdinalIgnoreCase) { - GVFSInstallerFileNamePrefix.ToUpperInvariant(), - VFSForGitInstallerFileNamePrefix.ToUpperInvariant() + GVFSInstallerFileNamePrefix, + VFSForGitInstallerFileNamePrefix }; foreach (string file in Directory.EnumerateFiles(downloadDirectory, "*", SearchOption.TopDirectoryOnly)) { - string[] components = Path.GetFileName(file).ToUpperInvariant().Split('.'); + string[] components = Path.GetFileName(file).Split('.'); int length = components.Length; if (length >= 2 && installerNames.Contains(components[0]) && extensions.Contains(components[length - 1])) { diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs index f6188b5732..3d4065878a 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs @@ -4,6 +4,11 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { + public static string[] InstallerExtensions() + { + return MacPlatform.InstallerExtensions; + } + public static bool IsElevated() { return MacPlatform.IsElevatedImplementation(); diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs index 56835e1127..9e23d11782 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs @@ -4,6 +4,11 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { + public static string[] InstallerExtensions() + { + return WindowsPlatform.InstallerExtensions; + } + public static bool IsElevated() { return WindowsPlatform.IsElevatedImplementation(); diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 94df57679e..7178a38fd6 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -109,7 +109,7 @@ private static void RemindUpgradeAvailable() int reminderFrequency = 10; int randomValue = random.Next(0, 100); - if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable()) + if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.InstallerExtensions())) { Console.WriteLine(Environment.NewLine + GVFSConstants.UpgradeVerbMessages.ReminderNotification); } diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs index 03568e5dc2..0434aae5ed 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs @@ -5,6 +5,8 @@ namespace GVFS.Platform.Mac { public partial class MacPlatform { + public static readonly string[] InstallerExtensions = { "dmg" }; + public static bool IsElevatedImplementation() { // TODO(Mac): Implement proper check diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs index ca9aac5ccb..1010b21c90 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs @@ -8,6 +8,8 @@ namespace GVFS.Platform.Windows { public partial class WindowsPlatform { + public static readonly string[] InstallerExtensions = { "exe" }; + private const int StillActive = 259; /* from Win32 STILL_ACTIVE */ private enum StdHandle From 5a22202b97e10d39643f5ee47287ab27efa382de Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 8 Oct 2018 14:33:03 -0400 Subject: [PATCH 119/244] Avoiding possible redundant initialisation of ConcurrentDictionary in FileBasedDictionary class. --- GVFS/GVFS.Common/FileBasedDictionary.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedDictionary.cs b/GVFS/GVFS.Common/FileBasedDictionary.cs index e02086a377..cccc72cebc 100644 --- a/GVFS/GVFS.Common/FileBasedDictionary.cs +++ b/GVFS/GVFS.Common/FileBasedDictionary.cs @@ -9,7 +9,7 @@ namespace GVFS.Common { public class FileBasedDictionary : FileBasedCollection { - private ConcurrentDictionary data = new ConcurrentDictionary(); + private ConcurrentDictionary data; private FileBasedDictionary( ITracer tracer, @@ -18,10 +18,7 @@ private FileBasedDictionary( IEqualityComparer keyComparer = null) : base(tracer, fileSystem, dataFilePath, collectionAppendsDirectlyToFile: false) { - if (keyComparer != null) - { - this.data = new ConcurrentDictionary(keyComparer); - } + this.data = new ConcurrentDictionary(keyComparer ?? EqualityComparer.Default); } public static bool TryCreate( From 4a2edc06b373b53491df3f93cf13b91532d9cb15 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 8 Oct 2018 12:43:40 -0700 Subject: [PATCH 120/244] Mac: New folder notifications --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 38 +++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 132ccd3830..6dec164680 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -105,6 +105,7 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const static PrjFS_Result HandleFileNotification( const MessageHeader* request, const char* path, + const char* fullPath_opt, bool isDirectory, PrjFS_NotificationType notificationType); @@ -598,6 +599,7 @@ static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) result = HandleFileNotification( requestHeader, request.path, + nullptr, // fullPath_opt requestHeader->messageType == MessageType_KtoU_NotifyDirectoryPreDelete, // isDirectory KUMessageTypeToNotificationType(static_cast(requestHeader->messageType))); break; @@ -608,9 +610,32 @@ static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) case MessageType_KtoU_NotifyDirectoryRenamed: case MessageType_KtoU_NotifyFileHardLinkCreated: { + string parentPath(request.path); + size_t lastDirSeparator = parentPath.find_last_of('/'); + if (lastDirSeparator != string::npos) + { + parentPath = parentPath.substr(0, lastDirSeparator); + + char parentFullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), parentPath.c_str(), parentFullPath); + + if (!IsBitSetInFileFlags(parentFullPath, FileFlags_IsInVirtualizationRoot)) + { + // TODO(Mac): Handle SetBitInFileFlags failures + SetBitInFileFlags(parentFullPath, FileFlags_IsInVirtualizationRoot, true); + + HandleFileNotification( + requestHeader, + parentPath.c_str(), + parentFullPath, + true, // isDirectory + PrjFS_NotificationType_NewFileCreated); + } + } + char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); - + // TODO(Mac): Handle SetBitInFileFlags failures SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); @@ -618,6 +643,7 @@ static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) result = HandleFileNotification( requestHeader, request.path, + fullPath, isDirectory, KUMessageTypeToNotificationType(static_cast(requestHeader->messageType))); break; @@ -838,6 +864,7 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const static PrjFS_Result HandleFileNotification( const MessageHeader* request, const char* path, + const char* fullPath_opt, bool isDirectory, PrjFS_NotificationType notificationType) { @@ -848,8 +875,13 @@ static PrjFS_Result HandleFileNotification( << " isDirectory: " << isDirectory << endl; #endif - char fullPath[PrjFSMaxPath]; - CombinePaths(s_virtualizationRootFullPath.c_str(), path, fullPath); + const char* fullPath = fullPath_opt; + char computedFullPath[PrjFSMaxPath]; + if (nullptr == fullPath) + { + CombinePaths(s_virtualizationRootFullPath.c_str(), path, computedFullPath); + fullPath = computedFullPath; + } PrjFSFileXAttrData xattrData = {}; GetXAttr(fullPath, PrjFSFileXAttrName, sizeof(PrjFSFileXAttrData), &xattrData); From 07a6003f0874673a14e97344ba96705a560a7b27 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 8 Oct 2018 13:48:33 -0700 Subject: [PATCH 121/244] Enable additional functional tests --- GVFS/GVFS.FunctionalTests/Categories.cs | 2 +- GVFS/GVFS.FunctionalTests/Program.cs | 2 +- .../EnlistmentPerFixture/GitFilesTests.cs | 10 ++++---- .../WorkingDirectoryTests.cs | 7 +++--- .../ModifiedPathsTests.cs | 2 +- .../MacFileSystemVirtualizer.cs | 25 +++++++++++-------- 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Categories.cs b/GVFS/GVFS.FunctionalTests/Categories.cs index 3d65d7d6bd..d262805196 100644 --- a/GVFS/GVFS.FunctionalTests/Categories.cs +++ b/GVFS/GVFS.FunctionalTests/Categories.cs @@ -15,7 +15,7 @@ public static class MacTODO // machines but not on the build agents public const string FailsOnBuildAgent = "FailsOnBuildAgent"; public const string NeedsLockHolder = "NeedsDotCoreLockHolder"; - public const string M2 = "M2_StaticViewGitCommands"; + public const string NeedsRenameOldPath = "NeedsRenameOldPath"; public const string M3 = "M3_AllGitCommands"; public const string M4 = "M4_All"; } diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index b467b28d5e..f7db4b9698 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -67,7 +67,7 @@ public static void Main(string[] args) { excludeCategories.Add(Categories.MacTODO.NeedsLockHolder); excludeCategories.Add(Categories.MacTODO.FailsOnBuildAgent); - excludeCategories.Add(Categories.MacTODO.M2); + excludeCategories.Add(Categories.MacTODO.NeedsRenameOldPath); excludeCategories.Add(Categories.MacTODO.M3); excludeCategories.Add(Categories.MacTODO.M4); excludeCategories.Add(Categories.WindowsOnly); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 3313fb8f68..479b608ccf 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -60,7 +60,6 @@ public void CreateHardLinkTest() } [TestCase, Order(3)] - [Category(Categories.MacTODO.M2)] public void CreateFileInFolderTest() { string folderName = "folder2"; @@ -81,7 +80,7 @@ public void CreateFileInFolderTest() } [TestCase, Order(4)] - [Category(Categories.MacTODO.M3)] + [Category(Categories.MacTODO.NeedsRenameOldPath)] public void RenameEmptyFolderTest() { string folderName = "folder3a"; @@ -102,7 +101,7 @@ public void RenameEmptyFolderTest() } [TestCase, Order(5)] - [Category(Categories.MacTODO.M2)] + [Category(Categories.MacTODO.NeedsRenameOldPath)] public void RenameFolderTest() { string folderName = "folder4a"; @@ -142,7 +141,7 @@ public void RenameFolderTest() } [TestCase, Order(6)] - [Category(Categories.MacTODO.M2)] + [Category(Categories.MacTODO.NeedsRenameOldPath)] public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() { if (this.fileSystem is PowerShellRunner) @@ -336,8 +335,9 @@ public void OverwrittenFileAddedToModifiedPathsAndSkipWorktreeBitCleared() this.VerifyWorktreeBit(fileToOverwriteEntry, LsFilesStatus.Cached); } + // WindowsOnly because Mac does not support SupersedeFile [TestCase, Order(15)] - [Category(Categories.MacTODO.M2)] + [Category(Categories.WindowsOnly)] public void SupersededFileAddedToModifiedPathsAndSkipWorktreeBitCleared() { string fileToSupersedeEntry = "GVFlt_FileOperationTest/WriteAndVerify.txt"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 0660f46406..88ff712170 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -418,7 +418,6 @@ public void FolderContentsProjectedAfterFolderCreateAndCheckout() [TestCase, Order(14)] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWithSameFolder() { // 3a55d3b760c87642424e834228a3408796501e7c is the commit prior to adding Test_EPF_MoveRenameFileTests @@ -442,9 +441,9 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + Properties.Settings.Default.Commitish); folder.ShouldBeADirectory(this.fileSystem); - (folder + @"\ChangeNestedUnhydratedFileNameCase\Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); - (folder + @"\ChangeUnhydratedFileName\Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); - (folder + @"\MoveUnhydratedFileToDotGitFolder\Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); + Path.Combine(folder, "ChangeNestedUnhydratedFileNameCase", "Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); + Path.Combine(folder, "ChangeUnhydratedFileName", "Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); + Path.Combine(folder, "MoveUnhydratedFileToDotGitFolder", "Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); } [TestCase, Order(15)] diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index f2f255180f..f8f64ce27d 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -77,7 +77,7 @@ public void DeletedTempFolderDeletesFilesFromModifiedFiles(FileSystemRunner file GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "Temp/", "Temp/temp1.txt", "Temp/temp2.txt"); } - [Category(Categories.MacTODO.M2)] + [Category(Categories.MacTODO.NeedsRenameOldPath)] [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) { diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 4b20c7e4ae..407aea43e9 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -525,17 +525,20 @@ private void OnNewFileCreated(string relativePath, bool isDirectory) { if (isDirectory) { - GitCommandLineParser gitCommand = new GitCommandLineParser(this.Context.Repository.GVFSLock.GetLockedGitCommand()); - if (gitCommand.IsValidGitCommand) - { - // TODO(Mac): Ensure that when git creates a folder all files\folders within that folder are written to disk - EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add("isDirectory", isDirectory); - this.Context.Tracer.RelatedWarning(metadata, $"{nameof(this.OnNewFileCreated)}: Git created a folder, currently an unsupported scenario on Mac"); - } - else - { - this.FileSystemCallbacks.OnFolderCreated(relativePath); + if (!relativePath.Equals(GVFSConstants.DotGit.Root, StringComparison.OrdinalIgnoreCase)) + { + GitCommandLineParser gitCommand = new GitCommandLineParser(this.Context.Repository.GVFSLock.GetLockedGitCommand()); + if (gitCommand.IsValidGitCommand) + { + // TODO(Mac): Ensure that when git creates a folder all files\folders within that folder are written to disk + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add("isDirectory", isDirectory); + this.Context.Tracer.RelatedWarning(metadata, $"{nameof(this.OnNewFileCreated)}: Git created a folder, currently an unsupported scenario on Mac"); + } + else + { + this.FileSystemCallbacks.OnFolderCreated(relativePath); + } } } else From 6fe4f037791bb645199caee840ec7aa8da4b49af Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Tue, 9 Oct 2018 11:56:11 -0400 Subject: [PATCH 122/244] Cleanup - Use Enlistment.GitBinPath when available, rather than invoking Platform. Updated GVFSHooksPlatforms to return string for InstallerExtensions, bcoz neither Mac nor Windows release ship with multiple installer extensions. --- GVFS/GVFS.Common/FileBasedDictionary.cs | 11 ++++++++--- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 7 ++++--- .../GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs | 4 ++-- .../HooksPlatform/GVFSHooksPlatform.Windows.cs | 4 ++-- GVFS/GVFS.Hooks/Program.cs | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs | 2 +- GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs | 2 +- GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 5 ++--- GVFS/GVFS/CommandLine/GVFSVerb.cs | 3 +-- 9 files changed, 22 insertions(+), 18 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedDictionary.cs b/GVFS/GVFS.Common/FileBasedDictionary.cs index cccc72cebc..393de7cb04 100644 --- a/GVFS/GVFS.Common/FileBasedDictionary.cs +++ b/GVFS/GVFS.Common/FileBasedDictionary.cs @@ -15,10 +15,10 @@ private FileBasedDictionary( ITracer tracer, PhysicalFileSystem fileSystem, string dataFilePath, - IEqualityComparer keyComparer = null) + IEqualityComparer keyComparer) : base(tracer, fileSystem, dataFilePath, collectionAppendsDirectlyToFile: false) { - this.data = new ConcurrentDictionary(keyComparer ?? EqualityComparer.Default); + this.data = new ConcurrentDictionary(keyComparer); } public static bool TryCreate( @@ -29,7 +29,12 @@ public static bool TryCreate( out string error, IEqualityComparer keyComparer = null) { - output = new FileBasedDictionary(tracer, fileSystem, dictionaryPath, keyComparer); + output = new FileBasedDictionary( + tracer, + fileSystem, + dictionaryPath, + keyComparer ?? EqualityComparer.Default); + if (!output.TryLoadFromDisk( output.TryParseAddLine, output.TryParseRemoveLine, diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index d26994bcc1..7e621a2495 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -14,12 +14,11 @@ public partial class ProductUpgrader private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; private const string VFSForGitInstallerFileNamePrefix = "VFSForGit"; - public static bool IsLocalUpgradeAvailable(string[] installerExtensions) + public static bool IsLocalUpgradeAvailable(string installerExtension) { string downloadDirectory = GetAssetDownloadsPath(); if (Directory.Exists(downloadDirectory)) { - HashSet extensions = new HashSet(installerExtensions, StringComparer.OrdinalIgnoreCase); HashSet installerNames = new HashSet(StringComparer.OrdinalIgnoreCase) { GVFSInstallerFileNamePrefix, @@ -30,7 +29,9 @@ public static bool IsLocalUpgradeAvailable(string[] installerExtensions) { string[] components = Path.GetFileName(file).Split('.'); int length = components.Length; - if (length >= 2 && installerNames.Contains(components[0]) && extensions.Contains(components[length - 1])) + if (length >= 2 && + installerNames.Contains(components[0]) && + installerExtension.Equals(components[length - 1], StringComparison.OrdinalIgnoreCase)) { return true; } diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs index 3d4065878a..e9e8b88735 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs @@ -4,9 +4,9 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { - public static string[] InstallerExtensions() + public static string InstallerExtension() { - return MacPlatform.InstallerExtensions; + return MacPlatform.InstallerExtension; } public static bool IsElevated() diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs index 9e23d11782..3df93424c6 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs @@ -4,9 +4,9 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { - public static string[] InstallerExtensions() + public static string InstallerExtension() { - return WindowsPlatform.InstallerExtensions; + return WindowsPlatform.InstallerExtension; } public static bool IsElevated() diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 7178a38fd6..c3646dcf51 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -109,7 +109,7 @@ private static void RemindUpgradeAvailable() int reminderFrequency = 10; int randomValue = random.Next(0, 100); - if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.InstallerExtensions())) + if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.InstallerExtension())) { Console.WriteLine(Environment.NewLine + GVFSConstants.UpgradeVerbMessages.ReminderNotification); } diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs index 0434aae5ed..dd123b4640 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs @@ -5,7 +5,7 @@ namespace GVFS.Platform.Mac { public partial class MacPlatform { - public static readonly string[] InstallerExtensions = { "dmg" }; + public static readonly string InstallerExtension = "dmg"; public static bool IsElevatedImplementation() { diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs index 1010b21c90..b0e6487ae0 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs @@ -8,7 +8,7 @@ namespace GVFS.Platform.Windows { public partial class WindowsPlatform { - public static readonly string[] InstallerExtensions = { "exe" }; + public static readonly string InstallerExtension = "exe"; private const int StillActive = 259; /* from Win32 STILL_ACTIVE */ diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index c8a0c0a52a..e752bc6254 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -52,8 +52,7 @@ protected override void Execute(GVFSEnlistment enlistment) GitVersion gitVersion = null; string error = null; - string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); - if (!string.IsNullOrEmpty(gitPath) && GitProcess.TryGetVersion(gitPath, out gitVersion, out error)) + if (!string.IsNullOrEmpty(enlistment.GitBinPath) && GitProcess.TryGetVersion(enlistment.GitBinPath, out gitVersion, out error)) { this.WriteMessage("git version " + gitVersion.ToString()); } @@ -62,7 +61,7 @@ protected override void Execute(GVFSEnlistment enlistment) this.WriteMessage("Could not determine git version. " + error); } - this.WriteMessage(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); + this.WriteMessage(enlistment.GitBinPath); this.WriteMessage(string.Empty); this.WriteMessage("Enlistment root: " + enlistment.EnlistmentRoot); this.WriteMessage("Cache Server: " + CacheServerResolver.GetCacheServerFromConfig(enlistment)); diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 94bd33752f..4f630ab312 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -619,9 +619,8 @@ private void CheckFileSystemSupportsRequiredFeatures(ITracer tracer, Enlistment private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out string version) { - string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); GitVersion gitVersion = null; - if (string.IsNullOrEmpty(gitPath) || !GitProcess.TryGetVersion(gitPath, out gitVersion, out string _)) + if (string.IsNullOrEmpty(enlistment.GitBinPath) || !GitProcess.TryGetVersion(enlistment.GitBinPath, out gitVersion, out string _)) { this.ReportErrorAndExit(tracer, "Error: Unable to retrieve the git version"); } From 19b90ccc52ae4342fd3fb30b4735e0f1360db769 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 8 Oct 2018 16:36:21 -0700 Subject: [PATCH 123/244] Notify provider of new ancestor folders, enable more tests, support git creating folder --- GVFS/GVFS.FunctionalTests/Categories.cs | 10 ++++- .../Tests/GitCommands/CheckoutTests.cs | 10 +---- .../Tests/GitCommands/GitCommandsTests.cs | 1 - .../Tests/GitCommands/RmTests.cs | 2 +- .../Tests/GitCommands/UpdateIndexTests.cs | 4 +- .../MacFileSystemVirtualizer.cs | 23 +++++++--- .../MacFileSystemVirtualizerTests.cs | 1 + .../Projection/GitIndexProjection.cs | 4 +- ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs | 1 + ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 44 +++++++++++++------ ProjFS.Mac/PrjFSLib/PrjFSLib.h | 1 + 11 files changed, 65 insertions(+), 36 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Categories.cs b/GVFS/GVFS.FunctionalTests/Categories.cs index d262805196..3ad24ce7d8 100644 --- a/GVFS/GVFS.FunctionalTests/Categories.cs +++ b/GVFS/GVFS.FunctionalTests/Categories.cs @@ -14,10 +14,18 @@ public static class MacTODO // The FailsOnBuildAgent category is for tests that pass on dev // machines but not on the build agents public const string FailsOnBuildAgent = "FailsOnBuildAgent"; + + // Tests that require the LockHolder project to be converted to .NET Core (#150) public const string NeedsLockHolder = "NeedsDotCoreLockHolder"; + + // Tests that require #356 (old paths to be delivered with rename notifications) public const string NeedsRenameOldPath = "NeedsRenameOldPath"; + + // Git related tests that are not yet passing on Mac public const string M3 = "M3_AllGitCommands"; - public const string M4 = "M4_All"; + + // Tests for GVFS features that are not required for correct git functionality + public const string M4 = "M4_GVFSFeatures"; } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 8457720bb1..011a3eb9ff 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -402,7 +402,6 @@ public void ResetMixedToCommitWithNewFileThenCheckoutNewBranchAndCheckoutCommitW // ReadFileAfterTryingToReadFileAtCommitWhereFileDoesNotExist is meant to exercise the NegativePathCache and its // behavior when projections change [TestCase] - [Category(Categories.MacTODO.M3)] public void ReadFileAfterTryingToReadFileAtCommitWhereFileDoesNotExist() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -612,7 +611,7 @@ public void CheckoutBranchWhileOutsideToolDoesNotAllowDeleteOfOpenRepoMetadata() } [TestCase] - [Category(Categories.MacTODO.M4)] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchWhileOutsideToolHasExclusiveReadHandleOnDatabasesFolder() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -798,9 +797,7 @@ public void SuccessfullyChecksOutDirectoryToFileToDirectory() this.ShouldNotExistOnDisk("d", "c"); } - // TODO(Mac): This test needs the fix for issue #264 [TestCase] - [Category(Categories.MacTODO.M3)] public void DeleteFileThenCheckout() { this.FolderShouldExistAndHaveFile("GitCommandsTests", "DeleteFileTests", "1", "#test"); @@ -816,7 +813,6 @@ public void DeleteFileThenCheckout() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutEditCheckoutWithoutFolderThenCheckoutWithMultipleFiles() { // Edit the file to get the entry in the modified paths database @@ -844,7 +840,6 @@ public void CreateAFolderThenCheckoutBranchWithFolder() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithDirectoryNameSameAsFile() { this.SetupForFileDirectoryTest(); @@ -852,21 +847,18 @@ public void CheckoutBranchWithDirectoryNameSameAsFile() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithDirectoryNameSameAsFileEnumerate() { this.RunFileDirectoryEnumerateTest("checkout"); } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithDirectoryNameSameAsFileWithRead() { this.RunFileDirectoryReadTest("checkout"); } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithDirectoryNameSameAsFileWithWrite() { this.RunFileDirectoryWriteTest("checkout"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 31ac5cb08c..3537c8113e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -1010,7 +1010,6 @@ public void UseAlias() } [TestCase] - [Category(Categories.MacTODO.M3)] public void RenameOnlyFileInFolder() { ControlGitRepo.Fetch("FunctionalTests/20170202_RenameTestMergeTarget"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs index 301cf48320..14443a24d0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs @@ -13,7 +13,7 @@ public RmTests() : base(enlistmentPerTest: false) // Mac(TODO): Something is triggering Readme.md to get created on disk before this // test validates that it's not present [TestCase] - [Category(Categories.MacTODO.M4)] + [Category(Categories.MacTODO.M3)] public void CanReadFileAfterGitRmDryRun() { this.ValidateGitCommand("status"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index d729822d0f..8240b32223 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -40,7 +40,7 @@ public void UpdateIndexRemoveFileOnDiskDontCheckStatus() } [TestCase] - [Category(Categories.MacTODO.M4)] + [Category(Categories.MacTODO.M3)] public void UpdateIndexRemoveAddFileOpenForWrite() { // TODO 940287: Remove this test and re-enable UpdateIndexRemoveFileOnDisk @@ -64,7 +64,7 @@ public void UpdateIndexRemoveAddFileOpenForWrite() } [TestCase] - [Category(Categories.MacTODO.M4)] + [Category(Categories.MacTODO.M3)] public void UpdateIndexWithCacheInfo() { // Update Protocol.md with the contents from blob 583f1... diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 407aea43e9..4a23c6f658 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -53,6 +53,9 @@ public static FSResult ResultToFSResult(Result result) case Result.EPathNotFound: return FSResult.FileOrPathNotFound; + case Result.EDirectoryNotEmpty: + return FSResult.DirectoryNotEmpty; + default: return FSResult.IOError; } @@ -526,17 +529,23 @@ private void OnNewFileCreated(string relativePath, bool isDirectory) if (isDirectory) { if (!relativePath.Equals(GVFSConstants.DotGit.Root, StringComparison.OrdinalIgnoreCase)) - { - GitCommandLineParser gitCommand = new GitCommandLineParser(this.Context.Repository.GVFSLock.GetLockedGitCommand()); + { + string lockedGitCommand = this.Context.Repository.GVFSLock.GetLockedGitCommand(); + GitCommandLineParser gitCommand = new GitCommandLineParser(lockedGitCommand); if (gitCommand.IsValidGitCommand) { - // TODO(Mac): Ensure that when git creates a folder all files\folders within that folder are written to disk - EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add("isDirectory", isDirectory); - this.Context.Tracer.RelatedWarning(metadata, $"{nameof(this.OnNewFileCreated)}: Git created a folder, currently an unsupported scenario on Mac"); + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(lockedGitCommand), lockedGitCommand); + metadata.Add(TracingConstants.MessageKey.InfoMessage, "Git command created new folder"); + this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.OnNewFileCreated)}_GitCreatedFolder", metadata); + + // Record this folder as expanded so that GitIndexProjection will re-expand the folder + // when the projection change completes. This is the closest we can get to converting this + // new folder to partial (as we do on Windows). + this.FileSystemCallbacks.OnPlaceholderFolderExpanded(relativePath); } else - { + { this.FileSystemCallbacks.OnFolderCreated(relativePath); } } diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 96c2d6e288..6cce64dfa5 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -27,6 +27,7 @@ public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { Result.Success, FSResult.Ok }, { Result.EFileNotFound, FSResult.FileOrPathNotFound }, { Result.EPathNotFound, FSResult.FileOrPathNotFound }, + { Result.EDirectoryNotEmpty, FSResult.DirectoryNotEmpty }, }; [TestCase] diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index c99808b744..3fca1fa073 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1393,9 +1393,9 @@ private void ReExpandFolder( { if (!existingFolderPlaceholders.Contains(childRelativePath)) { - this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); + this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); updatedPlaceholderList.TryAdd( - childRelativePath, + childRelativePath, new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); } } diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs index 698badb1e8..4bedbe0565 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs @@ -22,6 +22,7 @@ public enum Result : uint EIOError = 0x20000040, ENotAVirtualizationRoot = 0x20000080, EVirtualizationRootAlreadyExists = 0x20000100, + EDirectoryNotEmpty = 0x20000200, ENotYetImplemented = 0xFFFFFFFF, } diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 6dec164680..aa236bafd9 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ using std::pair; using std::queue; using std::set; using std::shared_ptr; +using std::stack; using std::string; typedef lock_guard mutex_lock; @@ -501,6 +503,8 @@ PrjFS_Result PrjFS_DeleteFile( case ENOENT: // A component of fullPath does not exist case ENOTDIR: // A component of fullPath is not a directory return PrjFS_Result_Success; + case ENOTEMPTY: + return PrjFS_Result_EDirectoryNotEmpty; default: return PrjFS_Result_EIOError; } @@ -610,27 +614,41 @@ static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) case MessageType_KtoU_NotifyDirectoryRenamed: case MessageType_KtoU_NotifyFileHardLinkCreated: { + // Walk up the directory tree and notify the provider about any directories + // not flagged as being in the root + stack> newFolderPaths; string parentPath(request.path); size_t lastDirSeparator = parentPath.find_last_of('/'); - if (lastDirSeparator != string::npos) + while (lastDirSeparator != string::npos && lastDirSeparator > 0) { parentPath = parentPath.substr(0, lastDirSeparator); - char parentFullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), parentPath.c_str(), parentFullPath); - - if (!IsBitSetInFileFlags(parentFullPath, FileFlags_IsInVirtualizationRoot)) + if (IsBitSetInFileFlags(parentFullPath, FileFlags_IsInVirtualizationRoot)) { - // TODO(Mac): Handle SetBitInFileFlags failures - SetBitInFileFlags(parentFullPath, FileFlags_IsInVirtualizationRoot, true); - - HandleFileNotification( - requestHeader, - parentPath.c_str(), - parentFullPath, - true, // isDirectory - PrjFS_NotificationType_NewFileCreated); + break; } + else + { + newFolderPaths.emplace(make_pair(parentPath, parentFullPath)); + lastDirSeparator = parentPath.find_last_of('/'); + } + } + + while (!newFolderPaths.empty()) + { + const pair& parentFolderPath = newFolderPaths.top(); + newFolderPaths.pop(); + + // TODO(Mac): Handle SetBitInFileFlags failures + SetBitInFileFlags(parentFolderPath.second.c_str(), FileFlags_IsInVirtualizationRoot, true); + + HandleFileNotification( + requestHeader, + parentFolderPath.first.c_str(), + parentFolderPath.second.c_str(), + true, // isDirectory + PrjFS_NotificationType_NewFileCreated); } char fullPath[PrjFSMaxPath]; diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.h b/ProjFS.Mac/PrjFSLib/PrjFSLib.h index ca2cfb0579..75cc0d496b 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.h +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.h @@ -33,6 +33,7 @@ typedef enum PrjFS_Result_EIOError = 0x20000040, PrjFS_Result_ENotAVirtualizationRoot = 0x20000080, PrjFS_Result_EVirtualizationRootAlreadyExists = 0x20000100, + PrjFS_Result_EDirectoryNotEmpty = 0x20000200, PrjFS_Result_ENotYetImplemented = 0xFFFFFFFF, From 5b8f17a9e18cd39fb697ce9887de6446dbfe8b8f Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Tue, 9 Oct 2018 12:59:01 -0400 Subject: [PATCH 124/244] Cleanups - made InstallerExtension strings const. Renamed InstallerExtension accessor to GetInstallerExtension. --- GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs | 2 +- GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs | 2 +- GVFS/GVFS.Hooks/Program.cs | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs | 2 +- GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs index e9e8b88735..2a459d964b 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs @@ -4,7 +4,7 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { - public static string InstallerExtension() + public static string GetInstallerExtension() { return MacPlatform.InstallerExtension; } diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs index 3df93424c6..ba5ce9b714 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs @@ -4,7 +4,7 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { - public static string InstallerExtension() + public static string GetInstallerExtension() { return WindowsPlatform.InstallerExtension; } diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index c3646dcf51..eab2ec0aea 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -109,7 +109,7 @@ private static void RemindUpgradeAvailable() int reminderFrequency = 10; int randomValue = random.Next(0, 100); - if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.InstallerExtension())) + if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.GetInstallerExtension())) { Console.WriteLine(Environment.NewLine + GVFSConstants.UpgradeVerbMessages.ReminderNotification); } diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs index dd123b4640..2cd456ee46 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs @@ -5,7 +5,7 @@ namespace GVFS.Platform.Mac { public partial class MacPlatform { - public static readonly string InstallerExtension = "dmg"; + public const string InstallerExtension = "dmg"; public static bool IsElevatedImplementation() { diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs index b0e6487ae0..6a87b81be9 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs @@ -8,7 +8,7 @@ namespace GVFS.Platform.Windows { public partial class WindowsPlatform { - public static readonly string InstallerExtension = "exe"; + public const string InstallerExtension = "exe"; private const int StillActive = 259; /* from Win32 STILL_ACTIVE */ From 9edf59826649daed32b02d1bca159d1c0b5f8c84 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 9 Oct 2018 11:40:42 -0700 Subject: [PATCH 125/244] Revert whitespace change --- GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 3fca1fa073..c99808b744 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1393,9 +1393,9 @@ private void ReExpandFolder( { if (!existingFolderPlaceholders.Contains(childRelativePath)) { - this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); + this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); updatedPlaceholderList.TryAdd( - childRelativePath, + childRelativePath, new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); } } From f642a7dfb840aa8b688d3476fe2f3f0011420de9 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 9 Oct 2018 14:19:07 -0700 Subject: [PATCH 126/244] Move call to pop --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index aa236bafd9..f7de1e7f91 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -638,7 +638,6 @@ static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) while (!newFolderPaths.empty()) { const pair& parentFolderPath = newFolderPaths.top(); - newFolderPaths.pop(); // TODO(Mac): Handle SetBitInFileFlags failures SetBitInFileFlags(parentFolderPath.second.c_str(), FileFlags_IsInVirtualizationRoot, true); @@ -649,6 +648,8 @@ static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) parentFolderPath.second.c_str(), true, // isDirectory PrjFS_NotificationType_NewFileCreated); + + newFolderPaths.pop(); } char fullPath[PrjFSMaxPath]; From 740bd76476368f9f11a97a053cb078d7d2994659 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Tue, 9 Oct 2018 18:00:52 -0700 Subject: [PATCH 127/244] Update build scripts to handle new Profiling configuration --- ProjFS.Mac/Scripts/Build.sh | 5 +++++ Scripts/Mac/BuildGVFSForMac.sh | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/ProjFS.Mac/Scripts/Build.sh b/ProjFS.Mac/Scripts/Build.sh index 333883c37b..9a1060ab9e 100755 --- a/ProjFS.Mac/Scripts/Build.sh +++ b/ProjFS.Mac/Scripts/Build.sh @@ -14,5 +14,10 @@ PROJFS=$SRCDIR/ProjFS.Mac xcodebuild -configuration $CONFIGURATION -workspace $PROJFS/PrjFS.xcworkspace build -scheme PrjFS -derivedDataPath $ROOTDIR/BuildOutput/ProjFS.Mac/Native || exit 1 +# If we're building the Profiling(Release) configuration, remove Profiling() for building .NET code +if [ "$CONFIGURATION" == "Profiling(Release)" ]; then + CONFIGURATION=Release +fi + dotnet restore $PROJFS/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj /p:Configuration=$CONFIGURATION /p:Platform=x64 --packages $PACKAGES || exit 1 dotnet build $PROJFS/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj /p:Configuration=$CONFIGURATION /p:Platform=x64 || exit 1 diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index 678bb89aaf..d6d07e4ad7 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -38,6 +38,11 @@ GITPATH="$(find $PACKAGES/gitformac.gvfs.installer/$GITVERSION -type f -name *.d # Now that we have a path containing the version number, generate GVFSConstants.GitVersion.cs $SCRIPTDIR/GenerateGitVersionConstants.sh "$GITPATH" $BUILDDIR || exit 1 +# If we're building the Profiling(Release) configuration, remove Profiling() for building .NET code +if [ "$CONFIGURATION" == "Profiling(Release)" ]; then + CONFIGURATION=Release +fi + DOTNETCONFIGURATION=$CONFIGURATION.Mac dotnet restore $SRCDIR/GVFS.sln /p:Configuration=$DOTNETCONFIGURATION --packages $PACKAGES || exit 1 dotnet build $SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --configuration $DOTNETCONFIGURATION /maxcpucount:1 || exit 1 From 1af261720a9440f5a5ee0e100e9b7072a8a35e17 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 9 Oct 2018 15:18:07 -0700 Subject: [PATCH 128/244] Mac: Handle projection changes where git deletes folders that are still in the projection --- .../Tests/GitCommands/CheckoutTests.cs | 1 - .../Projection/GitIndexProjection.cs | 57 +++++++++++++++---- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 25 ++++++-- 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 8457720bb1..351e6a7ef5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -892,7 +892,6 @@ public void CheckoutBranchDirectoryWithOneFileRead() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFileWrite() { this.RunFileDirectoryWriteTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index c99808b744..49384e3bcf 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1158,8 +1158,11 @@ private void UpdatePlaceholders() new HashSet(placeholderFoldersListCopy.Select(x => x.Path), StringComparer.OrdinalIgnoreCase) : null; - // Order the folders in decscending order so that we walk the tree from bottom up (ensuring child folders are deleted before - // their parents) + // Order the folders in decscending order so that we walk the tree from bottom up. + // Traversing the folders in this order: + // 1. Ensures child folders are deleted before their parents + // 2. Ensures that folders that have been deleted by git (but are still in the projection) are found before their + // parent folder is re-expanded (only applies on platforms where EnumerationExpandsDirectories is true) foreach (PlaceholderListDatabase.PlaceholderData folderPlaceholder in placeholderFoldersListCopy.OrderByDescending(x => x.Path)) { // Remove folder placeholders before re-expansion to ensure that projection changes that convert a folder to a file work @@ -1388,15 +1391,31 @@ private void ReExpandFolder( childRelativePath = relativeFolderPath + Path.DirectorySeparatorChar + childEntry.Name.GetString(); } - // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile if (childEntry.IsFolder) { if (!existingFolderPlaceholders.Contains(childRelativePath)) { - this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); + FileSystemResult result = this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); + switch (result.Result) + { + case FSResult.Ok: + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); + break; + + case FSResult.FileOrPathNotFound: + // Git command must have removed the folder being re-expanded (relativeFolderPath) + // Remove the folder from existingFolderPlaceholders so that its parent will create + // it again (when it's re-expanded) + existingFolderPlaceholders.Remove(relativeFolderPath); + return; + + default: + // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile + break; + } + } } else @@ -1406,10 +1425,26 @@ private void ReExpandFolder( FileData childFileData = childEntry as FileData; string sha = childFileData.Sha.ToString(); - this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, sha); - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); + FileSystemResult result = this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, sha); + switch (result.Result) + { + case FSResult.Ok: + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); + break; + + case FSResult.FileOrPathNotFound: + // Git command must have removed the folder being re-expanded (relativeFolderPath) + // Remove the folder from existingFolderPlaceholders so that its parent will create + // it again (when it's re-expanded) + existingFolderPlaceholders.Remove(relativeFolderPath); + return; + + default: + // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile + break; + } } } } diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 132ccd3830..2bd6636d07 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -323,23 +323,36 @@ PrjFS_Result PrjFS_WritePlaceholderFile( return PrjFS_Result_EInvalidArgs; } + PrjFS_Result result = PrjFS_Result_Invalid; PrjFSFileXAttrData fileXattrData = {}; char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); - // Mode "wbx" means - // - Create an empty file if none exists - // - Fail if a file already exists at this path - FILE* file = fopen(fullPath, "wbx"); + // Mode "wx" means: + // - "w": Open for writing. The stream is positioned at the beginning of the file. Create the file if it does not exist. + // - "x": If the file already exists, fopen() fails, and sets errno to EEXIST. + FILE* file = fopen(fullPath, "wx"); if (nullptr == file) { + switch(errno) + { + case ENOENT: // A directory component in fullPath does not exist or is a dangling symbolic link. + result = PrjFS_Result_EPathNotFound; + break; + case EEXIST: // The file already exists + default: + result = PrjFS_Result_EIOError; + break; + } + goto CleanupAndFail; } // Expand the file to the desired size if (ftruncate(fileno(file), fileSize)) { + result = PrjFS_Result_EIOError; goto CleanupAndFail; } @@ -354,12 +367,14 @@ PrjFS_Result PrjFS_WritePlaceholderFile( &fileXattrData, PrjFSFileXAttrName)) { + result = PrjFS_Result_EIOError; goto CleanupAndFail; } // TODO(Mac): Only call chmod if fileMode is different than the default file mode if (chmod(fullPath, fileMode)) { + result = PrjFS_Result_EIOError; goto CleanupAndFail; } @@ -375,7 +390,7 @@ PrjFS_Result PrjFS_WritePlaceholderFile( file = nullptr; } - return PrjFS_Result_EIOError; + return result; } PrjFS_Result PrjFS_WriteSymLink( From 8572e725024e0cc0ff95814b6793167093d0574b Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 10 Oct 2018 11:07:33 -0600 Subject: [PATCH 129/244] Add check-ignore and check-mailmap to commands that do not lock --- GVFS/GVFS.Hooks/Program.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index eab2ec0aea..64d40eceba 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -411,6 +411,8 @@ private static bool ShouldLock(string[] args) case "branch": case "cat-file": case "check-attr": + case "check-ignore": + case "check-mailmap": case "commit-graph": case "config": case "credential": From 5d62ef31e56f985029515b48de9676337f136eab Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 10 Oct 2018 10:16:59 -0700 Subject: [PATCH 130/244] Fix StyleCop issue --- GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 49384e3bcf..a1a6909865 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1415,7 +1415,6 @@ private void ReExpandFolder( // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile break; } - } } else From d57e79157b62025d0315f95848f3c65ca00c3b4e Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 10 Oct 2018 15:07:58 -0700 Subject: [PATCH 131/244] Add additional functional test and update PrjFS_WritePlaceholderDirectory to return a more specific error message --- .../Tests/GitCommands/CheckoutTests.cs | 47 +++++++++++++++++++ .../Tests/GitCommands/GitRepoTests.cs | 2 + ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 16 ++++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 351e6a7ef5..ac861a982e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -897,6 +897,53 @@ public void CheckoutBranchDirectoryWithOneFileWrite() this.RunFileDirectoryWriteTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); } + [TestCase] + public void CheckoutBranchDirectoryWithOneDeepFileWrite() + { + this.ControlGitRepo.Fetch(DeepDirectoryWithOneFile); + this.ControlGitRepo.Fetch(DeepDirectoryWithOneDifferentFile); + this.ValidateGitCommand($"checkout {DeepDirectoryWithOneFile}"); + this.FileShouldHaveContents( + "TestFile1\n", + "GitCommandsTests", + "CheckoutBranchDirectoryWithOneDeepFile", + "FolderDepth1", + "FolderDepth2", + "FolderDepth3", + "File1.txt"); + + // Edit the file and commit the change so that git will + // delete the file (and its parent directories) when + // changing branches + this.EditFile( + "Change file", + "GitCommandsTests", + "CheckoutBranchDirectoryWithOneDeepFile", + "FolderDepth1", + "FolderDepth2", + "FolderDepth3", + "File1.txt"); + this.ValidateGitCommand("add --all"); + this.RunGitCommand("commit -m \"Some change\""); + + this.ValidateGitCommand($"checkout {DeepDirectoryWithOneDifferentFile}"); + this.FileShouldHaveContents( + "TestFile2\n", + "GitCommandsTests", + "CheckoutBranchDirectoryWithOneDeepFile", + "FolderDepth1", + "FolderDepth2", + "FolderDepth3", + "File2.txt"); + this.ShouldNotExistOnDisk( + "GitCommandsTests", + "CheckoutBranchDirectoryWithOneDeepFile", + "FolderDepth1", + "FolderDepth2", + "FolderDepth3", + "File1.txt"); + } + private static void CopyIndexAndRename(string indexPath) { string tempIndexPath = indexPath + ".lock"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index 9200be7c55..d68e380959 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -19,6 +19,8 @@ public abstract class GitRepoTests protected const string DirectoryWithFileBeforeBranch = "FunctionalTests/20171025_DirectoryWithFileBefore"; protected const string DirectoryWithFileAfterBranch = "FunctionalTests/20171025_DirectoryWithFileAfter"; protected const string DirectoryWithDifferentFileAfterBranch = "FunctionalTests/20171025_DirectoryWithDifferentFile"; + protected const string DeepDirectoryWithOneFile = "FunctionalTests/20181010_DeepFolderOneFile"; + protected const string DeepDirectoryWithOneDifferentFile = "FunctionalTests/20181010_DeepFolderOneDifferentFile"; private bool enlistmentPerTest; diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 2bd6636d07..cddef78bbc 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -281,16 +281,29 @@ PrjFS_Result PrjFS_WritePlaceholderDirectory( return PrjFS_Result_EInvalidArgs; } + PrjFS_Result result = PrjFS_Result_Invalid; char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); if (mkdir(fullPath, 0777)) { + switch(errno) + { + // TODO(Mac): Return more specific error codes for other failure scenarios + case ENOENT: // A component of the path prefix does not exist or path is an empty string + result = PrjFS_Result_EPathNotFound; + break; + default: + result = PrjFS_Result_EIOError; + break; + } + goto CleanupAndFail; } if (!InitializeEmptyPlaceholder(fullPath)) { + result = PrjFS_Result_EIOError; goto CleanupAndFail; } @@ -298,7 +311,7 @@ PrjFS_Result PrjFS_WritePlaceholderDirectory( CleanupAndFail: // TODO: cleanup the directory on disk if needed - return PrjFS_Result_EIOError; + return result; } PrjFS_Result PrjFS_WritePlaceholderFile( @@ -337,6 +350,7 @@ PrjFS_Result PrjFS_WritePlaceholderFile( { switch(errno) { + // TODO(Mac): Return more specific error codes for other failure scenarios case ENOENT: // A directory component in fullPath does not exist or is a dangling symbolic link. result = PrjFS_Result_EPathNotFound; break; From 74743c0198d6bee78df8a1a2f8467072449af524 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 10 Oct 2018 15:35:54 -0700 Subject: [PATCH 132/244] Code cleanup --- .../Tests/GitCommands/CheckoutTests.cs | 8 +- .../Projection/GitIndexProjection.cs | 75 +++++++------------ 2 files changed, 33 insertions(+), 50 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index ac861a982e..eb04912966 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -900,9 +900,9 @@ public void CheckoutBranchDirectoryWithOneFileWrite() [TestCase] public void CheckoutBranchDirectoryWithOneDeepFileWrite() { - this.ControlGitRepo.Fetch(DeepDirectoryWithOneFile); - this.ControlGitRepo.Fetch(DeepDirectoryWithOneDifferentFile); - this.ValidateGitCommand($"checkout {DeepDirectoryWithOneFile}"); + this.ControlGitRepo.Fetch(GitRepoTests.DeepDirectoryWithOneFile); + this.ControlGitRepo.Fetch(GitRepoTests.DeepDirectoryWithOneDifferentFile); + this.ValidateGitCommand($"checkout {GitRepoTests.DeepDirectoryWithOneFile}"); this.FileShouldHaveContents( "TestFile1\n", "GitCommandsTests", @@ -926,7 +926,7 @@ public void CheckoutBranchDirectoryWithOneDeepFileWrite() this.ValidateGitCommand("add --all"); this.RunGitCommand("commit -m \"Some change\""); - this.ValidateGitCommand($"checkout {DeepDirectoryWithOneDifferentFile}"); + this.ValidateGitCommand($"checkout {GitRepoTests.DeepDirectoryWithOneDifferentFile}"); this.FileShouldHaveContents( "TestFile2\n", "GitCommandsTests", diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index a1a6909865..fb8edba06d 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1391,59 +1391,42 @@ private void ReExpandFolder( childRelativePath = relativeFolderPath + Path.DirectorySeparatorChar + childEntry.Name.GetString(); } - if (childEntry.IsFolder) + bool newChild = childEntry.IsFolder ? !existingFolderPlaceholders.Contains(childRelativePath) : !updatedPlaceholderList.ContainsKey(childRelativePath); + + if (newChild) { - if (!existingFolderPlaceholders.Contains(childRelativePath)) + FileSystemResult result; + string fileShaOrFolderValue; + if (childEntry.IsFolder) { - FileSystemResult result = this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); - switch (result.Result) - { - case FSResult.Ok: - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); - break; - - case FSResult.FileOrPathNotFound: - // Git command must have removed the folder being re-expanded (relativeFolderPath) - // Remove the folder from existingFolderPlaceholders so that its parent will create - // it again (when it's re-expanded) - existingFolderPlaceholders.Remove(relativeFolderPath); - return; - - default: - // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile - break; - } + fileShaOrFolderValue = PlaceholderListDatabase.PartialFolderValue; + result = this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); } - } - else - { - if (!updatedPlaceholderList.ContainsKey(childRelativePath)) + else { FileData childFileData = childEntry as FileData; - string sha = childFileData.Sha.ToString(); + fileShaOrFolderValue = childFileData.Sha.ToString(); + result = this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, fileShaOrFolderValue); + } - FileSystemResult result = this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, sha); - switch (result.Result) - { - case FSResult.Ok: - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); - break; - - case FSResult.FileOrPathNotFound: - // Git command must have removed the folder being re-expanded (relativeFolderPath) - // Remove the folder from existingFolderPlaceholders so that its parent will create - // it again (when it's re-expanded) - existingFolderPlaceholders.Remove(relativeFolderPath); - return; + switch (result.Result) + { + case FSResult.Ok: + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, fileShaOrFolderValue)); + break; - default: - // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile - break; - } + case FSResult.FileOrPathNotFound: + // Git command must have removed the folder being re-expanded (relativeFolderPath) + // Remove the folder from existingFolderPlaceholders so that its parent will create + // it again (when it's re-expanded) + existingFolderPlaceholders.Remove(relativeFolderPath); + return; + + default: + // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile + break; } } } From c3f03f800fbed0890ceabe54fcd86458b12c44c5 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 11 Oct 2018 10:02:56 -0400 Subject: [PATCH 133/244] Update GitForWindows to include tracing --- GVFS/GVFS.Build/GVFS.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Build/GVFS.props b/GVFS/GVFS.Build/GVFS.props index 2c39f75004..ef965a8c45 100644 --- a/GVFS/GVFS.Build/GVFS.props +++ b/GVFS/GVFS.Build/GVFS.props @@ -3,7 +3,7 @@ 0.2.173.2 - 2.20180814.4 + 2.20181011.3 From 34a54f91bdfe11547269139e37ed49323247f58a Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 10 Oct 2018 11:08:35 -0600 Subject: [PATCH 134/244] Add tests for creating placeholders --- .../GitCommands/CreatePlaceholderTests.cs | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs new file mode 100644 index 0000000000..ca0f371626 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs @@ -0,0 +1,73 @@ +using GVFS.FunctionalTests.Should; +using GVFS.FunctionalTests.Tools; +using NUnit.Framework; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; + +namespace GVFS.FunctionalTests.Tests.GitCommands +{ + [TestFixture] + [Category(Categories.GitCommands)] + public class CreatePlaceholderTests : GitRepoTests + { + private static readonly string FileToRead = Path.Combine("GVFS", "GVFS", "Program.cs"); + + public CreatePlaceholderTests() : base(enlistmentPerTest: true) + { + } + + [TestCase("check-attr")] + [TestCase("check-ignore")] + [TestCase("check-mailmap")] + [TestCase("diff-tree")] + [TestCase("fetch-pack")] + [TestCase("hash-object")] + [TestCase("index-pack")] + [TestCase("name-rev")] + [TestCase("notes")] + [TestCase("send-pack")] + [TestCase("rev-list")] + [TestCase("update-ref")] + public void AllowsPlaceholderCreationWhileGitCommandIsRunning(string commandToRun) + { + this.CheckPlaceholderCreation(commandToRun, shouldAllow: true); + } + + [TestCase("checkout-index")] + [TestCase("reset")] + [TestCase("update-index")] + public void BlocksPlaceholderCreationWhileGitCommandIsRunning(string commandToRun) + { + this.CheckPlaceholderCreation(commandToRun, shouldAllow: false); + } + + private void CheckPlaceholderCreation(string command, bool shouldAllow) + { + string eofCharacter = "\x04"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + eofCharacter = "\x1A"; + } + + this.EditFile($"Some new content for {command}.", "Protocol.md"); + ManualResetEventSlim resetEvent = GitHelpers.RunGitCommandWithWaitAndStdIn(this.Enlistment, resetTimeout: 3000, command: $"{command} --stdin", stdinToQuit: eofCharacter, processId: out _); + + if (shouldAllow) + { + this.FileContentsShouldMatch(FileToRead); + } + else + { + string virtualPath = Path.Combine(this.Enlistment.RepoRoot, FileToRead); + string controlPath = Path.Combine(this.ControlGitRepo.RootPath, FileToRead); + virtualPath.ShouldNotExistOnDisk(this.FileSystem); + controlPath.ShouldBeAFile(this.FileSystem); + } + + this.ValidateGitCommand("--no-optional-locks status"); + resetEvent.Wait(); + this.RunGitCommand("reset --hard"); + } + } +} From 7ff203c8bc8dd69ab0bb0fb7d15c31bce7821841 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 9 Oct 2018 10:45:06 -0600 Subject: [PATCH 135/244] Add test that adds untracked file in new folder 2 levels deep --- .../Tests/EnlistmentPerFixture/GitFilesTests.cs | 1 + .../Tests/GitCommands/GitCommandsTests.cs | 5 +++++ GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 3313fb8f68..b90943d12b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -155,6 +155,7 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "Folder/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "Folder/testfile"); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 31ac5cb08c..28ac308026 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -16,6 +16,8 @@ public class GitCommandsTests : GitRepoTests private const string EncodingFilename = "ريلٌأكتوبرûمارسأغسطسºٰٰۂْٗ۵ريلٌأك.txt"; private const string ContentWhenEditingFile = "// Adding a comment to the file"; private const string UnknownTestName = "Unknown"; + private const string TopLevelFolderToCreate = "level1"; + private const string SubFolderToCreate = "level2"; private static readonly string EditFilePath = Path.Combine("GVFS", "GVFS.Common", "GVFSContext.cs"); private static readonly string DeleteFilePath = Path.Combine("GVFS", "GVFS", "Program.cs"); @@ -1088,6 +1090,9 @@ private void CommitChangesSwitchBranchSwitchBack(Action fileSystemAction, [Calle private void CreateFile() { this.CreateFile("Some content here", Path.GetRandomFileName() + "tempFile.txt"); + this.CreateFolder(TopLevelFolderToCreate); + this.CreateFolder(Path.Combine(TopLevelFolderToCreate, SubFolderToCreate)); + this.CreateFile("File in new folder", Path.Combine(TopLevelFolderToCreate, SubFolderToCreate, Path.GetRandomFileName() + "folderFile.txt")); } private void EditFile() diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 0de0d127ca..5dbc77982e 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -117,7 +117,7 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); foreach (string gitPath in gitPaths) { - modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLineAddPrefix + gitPath)); + modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLineAddPrefix + gitPath, StringComparison.OrdinalIgnoreCase)); } } From 03fba043f131d376af455f058478c29289544554 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 9 Oct 2018 14:07:26 -0600 Subject: [PATCH 136/244] Update git for windows version --- GVFS/GVFS.Build/GVFS.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Build/GVFS.props b/GVFS/GVFS.Build/GVFS.props index 2c39f75004..ef965a8c45 100644 --- a/GVFS/GVFS.Build/GVFS.props +++ b/GVFS/GVFS.Build/GVFS.props @@ -3,7 +3,7 @@ 0.2.173.2 - 2.20180814.4 + 2.20181011.3 From 33f8566e0f89097be8b1f31b497caebdc6343f95 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 11 Oct 2018 13:09:34 -0600 Subject: [PATCH 137/244] Add try/catch/finally for the task running git command with stdin --- GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs index 333499ba0e..bc1d8dd54c 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs @@ -1,5 +1,6 @@ using GVFS.FunctionalTests.Properties; using GVFS.Tests.Should; +using NUnit.Framework; using System; using System.Collections.Generic; using System.Diagnostics; @@ -174,28 +175,37 @@ private static ManualResetEventSlim RunCommandWithWaitAndStdIn( { resetEvent.Wait(resetTimeout); - // Make sure to let the holding process end. - if (stdin != null) + try { - stdin.WriteLine(stdinToQuit); - stdin.Close(); - } - - if (holdingProcess != null) - { - bool holdingProcessHasExited = holdingProcess.WaitForExit(10000); - - if (!holdingProcess.HasExited) + // Make sure to let the holding process end. + if (stdin != null) { - holdingProcess.Kill(); + stdin.WriteLine(stdinToQuit); + stdin.Close(); } - holdingProcess.Dispose(); + if (holdingProcess != null) + { + bool holdingProcessHasExited = holdingProcess.WaitForExit(10000); - holdingProcessHasExited.ShouldBeTrue("Locking process did not exit in time."); - } + if (!holdingProcess.HasExited) + { + holdingProcess.Kill(); + } + + holdingProcess.Dispose(); - resetEvent.Set(); + holdingProcessHasExited.ShouldBeTrue("Locking process did not exit in time."); + } + } + catch (Exception ex) + { + Assert.Fail($"{nameof(RunCommandWithWaitAndStdIn)} exception closing stdin {ex.ToString()}"); + } + finally + { + resetEvent.Set(); + } }); return resetEvent; From a5c027e8425d77c3d374dce3c02f6c094620ba3d Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Thu, 11 Oct 2018 14:35:31 -0700 Subject: [PATCH 138/244] Address PR feedback and update MirrorProvider script too --- MirrorProvider/Scripts/Mac/Build.sh | 5 +++++ Scripts/Mac/BuildGVFSForMac.sh | 7 +++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/MirrorProvider/Scripts/Mac/Build.sh b/MirrorProvider/Scripts/Mac/Build.sh index a4b404697e..94617466d2 100755 --- a/MirrorProvider/Scripts/Mac/Build.sh +++ b/MirrorProvider/Scripts/Mac/Build.sh @@ -14,6 +14,11 @@ SLN=$SRCDIR/MirrorProvider/MirrorProvider.sln # Build the ProjFS kext and libraries $SRCDIR/ProjFS.Mac/Scripts/Build.sh $CONFIGURATION +# If we're building the Profiling(Release) configuration, remove Profiling() for building .NET code +if [ "$CONFIGURATION" == "Profiling(Release)" ]; then + CONFIGURATION=Release +fi + # Build the MirrorProvider dotnet restore $SLN /p:Configuration="$CONFIGURATION.Mac" --packages $ROOTDIR/packages dotnet build $SLN --configuration $CONFIGURATION.Mac diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index d6d07e4ad7..907854e4ad 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -43,10 +43,9 @@ if [ "$CONFIGURATION" == "Profiling(Release)" ]; then CONFIGURATION=Release fi -DOTNETCONFIGURATION=$CONFIGURATION.Mac -dotnet restore $SRCDIR/GVFS.sln /p:Configuration=$DOTNETCONFIGURATION --packages $PACKAGES || exit 1 -dotnet build $SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --configuration $DOTNETCONFIGURATION /maxcpucount:1 || exit 1 -dotnet publish $SRCDIR/GVFS.sln /p:Configuration=$DOTNETCONFIGURATION /p:Platform=x64 --runtime osx-x64 --framework netcoreapp2.1 --self-contained --output $PUBLISHDIR /maxcpucount:1 || exit 1 +dotnet restore $SRCDIR/GVFS.sln /p:Configuration=$CONFIGURATION.Mac --packages $PACKAGES || exit 1 +dotnet build $SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --configuration $CONFIGURATION.Mac /maxcpucount:1 || exit 1 +dotnet publish $SRCDIR/GVFS.sln /p:Configuration=$CONFIGURATION.Mac /p:Platform=x64 --runtime osx-x64 --framework netcoreapp2.1 --self-contained --output $PUBLISHDIR /maxcpucount:1 || exit 1 NATIVEDIR=$SRCDIR/GVFS/GVFS.Native.Mac xcodebuild -configuration $CONFIGURATION -workspace $NATIVEDIR/GVFS.Native.Mac.xcworkspace build -scheme GVFS.Native.Mac -derivedDataPath $ROOTDIR/BuildOutput/GVFS.Native.Mac || exit 1 From 23d457d7ad34251d935f416bbbf91e79db256293 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 11 Oct 2018 15:42:46 -0600 Subject: [PATCH 139/244] Fix commands that need additional parameters with stdin --- .../GitCommands/CreatePlaceholderTests.cs | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs index ca0f371626..3704e5e286 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs @@ -17,26 +17,27 @@ public CreatePlaceholderTests() : base(enlistmentPerTest: true) { } - [TestCase("check-attr")] - [TestCase("check-ignore")] - [TestCase("check-mailmap")] - [TestCase("diff-tree")] - [TestCase("fetch-pack")] - [TestCase("hash-object")] - [TestCase("index-pack")] - [TestCase("name-rev")] - [TestCase("notes")] - [TestCase("send-pack")] - [TestCase("rev-list")] - [TestCase("update-ref")] + [TestCase("check-attr --stdin --all")] + [TestCase("check-ignore --stdin")] + [TestCase("check-mailmap --stdin")] + [TestCase("diff-tree --stdin")] + [TestCase("hash-object --stdin")] + [TestCase("index-pack --stdin")] + [TestCase("name-rev --stdin")] + [TestCase("rev-list --stdin --quiet --all")] + [TestCase("update-ref --stdin")] public void AllowsPlaceholderCreationWhileGitCommandIsRunning(string commandToRun) { this.CheckPlaceholderCreation(commandToRun, shouldAllow: true); } - [TestCase("checkout-index")] - [TestCase("reset")] - [TestCase("update-index")] + [TestCase("checkout-index --stdin")] + [TestCase("fetch-pack --stdin URL")] + [TestCase("notes copy --stdin")] + [TestCase("reset --stdin")] + [TestCase("send-pack --stdin URL")] + [TestCase("update-index --stdin")] + [Category(Categories.WindowsOnly)] // Mac never blocks placeholder creation public void BlocksPlaceholderCreationWhileGitCommandIsRunning(string commandToRun) { this.CheckPlaceholderCreation(commandToRun, shouldAllow: false); @@ -51,7 +52,7 @@ private void CheckPlaceholderCreation(string command, bool shouldAllow) } this.EditFile($"Some new content for {command}.", "Protocol.md"); - ManualResetEventSlim resetEvent = GitHelpers.RunGitCommandWithWaitAndStdIn(this.Enlistment, resetTimeout: 3000, command: $"{command} --stdin", stdinToQuit: eofCharacter, processId: out _); + ManualResetEventSlim resetEvent = GitHelpers.RunGitCommandWithWaitAndStdIn(this.Enlistment, resetTimeout: 3000, command: $"{command}", stdinToQuit: eofCharacter, processId: out _); if (shouldAllow) { From ed46b0eb18c09b11237dcec0815cb49fa7b71b63 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 12 Oct 2018 00:37:09 +0000 Subject: [PATCH 140/244] GVFS.props: use release build with rebase fix --- GVFS/GVFS.Build/GVFS.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Build/GVFS.props b/GVFS/GVFS.Build/GVFS.props index ef965a8c45..48a82f3ffd 100644 --- a/GVFS/GVFS.Build/GVFS.props +++ b/GVFS/GVFS.Build/GVFS.props @@ -3,7 +3,7 @@ 0.2.173.2 - 2.20181011.3 + 2.20181012.4 From 730fb89d9dc232df11c3ba2708ab605d52d704dc Mon Sep 17 00:00:00 2001 From: Mike Tesch Date: Fri, 12 Oct 2018 14:24:27 -0400 Subject: [PATCH 141/244] Use a consistent date/time format for logging --- GVFS/GVFS.Common/Tracing/InProcEventListener.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/Tracing/InProcEventListener.cs b/GVFS/GVFS.Common/Tracing/InProcEventListener.cs index 2bf25dfa67..01afc70717 100644 --- a/GVFS/GVFS.Common/Tracing/InProcEventListener.cs +++ b/GVFS/GVFS.Common/Tracing/InProcEventListener.cs @@ -41,7 +41,7 @@ protected string GetLogString(string eventName, EventOpcode opcode, string jsonP { // Make a smarter guess (than 16 characters) about initial size to reduce allocations StringBuilder message = new StringBuilder(1024); - message.AppendFormat("[{0}] {1}", DateTime.Now, eventName); + message.AppendFormat("[{0:yyyy-MM-dd HH:mm:ss zzz}] {1}", DateTime.Now, eventName); if (opcode != 0) { From 5106a877037c1d31c46755612fbd3f8e0b9c4821 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 12 Oct 2018 11:29:20 -0700 Subject: [PATCH 142/244] Cleanup for PR feedback --- .../MacFileSystemVirtualizer.cs | 34 ++- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 197 +++++++++++++----- 2 files changed, 157 insertions(+), 74 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 4a23c6f658..5487614d65 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -528,26 +528,22 @@ private void OnNewFileCreated(string relativePath, bool isDirectory) { if (isDirectory) { - if (!relativePath.Equals(GVFSConstants.DotGit.Root, StringComparison.OrdinalIgnoreCase)) + string lockedGitCommand = this.Context.Repository.GVFSLock.GetLockedGitCommand(); + GitCommandLineParser gitCommand = new GitCommandLineParser(lockedGitCommand); + if (gitCommand.IsValidGitCommand) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(lockedGitCommand), lockedGitCommand); + metadata.Add(TracingConstants.MessageKey.InfoMessage, "Git command created new folder"); + this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.OnNewFileCreated)}_GitCreatedFolder", metadata); + + // Record this folder as expanded so that GitIndexProjection will re-expand the folder + // when the projection change completes. + this.FileSystemCallbacks.OnPlaceholderFolderExpanded(relativePath); + } + else { - string lockedGitCommand = this.Context.Repository.GVFSLock.GetLockedGitCommand(); - GitCommandLineParser gitCommand = new GitCommandLineParser(lockedGitCommand); - if (gitCommand.IsValidGitCommand) - { - EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add(nameof(lockedGitCommand), lockedGitCommand); - metadata.Add(TracingConstants.MessageKey.InfoMessage, "Git command created new folder"); - this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.OnNewFileCreated)}_GitCreatedFolder", metadata); - - // Record this folder as expanded so that GitIndexProjection will re-expand the folder - // when the projection change completes. This is the closest we can get to converting this - // new folder to partial (as we do on Windows). - this.FileSystemCallbacks.OnPlaceholderFolderExpanded(relativePath); - } - else - { - this.FileSystemCallbacks.OnFolderCreated(relativePath); - } + this.FileSystemCallbacks.OnFolderCreated(relativePath); } } else diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index f7de1e7f91..6efb30c9b5 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -100,17 +100,27 @@ static void CombinePaths(const char* root, const char* relative, char (&combined static errno_t SendKernelMessageResponse(uint64_t messageId, MessageType responseType); static errno_t RegisterVirtualizationRootPath(const char* path); +static PrjFS_Result MarkAllChildrenAsInRoot(const char* fullDirectoryPath); + static void HandleKernelRequest(void* messageMemory, uint32_t messageSize); static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path); +static PrjFS_Result HandleNewFileInRootNotification( + const MessageHeader* request, + const char* path, + const char* fullPath, + bool isDirectory, + PrjFS_NotificationType notificationType); static PrjFS_Result HandleFileNotification( const MessageHeader* request, const char* path, - const char* fullPath_opt, + const char* fullPath, bool isDirectory, PrjFS_NotificationType notificationType); +static void FindNewFoldersInRootAndNotifyProvider(const MessageHeader* request, const char* path); + static Message ParseMessageMemory(const void* messageMemory, uint32_t size); static void ClearMachNotification(mach_port_t port); @@ -269,7 +279,7 @@ PrjFS_Result PrjFS_ConvertDirectoryToVirtualizationRoot( return PrjFS_Result_EIOError; } - return PrjFS_Result_Success; + return MarkAllChildrenAsInRoot(virtualizationRootFullPath); } PrjFS_Result PrjFS_WritePlaceholderDirectory( @@ -600,10 +610,12 @@ static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) case MessageType_KtoU_NotifyFilePreDelete: case MessageType_KtoU_NotifyDirectoryPreDelete: { + char fullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); result = HandleFileNotification( requestHeader, request.path, - nullptr, // fullPath_opt + fullPath, requestHeader->messageType == MessageType_KtoU_NotifyDirectoryPreDelete, // isDirectory KUMessageTypeToNotificationType(static_cast(requestHeader->messageType))); break; @@ -614,52 +626,10 @@ static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) case MessageType_KtoU_NotifyDirectoryRenamed: case MessageType_KtoU_NotifyFileHardLinkCreated: { - // Walk up the directory tree and notify the provider about any directories - // not flagged as being in the root - stack> newFolderPaths; - string parentPath(request.path); - size_t lastDirSeparator = parentPath.find_last_of('/'); - while (lastDirSeparator != string::npos && lastDirSeparator > 0) - { - parentPath = parentPath.substr(0, lastDirSeparator); - char parentFullPath[PrjFSMaxPath]; - CombinePaths(s_virtualizationRootFullPath.c_str(), parentPath.c_str(), parentFullPath); - if (IsBitSetInFileFlags(parentFullPath, FileFlags_IsInVirtualizationRoot)) - { - break; - } - else - { - newFolderPaths.emplace(make_pair(parentPath, parentFullPath)); - lastDirSeparator = parentPath.find_last_of('/'); - } - } - - while (!newFolderPaths.empty()) - { - const pair& parentFolderPath = newFolderPaths.top(); - - // TODO(Mac): Handle SetBitInFileFlags failures - SetBitInFileFlags(parentFolderPath.second.c_str(), FileFlags_IsInVirtualizationRoot, true); - - HandleFileNotification( - requestHeader, - parentFolderPath.first.c_str(), - parentFolderPath.second.c_str(), - true, // isDirectory - PrjFS_NotificationType_NewFileCreated); - - newFolderPaths.pop(); - } - char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); - - // TODO(Mac): Handle SetBitInFileFlags failures - SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); - bool isDirectory = requestHeader->messageType == MessageType_KtoU_NotifyDirectoryRenamed; - result = HandleFileNotification( + result = HandleNewFileInRootNotification( requestHeader, request.path, fullPath, @@ -880,10 +850,33 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const return result; } +static PrjFS_Result HandleNewFileInRootNotification( + const MessageHeader* request, + const char* path, + const char* fullPath, + bool isDirectory, + PrjFS_NotificationType notificationType) +{ + // Whenever a new file shows up in the root, we need to check if its ancestor + // directories are flagged as in root. If they are not, flag them as in root and + // notify the provider + FindNewFoldersInRootAndNotifyProvider(request, path); + + // TODO(Mac): Handle SetBitInFileFlags failures + SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); + + return HandleFileNotification( + request, + path, + fullPath, + isDirectory, + notificationType); +} + static PrjFS_Result HandleFileNotification( const MessageHeader* request, const char* path, - const char* fullPath_opt, + const char* fullPath, bool isDirectory, PrjFS_NotificationType notificationType) { @@ -894,14 +887,6 @@ static PrjFS_Result HandleFileNotification( << " isDirectory: " << isDirectory << endl; #endif - const char* fullPath = fullPath_opt; - char computedFullPath[PrjFSMaxPath]; - if (nullptr == fullPath) - { - CombinePaths(s_virtualizationRootFullPath.c_str(), path, computedFullPath); - fullPath = computedFullPath; - } - PrjFSFileXAttrData xattrData = {}; GetXAttr(fullPath, PrjFSFileXAttrName, sizeof(PrjFSFileXAttrData), &xattrData); @@ -917,6 +902,47 @@ static PrjFS_Result HandleFileNotification( nullptr /* destinationRelativePath */); } +static void FindNewFoldersInRootAndNotifyProvider(const MessageHeader* request, const char* path) +{ + // Walk up the directory tree and notify the provider about any directories + // not flagged as being in the root + stack> newFolderPaths; + string parentPath(path); + size_t lastDirSeparator = parentPath.find_last_of('/'); + while (lastDirSeparator != string::npos && lastDirSeparator > 0) + { + parentPath = parentPath.substr(0, lastDirSeparator); + char parentFullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), parentPath.c_str(), parentFullPath); + if (IsBitSetInFileFlags(parentFullPath, FileFlags_IsInVirtualizationRoot)) + { + break; + } + else + { + newFolderPaths.emplace(make_pair(parentPath, parentFullPath)); + lastDirSeparator = parentPath.find_last_of('/'); + } + } + + while (!newFolderPaths.empty()) + { + const pair& parentFolderPath = newFolderPaths.top(); + + // TODO(Mac): Handle SetBitInFileFlags failures + SetBitInFileFlags(parentFolderPath.second.c_str(), FileFlags_IsInVirtualizationRoot, true); + + HandleFileNotification( + request, + parentFolderPath.first.c_str(), + parentFolderPath.second.c_str(), + true, // isDirectory + PrjFS_NotificationType_NewFileCreated); + + newFolderPaths.pop(); + } +} + static bool InitializeEmptyPlaceholder(const char* fullPath) { return @@ -1079,6 +1105,67 @@ static errno_t RegisterVirtualizationRootPath(const char* path) return static_cast(error); } +static PrjFS_Result MarkAllChildrenAsInRoot(const char* fullDirectoryPath) +{ + DIR* directory = nullptr; + PrjFS_Result result = PrjFS_Result_Success; + queue directoryRelativePaths; + directoryRelativePaths.push(""); + + char fullPathBuffer[PrjFSMaxPath]; + char relativePathBuffer[PrjFSMaxPath]; + + while (!directoryRelativePaths.empty()) + { + string directoryRelativePath(directoryRelativePaths.front()); + directoryRelativePaths.pop(); + + CombinePaths(fullDirectoryPath, directoryRelativePath.c_str(), fullPathBuffer); + DIR* directory = opendir(fullPathBuffer); + if (nullptr == directory) + { + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + dirent* dirEntry = readdir(directory); + while (dirEntry != nullptr) + { + bool entryIsDirectoryToUpdate = + dirEntry->d_type == DT_DIR && + 0 != strncmp(".", dirEntry->d_name, sizeof(dirEntry->d_name)) && + 0 != strncmp("..", dirEntry->d_name, sizeof(dirEntry->d_name)); + + if (entryIsDirectoryToUpdate || dirEntry->d_type == DT_LNK || dirEntry->d_type == DT_REG) + { + CombinePaths(directoryRelativePath.c_str(), dirEntry->d_name, relativePathBuffer); + CombinePaths(fullDirectoryPath, relativePathBuffer, fullPathBuffer); + if (!SetBitInFileFlags(fullPathBuffer, FileFlags_IsInVirtualizationRoot, true)) + { + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + if (entryIsDirectoryToUpdate) + { + directoryRelativePaths.emplace(relativePathBuffer); + } + } + + dirEntry = readdir(directory); + } + } + +CleanupAndReturn: + if (directory != nullptr) + { + closedir(directory); + } + + return result; + +} + static void ClearMachNotification(mach_port_t port) { struct { From c12e1f93483e2b87dd2b161f2e187f0a52f6389c Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Wed, 10 Oct 2018 15:41:21 -0700 Subject: [PATCH 143/244] Turn on 'gvfs prefetch --hydrate' on MacOS by providing a native OS implementation. --- .../FileSystem/IPlatformFileSystem.cs | 3 +- .../GVFS.Common/Prefetch/Jobs/ReadFilesJob.cs | 55 +----------------- .../EnlistmentPerFixture/PrefetchVerbTests.cs | 2 - GVFS/GVFS.Platform.Mac/MacFileSystem.cs | 34 +++++++++++ .../WindowsFileSystem.cs | 56 +++++++++++++++++++ .../Mock/FileSystem/MockPlatformFileSystem.cs | 5 ++ 6 files changed, 100 insertions(+), 55 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs b/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs index 93c8cb2e9e..c026c9fe2b 100644 --- a/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs +++ b/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs @@ -7,6 +7,7 @@ public interface IPlatformFileSystem void MoveAndOverwriteFile(string sourceFileName, string destinationFilename); void CreateHardLink(string newLinkFileName, string existingFileName); bool TryGetNormalizedPath(string path, out string normalizedPath, out string errorMessage); - void ChangeMode(string path, int mode); + void ChangeMode(string path, int mode); + bool HydrateFile(string fileName, byte[] buffer); } } diff --git a/GVFS/GVFS.Common/Prefetch/Jobs/ReadFilesJob.cs b/GVFS/GVFS.Common/Prefetch/Jobs/ReadFilesJob.cs index def598cfa1..44ef07ae52 100644 --- a/GVFS/GVFS.Common/Prefetch/Jobs/ReadFilesJob.cs +++ b/GVFS/GVFS.Common/Prefetch/Jobs/ReadFilesJob.cs @@ -1,9 +1,6 @@ using GVFS.Common.Tracing; -using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Collections.Generic; -using System.IO; -using System.Runtime.InteropServices; using System.Threading; namespace GVFS.Common.Prefetch.Jobs @@ -42,16 +39,8 @@ protected override void DoWork() while (this.availableBlobs.TryTake(out blobId, Timeout.Infinite)) { foreach (string path in this.blobIdToPaths[blobId]) - { - bool succeeded = false; - using (SafeFileHandle handle = NativeFileReader.Open(path)) - { - if (!handle.IsInvalid) - { - succeeded = NativeFileReader.ReadOneByte(handle, buffer); - } - } - + { + bool succeeded = GVFSPlatform.Instance.FileSystem.HydrateFile(path, buffer); if (succeeded) { Interlocked.Increment(ref this.readFileCount); @@ -64,7 +53,7 @@ protected override void DoWork() failedFilesCurrentThread++; this.HasFailures = true; } - } + } } activity.Stop( @@ -75,43 +64,5 @@ protected override void DoWork() }); } } - - private class NativeFileReader - { - private const uint GenericRead = 0x80000000; - private const uint OpenExisting = 3; - - public static SafeFileHandle Open(string fileName) - { - return CreateFile(fileName, GenericRead, (uint)(FileShare.ReadWrite | FileShare.Delete), 0, OpenExisting, 0, 0); - } - - public static unsafe bool ReadOneByte(SafeFileHandle handle, byte[] buffer) - { - int n = 0; - fixed (byte* p = buffer) - { - return ReadFile(handle, p, 1, &n, 0); - } - } - - [DllImport("kernel32", SetLastError = true, ThrowOnUnmappableChar = true, CharSet = CharSet.Unicode)] - private static extern unsafe SafeFileHandle CreateFile( - string fileName, - uint desiredAccess, - uint shareMode, - uint securityAttributes, - uint creationDisposition, - uint flagsAndAttributes, - int hemplateFile); - - [DllImport("kernel32", SetLastError = true)] - private static extern unsafe bool ReadFile( - SafeFileHandle file, - void* buffer, - int numberOfBytesToRead, - int* numberOfBytesRead, - int overlapped); - } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs index bcfa9cbd31..35c1b1c27f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs @@ -42,7 +42,6 @@ public void PrefetchByFileExtension() } [TestCase, Order(4)] - [Category(Categories.MacTODO.M4)] public void PrefetchByFileExtensionWithHydrate() { int expectedCount = 3; @@ -52,7 +51,6 @@ public void PrefetchByFileExtensionWithHydrate() } [TestCase, Order(5)] - [Category(Categories.MacTODO.M4)] public void PrefetchByFilesWithHydrateWhoseObjectsAreAlreadyDownloaded() { int expectedCount = 2; diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs index 96dec6a59b..ae40098478 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs @@ -38,10 +38,44 @@ public bool TryGetNormalizedPath(string path, out string normalizedPath, out str return MacFileSystem.TryGetNormalizedPathImplementation(path, out normalizedPath, out errorMessage); } + public bool HydrateFile(string fileName, byte[] buffer) + { + return NativeFileReader.TryReadFirstByteOfFile(fileName, buffer); + } + [DllImport("libc", EntryPoint = "chmod", SetLastError = true)] private static extern int Chmod(string pathname, int mode); [DllImport("libc", EntryPoint = "rename", SetLastError = true)] private static extern int Rename(string oldPath, string newPath); + + private class NativeFileReader + { + private const int ReadOnly = 0x0000; + + public static bool TryReadFirstByteOfFile(string fileName, byte[] buffer) + { + int fileDescriptor = Open(fileName, ReadOnly); + return TryReadOneByte(fileDescriptor, buffer); + } + + private static bool TryReadOneByte(int fileDescriptor, byte[] buffer) + { + int numBytes = Read(fileDescriptor, buffer, 1); + + if (numBytes == -1) + { + return false; + } + + return true; + } + + [DllImport("libc", EntryPoint = "open", SetLastError = true)] + private static extern int Open(string path, int flag); + + [DllImport("libc", EntryPoint = "read", SetLastError = true)] + private static extern int Read(int fd, [Out] byte[] buf, int count); + } } } diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystem.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystem.cs index 89aa726b40..217fda1f1c 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystem.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystem.cs @@ -1,5 +1,8 @@ using GVFS.Common; using GVFS.Common.FileSystem; +using Microsoft.Win32.SafeHandles; +using System.IO; +using System.Runtime.InteropServices; namespace GVFS.Platform.Windows { @@ -32,6 +35,59 @@ public void ChangeMode(string path, int mode) public bool TryGetNormalizedPath(string path, out string normalizedPath, out string errorMessage) { return WindowsFileSystem.TryGetNormalizedPathImplementation(path, out normalizedPath, out errorMessage); + } + + public bool HydrateFile(string fileName, byte[] buffer) + { + return NativeFileReader.TryReadFirstByteOfFile(fileName, buffer); + } + + private class NativeFileReader + { + private const uint GenericRead = 0x80000000; + private const uint OpenExisting = 3; + + public static bool TryReadFirstByteOfFile(string fileName, byte[] buffer) + { + using (SafeFileHandle handle = Open(fileName)) + { + if (!handle.IsInvalid) + { + return ReadOneByte(handle, buffer); + } + } + + return false; + } + + private static SafeFileHandle Open(string fileName) + { + return CreateFile(fileName, GenericRead, (uint)(FileShare.ReadWrite | FileShare.Delete), 0, OpenExisting, 0, 0); + } + + private static bool ReadOneByte(SafeFileHandle handle, byte[] buffer) + { + int bytesRead = 0; + return ReadFile(handle, buffer, 1, ref bytesRead, 0); + } + + [DllImport("kernel32", SetLastError = true, ThrowOnUnmappableChar = true, CharSet = CharSet.Unicode)] + private static extern SafeFileHandle CreateFile( + string fileName, + uint desiredAccess, + uint shareMode, + uint securityAttributes, + uint creationDisposition, + uint flagsAndAttributes, + int hemplateFile); + + [DllImport("kernel32", SetLastError = true)] + private static extern bool ReadFile( + SafeFileHandle file, + [Out] byte[] buffer, + int numberOfBytesToRead, + ref int numberOfBytesRead, + int overlapped); } } } diff --git a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs index 999f7efd90..ab00f82a65 100644 --- a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs +++ b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs @@ -32,6 +32,11 @@ public bool TryGetNormalizedPath(string path, out string normalizedPath, out str errorMessage = null; normalizedPath = path; return true; + } + + public bool HydrateFile(string fileName, byte[] buffer) + { + throw new NotSupportedException(); } } } From a5bb52796a50dec719f43f6e2ba099b570d15f83 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 15 Oct 2018 10:05:05 -0700 Subject: [PATCH 144/244] Additional cleanup for PR feedback --- .../MacFileSystemVirtualizer.cs | 7 +- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 165 ++++++++++-------- 2 files changed, 95 insertions(+), 77 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 5487614d65..dcc82468e9 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -538,7 +538,12 @@ private void OnNewFileCreated(string relativePath, bool isDirectory) this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.OnNewFileCreated)}_GitCreatedFolder", metadata); // Record this folder as expanded so that GitIndexProjection will re-expand the folder - // when the projection change completes. + // when the projection change completes. + // + // Git creates new folders when there are files that it needs to create. + // However, git will only create files that are in ModifiedPaths.dat. There could + // be other files in the projection (that were not created by git) and so VFS must re-expand the + // newly created folder to ensure that all files are written to disk. this.FileSystemCallbacks.OnPlaceholderFolderExpanded(relativePath); } else diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 6efb30c9b5..21e27e42ed 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -84,42 +84,43 @@ struct MutexAndUseCount typedef map FileMutexMap; // Function prototypes -static bool SetBitInFileFlags(const char* path, uint32_t bit, bool value); -static bool IsBitSetInFileFlags(const char* path, uint32_t bit); +static bool SetBitInFileFlags(const char* fullPath, uint32_t bit, bool value); +static bool IsBitSetInFileFlags(const char* fullPath, uint32_t bit); static bool InitializeEmptyPlaceholder(const char* fullPath); template static bool InitializeEmptyPlaceholder(const char* fullPath, TPlaceholder* data, const char* xattrName); -static bool AddXAttr(const char* path, const char* name, const void* value, size_t size); -static bool GetXAttr(const char* path, const char* name, size_t size, _Out_ void* value); +static bool AddXAttr(const char* fullPath, const char* name, const void* value, size_t size); +static bool GetXAttr(const char* fullPath, const char* name, size_t size, _Out_ void* value); static inline PrjFS_NotificationType KUMessageTypeToNotificationType(MessageType kuNotificationType); -static bool IsVirtualizationRoot(const char* path); +static bool IsVirtualizationRoot(const char* fullPath); static void CombinePaths(const char* root, const char* relative, char (&combined)[PrjFSMaxPath]); static errno_t SendKernelMessageResponse(uint64_t messageId, MessageType responseType); -static errno_t RegisterVirtualizationRootPath(const char* path); +static errno_t RegisterVirtualizationRootPath(const char* fullPath); -static PrjFS_Result MarkAllChildrenAsInRoot(const char* fullDirectoryPath); +static PrjFS_Result RecursivelyMarkAllChildrenAsInRoot(const char* fullDirectoryPath); static void HandleKernelRequest(void* messageMemory, uint32_t messageSize); -static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path); -static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path); -static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path); +static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* relativePath); +static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* relativePath); +static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* relativePath); static PrjFS_Result HandleNewFileInRootNotification( const MessageHeader* request, - const char* path, + const char* relativePath, const char* fullPath, bool isDirectory, PrjFS_NotificationType notificationType); static PrjFS_Result HandleFileNotification( const MessageHeader* request, - const char* path, + const char* relativePath, const char* fullPath, bool isDirectory, PrjFS_NotificationType notificationType); -static void FindNewFoldersInRootAndNotifyProvider(const MessageHeader* request, const char* path); +static void FindNewFoldersInRootAndNotifyProvider(const MessageHeader* request, const char* relativePath); +static bool IsDirEntChildDirectory(const dirent* directoryEntry); static Message ParseMessageMemory(const void* messageMemory, uint32_t size); @@ -279,7 +280,7 @@ PrjFS_Result PrjFS_ConvertDirectoryToVirtualizationRoot( return PrjFS_Result_EIOError; } - return MarkAllChildrenAsInRoot(virtualizationRootFullPath); + return RecursivelyMarkAllChildrenAsInRoot(virtualizationRootFullPath); } PrjFS_Result PrjFS_WritePlaceholderDirectory( @@ -415,6 +416,7 @@ PrjFS_Result PrjFS_WriteSymLink( goto CleanupAndFail; } + // TODO(Mac) #391: Handles failures of SetBitInFileFlags SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); return PrjFS_Result_Success; @@ -655,14 +657,14 @@ static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) free(messageMemory); } -static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path) +static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* relativePath) { #ifdef DEBUG - cout << "PrjFSLib.HandleEnumerateDirectoryRequest: " << path << endl; + cout << "PrjFSLib.HandleEnumerateDirectoryRequest: " << relativePath << endl; #endif char fullPath[PrjFSMaxPath]; - CombinePaths(s_virtualizationRootFullPath.c_str(), path, fullPath); + CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) { return PrjFS_Result_Success; @@ -680,7 +682,7 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request result = s_callbacks.EnumerateDirectory( 0 /* commandId */, - path, + relativePath, request->pid, request->procname); @@ -701,25 +703,25 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request return result; } -static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path) +static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* relativePath) { #ifdef DEBUG - cout << "PrjFSLib.HandleRecursivelyEnumerateDirectoryRequest: " << path << endl; + cout << "PrjFSLib.HandleRecursivelyEnumerateDirectoryRequest: " << relativePath << endl; #endif DIR* directory = nullptr; PrjFS_Result result = PrjFS_Result_Success; queue directoryRelativePaths; - directoryRelativePaths.push(path); + directoryRelativePaths.push(relativePath); // Walk each directory, expanding those that are found to be empty - char pathBuffer[PrjFSMaxPath]; + char path[PrjFSMaxPath]; while (!directoryRelativePaths.empty()) { string directoryRelativePath(directoryRelativePaths.front()); directoryRelativePaths.pop(); - CombinePaths(s_virtualizationRootFullPath.c_str(), directoryRelativePath.c_str(), pathBuffer); + CombinePaths(s_virtualizationRootFullPath.c_str(), directoryRelativePath.c_str(), path); PrjFS_Result result = HandleEnumerateDirectoryRequest(request, directoryRelativePath.c_str()); if (result != PrjFS_Result_Success) @@ -727,7 +729,7 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead goto CleanupAndReturn; } - DIR* directory = opendir(pathBuffer); + DIR* directory = opendir(path); if (nullptr == directory) { result = PrjFS_Result_EIOError; @@ -737,12 +739,10 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead dirent* dirEntry = readdir(directory); while (dirEntry != nullptr) { - if (dirEntry->d_type == DT_DIR && - 0 != strncmp(".", dirEntry->d_name, sizeof(dirEntry->d_name)) && - 0 != strncmp("..", dirEntry->d_name, sizeof(dirEntry->d_name))) + if (IsDirEntChildDirectory(dirEntry)) { - CombinePaths(directoryRelativePath.c_str(), dirEntry->d_name, pathBuffer); - directoryRelativePaths.emplace(pathBuffer); + CombinePaths(directoryRelativePath.c_str(), dirEntry->d_name, path); + directoryRelativePaths.emplace(path); } dirEntry = readdir(directory); @@ -758,14 +758,14 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead return result; } -static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path) +static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* relativePath) { #ifdef DEBUG - cout << "PrjFSLib.HandleHydrateFileRequest: " << path << endl; + cout << "PrjFSLib.HandleHydrateFileRequest: " << relativePath << endl; #endif char fullPath[PrjFSMaxPath]; - CombinePaths(s_virtualizationRootFullPath.c_str(), path, fullPath); + CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); PrjFSFileXAttrData xattrData = {}; if (!GetXAttr(fullPath, PrjFSFileXAttrName, sizeof(PrjFSFileXAttrData), &xattrData)) @@ -812,7 +812,7 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const result = s_callbacks.GetFileStream( 0 /* comandId */, - path, + relativePath, xattrData.providerId, xattrData.contentId, request->pid, @@ -852,37 +852,46 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const static PrjFS_Result HandleNewFileInRootNotification( const MessageHeader* request, - const char* path, + const char* relativePath, const char* fullPath, bool isDirectory, PrjFS_NotificationType notificationType) { +#ifdef DEBUG + cout + << "HandleNewFileInRootNotification: " << relativePath + << " notificationType: " << NotificationTypeToString(notificationType) + << " isDirectory: " << isDirectory << endl; +#endif + // Whenever a new file shows up in the root, we need to check if its ancestor // directories are flagged as in root. If they are not, flag them as in root and // notify the provider - FindNewFoldersInRootAndNotifyProvider(request, path); - - // TODO(Mac): Handle SetBitInFileFlags failures - SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); + FindNewFoldersInRootAndNotifyProvider(request, relativePath); - return HandleFileNotification( + PrjFS_Result result = HandleFileNotification( request, - path, + relativePath, fullPath, isDirectory, notificationType); + + // TODO(Mac) #391: Handle SetBitInFileFlags failures + SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); + + return result; } static PrjFS_Result HandleFileNotification( const MessageHeader* request, - const char* path, + const char* relativePath, const char* fullPath, bool isDirectory, PrjFS_NotificationType notificationType) { #ifdef DEBUG cout - << "PrjFSLib.HandleFileNotification: " << path + << "PrjFSLib.HandleFileNotification: " << relativePath << " notificationType: " << NotificationTypeToString(notificationType) << " isDirectory: " << isDirectory << endl; #endif @@ -892,7 +901,7 @@ static PrjFS_Result HandleFileNotification( return s_callbacks.NotifyOperation( 0 /* commandId */, - path, + relativePath, xattrData.providerId, xattrData.contentId, request->pid, @@ -902,12 +911,12 @@ static PrjFS_Result HandleFileNotification( nullptr /* destinationRelativePath */); } -static void FindNewFoldersInRootAndNotifyProvider(const MessageHeader* request, const char* path) +static void FindNewFoldersInRootAndNotifyProvider(const MessageHeader* request, const char* relativePath) { // Walk up the directory tree and notify the provider about any directories // not flagged as being in the root stack> newFolderPaths; - string parentPath(path); + string parentPath(relativePath); size_t lastDirSeparator = parentPath.find_last_of('/'); while (lastDirSeparator != string::npos && lastDirSeparator > 0) { @@ -928,9 +937,6 @@ static void FindNewFoldersInRootAndNotifyProvider(const MessageHeader* request, while (!newFolderPaths.empty()) { const pair& parentFolderPath = newFolderPaths.top(); - - // TODO(Mac): Handle SetBitInFileFlags failures - SetBitInFileFlags(parentFolderPath.second.c_str(), FileFlags_IsInVirtualizationRoot, true); HandleFileNotification( request, @@ -939,10 +945,21 @@ static void FindNewFoldersInRootAndNotifyProvider(const MessageHeader* request, true, // isDirectory PrjFS_NotificationType_NewFileCreated); + // TODO(Mac) #391: Handle SetBitInFileFlags failures + SetBitInFileFlags(parentFolderPath.second.c_str(), FileFlags_IsInVirtualizationRoot, true); + newFolderPaths.pop(); } } +static bool IsDirEntChildDirectory(const dirent* directoryEntry) +{ + return + directoryEntry->d_type == DT_DIR && + 0 != strncmp(".", directoryEntry->d_name, sizeof(directoryEntry->d_name)) && + 0 != strncmp("..", directoryEntry->d_name, sizeof(directoryEntry->d_name)); +} + static bool InitializeEmptyPlaceholder(const char* fullPath) { return @@ -968,10 +985,10 @@ static bool InitializeEmptyPlaceholder(const char* fullPath, TPlaceholder* data, return false; } -static bool IsVirtualizationRoot(const char* path) +static bool IsVirtualizationRoot(const char* fullPath) { PrjFSVirtualizationRootXAttrData data = {}; - if (GetXAttr(path, PrjFSVirtualizationRootXAttrName, sizeof(PrjFSVirtualizationRootXAttrData), &data)) + if (GetXAttr(fullPath, PrjFSVirtualizationRootXAttrName, sizeof(PrjFSVirtualizationRootXAttrData), &data)) { return true; } @@ -984,10 +1001,10 @@ static void CombinePaths(const char* root, const char* relative, char (&combined snprintf(combined, PrjFSMaxPath, "%s/%s", root, relative); } -static bool SetBitInFileFlags(const char* path, uint32_t bit, bool value) +static bool SetBitInFileFlags(const char* fullPath, uint32_t bit, bool value) { struct stat fileAttributes; - if (lstat(path, &fileAttributes)) + if (lstat(fullPath, &fileAttributes)) { return false; } @@ -1002,7 +1019,7 @@ static bool SetBitInFileFlags(const char* path, uint32_t bit, bool value) newValue = fileAttributes.st_flags & ~bit; } - if (lchflags(path, newValue)) + if (lchflags(fullPath, newValue)) { return false; } @@ -1010,10 +1027,10 @@ static bool SetBitInFileFlags(const char* path, uint32_t bit, bool value) return true; } -static bool IsBitSetInFileFlags(const char* path, uint32_t bit) +static bool IsBitSetInFileFlags(const char* fullPath, uint32_t bit) { struct stat fileAttributes; - if (lstat(path, &fileAttributes)) + if (lstat(fullPath, &fileAttributes)) { return false; } @@ -1021,9 +1038,9 @@ static bool IsBitSetInFileFlags(const char* path, uint32_t bit) return fileAttributes.st_flags & bit; } -static bool AddXAttr(const char* path, const char* name, const void* value, size_t size) +static bool AddXAttr(const char* fullPath, const char* name, const void* value, size_t size) { - if (setxattr(path, name, value, size, 0, 0)) + if (setxattr(fullPath, name, value, size, 0, 0)) { return false; } @@ -1031,9 +1048,9 @@ static bool AddXAttr(const char* path, const char* name, const void* value, size return true; } -static bool GetXAttr(const char* path, const char* name, size_t size, _Out_ void* value) +static bool GetXAttr(const char* fullPath, const char* name, size_t size, _Out_ void* value) { - if (getxattr(path, name, value, size, 0, 0) == size) + if (getxattr(fullPath, name, value, size, 0, 0) == size) { // TODO: also validate the magic number and format version. // It's easy to check their expected values, but we will need to decide what to do if they are incorrect. @@ -1089,39 +1106,39 @@ static errno_t SendKernelMessageResponse(uint64_t messageId, MessageType respons return callResult == kIOReturnSuccess ? 0 : EBADMSG; } -static errno_t RegisterVirtualizationRootPath(const char* path) +static errno_t RegisterVirtualizationRootPath(const char* fullPath) { uint64_t error = EBADMSG; uint32_t output_count = 1; - size_t pathSize = strlen(path) + 1; + size_t pathSize = strlen(fullPath) + 1; IOReturn callResult = IOConnectCallMethod( s_kernelServiceConnection, ProviderSelector_RegisterVirtualizationRootPath, nullptr, 0, // no scalar inputs - path, pathSize, // struct input + fullPath, pathSize, // struct input &error, &output_count, // scalar output nullptr, nullptr); // no struct output assert(callResult == kIOReturnSuccess); return static_cast(error); } -static PrjFS_Result MarkAllChildrenAsInRoot(const char* fullDirectoryPath) +static PrjFS_Result RecursivelyMarkAllChildrenAsInRoot(const char* fullDirectoryPath) { DIR* directory = nullptr; PrjFS_Result result = PrjFS_Result_Success; queue directoryRelativePaths; directoryRelativePaths.push(""); - char fullPathBuffer[PrjFSMaxPath]; - char relativePathBuffer[PrjFSMaxPath]; + char fullPath[PrjFSMaxPath]; + char relativePath[PrjFSMaxPath]; while (!directoryRelativePaths.empty()) { string directoryRelativePath(directoryRelativePaths.front()); directoryRelativePaths.pop(); - CombinePaths(fullDirectoryPath, directoryRelativePath.c_str(), fullPathBuffer); - DIR* directory = opendir(fullPathBuffer); + CombinePaths(fullDirectoryPath, directoryRelativePath.c_str(), fullPath); + DIR* directory = opendir(fullPath); if (nullptr == directory) { result = PrjFS_Result_EIOError; @@ -1131,16 +1148,12 @@ static PrjFS_Result MarkAllChildrenAsInRoot(const char* fullDirectoryPath) dirent* dirEntry = readdir(directory); while (dirEntry != nullptr) { - bool entryIsDirectoryToUpdate = - dirEntry->d_type == DT_DIR && - 0 != strncmp(".", dirEntry->d_name, sizeof(dirEntry->d_name)) && - 0 != strncmp("..", dirEntry->d_name, sizeof(dirEntry->d_name)); - + bool entryIsDirectoryToUpdate = IsDirEntChildDirectory(dirEntry); if (entryIsDirectoryToUpdate || dirEntry->d_type == DT_LNK || dirEntry->d_type == DT_REG) { - CombinePaths(directoryRelativePath.c_str(), dirEntry->d_name, relativePathBuffer); - CombinePaths(fullDirectoryPath, relativePathBuffer, fullPathBuffer); - if (!SetBitInFileFlags(fullPathBuffer, FileFlags_IsInVirtualizationRoot, true)) + CombinePaths(directoryRelativePath.c_str(), dirEntry->d_name, relativePath); + CombinePaths(fullDirectoryPath, relativePath, fullPath); + if (!SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true)) { result = PrjFS_Result_EIOError; goto CleanupAndReturn; @@ -1148,7 +1161,7 @@ static PrjFS_Result MarkAllChildrenAsInRoot(const char* fullDirectoryPath) if (entryIsDirectoryToUpdate) { - directoryRelativePaths.emplace(relativePathBuffer); + directoryRelativePaths.emplace(relativePath); } } From 5f9bab477551579680204faf8120aa70755b30bd Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Fri, 12 Oct 2018 12:52:20 -0400 Subject: [PATCH 145/244] - Updated gvfs config verb to support CRUD and List operations on settings. - Usage gvfs config [ --list | --delete ] --- GVFS/GVFS.Common/FileBasedDictionary.cs | 5 ++ GVFS/GVFS.Common/LocalGVFSConfig.cs | 112 +++++++++++++++++------- GVFS/GVFS/CommandLine/ConfigVerb.cs | 92 +++++++++++++++---- 3 files changed, 161 insertions(+), 48 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedDictionary.cs b/GVFS/GVFS.Common/FileBasedDictionary.cs index 393de7cb04..f5c553a956 100644 --- a/GVFS/GVFS.Common/FileBasedDictionary.cs +++ b/GVFS/GVFS.Common/FileBasedDictionary.cs @@ -106,6 +106,11 @@ public void RemoveAndFlush(TKey key) } } + public void GetAllKeysAndValues(out Dictionary keysAndValues) + { + keysAndValues = new Dictionary(this.data); + } + private void Flush() { this.WriteAndReplaceDataFile(this.GenerateDataLines); diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index 6094122fc3..2bebcf2864 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -1,6 +1,7 @@ using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using System; +using System.Collections.Generic; using System.IO; namespace GVFS.Common @@ -11,7 +12,7 @@ public class LocalGVFSConfig private readonly string configFile; private readonly PhysicalFileSystem fileSystem; private FileBasedDictionary allSettings; - + public LocalGVFSConfig() { string servicePath = Paths.GetServiceDataRoot(GVFSConstants.Service.ServiceName); @@ -21,37 +22,51 @@ public LocalGVFSConfig() this.fileSystem = new PhysicalFileSystem(); } + private delegate bool ConfigAction(); + + public bool TryGetAllConfig(out Dictionary allConfig, out string error, ITracer tracer) + { + Dictionary configCopy = null; + if (!this.TryPerformAction( + () => + { + this.allSettings.GetAllKeysAndValues(out configCopy); + return true; + }, + tracer, + out error)) + { + allConfig = null; + return false; + } + + allConfig = configCopy; + error = null; + return true; + } + public bool TryGetConfig( string name, out string value, out string error, ITracer tracer) { - if (!this.TryLoadSettings(tracer, out error)) + string valueFromDict = null; + if (!this.TryPerformAction( + () => + { + this.allSettings.TryGetValue(name, out valueFromDict); + return true; + }, + tracer, + out error)) { - error = $"Error getting config value {name}. {error}"; value = null; return false; } - - try - { - this.allSettings.TryGetValue(name, out value); - error = null; - return true; - } - catch (FileBasedCollectionException exception) - { - const string ErrorFormat = "Error getting config value for {0}. Config file {1}. {2}"; - if (tracer != null) - { - tracer.RelatedError(ErrorFormat, name, this.configFile, exception.ToString()); - } - error = string.Format(ErrorFormat, name, this.configFile, exception.Message); - value = null; - return false; - } + value = valueFromDict; + return true; } public bool TrySetConfig( @@ -60,30 +75,67 @@ public bool TrySetConfig( out string error, ITracer tracer) { - if (!this.TryLoadSettings(tracer, out error)) + if (!this.TryPerformAction( + () => + { + this.allSettings.SetValueAndFlush(name, value); + return true; + }, + tracer, + out error)) { error = $"Error setting config value {name}: {value}. {error}"; return false; } + return true; + } + + public bool TryRemoveConfig(string name, out string error, ITracer tracer) + { + if (!this.TryPerformAction( + () => + { + this.allSettings.RemoveAndFlush(name); + return true; + }, + tracer, + out error)) + { + error = $"Error removing config value {name}. {error}"; + return false; + } + + return true; + } + + private bool TryPerformAction(ConfigAction action, ITracer tracer, out string error) + { + if (!this.TryLoadSettings(tracer, out error)) + { + error = $"Error loading config settings."; + return false; + } + try { - this.allSettings.SetValueAndFlush(name, value); - error = null; - return true; + if (action()) + { + error = null; + return true; + } } catch (FileBasedCollectionException exception) { - const string ErrorFormat = "Error setting config value {0}: {1}. Config file {2}. {3}"; if (tracer != null) { - tracer.RelatedError(ErrorFormat, name, value, this.configFile, exception.ToString()); + tracer.RelatedError(exception.ToString()); } - error = string.Format(ErrorFormat, name, value, this.configFile, exception.Message); - value = null; - return false; + error = exception.Message; } + + return false; } private bool TryLoadSettings(ITracer tracer, out string error) diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index 6c64eb3d44..ea048c0ff7 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -1,6 +1,7 @@ using CommandLine; using GVFS.Common; using System; +using System.Collections.Generic; namespace GVFS.CommandLine { @@ -10,18 +11,34 @@ public class ConfigVerb : GVFSVerb.ForNoEnlistment private const string ConfigVerbName = "config"; private LocalGVFSConfig localConfig; + [Option( + "list", + SetName = "needNoKeys", + Default = false, + Required = false, + HelpText = "Show all settings")] + public bool List { get; set; } + + [Option( + "delete", + SetName = "needsKey", + Default = null, + Required = false, + HelpText = "Delete specified setting")] + public string KeyToDelete { get; set; } + [Value( - 0, - Required = true, - MetaName = "Setting name", - HelpText = "Name of setting that is to be set or read")] + 0, + Required = false, + MetaName = "Setting name", + HelpText = "Name of setting that is to be set, read or deleted")] public string Key { get; set; } [Value( - 1, - Required = false, - MetaName = "Setting value", - HelpText = "Value of setting to be set")] + 1, + Required = false, + MetaName = "Setting value", + HelpText = "Value of setting to be set")] public string Value { get; set; } protected override string VerbName @@ -34,27 +51,66 @@ public override void Execute() this.localConfig = new LocalGVFSConfig(); string error = null; - bool isRead = string.IsNullOrEmpty(this.Value); - - if (isRead) + if (this.List) { - string value = null; - if (this.localConfig.TryGetConfig(this.Key, out value, out error, tracer: null)) + Dictionary allSettings; + if (!this.localConfig.TryGetAllConfig(out allSettings, out error, tracer: null)) { - Console.WriteLine(value); + this.ReportErrorAndExit(error); } - else + + const string ConfigOutputFormat = "{0}={1}"; + foreach (KeyValuePair setting in allSettings) { - this.ReportErrorAndExit(error); + Console.WriteLine(ConfigOutputFormat, setting.Key, setting.Value); } + + return; } - else + + if (!string.IsNullOrEmpty(this.KeyToDelete)) { - if (!this.localConfig.TrySetConfig(this.Key, this.Value, out error, tracer: null)) + if (!this.localConfig.TryRemoveConfig(this.KeyToDelete, out error, tracer: null)) { this.ReportErrorAndExit(error); } + + return; + } + + bool keySpecified = !string.IsNullOrEmpty(this.Key); + if (keySpecified) + { + bool valueSpecified = !string.IsNullOrEmpty(this.Value); + if (valueSpecified) + { + if (!this.localConfig.TrySetConfig(this.Key, this.Value, out error, tracer: null)) + { + this.ReportErrorAndExit(error); + } + + return; + } + else + { + string valueRead = null; + if (!this.localConfig.TryGetConfig(this.Key, out valueRead, out error, tracer: null)) + { + this.ReportErrorAndExit(error); + } + else if (!string.IsNullOrEmpty(valueRead)) + { + Console.WriteLine(valueRead); + } + + return; + } } + + // this.PrintUsage(); + // RFC: There was no parsing error, CommandLine Parser would have handled it already + // if there were any. The issue happens when user types `gvfs config`. + // Should I print Usage or exit silently. } } } From 59d025005a8c37c8867a53ed0349ac1d046b6987 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 15 Oct 2018 12:07:15 -0400 Subject: [PATCH 146/244] incorporating review comments. Removed tracer. Style cleanups. adding l option for --list adding d option to --delete adding Functional tests --- GVFS/GVFS.Common/FileBasedDictionary.cs | 4 +- GVFS/GVFS.Common/LocalGVFSConfig.cs | 50 +++------ GVFS/GVFS.Common/ProductUpgrader.cs | 2 +- .../MultiEnlistmentTests/ConfigVerbTests.cs | 105 ++++++++++++++++++ .../GVFS.FunctionalTests/Tools/GVFSProcess.cs | 5 + GVFS/GVFS/CommandLine/ConfigVerb.cs | 35 +++--- 6 files changed, 147 insertions(+), 54 deletions(-) create mode 100644 GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs diff --git a/GVFS/GVFS.Common/FileBasedDictionary.cs b/GVFS/GVFS.Common/FileBasedDictionary.cs index f5c553a956..b4275a9b7c 100644 --- a/GVFS/GVFS.Common/FileBasedDictionary.cs +++ b/GVFS/GVFS.Common/FileBasedDictionary.cs @@ -106,9 +106,9 @@ public void RemoveAndFlush(TKey key) } } - public void GetAllKeysAndValues(out Dictionary keysAndValues) + public Dictionary GetAllKeysAndValues() { - keysAndValues = new Dictionary(this.data); + return new Dictionary(this.data); } private void Flush() diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index 2bebcf2864..a2382cb9da 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -22,18 +22,14 @@ public LocalGVFSConfig() this.fileSystem = new PhysicalFileSystem(); } - private delegate bool ConfigAction(); - - public bool TryGetAllConfig(out Dictionary allConfig, out string error, ITracer tracer) + public bool TryGetAllConfig(out Dictionary allConfig, out string error) { Dictionary configCopy = null; if (!this.TryPerformAction( () => { - this.allSettings.GetAllKeysAndValues(out configCopy); - return true; + configCopy = this.allSettings.GetAllKeysAndValues(); }, - tracer, out error)) { allConfig = null; @@ -48,20 +44,18 @@ public bool TryGetAllConfig(out Dictionary allConfig, out string public bool TryGetConfig( string name, out string value, - out string error, - ITracer tracer) + out string error) { string valueFromDict = null; if (!this.TryPerformAction( () => { this.allSettings.TryGetValue(name, out valueFromDict); - return true; }, - tracer, out error)) { value = null; + error = $"Error reading config {name}. {error}"; return false; } @@ -72,46 +66,41 @@ public bool TryGetConfig( public bool TrySetConfig( string name, string value, - out string error, - ITracer tracer) + out string error) { if (!this.TryPerformAction( () => { this.allSettings.SetValueAndFlush(name, value); - return true; }, - tracer, out error)) { - error = $"Error setting config value {name}: {value}. {error}"; + error = $"Error writing config {name}={value}. {error}"; return false; } return true; } - public bool TryRemoveConfig(string name, out string error, ITracer tracer) + public bool TryRemoveConfig(string name, out string error) { if (!this.TryPerformAction( () => { this.allSettings.RemoveAndFlush(name); - return true; }, - tracer, out error)) { - error = $"Error removing config value {name}. {error}"; + error = $"Error deleting config {name}. {error}"; return false; } return true; } - private bool TryPerformAction(ConfigAction action, ITracer tracer, out string error) + private bool TryPerformAction(Action action, out string error) { - if (!this.TryLoadSettings(tracer, out error)) + if (!this.TryLoadSettings(out error)) { error = $"Error loading config settings."; return false; @@ -119,32 +108,25 @@ private bool TryPerformAction(ConfigAction action, ITracer tracer, out string er try { - if (action()) - { - error = null; - return true; - } + action(); + error = null; + return true; } catch (FileBasedCollectionException exception) { - if (tracer != null) - { - tracer.RelatedError(exception.ToString()); - } - error = exception.Message; } return false; } - private bool TryLoadSettings(ITracer tracer, out string error) + private bool TryLoadSettings(out string error) { if (this.allSettings == null) { FileBasedDictionary config = null; if (FileBasedDictionary.TryCreate( - tracer: tracer, + tracer: null, dictionaryPath: this.configFile, fileSystem: this.fileSystem, output: out config, @@ -162,4 +144,4 @@ private bool TryLoadSettings(ITracer tracer, out string error) return true; } } -} +} \ No newline at end of file diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index 636278ea8c..9206b57532 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -273,7 +273,7 @@ public virtual bool TryLoadRingConfig(out string error) LocalGVFSConfig localConfig = new LocalGVFSConfig(); string ringConfig = null; - if (localConfig.TryGetConfig(GVFSConstants.LocalGVFSConfig.UpgradeRing, out ringConfig, out error, this.tracer)) + if (localConfig.TryGetConfig(GVFSConstants.LocalGVFSConfig.UpgradeRing, out ringConfig, out error)) { RingType ringType; diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs new file mode 100644 index 0000000000..6a87f20738 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs @@ -0,0 +1,105 @@ +using GVFS.FunctionalTests.Tools; +using GVFS.Tests.Should; +using NUnit.Framework; +using System.IO; + +namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests +{ + [TestFixture] + [NonParallelizable] + [Category(Categories.FullSuiteOnly)] + [Category(Categories.WindowsOnly)] + public class ConfigVerbTests : TestsWithMultiEnlistment + { + private const string ConfigFilePath = @"C:\ProgramData\GVFS\gvfs.config"; + + [OneTimeSetUp] + public void DeleteAllSettings() + { + if (File.Exists(ConfigFilePath)) + { + File.Delete(ConfigFilePath); + } + } + + [TestCase, Order(1)] + public void CreateValues() + { + this.RunConfigCommandAndCheckOutput("integerString 213", null); + this.RunConfigCommandAndCheckOutput("integerString", new string[] { "213" }); + + this.RunConfigCommandAndCheckOutput("floatString 213.15", null); + this.RunConfigCommandAndCheckOutput("floatString", new string[] { "213.15" }); + + this.RunConfigCommandAndCheckOutput("regularString foobar", null); + this.RunConfigCommandAndCheckOutput("regularString", new string[] { "foobar" }); + + this.RunConfigCommandAndCheckOutput("spacesString \"quick brown fox\"", null); + this.RunConfigCommandAndCheckOutput("spacesString", new string[] { "quick brown fox" }); + } + + [TestCase, Order(2)] + public void UpdateValues() + { + this.RunConfigCommandAndCheckOutput("integerString 314", null); + this.RunConfigCommandAndCheckOutput("integerString", new string[] { "314" }); + + this.RunConfigCommandAndCheckOutput("floatString 3.14159", null); + this.RunConfigCommandAndCheckOutput("floatString", new string[] { "3.14159" }); + + this.RunConfigCommandAndCheckOutput("regularString helloWorld!", null); + this.RunConfigCommandAndCheckOutput("regularString", new string[] { "helloWorld!" }); + + this.RunConfigCommandAndCheckOutput("spacesString \"jumped over lazy dog\"", null); + this.RunConfigCommandAndCheckOutput("spacesString", new string[] { "jumped over lazy dog" }); + } + + [TestCase, Order(3)] + public void ListValues() + { + string[] expectedSettings = new string[] + { + "integerString=314", + "floatString=3.1415", + "regularString=helloWorld!", + "spacesString=jumped over lazy dog" + }; + this.RunConfigCommandAndCheckOutput("--list", expectedSettings); + } + + [TestCase, Order(4)] + public void DeleteValues() + { + string[] expectedSettings = new string[] + { + "integerString", + "floatString", + "regularString", + "spacesString" + }; + foreach (string keyValue in expectedSettings) + { + this.RunConfigCommandAndCheckOutput($"--delete {keyValue}", null); + } + + this.RunConfigCommandAndCheckOutput($"--list", new string[] { string.Empty }); + } + + private void RunConfigCommandAndCheckOutput(string argument, string[] expectedOutput) + { + GVFSProcess gvfsProcess = new GVFSProcess( + GVFSTestConfig.PathToGVFS, + enlistmentRoot: null, + localCacheRoot: null); + + string result = gvfsProcess.RunConfigVerb(argument); + if (expectedOutput != null) + { + foreach (string output in expectedOutput) + { + result.ShouldContain(expectedOutput); + } + } + } + } +} diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs index 98939a8fb8..19fd6a287a 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs @@ -90,6 +90,11 @@ public string RunServiceVerb(string argument) return this.CallGVFS("service " + argument, failOnError: true); } + public string RunConfigVerb(string argument) + { + return this.CallGVFS("config " + argument, failOnError: true); + } + private string CallGVFS(string args, bool failOnError = false, string trace = null) { ProcessStartInfo processInfo = null; diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index ea048c0ff7..373391a777 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -12,17 +12,17 @@ public class ConfigVerb : GVFSVerb.ForNoEnlistment private LocalGVFSConfig localConfig; [Option( + 'l', "list", SetName = "needNoKeys", - Default = false, Required = false, HelpText = "Show all settings")] public bool List { get; set; } [Option( + 'd', "delete", SetName = "needsKey", - Default = null, Required = false, HelpText = "Delete specified setting")] public string KeyToDelete { get; set; } @@ -48,13 +48,20 @@ protected override string VerbName public override void Execute() { - this.localConfig = new LocalGVFSConfig(); + if (GVFSPlatform.Instance.IsUnderConstruction) + { + this.ReportErrorAndExit("`gvfs config` is not yet implemented on this operating system."); + } + this.localConfig = new LocalGVFSConfig(); + bool keySpecified = !string.IsNullOrEmpty(this.Key); + bool isDelete = !string.IsNullOrEmpty(this.KeyToDelete); string error = null; - if (this.List) + + if (this.List || (!keySpecified && !isDelete)) { Dictionary allSettings; - if (!this.localConfig.TryGetAllConfig(out allSettings, out error, tracer: null)) + if (!this.localConfig.TryGetAllConfig(out allSettings, out error)) { this.ReportErrorAndExit(error); } @@ -68,23 +75,22 @@ public override void Execute() return; } - if (!string.IsNullOrEmpty(this.KeyToDelete)) + if (isDelete) { - if (!this.localConfig.TryRemoveConfig(this.KeyToDelete, out error, tracer: null)) + if (!this.localConfig.TryRemoveConfig(this.KeyToDelete, out error)) { this.ReportErrorAndExit(error); } return; } - - bool keySpecified = !string.IsNullOrEmpty(this.Key); + if (keySpecified) { bool valueSpecified = !string.IsNullOrEmpty(this.Value); if (valueSpecified) { - if (!this.localConfig.TrySetConfig(this.Key, this.Value, out error, tracer: null)) + if (!this.localConfig.TrySetConfig(this.Key, this.Value, out error)) { this.ReportErrorAndExit(error); } @@ -94,7 +100,7 @@ public override void Execute() else { string valueRead = null; - if (!this.localConfig.TryGetConfig(this.Key, out valueRead, out error, tracer: null)) + if (!this.localConfig.TryGetConfig(this.Key, out valueRead, out error)) { this.ReportErrorAndExit(error); } @@ -106,11 +112,6 @@ public override void Execute() return; } } - - // this.PrintUsage(); - // RFC: There was no parsing error, CommandLine Parser would have handled it already - // if there were any. The issue happens when user types `gvfs config`. - // Should I print Usage or exit silently. } } -} +} \ No newline at end of file From 012ee31bb5ee806173f30b8da91d06d861ca66ca Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 9 Oct 2018 13:49:27 -0700 Subject: [PATCH 147/244] Mac: Update and delete placeholder functions should check if file is still a placeholder --- .../WorkingDirectoryTests.cs | 2 +- .../Tests/GitCommands/CheckoutTests.cs | 5 +- .../GitCommands/CherryPickConflictTests.cs | 1 - .../Tests/GitCommands/GitCommandsTests.cs | 1 - .../Tests/GitCommands/MergeConflictTests.cs | 1 - .../Tests/GitCommands/RebaseConflictTests.cs | 6 ++- .../MultiEnlistmentTests/SharedCacheTests.cs | 2 - .../MacFileSystemVirtualizer.cs | 3 ++ .../MacFileSystemVirtualizerTests.cs | 1 + ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs | 1 + ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 46 +++++++++++++++++-- ProjFS.Mac/PrjFSLib/PrjFSLib.h | 1 + 12 files changed, 56 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 88ff712170..18062718a3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -496,7 +496,7 @@ public void AllNullObjectRedownloaded() // TODO(Mac): Figure out why git for Mac is not requesting a redownload of the truncated object [TestCase, Order(17)] - [Category(Categories.MacTODO.M3)] + [Category(Categories.MacTODO.M4)] public void TruncatedObjectRedownloaded() { GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + this.Enlistment.Commitish); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index b9ceff67de..c234e15f54 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -495,7 +495,6 @@ public void CheckoutBranchWithOpenHandleBlockingRepoMetdataUpdate() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithOpenHandleBlockingProjectionDeleteAndRepoMetdataUpdate() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -561,7 +560,6 @@ public void CheckoutBranchWithStaleRepoMetadataTmpFileOnDisk() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWhileOutsideToolDoesNotAllowDeleteOfOpenRepoMetadata() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -610,8 +608,9 @@ public void CheckoutBranchWhileOutsideToolDoesNotAllowDeleteOfOpenRepoMetadata() } } + // WindowsOnly because the test depends on Windows specific file sharing behavior [TestCase] - [Category(Categories.MacTODO.M3)] + [Category(Categories.WindowsOnly)] public void CheckoutBranchWhileOutsideToolHasExclusiveReadHandleOnDatabasesFolder() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs index 5c3ceb7352..9009c4519a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs @@ -4,7 +4,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class CherryPickConflictTests : GitRepoTests { public CherryPickConflictTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index a6293327f3..4bf0c67935 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -552,7 +552,6 @@ public void RenameFileCommitChangesSwitchBranchSwitchBackTest() // MacOnly because renames of partial folders are blocked on Windows [TestCase] [Category(Categories.MacOnly)] - [Category(Categories.MacTODO.M3)] public void MoveFolderCommitChangesSwitchBranchSwitchBackTest() { this.CommitChangesSwitchBranchSwitchBack(fileSystemAction: this.MoveFolder); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs index 47472e0306..594d8aeb45 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs @@ -6,7 +6,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class MergeConflictTests : GitRepoTests { public MergeConflictTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs index 2744913b1e..a6beeabcc3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs @@ -4,7 +4,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class RebaseConflictTests : GitRepoTests { public RebaseConflictTests() : base(enlistmentPerTest: true) @@ -12,6 +11,7 @@ public RebaseConflictTests() : base(enlistmentPerTest: true) } [TestCase] + [Category(Categories.MacTODO.M3)] public void RebaseConflict() { this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); @@ -30,6 +30,7 @@ public void RebaseConflictWithFileReads() } [TestCase] + [Category(Categories.MacTODO.M3)] public void RebaseConflict_ThenAbort() { this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); @@ -39,6 +40,7 @@ public void RebaseConflict_ThenAbort() } [TestCase] + [Category(Categories.MacTODO.M3)] public void RebaseConflict_ThenSkip() { this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); @@ -48,6 +50,7 @@ public void RebaseConflict_ThenSkip() } [TestCase] + [Category(Categories.MacTODO.M3)] public void RebaseConflict_RemoveDeletedTheirsFile() { this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); @@ -56,6 +59,7 @@ public void RebaseConflict_RemoveDeletedTheirsFile() } [TestCase] + [Category(Categories.MacTODO.M3)] public void RebaseConflict_AddThenContinue() { this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index a47db87c39..4032fbc496 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -126,7 +126,6 @@ public void ParallelReadsInASharedCache() } [TestCase] - [Category(Categories.MacTODO.M3)] public void DeleteObjectsCacheAndCacheMappingBeforeMount() { GVFSFunctionalTestEnlistment enlistment1 = this.CloneAndMountEnlistment(); @@ -159,7 +158,6 @@ public void DeleteObjectsCacheAndCacheMappingBeforeMount() } [TestCase] - [Category(Categories.MacTODO.M3)] public void DeleteCacheDuringHydrations() { GVFSFunctionalTestEnlistment enlistment1 = this.CloneAndMountEnlistment(); diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index dcc82468e9..1494cb7bd2 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -56,6 +56,9 @@ public static FSResult ResultToFSResult(Result result) case Result.EDirectoryNotEmpty: return FSResult.DirectoryNotEmpty; + case Result.EVirtualizationInvalidOperation: + return FSResult.VirtualizationInvalidOperation; + default: return FSResult.IOError; } diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 6cce64dfa5..0973dd1c93 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -28,6 +28,7 @@ public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { Result.EFileNotFound, FSResult.FileOrPathNotFound }, { Result.EPathNotFound, FSResult.FileOrPathNotFound }, { Result.EDirectoryNotEmpty, FSResult.DirectoryNotEmpty }, + { Result.EVirtualizationInvalidOperation, FSResult.VirtualizationInvalidOperation }, }; [TestCase] diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs index 4bedbe0565..77fe0b0773 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/Result.cs @@ -23,6 +23,7 @@ public enum Result : uint ENotAVirtualizationRoot = 0x20000080, EVirtualizationRootAlreadyExists = 0x20000100, EDirectoryNotEmpty = 0x20000200, + EVirtualizationInvalidOperation = 0x20000400, ENotYetImplemented = 0xFFFFFFFF, } diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index b9262b7004..2cd709bd5e 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -91,6 +91,7 @@ static bool InitializeEmptyPlaceholder(const char* fullPath); template static bool InitializeEmptyPlaceholder(const char* fullPath, TPlaceholder* data, const char* xattrName); static bool AddXAttr(const char* fullPath, const char* name, const void* value, size_t size); static bool GetXAttr(const char* fullPath, const char* name, size_t size, _Out_ void* value); +static bool RemoveXAttr(const char* fullPath, const char* name); static inline PrjFS_NotificationType KUMessageTypeToNotificationType(MessageType kuNotificationType); @@ -524,7 +525,6 @@ PrjFS_Result PrjFS_DeleteFile( << hex << updateFlags << dec << ")" << endl; #endif - // TODO(Mac): Populate failure cause appropriately *failureCause = PrjFS_UpdateFailureCause_Invalid; if (nullptr == relativePath) @@ -533,10 +533,31 @@ PrjFS_Result PrjFS_DeleteFile( } // TODO(Mac): Ensure that races with hydration are handled properly - // TODO(Mac): Ensure file is not full before proceeding char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); + + struct stat path_stat; + stat(fullPath, &path_stat); + if (!(S_ISREG(path_stat.st_mode) || S_ISDIR(path_stat.st_mode))) + { + // Only files and directories can be deleted with PrjFS_DeleteFile + // Anything else should be treated as a full + *failureCause = PrjFS_UpdateFailureCause_FullFile; + return PrjFS_Result_EVirtualizationInvalidOperation; + } + + if (S_ISREG(path_stat.st_mode)) + { + // TODO(Mac): Determine if we need a similar check for directories as well + PrjFSFileXAttrData xattrData = {}; + if (!GetXAttr(fullPath, PrjFSFileXAttrName, sizeof(PrjFSFileXAttrData), &xattrData)) + { + *failureCause = PrjFS_UpdateFailureCause_FullFile; + return PrjFS_Result_EVirtualizationInvalidOperation; + } + } + if (0 != remove(fullPath)) { switch(errno) @@ -926,9 +947,9 @@ static PrjFS_Result HandleFileNotification( #endif PrjFSFileXAttrData xattrData = {}; - GetXAttr(fullPath, PrjFSFileXAttrName, sizeof(PrjFSFileXAttrData), &xattrData); + bool partialFile = GetXAttr(fullPath, PrjFSFileXAttrName, sizeof(PrjFSFileXAttrData), &xattrData); - return s_callbacks.NotifyOperation( + PrjFS_Result result = s_callbacks.NotifyOperation( 0 /* commandId */, relativePath, xattrData.providerId, @@ -938,6 +959,13 @@ static PrjFS_Result HandleFileNotification( isDirectory, notificationType, nullptr /* destinationRelativePath */); + + if (partialFile && PrjFS_NotificationType_FileModified == notificationType) + { + RemoveXAttr(fullPath, PrjFSFileXAttrName); + } + + return result; } static void FindNewFoldersInRootAndNotifyProvider(const MessageHeader* request, const char* relativePath) @@ -1090,6 +1118,16 @@ static bool GetXAttr(const char* fullPath, const char* name, size_t size, _Out_ return false; } +static bool RemoveXAttr(const char* fullPath, const char* name) +{ + if (removexattr(fullPath, name, XATTR_NOFOLLOW)) + { + return false; + } + + return true; +} + static inline PrjFS_NotificationType KUMessageTypeToNotificationType(MessageType kuNotificationType) { switch(kuNotificationType) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.h b/ProjFS.Mac/PrjFSLib/PrjFSLib.h index 75cc0d496b..001dff0a03 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.h +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.h @@ -34,6 +34,7 @@ typedef enum PrjFS_Result_ENotAVirtualizationRoot = 0x20000080, PrjFS_Result_EVirtualizationRootAlreadyExists = 0x20000100, PrjFS_Result_EDirectoryNotEmpty = 0x20000200, + PrjFS_Result_EVirtualizationInvalidOperation = 0x20000400, PrjFS_Result_ENotYetImplemented = 0xFFFFFFFF, From e05c9c70a962ee7566e2b6c8f2c911a8816869bb Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 16 Oct 2018 13:11:49 -0700 Subject: [PATCH 148/244] PR Feedback: Cleanup comments --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 2cd709bd5e..3b0012971d 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -542,7 +542,7 @@ PrjFS_Result PrjFS_DeleteFile( if (!(S_ISREG(path_stat.st_mode) || S_ISDIR(path_stat.st_mode))) { // Only files and directories can be deleted with PrjFS_DeleteFile - // Anything else should be treated as a full + // Anything else should be treated as a full file *failureCause = PrjFS_UpdateFailureCause_FullFile; return PrjFS_Result_EVirtualizationInvalidOperation; } @@ -962,6 +962,8 @@ static PrjFS_Result HandleFileNotification( if (partialFile && PrjFS_NotificationType_FileModified == notificationType) { + // PrjFS_NotificationType_FileModified is a post-modified FileOp event (that cannot be stopped + // by the provider) and so there's no need to check the result of the call to NotifyOperation RemoveXAttr(fullPath, PrjFSFileXAttrName); } From 383ab8d971db3711c3276a8c5dff1080009ceda1 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Tue, 16 Oct 2018 14:54:55 -0400 Subject: [PATCH 149/244] incorporating review comments - removed hard coded config file path, replaced WindowsOnly test category with MacTODO.M4 Functional test to verify that CommandLineParser throws error when mutually exclusive command line options are specified Updated alphabetical ordering of ConfigVerb in gvfs help output --- .../MultiEnlistmentTests/ConfigVerbTests.cs | 47 ++++++++++++------- GVFS/GVFS/Program.cs | 2 +- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs index 6a87f20738..a1d9b610f7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs @@ -1,24 +1,28 @@ using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; using NUnit.Framework; +using System; +using System.Diagnostics; using System.IO; namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests { [TestFixture] [NonParallelizable] - [Category(Categories.FullSuiteOnly)] - [Category(Categories.WindowsOnly)] + [Category(Categories.MacTODO.M4)] public class ConfigVerbTests : TestsWithMultiEnlistment { - private const string ConfigFilePath = @"C:\ProgramData\GVFS\gvfs.config"; - [OneTimeSetUp] public void DeleteAllSettings() { - if (File.Exists(ConfigFilePath)) + string configFilePath = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + "GVFS", + "gvfs.config"); + + if (File.Exists(configFilePath)) { - File.Delete(ConfigFilePath); + File.Delete(configFilePath); } } @@ -26,38 +30,38 @@ public void DeleteAllSettings() public void CreateValues() { this.RunConfigCommandAndCheckOutput("integerString 213", null); - this.RunConfigCommandAndCheckOutput("integerString", new string[] { "213" }); + this.RunConfigCommandAndCheckOutput("integerString", new[] { "213" }); this.RunConfigCommandAndCheckOutput("floatString 213.15", null); - this.RunConfigCommandAndCheckOutput("floatString", new string[] { "213.15" }); + this.RunConfigCommandAndCheckOutput("floatString", new[] { "213.15" }); this.RunConfigCommandAndCheckOutput("regularString foobar", null); - this.RunConfigCommandAndCheckOutput("regularString", new string[] { "foobar" }); + this.RunConfigCommandAndCheckOutput("regularString", new[] { "foobar" }); this.RunConfigCommandAndCheckOutput("spacesString \"quick brown fox\"", null); - this.RunConfigCommandAndCheckOutput("spacesString", new string[] { "quick brown fox" }); + this.RunConfigCommandAndCheckOutput("spacesString", new[] { "quick brown fox" }); } [TestCase, Order(2)] public void UpdateValues() { this.RunConfigCommandAndCheckOutput("integerString 314", null); - this.RunConfigCommandAndCheckOutput("integerString", new string[] { "314" }); + this.RunConfigCommandAndCheckOutput("integerString", new[] { "314" }); this.RunConfigCommandAndCheckOutput("floatString 3.14159", null); - this.RunConfigCommandAndCheckOutput("floatString", new string[] { "3.14159" }); + this.RunConfigCommandAndCheckOutput("floatString", new[] { "3.14159" }); this.RunConfigCommandAndCheckOutput("regularString helloWorld!", null); - this.RunConfigCommandAndCheckOutput("regularString", new string[] { "helloWorld!" }); + this.RunConfigCommandAndCheckOutput("regularString", new[] { "helloWorld!" }); this.RunConfigCommandAndCheckOutput("spacesString \"jumped over lazy dog\"", null); - this.RunConfigCommandAndCheckOutput("spacesString", new string[] { "jumped over lazy dog" }); + this.RunConfigCommandAndCheckOutput("spacesString", new[] { "jumped over lazy dog" }); } [TestCase, Order(3)] public void ListValues() { - string[] expectedSettings = new string[] + string[] expectedSettings = new[] { "integerString=314", "floatString=3.1415", @@ -70,19 +74,26 @@ public void ListValues() [TestCase, Order(4)] public void DeleteValues() { - string[] expectedSettings = new string[] + string[] settingsToDelete = new[] { "integerString", "floatString", "regularString", "spacesString" }; - foreach (string keyValue in expectedSettings) + foreach (string keyValue in settingsToDelete) { this.RunConfigCommandAndCheckOutput($"--delete {keyValue}", null); } - this.RunConfigCommandAndCheckOutput($"--list", new string[] { string.Empty }); + this.RunConfigCommandAndCheckOutput($"--list", new[] { string.Empty }); + } + + [TestCase, Order(5)] + public void ValidateMutuallyExclusiveOptions() + { + ProcessResult result = ProcessHelper.Run(GVFSTestConfig.PathToGVFS, "config --list --delete foo"); + result.Errors.ShouldContain(new[] { "ERROR", "list", "delete", "is not compatible with" }); } private void RunConfigCommandAndCheckOutput(string argument, string[] expectedOutput) diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index 4e52ee1d9f..43447e0612 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -18,6 +18,7 @@ public static void Main(string[] args) { typeof(CacheServerVerb), typeof(CloneVerb), + typeof(ConfigVerb), typeof(DehydrateVerb), typeof(DiagnoseVerb), typeof(LogVerb), @@ -28,7 +29,6 @@ public static void Main(string[] args) typeof(StatusVerb), typeof(UnmountVerb), typeof(UpgradeVerb), - typeof(ConfigVerb), }; int consoleWidth = 80; From ee516e72da26c7620897481036aa0176f0e8dc13 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 16 Oct 2018 10:11:29 -0400 Subject: [PATCH 150/244] Add WaitForBackgroundOperations call with checking modified paths --- .../Tests/WindowsUpdatePlaceholderTests.cs.cs | 28 ++++----- .../EnlistmentPerFixture/GitFilesTests.cs | 62 +++++++++---------- .../EnlistmentPerFixture/SymbolicLinkTests.cs | 26 ++++---- .../UpdatePlaceholderTests.cs | 8 +-- .../WorkingDirectoryTests.cs | 6 +- .../ModifiedPathsTests.cs | 9 +-- .../Tests/GitCommands/CheckoutTests.cs | 16 ++--- .../Tests/GitCommands/GitCommandsTests.cs | 6 +- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 15 +++-- 9 files changed, 88 insertions(+), 88 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsUpdatePlaceholderTests.cs.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsUpdatePlaceholderTests.cs.cs index be59d5b21c..94c602a271 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsUpdatePlaceholderTests.cs.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsUpdatePlaceholderTests.cs.cs @@ -62,7 +62,7 @@ public void LockToPreventDelete_SingleFile() this.GitCleanFile(TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); this.GitStatusShouldBeClean(OldCommitId); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); testFile1Path.ShouldNotExistOnDisk(this.fileSystem); this.GitCheckoutCommitId(NewFilesAndChangesCommitId); @@ -121,9 +121,9 @@ public void LockToPreventDelete_MultipleFiles() this.GitStatusShouldBeClean(OldCommitId); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile2Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile3Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventDelete/" + testFile2Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventDelete/" + testFile3Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); testFile2Path.ShouldNotExistOnDisk(this.fileSystem); testFile3Path.ShouldNotExistOnDisk(this.fileSystem); @@ -165,7 +165,7 @@ public void LockToPreventUpdate_SingleFile() this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); this.GitStatusShouldBeClean(OldCommitId); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1OldContents); this.GitCheckoutCommitId(NewFilesAndChangesCommitId); @@ -227,9 +227,9 @@ public void LockToPreventUpdate_MultipleFiles() this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); this.GitStatusShouldBeClean(OldCommitId); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2OldContents); testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3OldContents); testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4OldContents); @@ -323,12 +323,12 @@ public void LockToPreventUpdateAndDelete() this.GitStatusShouldBeClean(OldCommitId); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate1Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate2Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate3Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete1Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete2Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete3Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate1Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate2Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate3Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete1Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete2Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete3Name); testFileUpdate1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate1OldContents); testFileUpdate2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate2OldContents); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 8262b607fe..360bdb1140 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -25,17 +25,17 @@ public GitFilesTests(FileSystemRunner fileSystem) public void CreateFileTest() { string fileName = "file1.txt"; - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, fileName); this.fileSystem.WriteAllText(this.Enlistment.GetVirtualPathTo(fileName), "Some content here"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileName); this.Enlistment.GetVirtualPathTo(fileName).ShouldBeAFile(this.fileSystem).WithContents("Some content here"); string emptyFileName = "file1empty.txt"; - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, emptyFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, emptyFileName); this.fileSystem.CreateEmptyFile(this.Enlistment.GetVirtualPathTo(emptyFileName)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, emptyFileName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, emptyFileName); this.Enlistment.GetVirtualPathTo(emptyFileName).ShouldBeAFile(this.fileSystem); } @@ -44,18 +44,18 @@ public void CreateHardLinkTest() { string existingFileName = "fileToLinkTo.txt"; string existingFilePath = this.Enlistment.GetVirtualPathTo(existingFileName); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, existingFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, existingFileName); this.fileSystem.WriteAllText(existingFilePath, "Some content here"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, existingFileName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, existingFileName); existingFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); string newLinkFileName = "newHardLink.txt"; string newLinkFilePath = this.Enlistment.GetVirtualPathTo(newLinkFileName); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, newLinkFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, newLinkFileName); this.fileSystem.CreateHardLink(newLinkFilePath, existingFilePath); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, newLinkFileName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, newLinkFileName); newLinkFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); } @@ -67,7 +67,7 @@ public void CreateFileInFolderTest() string filePath = Path.Combine(folderName, fileName); this.Enlistment.GetVirtualPathTo(filePath).ShouldNotExistOnDisk(this.fileSystem); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, filePath); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, filePath); this.fileSystem.CreateDirectory(this.Enlistment.GetVirtualPathTo(folderName)); this.fileSystem.CreateEmptyFile(this.Enlistment.GetVirtualPathTo(filePath)); @@ -75,8 +75,8 @@ public void CreateFileInFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, folderName + "/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, folderName + "/" + fileName); } [TestCase, Order(4)] @@ -96,8 +96,8 @@ public void RenameEmptyFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedEntries); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, expectedModifiedEntries); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, folderName + "/"); } [TestCase, Order(5)] @@ -136,8 +136,8 @@ public void RenameFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedEntries); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, unexpectedModifiedEntries); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, expectedModifiedEntries); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, unexpectedModifiedEntries); } [TestCase, Order(6)] @@ -153,14 +153,14 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() this.fileSystem.CreateEmptyFile(Path.Combine(this.Enlistment.RepoRoot, "Folder", "testfile")); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "Folder/"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "Folder/testfile"); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, "Folder/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, "Folder/testfile"); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "folder/"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "folder/testfile"); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, "folder/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, "folder/testfile"); } [TestCase, Order(7)] @@ -187,7 +187,7 @@ public void ReadingFileDoesNotUpdateIndexOrModifiedPaths() afterUpdateResult.Output.StartsWith("S ").ShouldEqual(true); afterUpdateResult.Output.ShouldContain("ctime: 0:0", "mtime: 0:0", "size: 0\t"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToCheck); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, gitFileToCheck); } [TestCase, Order(8)] @@ -206,7 +206,7 @@ public void ModifiedFileWillGetAddedToModifiedPathsFile() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations did not complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToTest); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, gitFileToTest); this.VerifyWorktreeBit(gitFileToTest, LsFilesStatus.Cached); } @@ -222,8 +222,8 @@ public void RenamedFileAddedToModifiedPathsFile() this.Enlistment.GetVirtualPathTo(fileToRenameTargetEntry)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameTargetEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); @@ -242,8 +242,8 @@ public void RenamedFileAndOverwrittenTargetAddedToModifiedPathsFile() this.Enlistment.GetVirtualPathTo(fileToRenameTargetEntry)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameTargetEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); @@ -259,7 +259,7 @@ public void DeletedFileAddedToModifiedPathsFile() this.fileSystem.DeleteFile(this.Enlistment.GetVirtualPathTo(fileToDeleteEntry)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToDeleteEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToDeleteEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToDeleteEntry, LsFilesStatus.Cached); @@ -287,8 +287,8 @@ public void DeletedFolderAndChildrenAddedToToModifiedPathsFile() this.fileSystem.DeleteDirectory(this.Enlistment.GetVirtualPathTo(folderToDelete)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderToDelete + "/"); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, filesToDelete); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, folderToDelete + "/"); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, filesToDelete); // Verify skip-worktree cleared foreach (string file in filesToDelete) @@ -310,7 +310,7 @@ public void FileRenamedOutOfRepoAddedToModifiedPathsAndSkipWorktreeBitCleared() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); @@ -330,7 +330,7 @@ public void OverwrittenFileAddedToModifiedPathsAndSkipWorktreeBitCleared() fileToOverwriteVirtualPath.ShouldBeAFile(this.fileSystem).WithContents(testContents); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToOverwriteEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToOverwriteEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToOverwriteEntry, LsFilesStatus.Cached); @@ -350,7 +350,7 @@ public void SupersededFileAddedToModifiedPathsAndSkipWorktreeBitCleared() SupersedeFile(fileToSupersedePath, newContent).ShouldEqual(true); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToSupersedeEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToSupersedeEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToSupersedeEntry, LsFilesStatus.Cached); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs index 3b32e037b0..8a21955582 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs @@ -54,22 +54,22 @@ public void CheckoutBranchWithSymLinks() string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + TestFileName); string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFile2Name); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + TestFile2Name); string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); - GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + ChildLinkName); string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeTrue($"{grandChildLinkPath} should be a symlink"); grandChildLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); - GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolderName + "/" + GrandChildLinkName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + ChildFolderName + "/" + GrandChildLinkName); } [TestCase, Order(2)] @@ -86,18 +86,18 @@ public void CheckoutBranchWhereSymLinksChangeContentsAndTransitionToFile() string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + TestFileName); string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFile2Name); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + TestFile2Name); // In this branch childLinkPath has been changed to point to testFile2Path string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); - GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + ChildLinkName); // grandChildLinkPath should now be a file string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); @@ -108,7 +108,7 @@ public void CheckoutBranchWhereSymLinksChangeContentsAndTransitionToFile() string newGrandChildFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildFileName)); newGrandChildFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); this.bashRunner.IsSymbolicLink(newGrandChildFilePath).ShouldBeFalse($"{newGrandChildFilePath} should not be a symlink"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolderName + "/" + GrandChildFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + ChildFolderName + "/" + GrandChildFileName); } [TestCase, Order(3)] @@ -125,13 +125,13 @@ public void CheckoutBranchWhereFilesTransitionToSymLinks() string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); testFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeTrue($"{testFilePath} should be a symlink"); - GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFileName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + TestFileName); // There should be a new ChildFolder2Name directory string childFolder2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolder2Name)); this.bashRunner.IsSymbolicLink(childFolder2Path).ShouldBeFalse($"{childFolder2Path} should not be a symlink"); childFolder2Path.ShouldBeADirectory(this.bashRunner); - GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolder2Name); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + ChildFolder2Name); // The rest of the files are unchanged from FunctionalTests/20180925_SymLinksPart2 string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); @@ -141,7 +141,7 @@ public void CheckoutBranchWhereFilesTransitionToSymLinks() string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); - GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + ChildLinkName); string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeFalse($"{grandChildLinkPath} should not be a symlink"); @@ -166,11 +166,11 @@ public void CheckoutBranchWhereSymLinkTransistionsToFolderAndFolderTransitionsTo string linkNowADirectoryPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(linkNowADirectoryPath).ShouldBeFalse($"{linkNowADirectoryPath} should not be a symlink"); linkNowADirectoryPath.ShouldBeADirectory(this.bashRunner); - GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + ChildLinkName); string directoryNowALinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolder2Name)); this.bashRunner.IsSymbolicLink(directoryNowALinkPath).ShouldBeTrue($"{directoryNowALinkPath} should be a symlink"); - GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolder2Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.bashRunner, TestFolderName + "/" + ChildFolder2Name); } [TestCase, Order(5)] diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs index b279288a72..4b73cb864b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs @@ -79,7 +79,7 @@ public void LockWithFullShareUpdateAndDelete() this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate4Name); this.GitStatusShouldBeClean(OldCommitId); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate4Name); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate4Name); } testFileUpdate4Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate4OldContents); @@ -118,9 +118,9 @@ public void FileProjectedAfterPlaceholderDeleteFileAndCheckout() testFile2Path.ShouldNotExistOnDisk(this.fileSystem); testFile3Path.ShouldNotExistOnDisk(this.fileSystem); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/FileProjectedAfterPlaceholderDeleteFileAndCheckout/" + testFile1Name); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/FileProjectedAfterPlaceholderDeleteFileAndCheckout/" + testFile2Name); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/FileProjectedAfterPlaceholderDeleteFileAndCheckout/" + testFile3Name); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/FileProjectedAfterPlaceholderDeleteFileAndCheckout/" + testFile1Name); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/FileProjectedAfterPlaceholderDeleteFileAndCheckout/" + testFile2Name); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, TestParentFolderName + "/FileProjectedAfterPlaceholderDeleteFileAndCheckout/" + testFile3Name); this.GitCheckoutCommitId(NewFilesAndChangesCommitId); this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 88ff712170..70089ec4ee 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -401,7 +401,7 @@ public void FolderContentsProjectedAfterFolderCreateAndCheckout() GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout 54ea499de78eafb4dfd30b90e0bd4bcec26c4349"); // Confirm that no other test has created GVFlt_MultiThreadTest or put it in the modified files - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, folderName); string virtualFolderPath = this.Enlistment.GetVirtualPathTo(folderName); virtualFolderPath.ShouldNotExistOnDisk(this.fileSystem); @@ -427,7 +427,7 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith string folderName = "Test_EPF_MoveRenameFileTests"; string folder = this.Enlistment.GetVirtualPathTo(folderName); folder.ShouldNotExistOnDisk(this.fileSystem); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, folderName); // Confirm modified paths picks up renamed folder string newFolder = this.Enlistment.GetVirtualPathTo("newFolder"); @@ -435,7 +435,7 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith this.fileSystem.MoveDirectory(newFolder, folder); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, folderName + "/"); // Switch back to this.ControlGitRepo.Commitish and confirm that folder contents are correct GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + Properties.Settings.Default.Commitish); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index f8f64ce27d..ff8f4f9982 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -47,8 +47,7 @@ public void DeletedTempFileIsRemovedFromModifiedFiles(FileSystemRunner fileSyste fileSystem.DeleteFile(tempFile); tempFile.ShouldNotExistOnDisk(fileSystem); - this.Enlistment.UnmountGVFS(); - GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "temp.txt"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, fileSystem, "temp.txt"); } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] @@ -58,8 +57,7 @@ public void DeletedTempFolderIsRemovedFromModifiedFiles(FileSystemRunner fileSys fileSystem.DeleteDirectory(tempFolder); tempFolder.ShouldNotExistOnDisk(fileSystem); - this.Enlistment.UnmountGVFS(); - GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "Temp/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, fileSystem, "Temp/"); } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] @@ -73,8 +71,7 @@ public void DeletedTempFolderDeletesFilesFromModifiedFiles(FileSystemRunner file tempFile1.ShouldNotExistOnDisk(fileSystem); tempFile2.ShouldNotExistOnDisk(fileSystem); - this.Enlistment.UnmountGVFS(); - GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "Temp/", "Temp/temp1.txt", "Temp/temp2.txt"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, fileSystem, "Temp/", "Temp/temp1.txt", "Temp/temp2.txt"); } [Category(Categories.MacTODO.NeedsRenameOldPath)] diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index b9ceff67de..d3e36a0f49 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -186,11 +186,11 @@ public void CheckoutCommitWhereFileContentsChangeAfterRead() this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", fileName); // A read should not add the file to the modified paths - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, fileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.FileSystem, fileName); this.ValidateGitCommand("checkout FunctionalTests/20170206_Conflict_Source"); this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", fileName); - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, fileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.FileSystem, fileName); } [TestCase] @@ -206,11 +206,11 @@ public void CheckoutCommitWhereFileDeletedAfterRead() this.FileContentsShouldMatch(filePath); // A read should not add the file to the modified paths - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, fileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.FileSystem, fileName); this.ValidateGitCommand("checkout FunctionalTests/20170206_Conflict_Source"); this.ShouldNotExistOnDisk(filePath); - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, fileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.FileSystem, fileName); } [TestCase] @@ -226,7 +226,7 @@ public void CheckoutBranchAfterReadingFileAndVerifyContentsCorrect() this.FilesShouldMatchCheckoutOfSourceBranch(); // Verify modified paths contents - GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsContentsShouldEqual(this.Enlistment, this.FileSystem, "A .gitattributes" + GVFSHelpers.ModifiedPathsNewLine); } [TestCase] @@ -244,7 +244,7 @@ public void CheckoutBranchAfterReadingAllFilesAndVerifyContentsCorrect() .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, compareContent: true); // Verify modified paths contents - GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsContentsShouldEqual(this.Enlistment, this.FileSystem, "A .gitattributes" + GVFSHelpers.ModifiedPathsNewLine); } [TestCase] @@ -329,7 +329,7 @@ public void EditFileReadFileAndCheckoutConflict() this.FileContentsShouldMatch(editFilePath); this.Enlistment.GetVirtualPathTo(readFilePath).ShouldBeAFile(this.FileSystem).WithContents().ShouldNotEqual(originalReadFileContents); - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, Path.GetFileName(readFilePath)); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.FileSystem, Path.GetFileName(readFilePath)); } [TestCase] @@ -717,7 +717,7 @@ public void DeleteFolderAndChangeBranchToFolderWithDifferentCase() string folderName = "GVFlt_MultiThreadTest"; // Confirm that no other test has caused "GVFlt_MultiThreadTest" to be added to the modified paths database - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, folderName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.FileSystem, folderName); this.FolderShouldHaveCaseMatchingName(folderName, "GVFlt_MultiThreadTest"); this.DeleteFolder(folderName); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index a6293327f3..9d3b22ec4f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -491,7 +491,7 @@ public void MoveFileFromInsideRepoToOutsideRepoAndCommit() // Confirm that no other test has caused "Protocol.md" to be added to the modified paths string fileName = "Protocol.md"; - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, fileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.FileSystem, fileName); string controlTargetFolder = "MoveFileFromInsideRepoToOutsideRepoAndCommit_ControlTarget"; string gvfsTargetFolder = "MoveFileFromInsideRepoToOutsideRepoAndCommit_GVFSTarget"; @@ -992,7 +992,7 @@ public void EditFileNeedingUtf8Encoding() contents.ShouldEqual(expectedContents); // Confirm that the entry is not in the the modified paths database - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); + GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.FileSystem, relativeGitPath); this.ValidateGitCommand("status"); this.AppendAllText(ContentWhenEditingFile, virtualFile); @@ -1001,7 +1001,7 @@ public void EditFileNeedingUtf8Encoding() this.ValidateGitCommand("status"); // Confirm that the entry was added to the modified paths database - GVFSHelpers.ModifiedPathsShouldContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); + GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.FileSystem, relativeGitPath); } [TestCase] diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 5dbc77982e..47b1f315d9 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -102,16 +102,18 @@ public static string ReadAllTextFromWriteLockedFile(string filename) } } - public static void ModifiedPathsContentsShouldEqual(FileSystemRunner fileSystem, string dotGVFSRoot, string contents) + public static void ModifiedPathsContentsShouldEqual(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem, string contents) { - string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); + enlistment.WaitForBackgroundOperations().ShouldBeTrue("Background operations failed to complete."); + string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldEqual(contents); } - public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, string dotGVFSRoot, params string[] gitPaths) + public static void ModifiedPathsShouldContain(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem, params string[] gitPaths) { - string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); + enlistment.WaitForBackgroundOperations().ShouldBeTrue("Background operations failed to complete."); + string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); @@ -121,9 +123,10 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin } } - public static void ModifiedPathsShouldNotContain(FileSystemRunner fileSystem, string dotGVFSRoot, params string[] gitPaths) + public static void ModifiedPathsShouldNotContain(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem, params string[] gitPaths) { - string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); + enlistment.WaitForBackgroundOperations().ShouldBeTrue("Background operations failed to complete."); + string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); From 2e7de55ed491b81ead6075065a7b26581468de8d Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 17 Oct 2018 16:02:16 -0400 Subject: [PATCH 151/244] Move validation of background operations completing to common location --- .../Windows/Tests/JunctionAndSubstTests.cs | 2 +- .../EnlistmentPerFixture/GVFSLockTests.cs | 4 +-- .../EnlistmentPerFixture/GitFilesTests.cs | 36 +++++++++---------- .../UpdatePlaceholderTests.cs | 4 +-- .../WorkingDirectoryTests.cs | 2 +- .../ModifiedPathsTests.cs | 4 +-- .../Tools/GVFSFunctionalTestEnlistment.cs | 2 +- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 6 ++-- 8 files changed, 30 insertions(+), 30 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/JunctionAndSubstTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/JunctionAndSubstTests.cs index a5dcdc1433..f13712bec6 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/JunctionAndSubstTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/JunctionAndSubstTests.cs @@ -162,7 +162,7 @@ private void GitCommandWaitsForLock(string gitWorkingDirectory) ProcessResult statusWait = GitHelpers.InvokeGitAgainstGVFSRepo(gitWorkingDirectory, "status", cleanErrors: false); statusWait.Errors.ShouldContain(ExpectedStatusWaitingText); resetEvent.Set(); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations did not complete."); + this.Enlistment.WaitForBackgroundOperations(); } private void CreateSubstDrive(string path) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs index abe17b4950..92d8a05a70 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs @@ -94,13 +94,13 @@ private void OverwritingIndexShouldFail(string testFilePath) { string indexPath = this.Enlistment.GetVirtualPathTo(".git", "index"); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); byte[] indexContents = File.ReadAllBytes(indexPath); string testFileContents = "OverwriteIndexTest"; this.fileSystem.WriteAllText(testFilePath, testFileContents); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); this.RenameAndOverwrite(testFilePath, indexPath).ShouldBeFalse("GVFS should prevent renaming on top of index when GVFSLock is not held"); byte[] newIndexContents = File.ReadAllBytes(indexPath); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 360bdb1140..ad4bc8087c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -27,14 +27,14 @@ public void CreateFileTest() string fileName = "file1.txt"; GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, fileName); this.fileSystem.WriteAllText(this.Enlistment.GetVirtualPathTo(fileName), "Some content here"); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileName); this.Enlistment.GetVirtualPathTo(fileName).ShouldBeAFile(this.fileSystem).WithContents("Some content here"); string emptyFileName = "file1empty.txt"; GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, emptyFileName); this.fileSystem.CreateEmptyFile(this.Enlistment.GetVirtualPathTo(emptyFileName)); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, emptyFileName); this.Enlistment.GetVirtualPathTo(emptyFileName).ShouldBeAFile(this.fileSystem); } @@ -46,7 +46,7 @@ public void CreateHardLinkTest() string existingFilePath = this.Enlistment.GetVirtualPathTo(existingFileName); GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, existingFileName); this.fileSystem.WriteAllText(existingFilePath, "Some content here"); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, existingFileName); existingFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); @@ -54,7 +54,7 @@ public void CreateHardLinkTest() string newLinkFilePath = this.Enlistment.GetVirtualPathTo(newLinkFileName); GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, newLinkFileName); this.fileSystem.CreateHardLink(newLinkFilePath, existingFilePath); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, newLinkFileName); newLinkFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); } @@ -73,7 +73,7 @@ public void CreateFileInFolderTest() this.fileSystem.CreateEmptyFile(this.Enlistment.GetVirtualPathTo(filePath)); this.Enlistment.GetVirtualPathTo(filePath).ShouldBeAFile(this.fileSystem); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, folderName + "/"); GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, folderName + "/" + fileName); @@ -94,7 +94,7 @@ public void RenameEmptyFolderTest() this.fileSystem.CreateDirectory(this.Enlistment.GetVirtualPathTo(folderName)); this.fileSystem.MoveDirectory(this.Enlistment.GetVirtualPathTo(folderName), this.Enlistment.GetVirtualPathTo(renamedFolderName)); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, expectedModifiedEntries); GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, folderName + "/"); @@ -134,7 +134,7 @@ public void RenameFolderTest() this.fileSystem.MoveDirectory(this.Enlistment.GetVirtualPathTo(folderName), this.Enlistment.GetVirtualPathTo(renamedFolderName)); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, expectedModifiedEntries); GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, unexpectedModifiedEntries); @@ -151,13 +151,13 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "Folder")); this.fileSystem.CreateEmptyFile(Path.Combine(this.Enlistment.RepoRoot, "Folder", "testfile")); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, "Folder/"); GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, "Folder/testfile"); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, "folder/"); GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, this.fileSystem, "folder/testfile"); @@ -179,7 +179,7 @@ public void ReadingFileDoesNotUpdateIndexOrModifiedPaths() fileStreamToRead.ReadByte(); } - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations did not complete."); + this.Enlistment.WaitForBackgroundOperations(); ProcessResult afterUpdateResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "ls-files --debug -svmodc " + gitFileToCheck); afterUpdateResult.ShouldNotBeNull(); @@ -204,7 +204,7 @@ public void ModifiedFileWillGetAddedToModifiedPathsFile() this.fileSystem.FileExists(fileToCreate).ShouldEqual(true); resetEvent.Set(); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations did not complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, gitFileToTest); this.VerifyWorktreeBit(gitFileToTest, LsFilesStatus.Cached); @@ -220,7 +220,7 @@ public void RenamedFileAddedToModifiedPathsFile() this.fileSystem.MoveFile( this.Enlistment.GetVirtualPathTo(fileToRenameEntry), this.Enlistment.GetVirtualPathTo(fileToRenameTargetEntry)); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameEntry); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameTargetEntry); @@ -240,7 +240,7 @@ public void RenamedFileAndOverwrittenTargetAddedToModifiedPathsFile() this.fileSystem.ReplaceFile( this.Enlistment.GetVirtualPathTo(fileToRenameEntry), this.Enlistment.GetVirtualPathTo(fileToRenameTargetEntry)); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameEntry); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameTargetEntry); @@ -257,7 +257,7 @@ public void DeletedFileAddedToModifiedPathsFile() this.VerifyWorktreeBit(fileToDeleteEntry, LsFilesStatus.SkipWorktree); this.fileSystem.DeleteFile(this.Enlistment.GetVirtualPathTo(fileToDeleteEntry)); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToDeleteEntry); @@ -285,7 +285,7 @@ public void DeletedFolderAndChildrenAddedToToModifiedPathsFile() } this.fileSystem.DeleteDirectory(this.Enlistment.GetVirtualPathTo(folderToDelete)); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, folderToDelete + "/"); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, filesToDelete); @@ -308,7 +308,7 @@ public void FileRenamedOutOfRepoAddedToModifiedPathsAndSkipWorktreeBitCleared() this.fileSystem.MoveFile(fileToRenameVirtualPath, fileOutsideRepoPath); fileOutsideRepoPath.ShouldBeAFile(this.fileSystem).WithContents("lessData"); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToRenameEntry); @@ -326,7 +326,7 @@ public void OverwrittenFileAddedToModifiedPathsAndSkipWorktreeBitCleared() string testContents = $"Test contents for {nameof(this.OverwrittenFileAddedToModifiedPathsAndSkipWorktreeBitCleared)}"; this.fileSystem.WriteAllText(fileToOverwriteVirtualPath, testContents); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); fileToOverwriteVirtualPath.ShouldBeAFile(this.fileSystem).WithContents(testContents); @@ -348,7 +348,7 @@ public void SupersededFileAddedToModifiedPathsAndSkipWorktreeBitCleared() string newContent = $"{nameof(this.SupersededFileAddedToModifiedPathsAndSkipWorktreeBitCleared)} test new contents"; SupersedeFile(fileToSupersedePath, newContent).ShouldEqual(true); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, fileToSupersedeEntry); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs index 4b73cb864b..ca764d0dd0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs @@ -140,12 +140,12 @@ public void FullFilesDontAffectThePlaceholderDatabase() this.fileSystem.CreateEmptyFile(testFile); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ReadAllTextFromWriteLockedFile(placeholderDatabase).ShouldEqual(placeholdersBefore); this.fileSystem.DeleteFile(testFile); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ReadAllTextFromWriteLockedFile(placeholderDatabase).ShouldEqual(placeholdersBefore); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 70089ec4ee..7587d2c7e1 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -434,7 +434,7 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith this.fileSystem.CreateDirectory(newFolder); this.fileSystem.MoveDirectory(newFolder, folder); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); GVFSHelpers.ModifiedPathsShouldContain(this.Enlistment, this.fileSystem, folderName + "/"); // Switch back to this.ControlGitRepo.Commitish and confirm that folder contents are correct diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index ff8f4f9982..88d440a1d1 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -145,7 +145,7 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) this.Enlistment.UnmountGVFS(); this.Enlistment.MountGVFS(); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); string modifiedPathsDatabase = Path.Combine(this.Enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); @@ -192,7 +192,7 @@ public void ModifiedPathsCorrectAfterHardLinking(FileSystemRunner fileSystem) fileSystem.CreateHardLink(hardLinkOutsideRepoToFileInRepoPath, secondFileInRepoPath); hardLinkOutsideRepoToFileInRepoPath.ShouldBeAFile(fileSystem).WithContents(contents); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + this.Enlistment.WaitForBackgroundOperations(); string modifiedPathsDatabase = Path.Combine(this.Enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index bc28530b85..fed22ad960 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -218,7 +218,7 @@ public string Status(string trace = null) public bool WaitForBackgroundOperations(int maxWaitMilliseconds = DefaultMaxWaitMSForStatusCheck) { - return this.WaitForStatus(maxWaitMilliseconds, ZeroBackgroundOperations); + return this.WaitForStatus(maxWaitMilliseconds, ZeroBackgroundOperations).ShouldBeTrue("Background operations failed to complete."); } public bool WaitForLock(string lockCommand, int maxWaitMilliseconds = DefaultMaxWaitMSForStatusCheck) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 47b1f315d9..ae4f2fb2f3 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -104,7 +104,7 @@ public static string ReadAllTextFromWriteLockedFile(string filename) public static void ModifiedPathsContentsShouldEqual(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem, string contents) { - enlistment.WaitForBackgroundOperations().ShouldBeTrue("Background operations failed to complete."); + enlistment.WaitForBackgroundOperations(); string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldEqual(contents); @@ -112,7 +112,7 @@ public static void ModifiedPathsContentsShouldEqual(GVFSFunctionalTestEnlistment public static void ModifiedPathsShouldContain(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem, params string[] gitPaths) { - enlistment.WaitForBackgroundOperations().ShouldBeTrue("Background operations failed to complete."); + enlistment.WaitForBackgroundOperations(); string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); @@ -125,7 +125,7 @@ public static void ModifiedPathsShouldContain(GVFSFunctionalTestEnlistment enlis public static void ModifiedPathsShouldNotContain(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem, params string[] gitPaths) { - enlistment.WaitForBackgroundOperations().ShouldBeTrue("Background operations failed to complete."); + enlistment.WaitForBackgroundOperations(); string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); From 0b26d76ae9a0b603ee488a9554f2dcf1f4ba8ca3 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 17 Oct 2018 16:12:49 -0400 Subject: [PATCH 152/244] Use common method for getting the modified paths contents --- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index ae4f2fb2f3..d3075638d6 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -104,18 +104,13 @@ public static string ReadAllTextFromWriteLockedFile(string filename) public static void ModifiedPathsContentsShouldEqual(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem, string contents) { - enlistment.WaitForBackgroundOperations(); - string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); - modifiedPathsDatabase.ShouldBeAFile(fileSystem); - GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldEqual(contents); + string modifedPathsContents = GetModifiedPathsContents(enlistment, fileSystem); + modifedPathsContents.ShouldEqual(contents); } public static void ModifiedPathsShouldContain(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem, params string[] gitPaths) { - enlistment.WaitForBackgroundOperations(); - string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); - modifiedPathsDatabase.ShouldBeAFile(fileSystem); - string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); + string modifedPathsContents = GetModifiedPathsContents(enlistment, fileSystem); string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); foreach (string gitPath in gitPaths) { @@ -125,10 +120,7 @@ public static void ModifiedPathsShouldContain(GVFSFunctionalTestEnlistment enlis public static void ModifiedPathsShouldNotContain(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem, params string[] gitPaths) { - enlistment.WaitForBackgroundOperations(); - string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); - modifiedPathsDatabase.ShouldBeAFile(fileSystem); - string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); + string modifedPathsContents = GetModifiedPathsContents(enlistment, fileSystem); string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); foreach (string gitPath in gitPaths) { @@ -141,6 +133,14 @@ public static void ModifiedPathsShouldNotContain(GVFSFunctionalTestEnlistment en } } + private static string GetModifiedPathsContents(GVFSFunctionalTestEnlistment enlistment, FileSystemRunner fileSystem) + { + enlistment.WaitForBackgroundOperations(); + string modifiedPathsDatabase = Path.Combine(enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); + modifiedPathsDatabase.ShouldBeAFile(fileSystem); + return GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); + } + private static byte[] StringToShaBytes(string sha) { byte[] shaBytes = new byte[20]; From 86c41910d2919608f267d6239d82227fc96345ea Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 17 Oct 2018 18:12:50 -0400 Subject: [PATCH 153/244] Replaced CommandLineParser's Param set with our own option set check. Code cleanups - replaced text with const strings in Functional test, removed redundant code in Functional test. --- GVFS/GVFS.Common/LocalGVFSConfig.cs | 20 +-- .../MultiEnlistmentTests/ConfigVerbTests.cs | 129 ++++++++---------- GVFS/GVFS/CommandLine/ConfigVerb.cs | 47 +++++-- 3 files changed, 101 insertions(+), 95 deletions(-) diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index a2382cb9da..b109f71b77 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -26,10 +26,7 @@ public bool TryGetAllConfig(out Dictionary allConfig, out string { Dictionary configCopy = null; if (!this.TryPerformAction( - () => - { - configCopy = this.allSettings.GetAllKeysAndValues(); - }, + () => configCopy = this.allSettings.GetAllKeysAndValues(), out error)) { allConfig = null; @@ -48,10 +45,7 @@ public bool TryGetConfig( { string valueFromDict = null; if (!this.TryPerformAction( - () => - { - this.allSettings.TryGetValue(name, out valueFromDict); - }, + () => this.allSettings.TryGetValue(name, out valueFromDict), out error)) { value = null; @@ -69,10 +63,7 @@ public bool TrySetConfig( out string error) { if (!this.TryPerformAction( - () => - { - this.allSettings.SetValueAndFlush(name, value); - }, + () => this.allSettings.SetValueAndFlush(name, value), out error)) { error = $"Error writing config {name}={value}. {error}"; @@ -85,10 +76,7 @@ public bool TrySetConfig( public bool TryRemoveConfig(string name, out string error) { if (!this.TryPerformAction( - () => - { - this.allSettings.RemoveAndFlush(name); - }, + () => this.allSettings.RemoveAndFlush(name), out error)) { error = $"Error deleting config {name}. {error}"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs index a1d9b610f7..0c3569834a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs @@ -2,6 +2,7 @@ using GVFS.Tests.Should; using NUnit.Framework; using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -9,108 +10,96 @@ namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests { [TestFixture] [NonParallelizable] + [Category(Categories.FullSuiteOnly)] [Category(Categories.MacTODO.M4)] public class ConfigVerbTests : TestsWithMultiEnlistment { - [OneTimeSetUp] - public void DeleteAllSettings() + private const string IntegerSettingKey = "functionalTest_Integer"; + private const string FloatSettingKey = "functionalTest_Float"; + private const string RegularStringSettingKey = "functionalTest_RegularString"; + private const string SpacedStringSettingKey = "functionalTest_SpacedString"; + + private readonly Dictionary initialSettings = new Dictionary() { - string configFilePath = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), - "GVFS", - "gvfs.config"); + { IntegerSettingKey, "213" }, + { FloatSettingKey, "213.15" }, + { RegularStringSettingKey, "foobar" }, + { SpacedStringSettingKey, "quick brown fox" } + }; - if (File.Exists(configFilePath)) - { - File.Delete(configFilePath); - } - } + private readonly Dictionary updateSettings = new Dictionary() + { + { IntegerSettingKey, "32123" }, + { FloatSettingKey, "3.14159" }, + { RegularStringSettingKey, "helloWorld!" }, + { SpacedStringSettingKey, "jumped over lazy dog" } + }; [TestCase, Order(1)] - public void CreateValues() + public void CreateSettings() { - this.RunConfigCommandAndCheckOutput("integerString 213", null); - this.RunConfigCommandAndCheckOutput("integerString", new[] { "213" }); - - this.RunConfigCommandAndCheckOutput("floatString 213.15", null); - this.RunConfigCommandAndCheckOutput("floatString", new[] { "213.15" }); - - this.RunConfigCommandAndCheckOutput("regularString foobar", null); - this.RunConfigCommandAndCheckOutput("regularString", new[] { "foobar" }); - - this.RunConfigCommandAndCheckOutput("spacesString \"quick brown fox\"", null); - this.RunConfigCommandAndCheckOutput("spacesString", new[] { "quick brown fox" }); + this.ApplySettings(this.initialSettings); + this.ConfigShouldContainSettings(this.initialSettings); } [TestCase, Order(2)] - public void UpdateValues() + public void UpdateSettings() { - this.RunConfigCommandAndCheckOutput("integerString 314", null); - this.RunConfigCommandAndCheckOutput("integerString", new[] { "314" }); - - this.RunConfigCommandAndCheckOutput("floatString 3.14159", null); - this.RunConfigCommandAndCheckOutput("floatString", new[] { "3.14159" }); - - this.RunConfigCommandAndCheckOutput("regularString helloWorld!", null); - this.RunConfigCommandAndCheckOutput("regularString", new[] { "helloWorld!" }); - - this.RunConfigCommandAndCheckOutput("spacesString \"jumped over lazy dog\"", null); - this.RunConfigCommandAndCheckOutput("spacesString", new[] { "jumped over lazy dog" }); + this.ApplySettings(this.updateSettings); + this.ConfigShouldContainSettings(this.updateSettings); } [TestCase, Order(3)] - public void ListValues() + public void ListSettings() { - string[] expectedSettings = new[] - { - "integerString=314", - "floatString=3.1415", - "regularString=helloWorld!", - "spacesString=jumped over lazy dog" - }; - this.RunConfigCommandAndCheckOutput("--list", expectedSettings); + this.ConfigShouldContainSettings(this.updateSettings); } [TestCase, Order(4)] - public void DeleteValues() + public void DeleteSettings() { - string[] settingsToDelete = new[] - { - "integerString", - "floatString", - "regularString", - "spacesString" - }; - foreach (string keyValue in settingsToDelete) + List deletedLines = new List(); + foreach (KeyValuePair setting in this.updateSettings) { - this.RunConfigCommandAndCheckOutput($"--delete {keyValue}", null); + this.RunConfigCommand($"--delete {setting.Key}"); + deletedLines.Add(this.GetSettingLineInConfigFileFormat(setting)); } - this.RunConfigCommandAndCheckOutput($"--list", new[] { string.Empty }); + string allSettings = this.RunConfigCommand("--list"); + allSettings.ShouldNotContain(ignoreCase: true, unexpectedSubstrings: deletedLines.ToArray()); } - [TestCase, Order(5)] - public void ValidateMutuallyExclusiveOptions() + private void ConfigShouldContainSettings(Dictionary expectedSettings) { - ProcessResult result = ProcessHelper.Run(GVFSTestConfig.PathToGVFS, "config --list --delete foo"); - result.Errors.ShouldContain(new[] { "ERROR", "list", "delete", "is not compatible with" }); + List expectedLines = new List(); + foreach (KeyValuePair setting in expectedSettings) + { + expectedLines.Add(this.GetSettingLineInConfigFileFormat(setting)); + } + + string allSettings = this.RunConfigCommand("--list"); + allSettings.ShouldContain(expectedLines.ToArray()); } - private void RunConfigCommandAndCheckOutput(string argument, string[] expectedOutput) + private string GetSettingLineInConfigFileFormat(KeyValuePair setting) { - GVFSProcess gvfsProcess = new GVFSProcess( - GVFSTestConfig.PathToGVFS, - enlistmentRoot: null, - localCacheRoot: null); + return $"{setting.Key}={setting.Value}"; + } - string result = gvfsProcess.RunConfigVerb(argument); - if (expectedOutput != null) + private void ApplySettings(Dictionary settings) + { + foreach (KeyValuePair setting in settings) { - foreach (string output in expectedOutput) - { - result.ShouldContain(expectedOutput); - } + this.RunConfigCommand($"{ setting.Key } \"{ setting.Value }\""); } } + + private string RunConfigCommand(string argument) + { + ProcessResult result = ProcessHelper.Run(GVFSTestConfig.PathToGVFS, $"config { argument }"); + result.ExitCode.ShouldEqual(0, result.Errors); + + return result.Output; + } } } diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index 373391a777..3ef0dc2d08 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -14,7 +14,6 @@ public class ConfigVerb : GVFSVerb.ForNoEnlistment [Option( 'l', "list", - SetName = "needNoKeys", Required = false, HelpText = "Show all settings")] public bool List { get; set; } @@ -22,7 +21,6 @@ public class ConfigVerb : GVFSVerb.ForNoEnlistment [Option( 'd', "delete", - SetName = "needsKey", Required = false, HelpText = "Delete specified setting")] public string KeyToDelete { get; set; } @@ -31,7 +29,7 @@ public class ConfigVerb : GVFSVerb.ForNoEnlistment 0, Required = false, MetaName = "Setting name", - HelpText = "Name of setting that is to be set, read or deleted")] + HelpText = "Name of setting that is to be set or read")] public string Key { get; set; } [Value( @@ -54,11 +52,14 @@ public override void Execute() } this.localConfig = new LocalGVFSConfig(); - bool keySpecified = !string.IsNullOrEmpty(this.Key); - bool isDelete = !string.IsNullOrEmpty(this.KeyToDelete); string error = null; - if (this.List || (!keySpecified && !isDelete)) + if (this.IsMutuallyExclusiveOptionsSet(out error)) + { + this.ReportErrorAndExit(error); + } + + if (this.List) { Dictionary allSettings; if (!this.localConfig.TryGetAllConfig(out allSettings, out error)) @@ -75,7 +76,7 @@ public override void Execute() return; } - if (isDelete) + if (!string.IsNullOrEmpty(this.KeyToDelete)) { if (!this.localConfig.TryRemoveConfig(this.KeyToDelete, out error)) { @@ -84,8 +85,8 @@ public override void Execute() return; } - - if (keySpecified) + + if (!string.IsNullOrEmpty(this.Key)) { bool valueSpecified = !string.IsNullOrEmpty(this.Value); if (valueSpecified) @@ -113,5 +114,33 @@ public override void Execute() } } } + + private bool IsMutuallyExclusiveOptionsSet(out string consoleMessage) + { + bool deleteSpecified = !string.IsNullOrEmpty(this.KeyToDelete); + bool setOrReadSpecified = !string.IsNullOrEmpty(this.Key); + bool listSpecified = this.List; + + if (deleteSpecified && listSpecified) + { + consoleMessage = "You cannot delete and list settings at the same time."; + return true; + } + + if (setOrReadSpecified && listSpecified) + { + consoleMessage = "You cannot list all and view (or update) individual settings at the same time."; + return true; + } + + if (setOrReadSpecified && deleteSpecified) + { + consoleMessage = "You cannot delete a setting and view (or update) individual settings at the same time."; + return true; + } + + consoleMessage = null; + return false; + } } } \ No newline at end of file From 51a2b630e5921bb1c0ace8e78983a17203aa7708 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Thu, 18 Oct 2018 10:58:30 -0400 Subject: [PATCH 154/244] Ignore .vscode directory --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 856ab1e02b..c4a34004f6 100644 --- a/.gitignore +++ b/.gitignore @@ -223,3 +223,6 @@ ModelManifest.xml *.dll *.cab *.cer + +# VS Code private directory +.vscode/ From 5e713ae3d9c65b590e96293d75ccbc28a78bc664 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 5 Sep 2018 12:41:25 -0400 Subject: [PATCH 155/244] Use 'multi-pack-index' builtin instead of 'midx' to match upstream implementation --- GVFS/GVFS.Common/Git/GitObjects.cs | 26 +++---------------- GVFS/GVFS.Common/Git/GitProcess.cs | 6 ++--- .../EnlistmentPerFixture/PrefetchVerbTests.cs | 3 +-- .../PrefetchVerbWithoutSharedCacheTests.cs | 24 +---------------- GVFS/GVFS.Hooks/Program.cs | 2 +- .../FileSystemCallbacks.cs | 2 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 2 +- 7 files changed, 10 insertions(+), 55 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitObjects.cs b/GVFS/GVFS.Common/Git/GitObjects.cs index 22e6eab099..dac8f7d72d 100644 --- a/GVFS/GVFS.Common/Git/GitObjects.cs +++ b/GVFS/GVFS.Common/Git/GitObjects.cs @@ -155,33 +155,13 @@ public bool TryWriteMultiPackIndex(ITracer tracer, GVFSEnlistment enlistment, Ph using (ITracer activity = tracer.StartActivity(nameof(this.TryWriteMultiPackIndex), EventLevel.Informational, Keywords.Telemetry, metadata: null)) { GitProcess process = new GitProcess(enlistment); - GitProcess.Result result = process.WriteMultiPackIndex(enlistment.GitPackRoot); + GitProcess.Result result = process.WriteMultiPackIndex(enlistment.GitObjectsRoot); - if (!result.HasErrors) - { - string midxHash = result.Output.Trim(); - activity.RelatedInfo("Updated midx-head to hash {0}", midxHash); - - string expectedMidxHead = Path.Combine(enlistment.GitPackRoot, "midx-" + midxHash + ".midx"); - - List midxFiles = new List(); - - midxFiles.AddRange(fileSystem.GetFiles(enlistment.GitPackRoot, "midx-*.midx")); - midxFiles.AddRange(fileSystem.GetFiles(enlistment.GitPackRoot, "tmp_midx_*")); - - foreach (string midxFile in midxFiles) - { - if (!midxFile.Equals(expectedMidxHead, StringComparison.OrdinalIgnoreCase) && !fileSystem.TryDeleteFile(midxFile)) - { - activity.RelatedWarning("Failed to delete MIDX file {0}", midxFile); - } - } - } - else + if (result.HasErrors) { EventMetadata errorMetadata = new EventMetadata(); errorMetadata.Add("Operation", nameof(this.TryWriteMultiPackIndex)); - errorMetadata.Add("packDir", enlistment.GitPackRoot); + errorMetadata.Add("objectDir", enlistment.GitObjectsRoot); errorMetadata.Add("Errors", result.Errors); errorMetadata.Add("Output", result.Output.Length > 1024 ? result.Output.Substring(1024) : result.Output); activity.RelatedError(errorMetadata, result.Errors, Keywords.Telemetry); diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 916915ec24..7455b05c27 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -359,14 +359,12 @@ public Result IndexPack(string packfilePath, string idxOutputPath) /// /// Write a new multi-pack-index (MIDX) in the specified pack directory. /// - /// This will update the midx-head file to point to the new MIDX file. - /// /// If no new packfiles are found, then this is a no-op. /// - public Result WriteMultiPackIndex(string packDir) + public Result WriteMultiPackIndex(string objectDir) { // We override the config settings so we keep writing the MIDX file even if it is disabled for reads. - return this.InvokeGitAgainstDotGitFolder("-c core.midx=true midx --write --update-head --pack-dir \"" + packDir + "\""); + return this.InvokeGitAgainstDotGitFolder("-c core.multiPackIndex=true multi-pack-index write --object-dir=\"" + objectDir + "\""); } public Result RemoteAdd(string remoteName, string url) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs index 35c1b1c27f..1703560e40 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs @@ -137,9 +137,8 @@ private void PostFetchJobShouldComplete() } while (this.fileSystem.FileExists(postFetchLock)); - ProcessResult midxResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "midx --read --pack-dir=\"" + objectDir + "/pack\""); + ProcessResult midxResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "multi-pack-index verify --object-dir=\"" + objectDir + "\""); midxResult.ExitCode.ShouldEqual(0); - midxResult.Output.ShouldContain("4d494458"); // Header from midx file. ProcessResult graphResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "commit-graph read --object-dir=\"" + objectDir + "\""); graphResult.ExitCode.ShouldEqual(0); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs index dc573be9f5..72771a7d29 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs @@ -277,27 +277,6 @@ public void PrefetchCleansUpStaleTempPrefetchPacks() otherFilePath.ShouldBeAFile(this.fileSystem).WithContents(otherFileContents); } - [TestCase, Order(9)] - public void PostFetchJobCleansMidxFiles() - { - string packDir = Path.Combine(this.Enlistment.GetObjectRoot(this.fileSystem), "pack"); - string staleMidxFile = Path.Combine(packDir, "midx-FAKE.midx"); - string tmpMidxFile = Path.Combine(packDir, "tmp_midx_halted"); - - // Reset the pack directory so the prefetch definitely triggers MIDX computation - this.fileSystem.DeleteDirectory(packDir); - this.fileSystem.CreateDirectory(packDir); - - this.fileSystem.CreateEmptyFile(staleMidxFile); - this.fileSystem.CreateEmptyFile(tmpMidxFile); - - this.Enlistment.Prefetch("--commits"); - this.PostFetchJobShouldComplete(); - - this.fileSystem.FileExists(staleMidxFile).ShouldBeFalse(); - this.fileSystem.FileExists(tmpMidxFile).ShouldBeFalse(); - } - private void PackShouldHaveIdxFile(string pathPath) { string idxPath = Path.ChangeExtension(pathPath, ".idx"); @@ -378,9 +357,8 @@ private void PostFetchJobShouldComplete() Thread.Sleep(500); } - ProcessResult midxResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "midx --read --pack-dir=\"" + objectDir + "/pack\""); + ProcessResult midxResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "multi-pack-index verify --object-dir=\"" + objectDir + "\""); midxResult.ExitCode.ShouldEqual(0); - midxResult.Output.ShouldContain("4d494458"); // Header from midx file. ProcessResult graphResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "commit-graph read --object-dir=\"" + objectDir + "\""); graphResult.ExitCode.ShouldEqual(0); diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 64d40eceba..01f215a384 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -430,7 +430,7 @@ private static bool ShouldLock(string[] args) case "ls-files": case "ls-tree": case "merge-base": - case "midx": + case "multi-pack-index": case "name-rev": case "push": case "remote": diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index fbdb8657a1..b98006fdd3 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -568,7 +568,7 @@ private void PostFetchJob(List packIndexes) { this.context.Tracer.RelatedWarning( metadata: null, - message: PostFetchTelemetryKey + ": Failed to generate midx for new packfiles", + message: PostFetchTelemetryKey + ": Failed to generate multi-pack-index for new packfiles", keywords: Keywords.Telemetry); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 4f630ab312..bff2d32d02 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -81,7 +81,7 @@ public static bool TrySetRequiredGitConfigSettings(Enlistment enlistment) { "core.commitGraph", "true" }, { "core.fscache", "true" }, { "core.gvfs", "true" }, - { "core.midx", "true" }, + { "core.multiPackIndex", "true" }, { "core.preloadIndex", "true" }, { "core.safecrlf", "false" }, { "core.untrackedCache", "false" }, From 127e8b3fe46de47d82e513210ec650c34866ac18 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 24 Sep 2018 11:01:16 -0400 Subject: [PATCH 156/244] Run post-fetch job immediately on "gvfs mount" to ensure users have an up-to-date multi-pack-index file --- GVFS/GVFS.Mount/InProcessMount.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index 038d31c790..9bc7a95266 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -546,6 +546,9 @@ private void MountAndStartWorkingDirectoryCallbacks(CacheServerInfo cache) this.heartbeat = new HeartbeatThread(this.tracer, this.fileSystemCallbacks); this.heartbeat.Start(); + + // Launch a background job to compute the multi-pack-index. Will do nothing if up-to-date. + this.fileSystemCallbacks.LaunchPostFetchJob(packIndexes: new List()); } private void UnmountAndStopWorkingDirectoryCallbacks() From 66ad4902b2e31be6d361dfa45d0ad9f64ee1c199 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 24 Sep 2018 10:50:41 -0400 Subject: [PATCH 157/244] Update GitForWindows version to include new MIDX code --- GVFS/GVFS.Build/GVFS.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Build/GVFS.props b/GVFS/GVFS.Build/GVFS.props index 48a82f3ffd..fb04e5738f 100644 --- a/GVFS/GVFS.Build/GVFS.props +++ b/GVFS/GVFS.Build/GVFS.props @@ -3,7 +3,7 @@ 0.2.173.2 - 2.20181012.4 + 2.20181018.1 From f1bc061e073a709b7284fc72ce0c58f82f88d626 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Thu, 6 Sep 2018 10:42:31 -0700 Subject: [PATCH 158/244] Refactor to use a common authentication method --- GVFS/GVFS/CommandLine/CloneVerb.cs | 9 +++------ GVFS/GVFS/CommandLine/GVFSVerb.cs | 16 ++++++++++++++++ GVFS/GVFS/CommandLine/MountVerb.cs | 8 +++----- GVFS/GVFS/CommandLine/PrefetchVerb.cs | 8 +++----- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index d3861f9293..5a83ce830f 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -168,13 +168,10 @@ public override void Execute() this.Output.WriteLine(" Local Cache: " + resolvedLocalCacheRoot); this.Output.WriteLine(" Destination: " + enlistment.EnlistmentRoot); - string authErrorMessage = null; - if (!this.ShowStatusWhileRunning( - () => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage), - "Authenticating", - normalizedEnlistmentRootPath)) + string authErrorMessage; + if (!this.TryAuthenticate(tracer, enlistment, out authErrorMessage)) { - this.ReportErrorAndExit(tracer, "Cannot clone because authentication failed"); + this.ReportErrorAndExit(tracer, "Cannot clone because authentication failed: " + authErrorMessage); } RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 4f630ab312..523ebf8a7c 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -191,6 +191,22 @@ protected bool ShowStatusWhileRunning( return this.ShowStatusWhileRunning(action, message, gvfsLogEnlistmentRoot); } + protected bool TryAuthenticate(ITracer tracer, GVFSEnlistment enlistment, out string authErrorMessage) + { + string authError = null; + + bool result = this.ShowStatusWhileRunning( + () => + { + return enlistment.Authentication.TryRefreshCredentials(tracer, out authError); + }, + "Authenticating", + enlistment.EnlistmentRoot); + + authErrorMessage = authError; + return result; + } + protected void ReportErrorAndExit(ITracer tracer, ReturnCode exitCode, string error, params object[] args) { if (!string.IsNullOrEmpty(error)) diff --git a/GVFS/GVFS/CommandLine/MountVerb.cs b/GVFS/GVFS/CommandLine/MountVerb.cs index ad8e0a2f4b..d72f51f502 100644 --- a/GVFS/GVFS/CommandLine/MountVerb.cs +++ b/GVFS/GVFS/CommandLine/MountVerb.cs @@ -130,11 +130,9 @@ protected override void Execute(GVFSEnlistment enlistment) RetryConfig retryConfig = null; ServerGVFSConfig serverGVFSConfig = this.DownloadedGVFSConfig; if (!this.SkipVersionCheck) - { - string authErrorMessage = null; - if (!this.ShowStatusWhileRunning( - () => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage), - "Authenticating")) + { + string authErrorMessage; + if (!this.TryAuthenticate(tracer, enlistment, out authErrorMessage)) { this.Output.WriteLine(" WARNING: " + authErrorMessage); this.Output.WriteLine(" Mount will proceed, but new files cannot be accessed until GVFS can authenticate."); diff --git a/GVFS/GVFS/CommandLine/PrefetchVerb.cs b/GVFS/GVFS/CommandLine/PrefetchVerb.cs index acea80b6b9..40d492fcdb 100644 --- a/GVFS/GVFS/CommandLine/PrefetchVerb.cs +++ b/GVFS/GVFS/CommandLine/PrefetchVerb.cs @@ -102,13 +102,11 @@ protected override void Execute(GVFSEnlistment enlistment) CacheServerInfo cacheServer = this.ResolvedCacheServer; ServerGVFSConfig serverGVFSConfig = this.ServerGVFSConfig; if (!this.SkipVersionCheck) - { + { string authErrorMessage; - if (!this.ShowStatusWhileRunning( - () => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage), - "Authenticating")) + if (!this.TryAuthenticate(tracer, enlistment, out authErrorMessage)) { - this.ReportErrorAndExit(tracer, "Unable to prefetch because authentication failed"); + this.ReportErrorAndExit(tracer, "Unable to prefetch because authentication failed: " + authErrorMessage); } if (serverGVFSConfig == null) From 34c615deb98ef0bd421aaed5301b00dbc59b5cb3 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Thu, 6 Sep 2018 11:42:15 -0700 Subject: [PATCH 159/244] Attempt an anonymous query before requesting an auth token --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 31 ++++++++++++++++++++++- GVFS/GVFS.Common/Http/HttpRequestor.cs | 17 ++++++++++--- GVFS/GVFS.Mount/InProcessMount.cs | 3 ++- GVFS/GVFS/CommandLine/GVFSVerb.cs | 8 +++--- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index 30a872c7bc..b1c7437194 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -1,4 +1,5 @@ -using GVFS.Common.Tracing; +using GVFS.Common.Http; +using GVFS.Common.Tracing; using System; using System.Text; @@ -30,6 +31,8 @@ public bool IsBackingOff } } + public bool IsAnonymous { get; private set; } = true; + public void ConfirmCredentialsWorked(string usedCredential) { lock (this.gitAuthLock) @@ -71,6 +74,8 @@ public bool TryRefreshCredentials(ITracer tracer, out string errorMessage) public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out string errorMessage) { + this.IsAnonymous = false; + gitAuthString = this.cachedAuthString; if (this.cachedAuthString == null) { @@ -107,6 +112,7 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri } gitAuthString = this.cachedAuthString; + tracer.RelatedInfo("Received auth token"); } } @@ -114,6 +120,29 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri return true; } + public bool TryAnonymousQuery(ITracer tracer, GVFSEnlistment enlistment) + { + using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, new RetryConfig())) + { + ServerGVFSConfig gvfsConfig; + if (configRequestor.TryQueryGVFSConfig(out gvfsConfig)) + { + tracer.RelatedInfo("Anonymous query to /gvfs/config succeeded"); + + this.IsAnonymous = true; + return true; + } + + // TODO: We should not lump all errors together here. The query could have failed for a number of + // reasons unrelated to auth, so we still need to update TryQueryGVFSConfig to pass back a result + // indicating if the error was caused by a 401. But this is good enough for now to test the behavior. + tracer.RelatedInfo("Anonymous query to /gvfs/config failed"); + } + + this.IsAnonymous = false; + return false; + } + private DateTime GetNextAuthAttemptTime() { if (this.numberOfAttempts <= 1) diff --git a/GVFS/GVFS.Common/Http/HttpRequestor.cs b/GVFS/GVFS.Common/Http/HttpRequestor.cs index bd22348317..46c70d0e9d 100644 --- a/GVFS/GVFS.Common/Http/HttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/HttpRequestor.cs @@ -69,9 +69,10 @@ protected GitEndPointResponseData SendRequest( CancellationToken cancellationToken, MediaTypeWithQualityHeaderValue acceptType = null) { - string authString; + string authString = null; string errorMessage; - if (!this.authentication.TryGetCredentials(this.Tracer, out authString, out errorMessage)) + if (!this.authentication.IsAnonymous && + !this.authentication.TryGetCredentials(this.Tracer, out authString, out errorMessage)) { return new GitEndPointResponseData( HttpStatusCode.Unauthorized, @@ -152,7 +153,15 @@ protected GitEndPointResponseData SendRequest( errorMessage = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); int statusInt = (int)response.StatusCode; - if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.Redirect) + bool shouldRetry = ShouldRetry(response.StatusCode); + + if (response.StatusCode == HttpStatusCode.Unauthorized && + this.authentication.IsAnonymous) + { + shouldRetry = false; + errorMessage = "Anonymous request was rejected with a 401"; + } + else if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.Redirect) { this.authentication.Revoke(authString); if (!this.authentication.IsBackingOff) @@ -172,7 +181,7 @@ protected GitEndPointResponseData SendRequest( gitEndPointResponseData = new GitEndPointResponseData( response.StatusCode, new GitObjectsHttpException(response.StatusCode, errorMessage), - ShouldRetry(response.StatusCode), + shouldRetry, message: response, onResponseDisposed: () => availableConnections.Release()); } diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index 038d31c790..74dddf15e2 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -499,7 +499,8 @@ private void HandleUnmountRequest(NamedPipeServer.Connection connection) private void MountAndStartWorkingDirectoryCallbacks(CacheServerInfo cache) { string error; - if (!this.context.Enlistment.Authentication.TryRefreshCredentials(this.context.Tracer, out error)) + if (!this.context.Enlistment.Authentication.TryAnonymousQuery(this.context.Tracer, this.context.Enlistment) && + !this.context.Enlistment.Authentication.TryRefreshCredentials(this.context.Tracer, out error)) { this.FailMountAndExit("Failed to obtain git credentials: " + error); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 523ebf8a7c..9425307337 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -193,12 +193,14 @@ protected bool ShowStatusWhileRunning( protected bool TryAuthenticate(ITracer tracer, GVFSEnlistment enlistment, out string authErrorMessage) { - string authError = null; - + string authError = null; + bool result = this.ShowStatusWhileRunning( () => { - return enlistment.Authentication.TryRefreshCredentials(tracer, out authError); + return + enlistment.Authentication.TryAnonymousQuery(tracer, enlistment) || + enlistment.Authentication.TryRefreshCredentials(tracer, out authError); }, "Authenticating", enlistment.EnlistmentRoot); From d8fa542d78582115afb37e26cc166766865f0346 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Fri, 7 Sep 2018 09:28:17 -0700 Subject: [PATCH 160/244] Allow a verb to pass along its auth instance to another verb that it is executing --- GVFS/GVFS.Common/Enlistment.cs | 7 +++- GVFS/GVFS.Common/Git/GitAuthentication.cs | 5 +++ GVFS/GVFS/CommandLine/CloneVerb.cs | 4 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 49 +++++++++++++++++++++-- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.Common/Enlistment.cs b/GVFS/GVFS.Common/Enlistment.cs index b3aac8ad1a..f17e06585e 100644 --- a/GVFS/GVFS.Common/Enlistment.cs +++ b/GVFS/GVFS.Common/Enlistment.cs @@ -62,7 +62,7 @@ protected Enlistment( public string GitBinPath { get; } public string GVFSHooksRoot { get; } - public GitAuthentication Authentication { get; } + public GitAuthentication Authentication { get; private set; } public static string GetNewLogFileName(string logsRoot, string prefix) { @@ -90,5 +90,10 @@ public virtual GitProcess CreateGitProcess() { return new GitProcess(this); } + + public void ReuseExistingAuth(GitAuthentication authentication) + { + this.Authentication = authentication; + } } } \ No newline at end of file diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index b1c7437194..9191aa4269 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -122,6 +122,11 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri public bool TryAnonymousQuery(ITracer tracer, GVFSEnlistment enlistment) { + if (!this.IsAnonymous) + { + throw new InvalidOperationException("An anonymous request was already rejected"); + } + using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, new RetryConfig())) { ServerGVFSConfig gvfsConfig; diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 5a83ce830f..82e98e5b8d 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -205,7 +205,7 @@ public override void Execute() if (!this.NoPrefetch) { ReturnCode result = this.Execute( - fullEnlistmentRootPathParameter, + enlistment, verb => { verb.Commits = true; @@ -229,7 +229,7 @@ public override void Execute() else { this.Execute( - fullEnlistmentRootPathParameter, + enlistment, verb => { verb.SkipMountedCheck = true; diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 9425307337..88cd958a30 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -162,6 +162,32 @@ protected ReturnCode Execute( return verb.ReturnCode; } + protected ReturnCode Execute( + GVFSEnlistment enlistment, + Action configureVerb = null) + where TVerb : GVFSVerb.ForExistingEnlistment, new() + { + TVerb verb = new TVerb(); + verb.EnlistmentRootPathParameter = enlistment.EnlistmentRoot; + verb.ServiceName = this.ServiceName; + verb.Unattended = this.Unattended; + + if (configureVerb != null) + { + configureVerb(verb); + } + + try + { + verb.Execute(enlistment.Authentication); + } + catch (VerbAbortedException) + { + } + + return verb.ReturnCode; + } + protected bool ShowStatusWhileRunning( Func action, string message, @@ -197,10 +223,14 @@ protected bool TryAuthenticate(ITracer tracer, GVFSEnlistment enlistment, out st bool result = this.ShowStatusWhileRunning( () => - { - return - enlistment.Authentication.TryAnonymousQuery(tracer, enlistment) || - enlistment.Authentication.TryRefreshCredentials(tracer, out authError); + { + if (enlistment.Authentication.IsAnonymous && + enlistment.Authentication.TryAnonymousQuery(tracer, enlistment)) + { + return true; + } + + return enlistment.Authentication.TryRefreshCredentials(tracer, out authError); }, "Authenticating", enlistment.EnlistmentRoot); @@ -751,11 +781,22 @@ public ForExistingEnlistment(bool validateOrigin = true) : base(validateOrigin) public override string EnlistmentRootPathParameter { get; set; } public sealed override void Execute() + { + this.Execute(authentication: null); + } + + public void Execute(GitAuthentication authentication) { this.ValidatePathParameter(this.EnlistmentRootPathParameter); this.PreCreateEnlistment(); GVFSEnlistment enlistment = this.CreateEnlistment(this.EnlistmentRootPathParameter); + + if (authentication != null) + { + enlistment.ReuseExistingAuth(authentication); + } + this.Execute(enlistment); } From 927c3d61a8d6aa82e40e66558f8388c0af4e45c6 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Fri, 7 Sep 2018 10:31:26 -0700 Subject: [PATCH 161/244] Fixed up another entry point that requires auth --- GVFS/GVFS/CommandLine/CacheServerVerb.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/GVFS/GVFS/CommandLine/CacheServerVerb.cs b/GVFS/GVFS/CommandLine/CacheServerVerb.cs index 0209bf86ee..a2115b010a 100644 --- a/GVFS/GVFS/CommandLine/CacheServerVerb.cs +++ b/GVFS/GVFS/CommandLine/CacheServerVerb.cs @@ -42,6 +42,12 @@ protected override void Execute(GVFSEnlistment enlistment) using (ITracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "CacheVerb")) { + string authErrorMessage; + if (!this.TryAuthenticate(tracer, enlistment, out authErrorMessage)) + { + this.ReportErrorAndExit(tracer, "Authentication failed: " + authErrorMessage); + } + ServerGVFSConfig serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); From dc5bcec3f5197a07f18262c56f6ba95cb92869a5 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Mon, 24 Sep 2018 17:43:02 -0700 Subject: [PATCH 162/244] Add a single-step Initialize method --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 19 ++++++++++--------- GVFS/GVFS.Mount/InProcessMount.cs | 3 +-- GVFS/GVFS/CommandLine/GVFSVerb.cs | 11 +---------- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index 9191aa4269..e06c0bc19c 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -74,8 +74,6 @@ public bool TryRefreshCredentials(ITracer tracer, out string errorMessage) public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out string errorMessage) { - this.IsAnonymous = false; - gitAuthString = this.cachedAuthString; if (this.cachedAuthString == null) { @@ -120,19 +118,22 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri return true; } - public bool TryAnonymousQuery(ITracer tracer, GVFSEnlistment enlistment) + public bool Initialize(ITracer tracer, GVFSEnlistment enlistment, out string errorMessage) { - if (!this.IsAnonymous) - { - throw new InvalidOperationException("An anonymous request was already rejected"); - } + errorMessage = null; + return + this.TryAnonymousQuery(tracer, enlistment) || + this.TryRefreshCredentials(tracer, out errorMessage); + } + private bool TryAnonymousQuery(ITracer tracer, GVFSEnlistment enlistment) + { using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, new RetryConfig())) { ServerGVFSConfig gvfsConfig; if (configRequestor.TryQueryGVFSConfig(out gvfsConfig)) { - tracer.RelatedInfo("Anonymous query to /gvfs/config succeeded"); + tracer.RelatedInfo($"Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} succeeded"); this.IsAnonymous = true; return true; @@ -141,7 +142,7 @@ public bool TryAnonymousQuery(ITracer tracer, GVFSEnlistment enlistment) // TODO: We should not lump all errors together here. The query could have failed for a number of // reasons unrelated to auth, so we still need to update TryQueryGVFSConfig to pass back a result // indicating if the error was caused by a 401. But this is good enough for now to test the behavior. - tracer.RelatedInfo("Anonymous query to /gvfs/config failed"); + tracer.RelatedInfo($"Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} failed"); } this.IsAnonymous = false; diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index 74dddf15e2..c526848879 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -499,8 +499,7 @@ private void HandleUnmountRequest(NamedPipeServer.Connection connection) private void MountAndStartWorkingDirectoryCallbacks(CacheServerInfo cache) { string error; - if (!this.context.Enlistment.Authentication.TryAnonymousQuery(this.context.Tracer, this.context.Enlistment) && - !this.context.Enlistment.Authentication.TryRefreshCredentials(this.context.Tracer, out error)) + if (!this.context.Enlistment.Authentication.Initialize(this.context.Tracer, this.context.Enlistment, out error)) { this.FailMountAndExit("Failed to obtain git credentials: " + error); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 88cd958a30..3d568a3485 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -222,16 +222,7 @@ protected bool TryAuthenticate(ITracer tracer, GVFSEnlistment enlistment, out st string authError = null; bool result = this.ShowStatusWhileRunning( - () => - { - if (enlistment.Authentication.IsAnonymous && - enlistment.Authentication.TryAnonymousQuery(tracer, enlistment)) - { - return true; - } - - return enlistment.Authentication.TryRefreshCredentials(tracer, out authError); - }, + () => enlistment.Authentication.Initialize(tracer, enlistment, out authError), "Authenticating", enlistment.EnlistmentRoot); From ca076fac11ade1c2c7f11e16dbd5e70d4e220c73 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Mon, 24 Sep 2018 19:02:26 -0700 Subject: [PATCH 163/244] Pass the exisitng auth instance through to the constructor of Enlistment rather than swapping it out afterwards --- GVFS/GVFS.Common/DiskLayoutUpgrade.cs | 3 +- GVFS/GVFS.Common/Enlistment.cs | 5 ++-- GVFS/GVFS.Common/GVFSEnlistment.cs | 21 +++++++------- GVFS/GVFS.Common/Git/GitAuthentication.cs | 2 +- GVFS/GVFS.Common/Prefetch/GitEnlistment.cs | 6 ++-- GVFS/GVFS.Mount/InProcessMount.cs | 2 +- GVFS/GVFS.Mount/InProcessMountVerb.cs | 2 +- .../ProfilingEnvironment.cs | 2 +- .../Handlers/GetActiveRepoListHandler.cs | 2 +- .../GVFS.UnitTests/Git/GVFSGitObjectsTests.cs | 2 +- .../Mock/Common/MockGVFSEnlistment.cs | 4 +-- .../GVFS.UnitTests/Virtual/CommonRepoSetup.cs | 2 +- GVFS/GVFS/CommandLine/CloneVerb.cs | 3 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 15 ++++------ GVFS/GVFS/CommandLine/RepairVerb.cs | 3 +- GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs | 28 ++++++++----------- 16 files changed, 49 insertions(+), 53 deletions(-) diff --git a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs index 6655905950..c3f2be430b 100644 --- a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs +++ b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs @@ -174,7 +174,8 @@ protected bool TrySetGitConfig(ITracer tracer, string enlistmentRoot, Dictionary GVFSEnlistment enlistment = GVFSEnlistment.CreateFromDirectory( enlistmentRoot, GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(), - ProcessHelper.GetCurrentProcessLocation()); + ProcessHelper.GetCurrentProcessLocation(), + authentication: null); GitProcess git = enlistment.CreateGitProcess(); foreach (string key in configSettings.Keys) diff --git a/GVFS/GVFS.Common/Enlistment.cs b/GVFS/GVFS.Common/Enlistment.cs index f17e06585e..ca0142e6f8 100644 --- a/GVFS/GVFS.Common/Enlistment.cs +++ b/GVFS/GVFS.Common/Enlistment.cs @@ -12,7 +12,8 @@ protected Enlistment( string repoUrl, string gitBinPath, string gvfsHooksRoot, - bool flushFileBuffersForPacks) + bool flushFileBuffersForPacks, + GitAuthentication authentication) { if (string.IsNullOrWhiteSpace(gitBinPath)) { @@ -47,7 +48,7 @@ protected Enlistment( this.RepoUrl = originResult.Output.Trim(); } - this.Authentication = new GitAuthentication(gitProcess, this.RepoUrl); + this.Authentication = authentication ?? new GitAuthentication(gitProcess, this.RepoUrl); } public string EnlistmentRoot { get; } diff --git a/GVFS/GVFS.Common/GVFSEnlistment.cs b/GVFS/GVFS.Common/GVFSEnlistment.cs index f897dbfa8a..6bdc0b54fb 100644 --- a/GVFS/GVFS.Common/GVFSEnlistment.cs +++ b/GVFS/GVFS.Common/GVFSEnlistment.cs @@ -1,9 +1,8 @@ +using GVFS.Common.Git; using GVFS.Common.NamedPipes; using Newtonsoft.Json; using System; using System.IO; -using System.Security.AccessControl; -using System.Security.Principal; using System.Threading; namespace GVFS.Common @@ -19,14 +18,15 @@ public partial class GVFSEnlistment : Enlistment private string gvfsHooksVersion; // New enlistment - public GVFSEnlistment(string enlistmentRoot, string repoUrl, string gitBinPath, string gvfsHooksRoot) + public GVFSEnlistment(string enlistmentRoot, string repoUrl, string gitBinPath, string gvfsHooksRoot, GitAuthentication authentication) : base( enlistmentRoot, Path.Combine(enlistmentRoot, GVFSConstants.WorkingDirectoryRootName), repoUrl, gitBinPath, gvfsHooksRoot, - flushFileBuffersForPacks: true) + flushFileBuffersForPacks: true, + authentication: authentication) { this.NamedPipeName = GVFSPlatform.Instance.GetNamedPipeName(this.EnlistmentRoot); this.DotGVFSRoot = Path.Combine(this.EnlistmentRoot, GVFSConstants.DotGVFS.Root); @@ -37,12 +37,13 @@ public GVFSEnlistment(string enlistmentRoot, string repoUrl, string gitBinPath, } // Existing, configured enlistment - private GVFSEnlistment(string enlistmentRoot, string gitBinPath, string gvfsHooksRoot) + private GVFSEnlistment(string enlistmentRoot, string gitBinPath, string gvfsHooksRoot, GitAuthentication authentication) : this( enlistmentRoot, null, gitBinPath, - gvfsHooksRoot) + gvfsHooksRoot, + authentication) { } @@ -78,7 +79,7 @@ public string GVFSHooksVersion get { return this.gvfsHooksVersion; } } - public static GVFSEnlistment CreateWithoutRepoUrlFromDirectory(string directory, string gitBinRoot, string gvfsHooksRoot) + public static GVFSEnlistment CreateWithoutRepoUrlFromDirectory(string directory, string gitBinRoot, string gvfsHooksRoot, GitAuthentication authentication) { if (Directory.Exists(directory)) { @@ -89,13 +90,13 @@ public static GVFSEnlistment CreateWithoutRepoUrlFromDirectory(string directory, return null; } - return new GVFSEnlistment(enlistmentRoot, string.Empty, gitBinRoot, gvfsHooksRoot); + return new GVFSEnlistment(enlistmentRoot, string.Empty, gitBinRoot, gvfsHooksRoot, authentication); } return null; } - public static GVFSEnlistment CreateFromDirectory(string directory, string gitBinRoot, string gvfsHooksRoot) + public static GVFSEnlistment CreateFromDirectory(string directory, string gitBinRoot, string gvfsHooksRoot, GitAuthentication authentication) { if (Directory.Exists(directory)) { @@ -106,7 +107,7 @@ public static GVFSEnlistment CreateFromDirectory(string directory, string gitBin return null; } - return new GVFSEnlistment(enlistmentRoot, gitBinRoot, gvfsHooksRoot); + return new GVFSEnlistment(enlistmentRoot, gitBinRoot, gvfsHooksRoot, authentication); } return null; diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index e06c0bc19c..efeb34254b 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -118,7 +118,7 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri return true; } - public bool Initialize(ITracer tracer, GVFSEnlistment enlistment, out string errorMessage) + public bool TryInitialize(ITracer tracer, GVFSEnlistment enlistment, out string errorMessage) { errorMessage = null; return diff --git a/GVFS/GVFS.Common/Prefetch/GitEnlistment.cs b/GVFS/GVFS.Common/Prefetch/GitEnlistment.cs index 451860d3da..31a4889f01 100644 --- a/GVFS/GVFS.Common/Prefetch/GitEnlistment.cs +++ b/GVFS/GVFS.Common/Prefetch/GitEnlistment.cs @@ -1,5 +1,4 @@ -using GVFS.Common; -using System; +using System; using System.IO; namespace GVFS.Common.Prefetch @@ -13,7 +12,8 @@ private GitEnlistment(string repoRoot, string gitBinPath) null, gitBinPath, gvfsHooksRoot: null, - flushFileBuffersForPacks: false) + flushFileBuffersForPacks: false, + authentication: null) { this.GitObjectsRoot = Path.Combine(repoRoot, GVFSConstants.DotGit.Objects.Root); this.LocalObjectsRoot = this.GitObjectsRoot; diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index c526848879..c3a1e7ac2f 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -499,7 +499,7 @@ private void HandleUnmountRequest(NamedPipeServer.Connection connection) private void MountAndStartWorkingDirectoryCallbacks(CacheServerInfo cache) { string error; - if (!this.context.Enlistment.Authentication.Initialize(this.context.Tracer, this.context.Enlistment, out error)) + if (!this.context.Enlistment.Authentication.TryInitialize(this.context.Tracer, this.context.Enlistment, out error)) { this.FailMountAndExit("Failed to obtain git credentials: " + error); } diff --git a/GVFS/GVFS.Mount/InProcessMountVerb.cs b/GVFS/GVFS.Mount/InProcessMountVerb.cs index 71e835004b..10eabbd227 100644 --- a/GVFS/GVFS.Mount/InProcessMountVerb.cs +++ b/GVFS/GVFS.Mount/InProcessMountVerb.cs @@ -178,7 +178,7 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) GVFSEnlistment enlistment = null; try { - enlistment = GVFSEnlistment.CreateFromDirectory(enlistmentRootPath, gitBinPath, ProcessHelper.GetCurrentProcessLocation()); + enlistment = GVFSEnlistment.CreateFromDirectory(enlistmentRootPath, gitBinPath, ProcessHelper.GetCurrentProcessLocation(), authentication: null); if (enlistment == null) { this.ReportErrorAndExit( diff --git a/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs b/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs index 8f4d6a777f..348063e28d 100644 --- a/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs +++ b/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs @@ -27,7 +27,7 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) string gitBinPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); string hooksPath = ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSHooksExecutableName); - return GVFSEnlistment.CreateFromDirectory(enlistmentRootPath, gitBinPath, hooksPath); + return GVFSEnlistment.CreateFromDirectory(enlistmentRootPath, gitBinPath, hooksPath, authentication: null); } private GVFSContext CreateContext() diff --git a/GVFS/GVFS.Service/Handlers/GetActiveRepoListHandler.cs b/GVFS/GVFS.Service/Handlers/GetActiveRepoListHandler.cs index dbdf6025a0..81f1ed0dc8 100644 --- a/GVFS/GVFS.Service/Handlers/GetActiveRepoListHandler.cs +++ b/GVFS/GVFS.Service/Handlers/GetActiveRepoListHandler.cs @@ -76,7 +76,7 @@ private bool IsValidRepo(string repoRoot) try { - enlistment = GVFSEnlistment.CreateFromDirectory(repoRoot, gitBinPath, hooksPath); + enlistment = GVFSEnlistment.CreateFromDirectory(repoRoot, gitBinPath, hooksPath, authentication: null); } catch (InvalidRepoException) { diff --git a/GVFS/GVFS.UnitTests/Git/GVFSGitObjectsTests.cs b/GVFS/GVFS.UnitTests/Git/GVFSGitObjectsTests.cs index 016a6e3ce5..69a7a04150 100644 --- a/GVFS/GVFS.UnitTests/Git/GVFSGitObjectsTests.cs +++ b/GVFS/GVFS.UnitTests/Git/GVFSGitObjectsTests.cs @@ -138,7 +138,7 @@ private void AssertRetryableExceptionOnDownload( private GVFSGitObjects CreateTestableGVFSGitObjects(MockHttpGitObjects httpObjects, MockFileSystemWithCallbacks fileSystem) { MockTracer tracer = new MockTracer(); - GVFSEnlistment enlistment = new GVFSEnlistment(TestEnlistmentRoot, "https://fakeRepoUrl", "fakeGitBinPath", gvfsHooksRoot: null); + GVFSEnlistment enlistment = new GVFSEnlistment(TestEnlistmentRoot, "https://fakeRepoUrl", "fakeGitBinPath", gvfsHooksRoot: null, authentication: null); enlistment.InitializeCachePathsFromKey(TestLocalCacheRoot, TestObjecRoot); GitRepo repo = new GitRepo(tracer, enlistment, fileSystem, () => new MockLibGit2Repo(tracer)); diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockGVFSEnlistment.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockGVFSEnlistment.cs index 75e8028371..944522415c 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockGVFSEnlistment.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockGVFSEnlistment.cs @@ -9,7 +9,7 @@ public class MockGVFSEnlistment : GVFSEnlistment private MockGitProcess gitProcess; public MockGVFSEnlistment() - : base("mock:\\path", "mock://repoUrl", "mock:\\git", null) + : base("mock:\\path", "mock://repoUrl", "mock:\\git", gvfsHooksRoot: null, authentication: null) { this.GitObjectsRoot = "mock:\\path\\.git\\objects"; this.LocalObjectsRoot = this.GitObjectsRoot; @@ -17,7 +17,7 @@ public MockGVFSEnlistment() } public MockGVFSEnlistment(string enlistmentRoot, string repoUrl, string gitBinPath, string gvfsHooksRoot, MockGitProcess gitProcess) - : base(enlistmentRoot, repoUrl, gitBinPath, gvfsHooksRoot) + : base(enlistmentRoot, repoUrl, gitBinPath, gvfsHooksRoot, authentication: null) { this.gitProcess = gitProcess; } diff --git a/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs b/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs index ab97310392..9fd066f7e4 100644 --- a/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs +++ b/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs @@ -15,7 +15,7 @@ public CommonRepoSetup() MockTracer tracer = new MockTracer(); string enlistmentRoot = Path.Combine("mock:", "GVFS", "UnitTests", "Repo"); - GVFSEnlistment enlistment = new GVFSEnlistment(enlistmentRoot, "fake://repoUrl", "fake://gitBinPath", null); + GVFSEnlistment enlistment = new GVFSEnlistment(enlistmentRoot, "fake://repoUrl", "fake://gitBinPath", gvfsHooksRoot: null, authentication: null); enlistment.InitializeCachePathsFromKey("fake:\\gvfsSharedCache", "fakeCacheKey"); this.GitParentPath = enlistment.WorkingDirectoryRoot; diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 82e98e5b8d..04e9619235 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -310,7 +310,8 @@ private Result TryCreateEnlistment( normalizedEnlistementRootPath, this.RepositoryURL, gitBinPath, - hooksPath); + hooksPath, + authentication: null); return new Result(true); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 3d568a3485..fac32220e8 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -222,7 +222,7 @@ protected bool TryAuthenticate(ITracer tracer, GVFSEnlistment enlistment, out st string authError = null; bool result = this.ShowStatusWhileRunning( - () => enlistment.Authentication.Initialize(tracer, enlistment, out authError), + () => enlistment.Authentication.TryInitialize(tracer, enlistment, out authError), "Authenticating", enlistment.EnlistmentRoot); @@ -781,12 +781,7 @@ public void Execute(GitAuthentication authentication) this.ValidatePathParameter(this.EnlistmentRootPathParameter); this.PreCreateEnlistment(); - GVFSEnlistment enlistment = this.CreateEnlistment(this.EnlistmentRootPathParameter); - - if (authentication != null) - { - enlistment.ReuseExistingAuth(authentication); - } + GVFSEnlistment enlistment = this.CreateEnlistment(this.EnlistmentRootPathParameter, authentication); this.Execute(enlistment); } @@ -1038,7 +1033,7 @@ private void EnsureLocalCacheIsHealthy( } } - private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) + private GVFSEnlistment CreateEnlistment(string enlistmentRootPath, GitAuthentication authentication) { string gitBinPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); if (string.IsNullOrWhiteSpace(gitBinPath)) @@ -1065,11 +1060,11 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) { if (this.validateOriginURL) { - enlistment = GVFSEnlistment.CreateFromDirectory(enlistmentRootPath, gitBinPath, hooksPath); + enlistment = GVFSEnlistment.CreateFromDirectory(enlistmentRootPath, gitBinPath, hooksPath, authentication); } else { - enlistment = GVFSEnlistment.CreateWithoutRepoUrlFromDirectory(enlistmentRootPath, gitBinPath, hooksPath); + enlistment = GVFSEnlistment.CreateWithoutRepoUrlFromDirectory(enlistmentRootPath, gitBinPath, hooksPath, authentication); } if (enlistment == null) diff --git a/GVFS/GVFS/CommandLine/RepairVerb.cs b/GVFS/GVFS/CommandLine/RepairVerb.cs index 3111b6f440..a39627b318 100644 --- a/GVFS/GVFS/CommandLine/RepairVerb.cs +++ b/GVFS/GVFS/CommandLine/RepairVerb.cs @@ -42,7 +42,8 @@ public override void Execute() GVFSEnlistment enlistment = GVFSEnlistment.CreateWithoutRepoUrlFromDirectory( this.EnlistmentRootPathParameter, GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(), - hooksPath); + hooksPath, + authentication: null); if (enlistment == null) { diff --git a/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs b/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs index 58416e0cce..828c1c2353 100644 --- a/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs +++ b/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs @@ -43,17 +43,23 @@ public override IssueType HasIssue(List messages) return IssueType.Fixable; } - // At this point, we've confirmed that the repo url can be gotten, so we have to - // reinitialize the GitProcess with a valid repo url for 'git credential fill' - string repoUrl = null; + // We've validated the repo URL, so now make sure we can authenticate try { GVFSEnlistment enlistment = GVFSEnlistment.CreateFromDirectory( this.Enlistment.EnlistmentRoot, this.Enlistment.GitBinPath, - this.Enlistment.GVFSHooksRoot); - git = new GitProcess(enlistment); - repoUrl = enlistment.RepoUrl; + this.Enlistment.GVFSHooksRoot, + authentication: null); + + string authError; + if (!enlistment.Authentication.TryInitialize(this.Tracer, enlistment, out authError)) + { + messages.Add("Authentication failed. Run 'gvfs log' for more info."); + messages.Add(".git\\config is valid and remote 'origin' is set, but may have a typo:"); + messages.Add(result.Output.Trim()); + return IssueType.CantFix; + } } catch (InvalidRepoException) { @@ -61,16 +67,6 @@ public override IssueType HasIssue(List messages) return IssueType.CantFix; } - string username; - string password; - if (!git.TryGetCredentials(this.Tracer, repoUrl, out username, out password)) - { - messages.Add("Authentication failed. Run 'gvfs log' for more info."); - messages.Add(".git\\config is valid and remote 'origin' is set, but may have a typo:"); - messages.Add(result.Output.Trim()); - return IssueType.CantFix; - } - return IssueType.None; } From 5d8975ec4c92bbabdf4ba361b03d23f6942ffa5c Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Tue, 25 Sep 2018 08:31:30 -0700 Subject: [PATCH 164/244] Require an auth instance to be initialized before it can be used --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 82 +++++++++++++------ GVFS/GVFS.Common/Git/GitProcess.cs | 5 +- .../Git/GitAuthenticationTests.cs | 32 ++------ GVFS/GVFS/CommandLine/DehydrateVerb.cs | 3 +- 4 files changed, 70 insertions(+), 52 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index efeb34254b..c3e3522cb4 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -8,14 +8,17 @@ namespace GVFS.Common.Git public class GitAuthentication { private const double MaxBackoffSeconds = 30; + private readonly object gitAuthLock = new object(); + private readonly GitProcess git; + private readonly string repoUrl; + private int numberOfAttempts = 0; private DateTime lastAuthAttempt = DateTime.MinValue; private string cachedAuthString; - private GitProcess git; - private string repoUrl; + private bool isInitialized; public GitAuthentication(GitProcess git, string repoUrl) { @@ -57,10 +60,9 @@ public void Revoke(string usedCredential) if (this.cachedAuthString != null) { - // Wipe the username and password so we can try recovering if applicable. this.cachedAuthString = null; - this.git.RevokeCredential(this.repoUrl); + this.UpdateBackoff(); } } @@ -74,39 +76,28 @@ public bool TryRefreshCredentials(ITracer tracer, out string errorMessage) public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out string errorMessage) { + if (!this.isInitialized) + { + throw new InvalidOperationException("This auth instance must be initialized before it can be used"); + } + gitAuthString = this.cachedAuthString; - if (this.cachedAuthString == null) + if (gitAuthString == null) { lock (this.gitAuthLock) { if (this.cachedAuthString == null) { - string gitUsername; - string gitPassword; - if (this.IsBackingOff) { - gitAuthString = null; errorMessage = "Auth failed. No retries will be made until: " + this.GetNextAuthAttemptTime(); return false; } - if (!this.git.TryGetCredentials(tracer, this.repoUrl, out gitUsername, out gitPassword)) + if (!this.TryCallGitCredential(tracer, out errorMessage)) { - gitAuthString = null; - errorMessage = "Authentication failed."; - this.UpdateBackoff(); return false; } - - if (!string.IsNullOrEmpty(gitUsername) && !string.IsNullOrEmpty(gitPassword)) - { - this.cachedAuthString = Convert.ToBase64String(Encoding.ASCII.GetBytes(gitUsername + ":" + gitPassword)); - } - else - { - this.cachedAuthString = string.Empty; - } } gitAuthString = this.cachedAuthString; @@ -121,9 +112,26 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri public bool TryInitialize(ITracer tracer, GVFSEnlistment enlistment, out string errorMessage) { errorMessage = null; - return - this.TryAnonymousQuery(tracer, enlistment) || - this.TryRefreshCredentials(tracer, out errorMessage); + + if (this.TryAnonymousQuery(tracer, enlistment) || + this.TryCallGitCredential(tracer, out errorMessage)) + { + this.isInitialized = true; + return true; + } + + return false; + } + + public bool TryInitializeAndRequireAuth(ITracer tracer, out string errorMessage) + { + if (this.TryCallGitCredential(tracer, out errorMessage)) + { + this.isInitialized = true; + return true; + } + + return false; } private bool TryAnonymousQuery(ITracer tracer, GVFSEnlistment enlistment) @@ -165,5 +173,29 @@ private void UpdateBackoff() this.lastAuthAttempt = DateTime.Now; this.numberOfAttempts++; } + + private bool TryCallGitCredential(ITracer tracer, out string errorMessage) + { + string gitUsername; + string gitPassword; + if (!this.git.TryGetCredentials(tracer, this.repoUrl, out gitUsername, out gitPassword, out errorMessage)) + { + this.UpdateBackoff(); + return false; + } + + if (!string.IsNullOrEmpty(gitUsername) && !string.IsNullOrEmpty(gitPassword)) + { + this.cachedAuthString = Convert.ToBase64String(Encoding.ASCII.GetBytes(gitUsername + ":" + gitPassword)); + } + else + { + // TODO: How can we get back an empty username/password? This may have been an early attempt to "handle" anonymous auth + // and if so, it should be treated as an error here. + this.cachedAuthString = string.Empty; + } + + return true; + } } } diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 916915ec24..6460b530ff 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -120,10 +120,12 @@ public virtual bool TryGetCredentials( ITracer tracer, string repoUrl, out string username, - out string password) + out string password, + out string errorMessage) { username = null; password = null; + errorMessage = null; using (ITracer activity = tracer.StartActivity("TryGetCredentials", EventLevel.Informational)) { @@ -139,6 +141,7 @@ public virtual bool TryGetCredentials( errorData, "Git could not get credentials: " + gitCredentialOutput.Errors, Keywords.Network | Keywords.Telemetry); + errorMessage = gitCredentialOutput.Errors; return false; } diff --git a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs index bc22fd7c13..1e49c494e7 100644 --- a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs +++ b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs @@ -16,6 +16,7 @@ public void AuthShouldBackoffAfterFirstRetryFailure() MockGitProcess gitProcess = this.GetGitProcess(); GitAuthentication dut = new GitAuthentication(gitProcess, "mock://repoUrl"); + dut.TryInitializeAndRequireAuth(tracer, out _); string authString; string error; @@ -40,6 +41,7 @@ public void BackoffIsNotInEffectAfterSuccess() MockGitProcess gitProcess = this.GetGitProcess(); GitAuthentication dut = new GitAuthentication(gitProcess, "mock://repoUrl"); + dut.TryInitializeAndRequireAuth(tracer, out _); string authString; string error; @@ -61,6 +63,7 @@ public void ContinuesToBackoffIfTryGetCredentialsFails() MockGitProcess gitProcess = this.GetGitProcess(); GitAuthentication dut = new GitAuthentication(gitProcess, "mock://repoUrl"); + dut.TryInitializeAndRequireAuth(tracer, out _); string authString; string error; @@ -78,30 +81,6 @@ public void ContinuesToBackoffIfTryGetCredentialsFails() dut.IsBackingOff.ShouldEqual(true, "Should continue to backoff if failed to get credentials"); } - [TestCase] - public void GitProcessFailuresAreRetried() - { - MockTracer tracer = new MockTracer(); - MockGitProcess gitProcess = this.GetGitProcess(); - - GitAuthentication dut = new GitAuthentication(gitProcess, "mock://repoUrl"); - - string authString; - string error; - - gitProcess.ShouldFail = true; - - dut.TryGetCredentials(tracer, out authString, out error).ShouldEqual(false, "Succeeded despite GitProcess returning failure"); - - // Reboke should be a no-op as valid credentials have not been stored - dut.Revoke(authString); - dut.IsBackingOff.ShouldEqual(false, "Should not backoff if there were no credentials to revoke"); - - gitProcess.ShouldFail = false; - - dut.TryGetCredentials(tracer, out authString, out error).ShouldEqual(true, "Failed to get credential on retry"); - } - [TestCase] public void TwoThreadsFailAtOnceStillRetriesOnce() { @@ -109,7 +88,8 @@ public void TwoThreadsFailAtOnceStillRetriesOnce() MockGitProcess gitProcess = this.GetGitProcess(); GitAuthentication dut = new GitAuthentication(gitProcess, "mock://repoUrl"); - + dut.TryInitializeAndRequireAuth(tracer, out _); + string authString; string error; @@ -133,6 +113,7 @@ public void TwoThreadsInterleavingFailuresStillRetriesOnce() MockGitProcess gitProcess = this.GetGitProcess(); GitAuthentication dut = new GitAuthentication(gitProcess, "mock://repoUrl"); + dut.TryInitializeAndRequireAuth(tracer, out _); string thread1Auth; string thread2Auth; @@ -162,6 +143,7 @@ public void TwoThreadsInterleavingFailuresShouldntStompASuccess() MockGitProcess gitProcess = this.GetGitProcess(); GitAuthentication dut = new GitAuthentication(gitProcess, "mock://repoUrl"); + dut.TryInitializeAndRequireAuth(tracer, out _); string thread1Auth; string thread2Auth; diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index 742e8c2716..c26c0e56db 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -344,7 +344,8 @@ private bool TryDownloadGitObjects(ITracer tracer, GVFSEnlistment enlistment, Re { string errorMessage = null; - if (!this.ShowStatusWhileRunning( + if (!this.TryAuthenticate(tracer, enlistment, out errorMessage) || + !this.ShowStatusWhileRunning( () => { CacheServerInfo cacheServer = new CacheServerInfo(enlistment.RepoUrl, null); From 4d215d4ab2e8253940e2ebfdb9bf31143942d143 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Tue, 25 Sep 2018 08:36:43 -0700 Subject: [PATCH 165/244] Fix up FastFetch to initialize auth before making any queries --- GVFS/FastFetch/FastFetchVerb.cs | 11 +++++++++-- GVFS/GVFS.Common/Git/GitAuthentication.cs | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index cd37e371bc..bd3493e1b6 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -186,7 +186,7 @@ private int ExecuteWithExitCode() Console.WriteLine("Must be run within a git repo"); return ExitFailure; } - + string commitish = this.Commit ?? this.Branch; if (string.IsNullOrWhiteSpace(commitish)) { @@ -231,10 +231,17 @@ private int ExecuteWithExitCode() { "TargetCommitish", commitish }, { "Checkout", this.Checkout }, }); + + string error; + if (!enlistment.Authentication.TryInitialize(tracer, enlistment, out error)) + { + tracer.RelatedError(error); + Console.WriteLine(error); + return ExitFailure; + } RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); BlobPrefetcher prefetcher = this.GetFolderPrefetcher(tracer, enlistment, cacheServer, retryConfig); - string error; if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, prefetcher.FolderList, out error)) { tracer.RelatedError(error); diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index c3e3522cb4..ebfc144a06 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -109,7 +109,7 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri return true; } - public bool TryInitialize(ITracer tracer, GVFSEnlistment enlistment, out string errorMessage) + public bool TryInitialize(ITracer tracer, Enlistment enlistment, out string errorMessage) { errorMessage = null; @@ -134,7 +134,7 @@ public bool TryInitializeAndRequireAuth(ITracer tracer, out string errorMessage) return false; } - private bool TryAnonymousQuery(ITracer tracer, GVFSEnlistment enlistment) + private bool TryAnonymousQuery(ITracer tracer, Enlistment enlistment) { using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, new RetryConfig())) { From 11c98fe513f0b7a6d3b15dd6772bc732451449e7 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Tue, 25 Sep 2018 09:33:59 -0700 Subject: [PATCH 166/244] Fail to initialize if there were unrelated errors during the anonymous auth attempt --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 55 ++++++++++++++------ GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs | 10 +++- GVFS/GVFS/CommandLine/GVFSVerb.cs | 2 +- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index ebfc144a06..28527abcd2 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -1,6 +1,7 @@ using GVFS.Common.Http; using GVFS.Common.Tracing; using System; +using System.Net; using System.Text; namespace GVFS.Common.Git @@ -101,7 +102,6 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri } gitAuthString = this.cachedAuthString; - tracer.RelatedInfo("Received auth token"); } } @@ -113,14 +113,24 @@ public bool TryInitialize(ITracer tracer, Enlistment enlistment, out string erro { errorMessage = null; - if (this.TryAnonymousQuery(tracer, enlistment) || - this.TryCallGitCredential(tracer, out errorMessage)) + bool queryFailed; + bool requiresAuth; + + this.AttemptAnonymousQuery(tracer, enlistment, out queryFailed, out requiresAuth); + if (queryFailed) { - this.isInitialized = true; - return true; + errorMessage = $"Unable to determine if authentication is required"; + return false; } - return false; + if (requiresAuth && + !this.TryCallGitCredential(tracer, out errorMessage)) + { + return false; + } + + this.isInitialized = true; + return true; } public bool TryInitializeAndRequireAuth(ITracer tracer, out string errorMessage) @@ -134,27 +144,38 @@ public bool TryInitializeAndRequireAuth(ITracer tracer, out string errorMessage) return false; } - private bool TryAnonymousQuery(ITracer tracer, Enlistment enlistment) + private void AttemptAnonymousQuery(ITracer tracer, Enlistment enlistment, out bool queryFailed, out bool requiresAuth) { using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, new RetryConfig())) { ServerGVFSConfig gvfsConfig; - if (configRequestor.TryQueryGVFSConfig(out gvfsConfig)) + HttpStatusCode httpError; + if (configRequestor.TryQueryGVFSConfig(out gvfsConfig, out httpError)) { - tracer.RelatedInfo($"Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} succeeded"); + tracer.RelatedInfo($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} succeeded"); this.IsAnonymous = true; - return true; + + queryFailed = false; + requiresAuth = false; } + else if (httpError == HttpStatusCode.Unauthorized) + { + tracer.RelatedInfo($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} rejected with 401"); - // TODO: We should not lump all errors together here. The query could have failed for a number of - // reasons unrelated to auth, so we still need to update TryQueryGVFSConfig to pass back a result - // indicating if the error was caused by a 401. But this is good enough for now to test the behavior. - tracer.RelatedInfo($"Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} failed"); - } + this.IsAnonymous = false; - this.IsAnonymous = false; - return false; + queryFailed = false; + requiresAuth = true; + } + else + { + tracer.RelatedError($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} failed with {httpError}"); + + queryFailed = true; + requiresAuth = false; + } + } } private DateTime GetNextAuthAttemptTime() diff --git a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs index 8c6e450c12..88a780d3c1 100644 --- a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs @@ -1,6 +1,7 @@ using GVFS.Common.Tracing; using Newtonsoft.Json; using System; +using System.Net; using System.Net.Http; using System.Threading; @@ -16,9 +17,10 @@ public ConfigHttpRequestor(ITracer tracer, Enlistment enlistment, RetryConfig re this.repoUrl = enlistment.RepoUrl; } - public bool TryQueryGVFSConfig(out ServerGVFSConfig serverGVFSConfig) + public bool TryQueryGVFSConfig(out ServerGVFSConfig serverGVFSConfig, out HttpStatusCode httpError) { serverGVFSConfig = null; + httpError = default(HttpStatusCode); Uri gvfsConfigEndpoint; string gvfsConfigEndpointString = this.repoUrl + GVFSConstants.Endpoints.GVFSConfig; @@ -75,6 +77,12 @@ public bool TryQueryGVFSConfig(out ServerGVFSConfig serverGVFSConfig) return true; } + GitObjectsHttpException httpException = output.Error as GitObjectsHttpException; + if (httpException != null) + { + httpError = httpException.StatusCode; + } + return false; } } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index fac32220e8..c23b156fc5 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -291,7 +291,7 @@ protected ServerGVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlist { using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, retryConfig)) { - return configRequestor.TryQueryGVFSConfig(out serverGVFSConfig); + return configRequestor.TryQueryGVFSConfig(out serverGVFSConfig, out _); } }, "Querying remote for config", From 9a4d2b340dbc7cc08f2810bede7dea36c1614d3b Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Tue, 25 Sep 2018 14:42:16 -0700 Subject: [PATCH 167/244] Cleaned up logging and don't report an error on a failed anonymous query --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 77 ++++++++++++-------- GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs | 12 ++- GVFS/GVFS/CommandLine/GVFSVerb.cs | 3 +- 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index 28527abcd2..c98c528155 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -111,30 +111,38 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri public bool TryInitialize(ITracer tracer, Enlistment enlistment, out string errorMessage) { - errorMessage = null; + if (this.isInitialized) + { + throw new InvalidOperationException("Already initialized"); + } - bool queryFailed; - bool requiresAuth; + errorMessage = null; - this.AttemptAnonymousQuery(tracer, enlistment, out queryFailed, out requiresAuth); - if (queryFailed) + bool isAnonymous; + if (!this.TryAnonymousQuery(tracer, enlistment, out isAnonymous)) { errorMessage = $"Unable to determine if authentication is required"; return false; } - if (requiresAuth && + if (isAnonymous && !this.TryCallGitCredential(tracer, out errorMessage)) { return false; } + this.IsAnonymous = isAnonymous; this.isInitialized = true; return true; } public bool TryInitializeAndRequireAuth(ITracer tracer, out string errorMessage) { + if (this.isInitialized) + { + throw new InvalidOperationException("Already initialized"); + } + if (this.TryCallGitCredential(tracer, out errorMessage)) { this.isInitialized = true; @@ -144,38 +152,49 @@ public bool TryInitializeAndRequireAuth(ITracer tracer, out string errorMessage) return false; } - private void AttemptAnonymousQuery(ITracer tracer, Enlistment enlistment, out bool queryFailed, out bool requiresAuth) + private bool TryAnonymousQuery(ITracer tracer, Enlistment enlistment, out bool isAnonymous) { - using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, new RetryConfig())) + bool querySucceeded; + using (ITracer anonymousTracer = tracer.StartActivity("AttemptAnonymousAuth", EventLevel.Informational)) { - ServerGVFSConfig gvfsConfig; - HttpStatusCode httpError; - if (configRequestor.TryQueryGVFSConfig(out gvfsConfig, out httpError)) - { - tracer.RelatedInfo($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} succeeded"); - - this.IsAnonymous = true; + HttpStatusCode? httpStatus; - queryFailed = false; - requiresAuth = false; - } - else if (httpError == HttpStatusCode.Unauthorized) + using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(anonymousTracer, enlistment, new RetryConfig())) { - tracer.RelatedInfo($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} rejected with 401"); + ServerGVFSConfig gvfsConfig; + const bool LogErrors = false; + if (configRequestor.TryQueryGVFSConfig(LogErrors, out gvfsConfig, out httpStatus)) + { + anonymousTracer.RelatedInfo($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} succeeded"); - this.IsAnonymous = false; + querySucceeded = true; + isAnonymous = true; + } + else if (httpStatus == HttpStatusCode.Unauthorized) + { + anonymousTracer.RelatedInfo($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} rejected with 401"); - queryFailed = false; - requiresAuth = true; - } - else - { - tracer.RelatedError($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} failed with {httpError}"); + querySucceeded = true; + isAnonymous = false; + } + else + { + anonymousTracer.RelatedError($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} failed with {httpStatus}"); - queryFailed = true; - requiresAuth = false; + querySucceeded = false; + isAnonymous = false; + } } + + anonymousTracer.Stop(new EventMetadata + { + { "HttpStatus", httpStatus.HasValue ? httpStatus.ToString() : "None" }, + { "QuerySucceeded", querySucceeded }, + { "IsAnonymous", isAnonymous }, + }); } + + return querySucceeded; } private DateTime GetNextAuthAttemptTime() diff --git a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs index 88a780d3c1..b77c258e58 100644 --- a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs @@ -17,10 +17,10 @@ public ConfigHttpRequestor(ITracer tracer, Enlistment enlistment, RetryConfig re this.repoUrl = enlistment.RepoUrl; } - public bool TryQueryGVFSConfig(out ServerGVFSConfig serverGVFSConfig, out HttpStatusCode httpError) + public bool TryQueryGVFSConfig(bool logErrors, out ServerGVFSConfig serverGVFSConfig, out HttpStatusCode? httpStatus) { serverGVFSConfig = null; - httpError = default(HttpStatusCode); + httpStatus = null; Uri gvfsConfigEndpoint; string gvfsConfigEndpointString = this.repoUrl + GVFSConstants.Endpoints.GVFSConfig; @@ -41,7 +41,11 @@ public bool TryQueryGVFSConfig(out ServerGVFSConfig serverGVFSConfig, out HttpSt long requestId = HttpRequestor.GetNewRequestId(); RetryWrapper retrier = new RetryWrapper(this.RetryConfig.MaxAttempts, CancellationToken.None); - retrier.OnFailure += RetryWrapper.StandardErrorHandler(this.Tracer, requestId, "QueryGvfsConfig"); + + if (logErrors) + { + retrier.OnFailure += RetryWrapper.StandardErrorHandler(this.Tracer, requestId, "QueryGvfsConfig"); + } RetryWrapper.InvocationResult output = retrier.Invoke( tryCount => @@ -80,7 +84,7 @@ public bool TryQueryGVFSConfig(out ServerGVFSConfig serverGVFSConfig, out HttpSt GitObjectsHttpException httpException = output.Error as GitObjectsHttpException; if (httpException != null) { - httpError = httpException.StatusCode; + httpStatus = httpException.StatusCode; } return false; diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index c23b156fc5..9b9ccef01c 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -291,7 +291,8 @@ protected ServerGVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlist { using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, retryConfig)) { - return configRequestor.TryQueryGVFSConfig(out serverGVFSConfig, out _); + const bool LogErrors = true; + return configRequestor.TryQueryGVFSConfig(LogErrors, out serverGVFSConfig, out _); } }, "Querying remote for config", From 29104cd20b995d40aa1484d0dbf2090c26bef6b4 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Tue, 25 Sep 2018 15:58:38 -0700 Subject: [PATCH 168/244] Fixed failing dehydrate scenario, reduced tracing chatter --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 12 +++--------- GVFS/GVFS/CommandLine/DehydrateVerb.cs | 11 ++++++++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index c98c528155..e6891732ac 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -125,7 +125,7 @@ public bool TryInitialize(ITracer tracer, Enlistment enlistment, out string erro return false; } - if (isAnonymous && + if (!isAnonymous && !this.TryCallGitCredential(tracer, out errorMessage)) { return false; @@ -135,7 +135,7 @@ public bool TryInitialize(ITracer tracer, Enlistment enlistment, out string erro this.isInitialized = true; return true; } - + public bool TryInitializeAndRequireAuth(ITracer tracer, out string errorMessage) { if (this.isInitialized) @@ -165,22 +165,16 @@ private bool TryAnonymousQuery(ITracer tracer, Enlistment enlistment, out bool i const bool LogErrors = false; if (configRequestor.TryQueryGVFSConfig(LogErrors, out gvfsConfig, out httpStatus)) { - anonymousTracer.RelatedInfo($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} succeeded"); - querySucceeded = true; isAnonymous = true; } else if (httpStatus == HttpStatusCode.Unauthorized) { - anonymousTracer.RelatedInfo($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} rejected with 401"); - querySucceeded = true; isAnonymous = false; } else { - anonymousTracer.RelatedError($"GitAuthentication.AttemptAnonymousQuery: Anonymous query to {GVFSConstants.Endpoints.GVFSConfig} failed with {httpStatus}"); - querySucceeded = false; isAnonymous = false; } @@ -188,7 +182,7 @@ private bool TryAnonymousQuery(ITracer tracer, Enlistment enlistment, out bool i anonymousTracer.Stop(new EventMetadata { - { "HttpStatus", httpStatus.HasValue ? httpStatus.ToString() : "None" }, + { "HttpStatus", httpStatus.HasValue ? ((int)httpStatus).ToString() : "None" }, { "QuerySucceeded", querySucceeded }, { "IsAnonymous", isAnonymous }, }); diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index c26c0e56db..e58c538993 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -90,7 +90,7 @@ of your enlistment's src folder. this.Output.WriteLine(); this.Unmount(tracer); - + string error; if (!DiskLayoutUpgrade.TryCheckDiskLayoutVersion(tracer, enlistment.EnlistmentRoot, out error)) { @@ -103,6 +103,12 @@ of your enlistment's src folder. this.ReportErrorAndExit(tracer, "Failed to determine GVFS timeout and max retries: " + error); } + string errorMessage; + if (!this.TryAuthenticate(tracer, enlistment, out errorMessage)) + { + this.ReportErrorAndExit(tracer, errorMessage); + } + // Local cache and objects paths are required for TryDownloadGitObjects this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, serverGVFSConfig: null, cacheServer: null); @@ -344,8 +350,7 @@ private bool TryDownloadGitObjects(ITracer tracer, GVFSEnlistment enlistment, Re { string errorMessage = null; - if (!this.TryAuthenticate(tracer, enlistment, out errorMessage) || - !this.ShowStatusWhileRunning( + if (!this.ShowStatusWhileRunning( () => { CacheServerInfo cacheServer = new CacheServerInfo(enlistment.RepoUrl, null); From 13d7e887a2e2820437dc7bd44f2f5efdb565ea0d Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Tue, 25 Sep 2018 16:04:14 -0700 Subject: [PATCH 169/244] Remove unused code --- GVFS/GVFS.Common/Enlistment.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/GVFS/GVFS.Common/Enlistment.cs b/GVFS/GVFS.Common/Enlistment.cs index ca0142e6f8..599cf00a68 100644 --- a/GVFS/GVFS.Common/Enlistment.cs +++ b/GVFS/GVFS.Common/Enlistment.cs @@ -63,7 +63,7 @@ protected Enlistment( public string GitBinPath { get; } public string GVFSHooksRoot { get; } - public GitAuthentication Authentication { get; private set; } + public GitAuthentication Authentication { get; } public static string GetNewLogFileName(string logsRoot, string prefix) { @@ -91,10 +91,5 @@ public virtual GitProcess CreateGitProcess() { return new GitProcess(this); } - - public void ReuseExistingAuth(GitAuthentication authentication) - { - this.Authentication = authentication; - } } } \ No newline at end of file From a73bd7d68edce890fe671d07788f8b25aba5f98d Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Fri, 5 Oct 2018 11:19:10 -0700 Subject: [PATCH 170/244] Don't allow switching between anonymous/authenticated, and treat empty creds as error --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index e6891732ac..e01f2d1701 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -82,6 +82,11 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri throw new InvalidOperationException("This auth instance must be initialized before it can be used"); } + if (this.IsAnonymous) + { + throw new NotSupportedException("This session succeeded with anonymous auth, cannot switch to authenticated without restarting"); + } + gitAuthString = this.cachedAuthString; if (gitAuthString == null) { @@ -224,9 +229,8 @@ private bool TryCallGitCredential(ITracer tracer, out string errorMessage) } else { - // TODO: How can we get back an empty username/password? This may have been an early attempt to "handle" anonymous auth - // and if so, it should be treated as an error here. - this.cachedAuthString = string.Empty; + errorMessage = "Got back empty credentials from git"; + return false; } return true; From 065591695b555c1c0503e0812d5c00fd06f2082f Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Fri, 5 Oct 2018 11:38:48 -0700 Subject: [PATCH 171/244] Removed invalid exception --- GVFS/GVFS.Common/Git/GitAuthentication.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index e01f2d1701..63d3eeedcd 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -82,11 +82,6 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri throw new InvalidOperationException("This auth instance must be initialized before it can be used"); } - if (this.IsAnonymous) - { - throw new NotSupportedException("This session succeeded with anonymous auth, cannot switch to authenticated without restarting"); - } - gitAuthString = this.cachedAuthString; if (gitAuthString == null) { From 4c843f61288dde7375838cce9ba1c2994757090f Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Thu, 18 Oct 2018 11:08:57 -0700 Subject: [PATCH 172/244] Set httpStatus to 200 in the success case --- GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs index b77c258e58..1adbef553e 100644 --- a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs @@ -78,6 +78,7 @@ public bool TryQueryGVFSConfig(bool logErrors, out ServerGVFSConfig serverGVFSCo if (output.Succeeded) { serverGVFSConfig = output.Result; + httpStatus = HttpStatusCode.OK; return true; } From 12751449601eff7d1ccf3376ccd4aa80c9ca63a0 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Thu, 18 Oct 2018 14:11:10 -0400 Subject: [PATCH 173/244] Cleanup ConfigVerb- removed multiple returns with nested if else. Rephrased delete help text ConfigVerbTests - Delete functional test config settings before starting the tests. Removed GVFSProcess.RunConfigVerb() method. Display actionable error message when user specifies no config option Made ConfigVerbTests parallelizable --- .../MultiEnlistmentTests/ConfigVerbTests.cs | 2 +- .../GVFS.FunctionalTests/Tools/GVFSProcess.cs | 5 ----- GVFS/GVFS/CommandLine/ConfigVerb.cs | 20 +++++++------------ 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs index 0c3569834a..ee76cbb6e9 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs @@ -9,7 +9,6 @@ namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests { [TestFixture] - [NonParallelizable] [Category(Categories.FullSuiteOnly)] [Category(Categories.MacTODO.M4)] public class ConfigVerbTests : TestsWithMultiEnlistment @@ -38,6 +37,7 @@ public class ConfigVerbTests : TestsWithMultiEnlistment [TestCase, Order(1)] public void CreateSettings() { + this.DeleteSettings(); this.ApplySettings(this.initialSettings); this.ConfigShouldContainSettings(this.initialSettings); } diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs index 19fd6a287a..98939a8fb8 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs @@ -90,11 +90,6 @@ public string RunServiceVerb(string argument) return this.CallGVFS("service " + argument, failOnError: true); } - public string RunConfigVerb(string argument) - { - return this.CallGVFS("config " + argument, failOnError: true); - } - private string CallGVFS(string args, bool failOnError = false, string trace = null) { ProcessStartInfo processInfo = null; diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index 3ef0dc2d08..20ff070707 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -22,7 +22,7 @@ public class ConfigVerb : GVFSVerb.ForNoEnlistment 'd', "delete", Required = false, - HelpText = "Delete specified setting")] + HelpText = "Name of setting to delete")] public string KeyToDelete { get; set; } [Value( @@ -72,21 +72,15 @@ public override void Execute() { Console.WriteLine(ConfigOutputFormat, setting.Key, setting.Value); } - - return; } - - if (!string.IsNullOrEmpty(this.KeyToDelete)) + else if (!string.IsNullOrEmpty(this.KeyToDelete)) { if (!this.localConfig.TryRemoveConfig(this.KeyToDelete, out error)) { this.ReportErrorAndExit(error); } - - return; } - - if (!string.IsNullOrEmpty(this.Key)) + else if (!string.IsNullOrEmpty(this.Key)) { bool valueSpecified = !string.IsNullOrEmpty(this.Value); if (valueSpecified) @@ -95,8 +89,6 @@ public override void Execute() { this.ReportErrorAndExit(error); } - - return; } else { @@ -109,10 +101,12 @@ public override void Execute() { Console.WriteLine(valueRead); } - - return; } } + else + { + this.ReportErrorAndExit("You must specify an option. Run `gvfs config --help` for details."); + } } private bool IsMutuallyExclusiveOptionsSet(out string consoleMessage) From 6643b70bd1ba27d1f7037eb3b2da6719c1880373 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Thu, 18 Oct 2018 10:59:43 -0400 Subject: [PATCH 174/244] Create init scripts to set up dev environment in cmd/ps --- Scripts/Macros.cmd.txt | 7 ++++++ Scripts/Macros.ps1 | 7 ++++++ init.cmd | 55 ++++++++++++++++++++++++++++++++++++++++++ init.ps1 | 18 ++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 Scripts/Macros.cmd.txt create mode 100644 Scripts/Macros.ps1 create mode 100644 init.cmd create mode 100644 init.ps1 diff --git a/Scripts/Macros.cmd.txt b/Scripts/Macros.cmd.txt new file mode 100644 index 0000000000..68c15c385e --- /dev/null +++ b/Scripts/Macros.cmd.txt @@ -0,0 +1,7 @@ +b=%VFS_SCRIPTSDIR%\BuildGVFSForWindows.bat $* +br=%VFS_SCRIPTSDIR%\BuildGVFSForWindows.bat Release $* +ut=%VFS_SCRIPTSDIR%\RunUnitTests.bat $* +utr=%VFS_SCRIPTSDIR%\RunUnitTests.bat Release $* +ft=%VFS_SCRIPTSDIR%\RunFunctionalTests.bat $* +ftr=%VFS_SCRIPTSDIR%\RunFunctionalTests.bat Release $* +scorch=%VFS_SCRIPTSDIR%\NukeBuildOutputs.bat $* diff --git a/Scripts/Macros.ps1 b/Scripts/Macros.ps1 new file mode 100644 index 0000000000..57ab73cae8 --- /dev/null +++ b/Scripts/Macros.ps1 @@ -0,0 +1,7 @@ +function global:b { & (Join-Path $Env:VFS_SCRIPTSDIR "BuildGVFSForWindows.bat") @args } +function global:br { & (Join-Path $Env:VFS_SCRIPTSDIR "BuildGVFSForWindows.bat") Release @args } +function global:ut { & (Join-Path $Env:VFS_SCRIPTSDIR "RunUnitTests.bat") @args } +function global:utr { & (Join-Path $Env:VFS_SCRIPTSDIR "RunUnitTests.bat") Release @args } +function global:ft { & (Join-Path $Env:VFS_SCRIPTSDIR "RunFunctionalTests.bat") @args } +function global:ftr { & (Join-Path $Env:VFS_SCRIPTSDIR "RunFunctionalTests.bat") Release @args } +function global:scorch { & (Join-Path $Env:VFS_SCRIPTSDIR "NukeBuildOutputs.bat") @args } diff --git a/init.cmd b/init.cmd new file mode 100644 index 0000000000..eac963065a --- /dev/null +++ b/init.cmd @@ -0,0 +1,55 @@ +@ECHO OFF + +REM Don't run inside of devenv shells +IF NOT "%VSCMD_VER%"=="" ( + ECHO ERROR: Do not run from a Developer Command prompt. + ECHO Use a plain shell and try again. + EXIT /b 10 +) + +REM Don't run twice +IF NOT "%VFS_DEVSHELL%"=="" ( + ECHO ERROR: This shell is already a VFS for Git developer shell. + EXIT /b 20 +) + +REM Set environment variables for interesting paths that scripts might need access to. +PUSHD %~dp0 +SET VFS_SRCDIR=%CD% + +CALL :RESOLVEPATH "%VFS_SRCDIR%\Scripts" +SET VFS_SCRIPTSDIR=%_PARSED_PATH_% + +CALL :RESOLVEPATH "%VFS_SRCDIR%\.." +SET VFS_ENLISTMENTDIR=%_PARSED_PATH_% + +CALL :RESOLVEPATH "%VFS_ENLISTMENTDIR%\BuildOutput" +SET VFS_OUTPUTDIR=%_PARSED_PATH_% + +CALL :RESOLVEPATH "%VFS_ENLISTMENTDIR%\packages" +SET VFS_PACKAGESDIR=%_PARSED_PATH_% + +CALL :RESOLVEPATH "%VFS_ENLISTMENTDIR%\.tools" +SET VFS_TOOLSDIR=%_PARSED_PATH_% + +REM Clean up +SET _PARSED_PATH_= + +REM Mark this as a dev shell and load the macros +SET VFS_DEVSHELL=true +TITLE VFS ^for Git Developer Shell ^(%VFS_SRCDIR%^) +doskey /MACROFILE=.\Scripts\Macros.cmd.txt + +ECHO ============================================================= +ECHO * Welcome to the VFS for Git developer shell * +ECHO * * +ECHO * Build: b Build release: br * +ECHO * Run unit tests: ut Unit test release: utr * +ECHO * Run functional tests: ft Functional test release: ftr * +ECHO ============================================================= + +GOTO :EOF + +:RESOLVEPATH +SET "_PARSED_PATH_=%~f1" +GOTO :EOF diff --git a/init.ps1 b/init.ps1 new file mode 100644 index 0000000000..dad733c8b0 --- /dev/null +++ b/init.ps1 @@ -0,0 +1,18 @@ +# Delegate the work to init.cmd and save off the environment it creates to a temp file. +$cwd = (Get-Location) +$tmp = [IO.Path]::GetTempFileName() +cmd.exe /c "$PSScriptRoot\init.cmd && set>$tmp" + +# Read the env vars from the temp file and set it into this shell. +$lines = [System.IO.File]::ReadAllLines("$tmp") +Set-Location Env: +foreach ($line in $lines) { + $envVar = $line.Split('=') + Set-Item -Path $envVar[0] -Value $envVar[1] +} +Set-Location $cwd + +# Set up macros. +& "$PSScriptRoot\Scripts\Macros.ps1" + +$Host.UI.RawUI.WindowTitle = "VFS for Git Developer Shell ($Env:VFS_SRCDIR)" From 93ca1b11df7ce5c2a1372f83cb48e759b55fb860 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Thu, 18 Oct 2018 11:00:16 -0400 Subject: [PATCH 175/244] Enforce running init before other scripts. Remove relative paths from scripts. --- Scripts/BuildGVFSForWindows.bat | 8 ++++--- Scripts/EnsureVfsDevShell.bat | 14 ++++++++++++ Scripts/NukeBuildOutputs.bat | 11 ++++----- Scripts/ReinstallGVFS.bat | 13 ++++++----- Scripts/RestorePackages.bat | 10 +++++---- Scripts/RunFunctionalTests.bat | 40 +++++++++++++++++---------------- Scripts/RunUnitTests.bat | 6 +++-- Scripts/SetupDevService.bat | 4 +++- Scripts/StopAllServices.bat | 7 ++++-- Scripts/UninstallGVFS.bat | 7 ++++-- 10 files changed, 77 insertions(+), 43 deletions(-) create mode 100644 Scripts/EnsureVfsDevShell.bat diff --git a/Scripts/BuildGVFSForWindows.bat b/Scripts/BuildGVFSForWindows.bat index ac3ec7a1d3..810d7abeca 100644 --- a/Scripts/BuildGVFSForWindows.bat +++ b/Scripts/BuildGVFSForWindows.bat @@ -1,12 +1,14 @@ @ECHO OFF SETLOCAL +CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 + IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") IF "%2"=="" (SET "GVFSVersion=0.2.173.2") ELSE (SET "GVFSVersion=%2") SET SolutionConfiguration=%Configuration%.Windows -SET nuget="%~dp0\..\..\.tools\nuget.exe" +SET nuget="%VFS_TOOLSDIR%\nuget.exe" IF NOT EXIST %nuget% ( mkdir %nuget%\.. powershell -ExecutionPolicy Bypass -Command "Invoke-WebRequest 'https://dist.nuget.org/win-x86-commandline/latest/nuget.exe' -OutFile %nuget%" @@ -15,7 +17,7 @@ IF NOT EXIST %nuget% ( :: Acquire vswhere to find dev15 installations reliably. SET vswherever=2.5.2 %nuget% install vswhere -Version %vswherever% || exit /b 1 -SET vswhere=%~dp0..\..\packages\vswhere.%vswherever%\tools\vswhere.exe +SET vswhere=%VFS_PACKAGESDIR%\vswhere.%vswherever%\tools\vswhere.exe :: Use vswhere to find the latest VS installation (including prerelease installations) with the msbuild component. :: See https://github.com/Microsoft/vswhere/wiki/Find-MSBuild @@ -35,6 +37,6 @@ IF NOT EXIST %msbuild% ( exit /b 1 ) -%msbuild% %~dp0\..\GVFS.sln /p:GVFSVersion=%GVFSVersion% /p:Configuration=%SolutionConfiguration% /p:Platform=x64 || exit /b 1 +%msbuild% %VFS_SRCDIR%\GVFS.sln /p:GVFSVersion=%GVFSVersion% /p:Configuration=%SolutionConfiguration% /p:Platform=x64 || exit /b 1 ENDLOCAL \ No newline at end of file diff --git a/Scripts/EnsureVfsDevShell.bat b/Scripts/EnsureVfsDevShell.bat new file mode 100644 index 0000000000..547bca067e --- /dev/null +++ b/Scripts/EnsureVfsDevShell.bat @@ -0,0 +1,14 @@ +@ECHO OFF + +REM Maintain compat for current build definitions while we move to YAML by +REM automatically initing when this is running on a build agent. +IF "%TF_BUILD%"=="true" ( + CALL %~dp0/../init.cmd +) +REM Delete this block when we have moved to YAML-based pipelines. + +IF NOT "%VFS_DEVSHELL%"=="true" ( + ECHO ERROR: This shell is not a VFS for Git developer shell. + ECHO Run init.cmd or init.ps1 at the root of the repository. + EXIT /b 10 +) diff --git a/Scripts/NukeBuildOutputs.bat b/Scripts/NukeBuildOutputs.bat index c6f4b98e42..d7ae5f1e0c 100644 --- a/Scripts/NukeBuildOutputs.bat +++ b/Scripts/NukeBuildOutputs.bat @@ -1,4 +1,5 @@ @ECHO OFF +CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 taskkill /f /im GVFS.Mount.exe 2>&1 verify >nul @@ -17,18 +18,18 @@ IF EXIST C:\Repos\GVFSPerfTest ( ECHO no perf test enlistment found ) -IF EXIST %~dp0\..\..\BuildOutput ( +IF EXIST %VFS_OUTPUTDIR% ( ECHO deleting build outputs - rmdir /s /q %~dp0\..\..\BuildOutput + rmdir /s /q %VFS_OUTPUTDIR% ) ELSE ( ECHO no build outputs found ) -IF EXIST %~dp0\..\..\packages ( +IF EXIST %VFS_PACKAGESDIR% ( ECHO deleting packages - rmdir /s /q %~dp0\..\..\packages + rmdir /s /q %VFS_PACKAGESDIR% ) ELSE ( ECHO no packages found ) -call %~dp0\StopAllServices.bat +call %VFS_SCRIPTSDIR%\StopAllServices.bat diff --git a/Scripts/ReinstallGVFS.bat b/Scripts/ReinstallGVFS.bat index a735398e4c..181d223a10 100644 --- a/Scripts/ReinstallGVFS.bat +++ b/Scripts/ReinstallGVFS.bat @@ -1,6 +1,9 @@ +@ECHO OFF +CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 + IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") -call %~dp0\UninstallGVFS.bat +call %VFS_SCRIPTSDIR%\UninstallGVFS.bat if not exist "c:\Program Files\Git" goto :noGit for /F "delims=" %%g in ('dir "c:\Program Files\Git\unins*.exe" /B /S /O:-D') do %%g /VERYSILENT /SUPPRESSMSGBOXES /NORESTART & goto :deleteGit @@ -12,8 +15,8 @@ rmdir /q/s "c:\Program Files\Git" REM This is a hacky way to sleep for 2 seconds in a non-interactive window. The timeout command does not work if it can't redirect stdin. ping 1.1.1.1 -n 1 -w 2000 >NUL -call %~dp0\StopService.bat gvflt -call %~dp0\StopService.bat prjflt +call %VFS_SCRIPTSDIR%\StopService.bat gvflt +call %VFS_SCRIPTSDIR%\StopService.bat prjflt if not exist c:\Windows\System32\drivers\gvflt.sys goto :removePrjFlt del c:\Windows\System32\drivers\gvflt.sys @@ -25,5 +28,5 @@ verify >nul del c:\Windows\System32\drivers\PrjFlt.sys :runInstallers -call %~dp0\..\..\BuildOutput\GVFS.Build\InstallG4W.bat -call %~dp0\..\..\BuildOutput\GVFS.Build\InstallGVFS.bat +call %VFS_OUTPUTDIR%\GVFS.Build\InstallG4W.bat +call %VFS_OUTPUTDIR%\GVFS.Build\InstallGVFS.bat diff --git a/Scripts/RestorePackages.bat b/Scripts/RestorePackages.bat index a9b7e6c6c0..ae6832e44c 100644 --- a/Scripts/RestorePackages.bat +++ b/Scripts/RestorePackages.bat @@ -1,15 +1,17 @@ @ECHO OFF +CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 + SETLOCAL -SET nuget="%~dp0\..\..\.tools\nuget.exe" +SET nuget="%VFS_TOOLSDIR%\nuget.exe" IF NOT EXIST %nuget% ( mkdir %nuget%\.. powershell -ExecutionPolicy Bypass -Command "Invoke-WebRequest 'https://dist.nuget.org/win-x86-commandline/latest/nuget.exe' -OutFile %nuget%" ) -%nuget% restore %~dp0\..\GVFS.sln || exit /b 1 +%nuget% restore %VFS_SRCDIR%\GVFS.sln || exit /b 1 -dotnet restore %~dp0\..\GVFS\GVFS.Common\GVFS.Common.csproj || exit /b 1 -dotnet restore %~dp0\..\GVFS\GVFS.Virtualization\GVFS.Virtualization.csproj || exit /b 1 +dotnet restore %VFS_SRCDIR%\GVFS\GVFS.Common\GVFS.Common.csproj || exit /b 1 +dotnet restore %VFS_SRCDIR%\GVFS\GVFS.Virtualization\GVFS.Virtualization.csproj || exit /b 1 ENDLOCAL \ No newline at end of file diff --git a/Scripts/RunFunctionalTests.bat b/Scripts/RunFunctionalTests.bat index 9f1330906d..0e478ad7c0 100644 --- a/Scripts/RunFunctionalTests.bat +++ b/Scripts/RunFunctionalTests.bat @@ -1,4 +1,6 @@ @ECHO OFF +CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 + IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") SETLOCAL @@ -7,24 +9,24 @@ SET PATH=C:\Program Files\GVFS;C:\Program Files\Git\cmd;%PATH% if not "%2"=="--test-gvfs-on-path" goto :startFunctionalTests REM Force GVFS.FunctionalTests.exe to use the installed version of GVFS -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GitHooksLoader.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.Hooks.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.ReadObjectHook.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.VirtualFileSystemHook.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.Mount.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.Service.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.Service.UI.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GitHooksLoader.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.Hooks.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.ReadObjectHook.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.VirtualFileSystemHook.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.Mount.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.Service.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.Service.UI.exe REM Same for GVFS.FunctionalTests.Windows.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GitHooksLoader.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.Hooks.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.ReadObjectHook.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.VirtualFileSystemHook.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.Mount.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.Service.exe -del %~dp0\..\..\BuildOutput\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.Service.UI.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GitHooksLoader.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.Hooks.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.ReadObjectHook.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.VirtualFileSystemHook.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.Mount.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.Service.exe +del %VFS_OUTPUTDIR%\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.Service.UI.exe echo PATH = %PATH% echo gvfs location: @@ -35,12 +37,12 @@ echo git location: where git :startFunctionalTests -dotnet %~dp0\..\..\BuildOutput\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.FunctionalTests.dll %2 %3 %4 %5 || goto :endFunctionalTests -%~dp0\..\..\BuildOutput\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.FunctionalTests.Windows.exe --windows-only %2 %3 %4 %5 || goto :endFunctionalTests +dotnet %VFS_OUTPUTDIR%\GVFS.FunctionalTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.FunctionalTests.dll %2 %3 %4 %5 || goto :endFunctionalTests +%VFS_OUTPUTDIR%\GVFS.FunctionalTests.Windows\bin\x64\%Configuration%\GVFS.FunctionalTests.Windows.exe --windows-only %2 %3 %4 %5 || goto :endFunctionalTests :endFunctionalTests set error=%errorlevel% -call %~dp0\StopAllServices.bat +call %VFS_SCRIPTSDIR%\StopAllServices.bat exit /b %error% \ No newline at end of file diff --git a/Scripts/RunUnitTests.bat b/Scripts/RunUnitTests.bat index 223780852f..5cbcf83755 100644 --- a/Scripts/RunUnitTests.bat +++ b/Scripts/RunUnitTests.bat @@ -1,9 +1,11 @@ @ECHO OFF +CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 + IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") set RESULT=0 -%~dp0\..\..\BuildOutput\GVFS.UnitTests.Windows\bin\x64\%Configuration%\GVFS.UnitTests.Windows.exe || set RESULT=1 -dotnet %~dp0\..\..\BuildOutput\GVFS.UnitTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.UnitTests.dll || set RESULT=1 +%VFS_OUTPUTDIR%\GVFS.UnitTests.Windows\bin\x64\%Configuration%\GVFS.UnitTests.Windows.exe || set RESULT=1 +dotnet %VFS_OUTPUTDIR%\GVFS.UnitTests\bin\x64\%Configuration%\netcoreapp2.1\GVFS.UnitTests.dll || set RESULT=1 exit /b %RESULT% \ No newline at end of file diff --git a/Scripts/SetupDevService.bat b/Scripts/SetupDevService.bat index 317207e3e8..d2b4a28bf2 100644 --- a/Scripts/SetupDevService.bat +++ b/Scripts/SetupDevService.bat @@ -1,4 +1,6 @@ @ECHO OFF +CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 + IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") -sc create %Configuration%.GVFS.Service binPath=%~dp0\..\..\BuildOutput\GVFS.Service\bin\x64\%Configuration%\GVFS.Service.exe \ No newline at end of file +sc create %Configuration%.GVFS.Service binPath=%VFS_OUTPUTDIR%\GVFS.Service\bin\x64\%Configuration%\GVFS.Service.exe \ No newline at end of file diff --git a/Scripts/StopAllServices.bat b/Scripts/StopAllServices.bat index 8d80e18dcf..9139a6813b 100644 --- a/Scripts/StopAllServices.bat +++ b/Scripts/StopAllServices.bat @@ -1,2 +1,5 @@ -call %~dp0\StopService.bat GVFS.Service -call %~dp0\StopService.bat Test.GVFS.Service +@ECHO OFF +CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 + +call %VFS_SCRIPTSDIR%\StopService.bat GVFS.Service +call %VFS_SCRIPTSDIR%\StopService.bat Test.GVFS.Service diff --git a/Scripts/UninstallGVFS.bat b/Scripts/UninstallGVFS.bat index 6a0451b500..721a9893e6 100644 --- a/Scripts/UninstallGVFS.bat +++ b/Scripts/UninstallGVFS.bat @@ -1,14 +1,17 @@ +@ECHO OFF +CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 + taskkill /F /T /FI "IMAGENAME eq git.exe" taskkill /F /T /FI "IMAGENAME eq GVFS.exe" taskkill /F /T /FI "IMAGENAME eq GVFS.Mount.exe" if not exist "c:\Program Files\GVFS" goto :end -call %~dp0\StopAllServices.bat +call %VFS_SCRIPTSDIR%\StopAllServices.bat REM The GVFS uninstaller will not remove prjflt, and so we must remove it ourselves first. If we don't, the non-inbox ProjFS REM will cause problems next time the inbox ProjFS is enabled -call %~dp0\StopService.bat prjflt +call %VFS_SCRIPTSDIR%\StopService.bat prjflt rundll32.exe SETUPAPI.DLL,InstallHinfSection DefaultUninstall 128 C:\Program Files\GVFS\Filter\prjflt.inf REM Find the latest uninstaller file by date and run it. Goto the next step after a single execution. From 8d4596a170d564f676fce5ecec1b1e28ef706d40 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Tue, 16 Oct 2018 17:43:30 -0700 Subject: [PATCH 176/244] Close file descriptors when done with them as part of prefetch --hydrate Unload the kext before installing Git to avoid a kernel panic on build agents --- GVFS/GVFS.Platform.Mac/MacFileSystem.cs | 18 ++++++++++++++++-- Scripts/Mac/PrepFunctionalTests.sh | 3 +++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs index ae40098478..1bb5b8ccee 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs @@ -55,8 +55,19 @@ private class NativeFileReader public static bool TryReadFirstByteOfFile(string fileName, byte[] buffer) { - int fileDescriptor = Open(fileName, ReadOnly); - return TryReadOneByte(fileDescriptor, buffer); + int fileDescriptor = 1; + bool readStatus; + try + { + fileDescriptor = Open(fileName, ReadOnly); + readStatus = TryReadOneByte(fileDescriptor, buffer); + } + finally + { + Close(fileDescriptor); + } + + return readStatus; } private static bool TryReadOneByte(int fileDescriptor, byte[] buffer) @@ -74,6 +85,9 @@ private static bool TryReadOneByte(int fileDescriptor, byte[] buffer) [DllImport("libc", EntryPoint = "open", SetLastError = true)] private static extern int Open(string path, int flag); + [DllImport("libc", EntryPoint = "close", SetLastError = true)] + private static extern int Close(int fd); + [DllImport("libc", EntryPoint = "read", SetLastError = true)] private static extern int Read(int fd, [Out] byte[] buf, int count); } diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index fb35bde658..d7d9c3cfa1 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -2,6 +2,9 @@ SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) +# Ensure the kext isn't loaded before installing Git +$SCRIPTDIR/../../ProjFS.Mac/Scripts/UnloadPrjFSKext.sh + # Install GVFS-aware Git (that was downloaded by the build script) GITVERSION="$($SCRIPTDIR/GetGitVersionNumber.sh)" ROOTDIR=$SCRIPTDIR/../../.. From e4f2b49487e281867bd2d6568923475e8707dd37 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 19 Oct 2018 09:34:08 -0400 Subject: [PATCH 177/244] CommitPrefetcher: Delete MIDX when deleting bad packs --- GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs index ebe943ee01..aea6a5b1ac 100644 --- a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs @@ -113,6 +113,20 @@ public static bool TryGetMaxGoodPrefetchTimestamp( const int MaxDeleteRetries = 200; // 200 * IoFailureRetryDelayMS (50ms) = 10 seconds const int RetryLoggingThreshold = 40; // 40 * IoFailureRetryDelayMS (50ms) = 2 seconds + // Before we delete _any_ pack-files, we need to delete the multi-pack-index, which + // may refer to those packs. + + EventMetadata metadata = new EventMetadata(); + string midxPath = Path.Combine(enlistment.GitPackRoot, "multi-pack-index"); + metadata.Add("path", midxPath); + metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting multi-pack-index"); + tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteMultiPack_index", metadata); + if (!fileSystem.TryWaitForDelete(tracer, midxPath, IoFailureRetryDelayMS, MaxDeleteRetries, RetryLoggingThreshold)) + { + error = $"Unable to delete {midxPath}"; + return false; + } + // Delete packs and indexes in reverse order so that if prefetch is killed, subseqeuent prefetch commands will // find the right starting spot. for (int i = orderedPacks.Count - 1; i >= firstBadPack; --i) @@ -120,7 +134,7 @@ public static bool TryGetMaxGoodPrefetchTimestamp( string packPath = orderedPacks[i].Path; string idxPath = Path.ChangeExtension(packPath, ".idx"); - EventMetadata metadata = new EventMetadata(); + metadata = new EventMetadata(); metadata.Add("path", idxPath); metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(TryGetMaxGoodPrefetchTimestamp)} deleting bad idx file"); tracer.RelatedEvent(EventLevel.Informational, $"{nameof(TryGetMaxGoodPrefetchTimestamp)}_DeleteBadIdx", metadata); From 97fa3099da5fc5cc09e169fb30812558a7fa390f Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Fri, 19 Oct 2018 10:49:58 -0400 Subject: [PATCH 178/244] Added OneTimeSetup method to clear all FunctionalTest configs before starting the tests Added test-method to test read of a single setting (vs list). Cleanups - removed spacing around curly braces --- .../MultiEnlistmentTests/ConfigVerbTests.cs | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs index ee76cbb6e9..352591dce7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ConfigVerbTests.cs @@ -34,10 +34,16 @@ public class ConfigVerbTests : TestsWithMultiEnlistment { SpacedStringSettingKey, "jumped over lazy dog" } }; + [OneTimeSetUp] + public void ResetTestConfig() + { + this.DeleteSettings(this.initialSettings); + this.DeleteSettings(this.updateSettings); + } + [TestCase, Order(1)] public void CreateSettings() { - this.DeleteSettings(); this.ApplySettings(this.initialSettings); this.ConfigShouldContainSettings(this.initialSettings); } @@ -56,12 +62,23 @@ public void ListSettings() } [TestCase, Order(4)] + public void ReadSingleSetting() + { + foreach (KeyValuePair setting in this.updateSettings) + { + string value = this.RunConfigCommand($"{setting.Key}"); + value.TrimEnd(Environment.NewLine.ToCharArray()).ShouldEqual($"{setting.Value}"); + } + } + + [TestCase, Order(5)] public void DeleteSettings() { + this.DeleteSettings(this.updateSettings); + List deletedLines = new List(); foreach (KeyValuePair setting in this.updateSettings) { - this.RunConfigCommand($"--delete {setting.Key}"); deletedLines.Add(this.GetSettingLineInConfigFileFormat(setting)); } @@ -69,6 +86,15 @@ public void DeleteSettings() allSettings.ShouldNotContain(ignoreCase: true, unexpectedSubstrings: deletedLines.ToArray()); } + private void DeleteSettings(Dictionary settings) + { + List deletedLines = new List(); + foreach (KeyValuePair setting in settings) + { + this.RunConfigCommand($"--delete {setting.Key}"); + } + } + private void ConfigShouldContainSettings(Dictionary expectedSettings) { List expectedLines = new List(); @@ -90,13 +116,13 @@ private void ApplySettings(Dictionary settings) { foreach (KeyValuePair setting in settings) { - this.RunConfigCommand($"{ setting.Key } \"{ setting.Value }\""); + this.RunConfigCommand($"{setting.Key} \"{setting.Value}\""); } } private string RunConfigCommand(string argument) { - ProcessResult result = ProcessHelper.Run(GVFSTestConfig.PathToGVFS, $"config { argument }"); + ProcessResult result = ProcessHelper.Run(GVFSTestConfig.PathToGVFS, $"config {argument}"); result.ExitCode.ShouldEqual(0, result.Errors); return result.Output; From c3b928208fa28534d2ab52968832054361beac31 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 19 Oct 2018 13:00:09 -0400 Subject: [PATCH 179/244] Use IsUnderConstruction to avoid threadabort on mac --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index b98006fdd3..e209e3fa2f 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -198,7 +198,11 @@ public void Stop() lock (this.postFetchJobLock) { // TODO(Mac): System.PlatformNotSupportedException: Thread abort is not supported on this platform - this.postFetchJobThread?.Abort(); + + if (!GVFSPlatform.Instance.IsUnderConstruction) + { + this.postFetchJobThread?.Abort(); + } } // Shutdown the GitStatusCache before other From d7698d09e52813f77ea571b4f5cbf36a5c4a465f Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 18 Oct 2018 09:37:29 -0700 Subject: [PATCH 180/244] Add untracked files from reset mixed to ModifiedPaths.dat --- GVFS/GVFS.Common/PlaceholderListDatabase.cs | 52 +++++++++++++++++++ .../Tests/GitCommands/CheckoutTests.cs | 3 -- .../Tests/GitCommands/GitCommandsTests.cs | 2 - .../Tests/GitCommands/ResetMixedTests.cs | 1 - .../Tests/GitCommands/ResetSoftTests.cs | 1 - .../Tests/GitCommands/UpdateIndexTests.cs | 8 +-- .../GitIndexProjection.GitIndexParser.cs | 26 ++++++++-- .../Projection/GitIndexProjection.cs | 20 ++++++- 8 files changed, 97 insertions(+), 16 deletions(-) diff --git a/GVFS/GVFS.Common/PlaceholderListDatabase.cs b/GVFS/GVFS.Common/PlaceholderListDatabase.cs index 5531fd0c0f..819331d91c 100644 --- a/GVFS/GVFS.Common/PlaceholderListDatabase.cs +++ b/GVFS/GVFS.Common/PlaceholderListDatabase.cs @@ -167,6 +167,58 @@ public void GetAllEntries(out List filePlaceholders, out List

placeholderDataByPath, + out List folderPlaceholders) + { + try + { + Dictionary filePlaceholdersFromDiskByPath = + new Dictionary(Math.Max(1, this.EstimatedCount), StringComparer.Ordinal); + + // This version of GetAllEntries was designed to be used when some file placeholders will need to be removed. Allocate folderPlaceholdersFromDisk + // with enough space for all entries so that the caller can use folderPlaceholders as the list of placeholders to keep (moving items from placeholderDataByPath + // to folderPlaceholders, and then calling WriteAllEntriesAndFlush with folderPlaceholders) + List folderPlaceholdersFromDisk = new List(Math.Max(1, this.EstimatedCount)); + + string error; + if (!this.TryLoadFromDisk( + this.TryParseAddLine, + this.TryParseRemoveLine, + (key, value) => + { + if (value == PartialFolderValue || value == ExpandedFolderValue) + { + folderPlaceholdersFromDisk.Add(new PlaceholderData(path: key, fileShaOrFolderValue: value)); + } + else + { + filePlaceholdersFromDiskByPath[key] = new PlaceholderData(path: key, fileShaOrFolderValue: value); + } + }, + out error, + () => + { + if (this.placeholderDataEntries != null) + { + throw new InvalidOperationException("PlaceholderListDatabase should always flush queue placeholders using WriteAllEntriesAndFlush before calling GetAllEntries again."); + } + + this.placeholderDataEntries = new List(); + })) + { + throw new InvalidDataException(error); + } + + placeholderDataByPath = filePlaceholdersFromDiskByPath; + folderPlaceholders = folderPlaceholdersFromDisk; + } + catch (Exception e) + { + throw new FileBasedCollectionException(e); + } + } + public void WriteAllEntriesAndFlush(IEnumerable updatedPlaceholders) { try diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 5d10b5792f..9a5383f1dd 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -380,7 +380,6 @@ public void ModifyAndCheckoutFirstOfSeveralFilesWhoseNamesAppearBeforeDot() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetMixedToCommitWithNewFileThenCheckoutNewBranchAndCheckoutCommitWithNewFile() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -668,7 +667,6 @@ public void CheckoutBranchWhileOutsideToolHasExclusiveReadHandleOnDatabasesFolde } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetMixedTwiceThenCheckoutWithChanges() { this.ControlGitRepo.Fetch("FunctionalTests/20171219_MultipleFileEdits"); @@ -688,7 +686,6 @@ public void ResetMixedTwiceThenCheckoutWithChanges() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetMixedTwiceThenCheckoutWithRemovedFiles() { this.ControlGitRepo.Fetch("FunctionalTests/20180102_MultipleFileDeletes"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 6755a3846b..99da0d9dce 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -727,7 +727,6 @@ public void ResetSoft() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetMixed() { this.ValidateGitCommand("checkout -b tests/functional/ResetMixed"); @@ -735,7 +734,6 @@ public void ResetMixed() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetMixed2() { this.ValidateGitCommand("checkout -b tests/functional/ResetMixed2"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs index ed90a670a4..d0b87606ef 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs @@ -5,7 +5,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class ResetMixedTests : GitRepoTests { public ResetMixedTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs index e79cdf99b4..8aaed0884e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs @@ -50,7 +50,6 @@ public void ResetSoftThenCheckoutNoConflicts() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetSoftThenResetHeadThenCheckoutNoConflicts() { this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index 8240b32223..feb648353b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -40,7 +40,6 @@ public void UpdateIndexRemoveFileOnDiskDontCheckStatus() } [TestCase] - [Category(Categories.MacTODO.M3)] public void UpdateIndexRemoveAddFileOpenForWrite() { // TODO 940287: Remove this test and re-enable UpdateIndexRemoveFileOnDisk @@ -53,7 +52,11 @@ public void UpdateIndexRemoveAddFileOpenForWrite() this.FilesShouldMatchCheckoutOfTargetBranch(); // Open Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt for write so that it's added to the modified paths database - using (FileStream stream = File.Open(Path.Combine(this.Enlistment.RepoRoot, @"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"), FileMode.Open, FileAccess.Write)) + using ( + FileStream stream = File.Open( + Path.Combine(this.Enlistment.RepoRoot, "Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"), + FileMode.Open, + FileAccess.Write)) { // TODO 940287: Remove this File.Open once update-index --add\--remove are working as expected } @@ -64,7 +67,6 @@ public void UpdateIndexRemoveAddFileOpenForWrite() } [TestCase] - [Category(Categories.MacTODO.M3)] public void UpdateIndexWithCacheInfo() { // Update Protocol.md with the contents from blob 583f1... diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 4893ede463..d6f6172794 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -2,6 +2,7 @@ using GVFS.Common.Tracing; using GVFS.Virtualization.Background; using System; +using System.Collections.Generic; using System.IO; using System.Linq; @@ -69,14 +70,21 @@ public void RebuildProjection(ITracer tracer, Stream indexStream) } } - public FileSystemTaskResult AddMissingModifiedFiles(ITracer tracer, Stream indexStream) + public FileSystemTaskResult AddMissingModifiedFiles( + ITracer tracer, + Stream indexStream, + Dictionary placeholderDataByPath, + List placeholderDataToKeep) { if (this.projection == null) { throw new InvalidOperationException($"{nameof(this.projection)} cannot be null when calling {nameof(AddMissingModifiedFiles)}"); } - return this.ParseIndex(tracer, indexStream, this.AddToModifiedFiles); + return this.ParseIndex( + tracer, + indexStream, + (data) => this.AddToModifiedFiles(data, placeholderDataByPath, placeholderDataToKeep)); } private static FileSystemTaskResult ValidateIndexEntry(GitIndexEntry data) @@ -105,18 +113,28 @@ private FileSystemTaskResult AddToProjection(GitIndexEntry data) return FileSystemTaskResult.Success; } - private FileSystemTaskResult AddToModifiedFiles(GitIndexEntry data) + private FileSystemTaskResult AddToModifiedFiles( + GitIndexEntry data, + Dictionary placeholderDataByPath, + List placeholderDataToKeep) { + data.ParsePath(); + if (!data.SkipWorktree) { // A git command (e.g. 'git reset --mixed') may have cleared a file's skip worktree bit without // triggering an update to the projection. Ensure this file is in GVFS's modified files database - data.ParsePath(); return this.projection.AddModifiedPath(data.GetFullPath()); } else { data.ClearLastParent(); + + PlaceholderListDatabase.PlaceholderData placeholder; + if (placeholderDataByPath.Remove(data.GetFullPath(), out placeholder)) + { + placeholderDataToKeep.Add(placeholder); + } } return FileSystemTaskResult.Success; diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index fb8edba06d..f1d2680adb 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -160,10 +160,13 @@ void IProfilerOnlyIndexProjection.ForceRebuildProjection() ///

void IProfilerOnlyIndexProjection.ForceAddMissingModifiedPaths(ITracer tracer) { + Dictionary placeholderDataByPath = new Dictionary(StringComparer.Ordinal); + List placeholderDataToKeep = new List(); + using (FileStream indexStream = new FileStream(this.indexPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, IndexFileStreamBufferSize)) { // Not checking the FileSystemTaskResult here because this is only for profiling - this.indexParser.AddMissingModifiedFiles(tracer, indexStream); + this.indexParser.AddMissingModifiedFiles(tracer, indexStream, placeholderDataByPath, placeholderDataToKeep); } } @@ -529,12 +532,25 @@ public FileSystemTaskResult AddMissingModifiedFiles() { if (this.modifiedFilesInvalid) { - FileSystemTaskResult result = this.indexParser.AddMissingModifiedFiles(this.context.Tracer, this.indexFileStream); + Dictionary placeholderDataByPath; + List placeholdersToKeep; + this.placeholderList.GetAllEntries(out placeholderDataByPath, folderPlaceholders: out placeholdersToKeep); + + FileSystemTaskResult result = this.indexParser.AddMissingModifiedFiles(this.context.Tracer, this.indexFileStream, placeholderDataByPath, placeholdersToKeep); if (result == FileSystemTaskResult.Success) { this.modifiedFilesInvalid = false; } + // Any paths that are not in the the index must be added to ModifiedPaths.dat so that git + // reports them as untracked + foreach (string path in placeholderDataByPath.Keys) + { + this.AddModifiedPath(path); + } + + this.placeholderList.WriteAllEntriesAndFlush(placeholdersToKeep); + return result; } } From 7643cfd9613b1c1dd01fb9810cbb79bb65c27163 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 19 Oct 2018 12:02:15 -0700 Subject: [PATCH 181/244] Don't use .NET Core specific Remove --- .../Projection/GitIndexProjection.GitIndexParser.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index d6f6172794..53a40e3f37 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -130,10 +130,12 @@ private FileSystemTaskResult AddToModifiedFiles( { data.ClearLastParent(); + string placeholderPath = data.GetFullPath(); PlaceholderListDatabase.PlaceholderData placeholder; - if (placeholderDataByPath.Remove(data.GetFullPath(), out placeholder)) + if (placeholderDataByPath.TryGetValue(placeholderPath, out placeholder)) { placeholderDataToKeep.Add(placeholder); + placeholderDataByPath.Remove(placeholderPath); } } From f8de64885c476e54ba2b9dc97cc090b5af70fb0b Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 22 Oct 2018 14:45:12 -0700 Subject: [PATCH 182/244] PR Feedback: Cleanup and ensure paths correct on all platforms --- GVFS/GVFS.Common/PlaceholderListDatabase.cs | 29 +----- .../Projection/GitIndexEntryTests.cs | 4 +- .../GitIndexProjection.GitIndexEntry.cs | 8 +- .../GitIndexProjection.GitIndexParser.cs | 92 +++++++++++++------ .../Projection/GitIndexProjection.cs | 27 ++---- 5 files changed, 86 insertions(+), 74 deletions(-) diff --git a/GVFS/GVFS.Common/PlaceholderListDatabase.cs b/GVFS/GVFS.Common/PlaceholderListDatabase.cs index 819331d91c..a4750cf110 100644 --- a/GVFS/GVFS.Common/PlaceholderListDatabase.cs +++ b/GVFS/GVFS.Common/PlaceholderListDatabase.cs @@ -167,51 +167,30 @@ public void GetAllEntries(out List filePlaceholders, out List

placeholderDataByPath, - out List folderPlaceholders) + public void GetAllFileEntries(out Dictionary filePlaceholdersByPath) { try { Dictionary filePlaceholdersFromDiskByPath = new Dictionary(Math.Max(1, this.EstimatedCount), StringComparer.Ordinal); - // This version of GetAllEntries was designed to be used when some file placeholders will need to be removed. Allocate folderPlaceholdersFromDisk - // with enough space for all entries so that the caller can use folderPlaceholders as the list of placeholders to keep (moving items from placeholderDataByPath - // to folderPlaceholders, and then calling WriteAllEntriesAndFlush with folderPlaceholders) - List folderPlaceholdersFromDisk = new List(Math.Max(1, this.EstimatedCount)); - string error; if (!this.TryLoadFromDisk( this.TryParseAddLine, this.TryParseRemoveLine, (key, value) => { - if (value == PartialFolderValue || value == ExpandedFolderValue) - { - folderPlaceholdersFromDisk.Add(new PlaceholderData(path: key, fileShaOrFolderValue: value)); - } - else + if (value != PartialFolderValue && value != ExpandedFolderValue) { filePlaceholdersFromDiskByPath[key] = new PlaceholderData(path: key, fileShaOrFolderValue: value); } }, - out error, - () => - { - if (this.placeholderDataEntries != null) - { - throw new InvalidOperationException("PlaceholderListDatabase should always flush queue placeholders using WriteAllEntriesAndFlush before calling GetAllEntries again."); - } - - this.placeholderDataEntries = new List(); - })) + out error)) { throw new InvalidDataException(error); } - placeholderDataByPath = filePlaceholdersFromDiskByPath; - folderPlaceholders = folderPlaceholdersFromDisk; + filePlaceholdersByPath = filePlaceholdersFromDiskByPath; } catch (Exception e) { diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs index 58e50ea8f2..1a03f3b3d8 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs @@ -2,6 +2,7 @@ using GVFS.UnitTests.Mock.Common; using NUnit.Framework; using System; +using System.IO; using System.Text; using static GVFS.Virtualization.Projection.GitIndexProjection; @@ -161,7 +162,8 @@ private void TestPathParts(GitIndexEntry indexEntry, string[] pathParts, bool ha } indexEntry.GetChildName().GetString().ShouldEqual(pathParts[pathParts.Length - 1]); - indexEntry.GetFullPath().ShouldEqual(string.Join("/", pathParts)); + indexEntry.GetGitPath().ShouldEqual(string.Join("/", pathParts)); + indexEntry.GetRelativePath().ShouldEqual(string.Join(Path.DirectorySeparatorChar, pathParts)); } } } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 8cd38e283d..4ff1fc4e16 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -1,4 +1,5 @@ using GVFS.Common; +using System.IO; using System.Linq; namespace GVFS.Virtualization.Projection @@ -140,10 +141,15 @@ public LazyUTF8String GetChildName() return this.PathParts[this.NumParts - 1]; } - public string GetFullPath() + public string GetGitPath() { return string.Join(GVFSConstants.GitPathSeparatorString, this.PathParts.Take(this.NumParts).Select(x => x.GetString())); } + + public string GetRelativePath() + { + return string.Join(Path.DirectorySeparatorChar, this.PathParts.Take(this.NumParts).Select(x => x.GetString())); + } private unsafe bool RangeContains(byte* bufferPtr, int count, byte value) { diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 53a40e3f37..fcf36d012e 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -62,29 +62,50 @@ public void RebuildProjection(ITracer tracer, Stream indexStream) } this.projection.ClearProjectionCaches(); - FileSystemTaskResult result = this.ParseIndex(tracer, indexStream, this.AddToProjection); + FileSystemTaskResult result = this.ParseIndex(tracer, indexStream, this.AddIndexEntryToProjection); if (result != FileSystemTaskResult.Success) { // RebuildProjection should always result in FileSystemTaskResult.Success (or a thrown exception) - throw new InvalidOperationException($"{nameof(RebuildProjection)}: {nameof(GitIndexParser.ParseIndex)} failed to {nameof(this.AddToProjection)}"); + throw new InvalidOperationException($"{nameof(RebuildProjection)}: {nameof(GitIndexParser.ParseIndex)} failed to {nameof(this.AddIndexEntryToProjection)}"); } } - public FileSystemTaskResult AddMissingModifiedFiles( + public FileSystemTaskResult AddMissingModifiedFilesAndRemoveThemFromPlaceholderList( ITracer tracer, - Stream indexStream, - Dictionary placeholderDataByPath, - List placeholderDataToKeep) + Stream indexStream) { if (this.projection == null) { - throw new InvalidOperationException($"{nameof(this.projection)} cannot be null when calling {nameof(AddMissingModifiedFiles)}"); + throw new InvalidOperationException($"{nameof(this.projection)} cannot be null when calling {nameof(AddMissingModifiedFilesAndRemoveThemFromPlaceholderList)}"); } - return this.ParseIndex( + Dictionary filePlaceholders; + this.projection.placeholderList.GetAllFileEntries(out filePlaceholders); + + FileSystemTaskResult result = this.ParseIndex( tracer, indexStream, - (data) => this.AddToModifiedFiles(data, placeholderDataByPath, placeholderDataToKeep)); + (data) => this.AdjustModifedPathsAndPlaceholdersForIndexEntry(data, filePlaceholders)); + + if (result != FileSystemTaskResult.Success) + { + return result; + } + + // Any paths that were not found in the index need to be added to ModifiedPaths + // and removed from the placeholder list + foreach (string path in filePlaceholders.Keys) + { + result = this.projection.AddModifiedPath(path); + if (result != FileSystemTaskResult.Success) + { + return result; + } + + this.projection.RemoveFromPlaceholderList(path); + } + + return FileSystemTaskResult.Success; } private static FileSystemTaskResult ValidateIndexEntry(GitIndexEntry data) @@ -97,7 +118,7 @@ private static FileSystemTaskResult ValidateIndexEntry(GitIndexEntry data) return FileSystemTaskResult.Success; } - private FileSystemTaskResult AddToProjection(GitIndexEntry data) + private FileSystemTaskResult AddIndexEntryToProjection(GitIndexEntry data) { // Never want to project the common ancestor even if the skip worktree bit is on if ((data.MergeState != MergeStage.CommonAncestor && data.SkipWorktree) || data.MergeState == MergeStage.Yours) @@ -113,33 +134,50 @@ private FileSystemTaskResult AddToProjection(GitIndexEntry data) return FileSystemTaskResult.Success; } - private FileSystemTaskResult AddToModifiedFiles( - GitIndexEntry data, - Dictionary placeholderDataByPath, - List placeholderDataToKeep) + ///

+ /// Adjusts the modifed paths and placeholders list for an index entry. + /// + /// Index entry + /// + /// Dictionary of file placeholders. AdjustModifedPathsAndPlaceholdersForIndexEntry will + /// remove enties from filePlaceholders as they are found in the index. After + /// AdjustModifedPathsAndPlaceholdersForIndexEntry is called for all entries in the index + /// filePlaceholders will contain only those placeholders that are not in the index. + /// + private FileSystemTaskResult AdjustModifedPathsAndPlaceholdersForIndexEntry( + GitIndexEntry gitIndexEntry, + Dictionary filePlaceholders) { - data.ParsePath(); + gitIndexEntry.ParsePath(); + string placeholderRelativePath = gitIndexEntry.GetRelativePath(); + + FileSystemTaskResult result = FileSystemTaskResult.Success; - if (!data.SkipWorktree) + if (!gitIndexEntry.SkipWorktree) { // A git command (e.g. 'git reset --mixed') may have cleared a file's skip worktree bit without - // triggering an update to the projection. Ensure this file is in GVFS's modified files database - return this.projection.AddModifiedPath(data.GetFullPath()); + // triggering an update to the projection. If git cleared the skip-worktree bit then git will + // be responsible for updating the file and we need to: + // - Ensure this file is in GVFS's modified files database + // - Remove this path from the placeholders list (if present) + result = this.projection.AddModifiedPath(placeholderRelativePath); + + if (result == FileSystemTaskResult.Success) + { + if (filePlaceholders.Remove(placeholderRelativePath)) + { + this.projection.RemoveFromPlaceholderList(placeholderRelativePath); + } + } } else { - data.ClearLastParent(); + gitIndexEntry.ClearLastParent(); - string placeholderPath = data.GetFullPath(); - PlaceholderListDatabase.PlaceholderData placeholder; - if (placeholderDataByPath.TryGetValue(placeholderPath, out placeholder)) - { - placeholderDataToKeep.Add(placeholder); - placeholderDataByPath.Remove(placeholderPath); - } + filePlaceholders.Remove(placeholderRelativePath); } - return FileSystemTaskResult.Success; + return result; } /// diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index f1d2680adb..6b18b89596 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -160,13 +160,10 @@ void IProfilerOnlyIndexProjection.ForceRebuildProjection() /// void IProfilerOnlyIndexProjection.ForceAddMissingModifiedPaths(ITracer tracer) { - Dictionary placeholderDataByPath = new Dictionary(StringComparer.Ordinal); - List placeholderDataToKeep = new List(); - using (FileStream indexStream = new FileStream(this.indexPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, IndexFileStreamBufferSize)) { // Not checking the FileSystemTaskResult here because this is only for profiling - this.indexParser.AddMissingModifiedFiles(tracer, indexStream, placeholderDataByPath, placeholderDataToKeep); + this.indexParser.AddMissingModifiedFilesAndRemoveThemFromPlaceholderList(tracer, indexStream); } } @@ -532,25 +529,15 @@ public FileSystemTaskResult AddMissingModifiedFiles() { if (this.modifiedFilesInvalid) { - Dictionary placeholderDataByPath; - List placeholdersToKeep; - this.placeholderList.GetAllEntries(out placeholderDataByPath, folderPlaceholders: out placeholdersToKeep); - - FileSystemTaskResult result = this.indexParser.AddMissingModifiedFiles(this.context.Tracer, this.indexFileStream, placeholderDataByPath, placeholdersToKeep); + FileSystemTaskResult result = this.indexParser.AddMissingModifiedFilesAndRemoveThemFromPlaceholderList( + this.context.Tracer, + this.indexFileStream); + if (result == FileSystemTaskResult.Success) { this.modifiedFilesInvalid = false; } - // Any paths that are not in the the index must be added to ModifiedPaths.dat so that git - // reports them as untracked - foreach (string path in placeholderDataByPath.Keys) - { - this.AddModifiedPath(path); - } - - this.placeholderList.WriteAllEntriesAndFlush(placeholdersToKeep); - return result; } } @@ -702,7 +689,7 @@ private void AddItemFromIndexEntry(GitIndexEntry indexEntry) // TODO(Mac): The line below causes a conversion from LazyUTF8String to .NET string. // Measure the perf and memory overhead of performing this conversion, and determine if we need // a way to keep the path as LazyUTF8String[] - this.nonDefaultFileTypesAndModes.Add(indexEntry.GetFullPath(), indexEntry.TypeAndMode); + this.nonDefaultFileTypesAndModes.Add(indexEntry.GetGitPath(), indexEntry.TypeAndMode); } } } @@ -809,7 +796,7 @@ private FolderData AddFileToTree(GitIndexEntry indexEntry) parentFolderName = this.rootFolderData.Name.GetString(); } - string gitPath = indexEntry.GetFullPath(); + string gitPath = indexEntry.GetGitPath(); EventMetadata metadata = CreateEventMetadata(); metadata.Add("gitPath", gitPath); From cbdaeaa15ef70df32c7784980015f992ee5022dc Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 22 Oct 2018 15:29:51 -0700 Subject: [PATCH 183/244] Fix build break on Windows --- .../Projection/GitIndexProjection.GitIndexEntry.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 4ff1fc4e16..bf87bc398d 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -19,6 +19,8 @@ internal class GitIndexEntry private const int MaxParts = MaxPathBufferSize / 2; private const byte PathSeparatorCode = 0x2F; + private static readonly string PathSeparatorString = Path.DirectorySeparatorChar.ToString(); + private int previousFinalSeparatorIndex = int.MaxValue; public GitIndexEntry() @@ -148,7 +150,7 @@ public string GetGitPath() public string GetRelativePath() { - return string.Join(Path.DirectorySeparatorChar, this.PathParts.Take(this.NumParts).Select(x => x.GetString())); + return string.Join(PathSeparatorString, this.PathParts.Take(this.NumParts).Select(x => x.GetString())); } private unsafe bool RangeContains(byte* bufferPtr, int count, byte value) From 6e697e06683cfd069e8aa3e84b82946ed582f492 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 22 Oct 2018 16:04:10 -0700 Subject: [PATCH 184/244] Fix another Windows build break --- .../Virtualization/Projection/GitIndexEntryTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs index 1a03f3b3d8..b491cff3ab 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs @@ -163,7 +163,7 @@ private void TestPathParts(GitIndexEntry indexEntry, string[] pathParts, bool ha indexEntry.GetChildName().GetString().ShouldEqual(pathParts[pathParts.Length - 1]); indexEntry.GetGitPath().ShouldEqual(string.Join("/", pathParts)); - indexEntry.GetRelativePath().ShouldEqual(string.Join(Path.DirectorySeparatorChar, pathParts)); + indexEntry.GetRelativePath().ShouldEqual(string.Join(Path.DirectorySeparatorChar.ToString(), pathParts)); } } } From e1da84bbdddd9df48ec6d0a7662acb8a928d077c Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Mon, 22 Oct 2018 16:49:42 -0700 Subject: [PATCH 185/244] Fix repositoryPath to work on all platforms --- nuget.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuget.config b/nuget.config index 2616ca0460..dcdfdfb432 100644 --- a/nuget.config +++ b/nuget.config @@ -1,7 +1,7 @@  - + From 986d7ea3c6e89f9d3b82e7f2d22fa1221107349f Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 23 Oct 2018 09:39:24 -0700 Subject: [PATCH 186/244] Add functional test the fails on Windows without fix --- .../Tests/GitCommands/ResetMixedTests.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs index d0b87606ef..1b8f9cc0a6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs @@ -19,6 +19,15 @@ public void ResetMixed() this.FilesShouldMatchCheckoutOfTargetBranch(); } + [TestCase] + public void ResetMixedAfterPrefetch() + { + this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); + this.Enlistment.Prefetch("--files * --hydrate"); + this.ValidateGitCommand("reset --mixed HEAD~1"); + this.FilesShouldMatchCheckoutOfTargetBranch(); + } + [TestCase] public void ResetMixedAndCheckoutNewBranch() { From 7343743a465e3e0053c48259b8d7c28c1199d9ff Mon Sep 17 00:00:00 2001 From: John Briggs Date: Fri, 19 Oct 2018 12:11:30 -0400 Subject: [PATCH 187/244] Fix casing in EnsureVFSDevShell.bat --- Scripts/BuildGVFSForWindows.bat | 2 +- Scripts/{EnsureVfsDevShell.bat => EnsureVFSDevShell.bat} | 0 Scripts/NukeBuildOutputs.bat | 2 +- Scripts/ReinstallGVFS.bat | 2 +- Scripts/RestorePackages.bat | 2 +- Scripts/RunFunctionalTests.bat | 2 +- Scripts/RunUnitTests.bat | 2 +- Scripts/SetupDevService.bat | 2 +- Scripts/StopAllServices.bat | 2 +- Scripts/UninstallGVFS.bat | 2 +- 10 files changed, 9 insertions(+), 9 deletions(-) rename Scripts/{EnsureVfsDevShell.bat => EnsureVFSDevShell.bat} (100%) diff --git a/Scripts/BuildGVFSForWindows.bat b/Scripts/BuildGVFSForWindows.bat index 810d7abeca..23c73026d3 100644 --- a/Scripts/BuildGVFSForWindows.bat +++ b/Scripts/BuildGVFSForWindows.bat @@ -1,7 +1,7 @@ @ECHO OFF SETLOCAL -CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 +CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") IF "%2"=="" (SET "GVFSVersion=0.2.173.2") ELSE (SET "GVFSVersion=%2") diff --git a/Scripts/EnsureVfsDevShell.bat b/Scripts/EnsureVFSDevShell.bat similarity index 100% rename from Scripts/EnsureVfsDevShell.bat rename to Scripts/EnsureVFSDevShell.bat diff --git a/Scripts/NukeBuildOutputs.bat b/Scripts/NukeBuildOutputs.bat index d7ae5f1e0c..77b5b8a373 100644 --- a/Scripts/NukeBuildOutputs.bat +++ b/Scripts/NukeBuildOutputs.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 +CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 taskkill /f /im GVFS.Mount.exe 2>&1 verify >nul diff --git a/Scripts/ReinstallGVFS.bat b/Scripts/ReinstallGVFS.bat index 181d223a10..b604160041 100644 --- a/Scripts/ReinstallGVFS.bat +++ b/Scripts/ReinstallGVFS.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 +CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") diff --git a/Scripts/RestorePackages.bat b/Scripts/RestorePackages.bat index ae6832e44c..5c01dd8ef3 100644 --- a/Scripts/RestorePackages.bat +++ b/Scripts/RestorePackages.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 +CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 SETLOCAL diff --git a/Scripts/RunFunctionalTests.bat b/Scripts/RunFunctionalTests.bat index 0e478ad7c0..51e3d00104 100644 --- a/Scripts/RunFunctionalTests.bat +++ b/Scripts/RunFunctionalTests.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 +CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") diff --git a/Scripts/RunUnitTests.bat b/Scripts/RunUnitTests.bat index 5cbcf83755..ec834d06c8 100644 --- a/Scripts/RunUnitTests.bat +++ b/Scripts/RunUnitTests.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 +CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") diff --git a/Scripts/SetupDevService.bat b/Scripts/SetupDevService.bat index d2b4a28bf2..021103853b 100644 --- a/Scripts/SetupDevService.bat +++ b/Scripts/SetupDevService.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 +CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") diff --git a/Scripts/StopAllServices.bat b/Scripts/StopAllServices.bat index 9139a6813b..3b45879091 100644 --- a/Scripts/StopAllServices.bat +++ b/Scripts/StopAllServices.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 +CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 call %VFS_SCRIPTSDIR%\StopService.bat GVFS.Service call %VFS_SCRIPTSDIR%\StopService.bat Test.GVFS.Service diff --git a/Scripts/UninstallGVFS.bat b/Scripts/UninstallGVFS.bat index 721a9893e6..b77ae76df7 100644 --- a/Scripts/UninstallGVFS.bat +++ b/Scripts/UninstallGVFS.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVfsDevShell.bat || EXIT /b 10 +CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 taskkill /F /T /FI "IMAGENAME eq git.exe" taskkill /F /T /FI "IMAGENAME eq GVFS.exe" From f27cec14de481d72b6f409687feb07e4bad309a5 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Fri, 19 Oct 2018 15:03:22 -0400 Subject: [PATCH 188/244] Trim down the list of macros and allow running specific functional tests --- Scripts/Macros.cmd.txt | 11 ++++------- Scripts/Macros.ps1 | 11 ++++------- init.cmd | 15 ++++++++------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/Scripts/Macros.cmd.txt b/Scripts/Macros.cmd.txt index 68c15c385e..58c57c01b2 100644 --- a/Scripts/Macros.cmd.txt +++ b/Scripts/Macros.cmd.txt @@ -1,7 +1,4 @@ -b=%VFS_SCRIPTSDIR%\BuildGVFSForWindows.bat $* -br=%VFS_SCRIPTSDIR%\BuildGVFSForWindows.bat Release $* -ut=%VFS_SCRIPTSDIR%\RunUnitTests.bat $* -utr=%VFS_SCRIPTSDIR%\RunUnitTests.bat Release $* -ft=%VFS_SCRIPTSDIR%\RunFunctionalTests.bat $* -ftr=%VFS_SCRIPTSDIR%\RunFunctionalTests.bat Release $* -scorch=%VFS_SCRIPTSDIR%\NukeBuildOutputs.bat $* +b=%VFS_SCRIPTSDIR%\BuildGVFSForWindows.bat Debug $* +ftest=%VFS_SCRIPTSDIR%\RunFunctionalTests.bat Debug --full-suite --test=$* +ig=%VFS_OUTPUTDIR%\GVFS.Build\InstallG4W.bat +iv=%VFS_OUTPUTDIR%\GVFS.Build\InstallGVFS.bat diff --git a/Scripts/Macros.ps1 b/Scripts/Macros.ps1 index 57ab73cae8..808190f815 100644 --- a/Scripts/Macros.ps1 +++ b/Scripts/Macros.ps1 @@ -1,7 +1,4 @@ -function global:b { & (Join-Path $Env:VFS_SCRIPTSDIR "BuildGVFSForWindows.bat") @args } -function global:br { & (Join-Path $Env:VFS_SCRIPTSDIR "BuildGVFSForWindows.bat") Release @args } -function global:ut { & (Join-Path $Env:VFS_SCRIPTSDIR "RunUnitTests.bat") @args } -function global:utr { & (Join-Path $Env:VFS_SCRIPTSDIR "RunUnitTests.bat") Release @args } -function global:ft { & (Join-Path $Env:VFS_SCRIPTSDIR "RunFunctionalTests.bat") @args } -function global:ftr { & (Join-Path $Env:VFS_SCRIPTSDIR "RunFunctionalTests.bat") Release @args } -function global:scorch { & (Join-Path $Env:VFS_SCRIPTSDIR "NukeBuildOutputs.bat") @args } +function global:b { & (Join-Path $Env:VFS_SCRIPTSDIR "BuildGVFSForWindows.bat") Debug $args } +function global:ftest { & (Join-Path $Env:VFS_SCRIPTSDIR "RunFunctionalTests.bat" ) Debug --full-suite "--test=$args" } +function global:ig { & (Join-Path $Env:VFS_OUTPUTDIR "GVFS.Build\InstallG4W.bat ") } +function global:iv { & (Join-Path $Env:VFS_OUTPUTDIR "GVFS.Build\InstallGVFS.bat ") } diff --git a/init.cmd b/init.cmd index eac963065a..09a9dad4fd 100644 --- a/init.cmd +++ b/init.cmd @@ -40,13 +40,14 @@ SET VFS_DEVSHELL=true TITLE VFS ^for Git Developer Shell ^(%VFS_SRCDIR%^) doskey /MACROFILE=.\Scripts\Macros.cmd.txt -ECHO ============================================================= -ECHO * Welcome to the VFS for Git developer shell * -ECHO * * -ECHO * Build: b Build release: br * -ECHO * Run unit tests: ut Unit test release: utr * -ECHO * Run functional tests: ft Functional test release: ftr * -ECHO ============================================================= +ECHO ============================================== +ECHO * Welcome to the VFS for Git developer shell * +ECHO * * +ECHO * Build: b * +ECHO * Install Git: ig * +ECHO * Install VFS for Git: iv * +ECHO * Run functional test: ftest [test name(s)] * +ECHO ============================================== GOTO :EOF From 01f78b675ffb3b91ad1a849f897e33698f0ae2ad Mon Sep 17 00:00:00 2001 From: John Briggs Date: Fri, 19 Oct 2018 15:26:48 -0400 Subject: [PATCH 189/244] Just test for the agent flag being set to something --- Scripts/EnsureVFSDevShell.bat | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Scripts/EnsureVFSDevShell.bat b/Scripts/EnsureVFSDevShell.bat index 547bca067e..078c85e66c 100644 --- a/Scripts/EnsureVFSDevShell.bat +++ b/Scripts/EnsureVFSDevShell.bat @@ -2,8 +2,10 @@ REM Maintain compat for current build definitions while we move to YAML by REM automatically initing when this is running on a build agent. -IF "%TF_BUILD%"=="true" ( - CALL %~dp0/../init.cmd +IF NOT "%TF_BUILD%"=="" ( + IF "%VFS_DEVSHELL%"=="" ( + CALL %~dp0/../init.cmd + ) ) REM Delete this block when we have moved to YAML-based pipelines. From 30eba6329f838a49e4e54f38a3a16ff549fc2658 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Fri, 19 Oct 2018 16:51:19 -0400 Subject: [PATCH 190/244] Allow running from devenv prompts, put nuget on the path --- init.cmd | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/init.cmd b/init.cmd index 09a9dad4fd..25522837f2 100644 --- a/init.cmd +++ b/init.cmd @@ -1,12 +1,5 @@ @ECHO OFF -REM Don't run inside of devenv shells -IF NOT "%VSCMD_VER%"=="" ( - ECHO ERROR: Do not run from a Developer Command prompt. - ECHO Use a plain shell and try again. - EXIT /b 10 -) - REM Don't run twice IF NOT "%VFS_DEVSHELL%"=="" ( ECHO ERROR: This shell is already a VFS for Git developer shell. @@ -32,6 +25,8 @@ SET VFS_PACKAGESDIR=%_PARSED_PATH_% CALL :RESOLVEPATH "%VFS_ENLISTMENTDIR%\.tools" SET VFS_TOOLSDIR=%_PARSED_PATH_% +SET PATH=%VFS_TOOLSDIR%;%PATH% + REM Clean up SET _PARSED_PATH_= From f4b5868bd79305ab4a40a8b9eeead3e77c5d96be Mon Sep 17 00:00:00 2001 From: John Briggs Date: Tue, 23 Oct 2018 13:35:30 -0400 Subject: [PATCH 191/244] Remove convenience macros from init --- Scripts/Macros.cmd.txt | 4 ---- Scripts/Macros.ps1 | 4 ---- init.cmd | 22 ---------------------- init.ps1 | 18 ------------------ 4 files changed, 48 deletions(-) delete mode 100644 Scripts/Macros.cmd.txt delete mode 100644 Scripts/Macros.ps1 delete mode 100644 init.ps1 diff --git a/Scripts/Macros.cmd.txt b/Scripts/Macros.cmd.txt deleted file mode 100644 index 58c57c01b2..0000000000 --- a/Scripts/Macros.cmd.txt +++ /dev/null @@ -1,4 +0,0 @@ -b=%VFS_SCRIPTSDIR%\BuildGVFSForWindows.bat Debug $* -ftest=%VFS_SCRIPTSDIR%\RunFunctionalTests.bat Debug --full-suite --test=$* -ig=%VFS_OUTPUTDIR%\GVFS.Build\InstallG4W.bat -iv=%VFS_OUTPUTDIR%\GVFS.Build\InstallGVFS.bat diff --git a/Scripts/Macros.ps1 b/Scripts/Macros.ps1 deleted file mode 100644 index 808190f815..0000000000 --- a/Scripts/Macros.ps1 +++ /dev/null @@ -1,4 +0,0 @@ -function global:b { & (Join-Path $Env:VFS_SCRIPTSDIR "BuildGVFSForWindows.bat") Debug $args } -function global:ftest { & (Join-Path $Env:VFS_SCRIPTSDIR "RunFunctionalTests.bat" ) Debug --full-suite "--test=$args" } -function global:ig { & (Join-Path $Env:VFS_OUTPUTDIR "GVFS.Build\InstallG4W.bat ") } -function global:iv { & (Join-Path $Env:VFS_OUTPUTDIR "GVFS.Build\InstallGVFS.bat ") } diff --git a/init.cmd b/init.cmd index 25522837f2..8eb6b786a7 100644 --- a/init.cmd +++ b/init.cmd @@ -1,11 +1,5 @@ @ECHO OFF -REM Don't run twice -IF NOT "%VFS_DEVSHELL%"=="" ( - ECHO ERROR: This shell is already a VFS for Git developer shell. - EXIT /b 20 -) - REM Set environment variables for interesting paths that scripts might need access to. PUSHD %~dp0 SET VFS_SRCDIR=%CD% @@ -25,25 +19,9 @@ SET VFS_PACKAGESDIR=%_PARSED_PATH_% CALL :RESOLVEPATH "%VFS_ENLISTMENTDIR%\.tools" SET VFS_TOOLSDIR=%_PARSED_PATH_% -SET PATH=%VFS_TOOLSDIR%;%PATH% - REM Clean up SET _PARSED_PATH_= -REM Mark this as a dev shell and load the macros -SET VFS_DEVSHELL=true -TITLE VFS ^for Git Developer Shell ^(%VFS_SRCDIR%^) -doskey /MACROFILE=.\Scripts\Macros.cmd.txt - -ECHO ============================================== -ECHO * Welcome to the VFS for Git developer shell * -ECHO * * -ECHO * Build: b * -ECHO * Install Git: ig * -ECHO * Install VFS for Git: iv * -ECHO * Run functional test: ftest [test name(s)] * -ECHO ============================================== - GOTO :EOF :RESOLVEPATH diff --git a/init.ps1 b/init.ps1 deleted file mode 100644 index dad733c8b0..0000000000 --- a/init.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -# Delegate the work to init.cmd and save off the environment it creates to a temp file. -$cwd = (Get-Location) -$tmp = [IO.Path]::GetTempFileName() -cmd.exe /c "$PSScriptRoot\init.cmd && set>$tmp" - -# Read the env vars from the temp file and set it into this shell. -$lines = [System.IO.File]::ReadAllLines("$tmp") -Set-Location Env: -foreach ($line in $lines) { - $envVar = $line.Split('=') - Set-Item -Path $envVar[0] -Value $envVar[1] -} -Set-Location $cwd - -# Set up macros. -& "$PSScriptRoot\Scripts\Macros.ps1" - -$Host.UI.RawUI.WindowTitle = "VFS for Git Developer Shell ($Env:VFS_SRCDIR)" From c08f0c7c8aec890dde93e69e2fb94d13c4aec8ab Mon Sep 17 00:00:00 2001 From: John Briggs Date: Tue, 23 Oct 2018 13:35:57 -0400 Subject: [PATCH 192/244] Move init script --- init.cmd => Scripts/InitializeEnvironment.bat | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename init.cmd => Scripts/InitializeEnvironment.bat (100%) diff --git a/init.cmd b/Scripts/InitializeEnvironment.bat similarity index 100% rename from init.cmd rename to Scripts/InitializeEnvironment.bat From 4821f22a449b029d2c4ac2cddeece5e6234059b8 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Tue, 23 Oct 2018 15:44:56 -0400 Subject: [PATCH 193/244] Remove *DevService.bat scripts --- Scripts/SetupDevService.bat | 6 ------ Scripts/StartDevService.bat | 4 ---- Scripts/StopDevService.bat | 4 ---- 3 files changed, 14 deletions(-) delete mode 100644 Scripts/SetupDevService.bat delete mode 100644 Scripts/StartDevService.bat delete mode 100644 Scripts/StopDevService.bat diff --git a/Scripts/SetupDevService.bat b/Scripts/SetupDevService.bat deleted file mode 100644 index 021103853b..0000000000 --- a/Scripts/SetupDevService.bat +++ /dev/null @@ -1,6 +0,0 @@ -@ECHO OFF -CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 - -IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") - -sc create %Configuration%.GVFS.Service binPath=%VFS_OUTPUTDIR%\GVFS.Service\bin\x64\%Configuration%\GVFS.Service.exe \ No newline at end of file diff --git a/Scripts/StartDevService.bat b/Scripts/StartDevService.bat deleted file mode 100644 index d3675c0c15..0000000000 --- a/Scripts/StartDevService.bat +++ /dev/null @@ -1,4 +0,0 @@ -@ECHO OFF -IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") - -sc start %Configuration%.GVFS.Service --servicename=%Configuration%.GVFS.Service \ No newline at end of file diff --git a/Scripts/StopDevService.bat b/Scripts/StopDevService.bat deleted file mode 100644 index d0d779ad4d..0000000000 --- a/Scripts/StopDevService.bat +++ /dev/null @@ -1,4 +0,0 @@ -@ECHO OFF -IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") - -sc stop %Configuration%.GVFS.Service \ No newline at end of file From 32fc0cf741aa6ab0f44c29b62cdc7c096499cfde Mon Sep 17 00:00:00 2001 From: John Briggs Date: Tue, 23 Oct 2018 15:45:41 -0400 Subject: [PATCH 194/244] Make Windows scripts init the environment --- Scripts/BuildGVFSForWindows.bat | 2 +- Scripts/EnsureVFSDevShell.bat | 16 ---------------- Scripts/InitializeEnvironment.bat | 17 ++++++----------- Scripts/NukeBuildOutputs.bat | 2 +- Scripts/ReinstallGVFS.bat | 2 +- Scripts/RestorePackages.bat | 2 +- Scripts/RunFunctionalTests.bat | 2 +- Scripts/RunUnitTests.bat | 2 +- Scripts/StopAllServices.bat | 2 +- Scripts/UninstallGVFS.bat | 2 +- 10 files changed, 14 insertions(+), 35 deletions(-) delete mode 100644 Scripts/EnsureVFSDevShell.bat diff --git a/Scripts/BuildGVFSForWindows.bat b/Scripts/BuildGVFSForWindows.bat index 23c73026d3..966e7b4d04 100644 --- a/Scripts/BuildGVFSForWindows.bat +++ b/Scripts/BuildGVFSForWindows.bat @@ -1,7 +1,7 @@ @ECHO OFF SETLOCAL -CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 +CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") IF "%2"=="" (SET "GVFSVersion=0.2.173.2") ELSE (SET "GVFSVersion=%2") diff --git a/Scripts/EnsureVFSDevShell.bat b/Scripts/EnsureVFSDevShell.bat deleted file mode 100644 index 078c85e66c..0000000000 --- a/Scripts/EnsureVFSDevShell.bat +++ /dev/null @@ -1,16 +0,0 @@ -@ECHO OFF - -REM Maintain compat for current build definitions while we move to YAML by -REM automatically initing when this is running on a build agent. -IF NOT "%TF_BUILD%"=="" ( - IF "%VFS_DEVSHELL%"=="" ( - CALL %~dp0/../init.cmd - ) -) -REM Delete this block when we have moved to YAML-based pipelines. - -IF NOT "%VFS_DEVSHELL%"=="true" ( - ECHO ERROR: This shell is not a VFS for Git developer shell. - ECHO Run init.cmd or init.ps1 at the root of the repository. - EXIT /b 10 -) diff --git a/Scripts/InitializeEnvironment.bat b/Scripts/InitializeEnvironment.bat index 8eb6b786a7..eaa1d66922 100644 --- a/Scripts/InitializeEnvironment.bat +++ b/Scripts/InitializeEnvironment.bat @@ -2,22 +2,17 @@ REM Set environment variables for interesting paths that scripts might need access to. PUSHD %~dp0 -SET VFS_SRCDIR=%CD% +SET VFS_SCRIPTSDIR=%CD% -CALL :RESOLVEPATH "%VFS_SRCDIR%\Scripts" -SET VFS_SCRIPTSDIR=%_PARSED_PATH_% +CALL :RESOLVEPATH "%VFS_SCRIPTSDIR%\.." +SET VFS_SRCDIR=%_PARSED_PATH_% CALL :RESOLVEPATH "%VFS_SRCDIR%\.." SET VFS_ENLISTMENTDIR=%_PARSED_PATH_% -CALL :RESOLVEPATH "%VFS_ENLISTMENTDIR%\BuildOutput" -SET VFS_OUTPUTDIR=%_PARSED_PATH_% - -CALL :RESOLVEPATH "%VFS_ENLISTMENTDIR%\packages" -SET VFS_PACKAGESDIR=%_PARSED_PATH_% - -CALL :RESOLVEPATH "%VFS_ENLISTMENTDIR%\.tools" -SET VFS_TOOLSDIR=%_PARSED_PATH_% +SET VFS_OUTPUTDIR=%VFS_ENLISTMENTDIR%\BuildOutput +SET VFS_PACKAGESDIR=%VFS_ENLISTMENTDIR%\packages +SET VFS_TOOLSDIR=%VFS_ENLISTMENTDIR%\.tools REM Clean up SET _PARSED_PATH_= diff --git a/Scripts/NukeBuildOutputs.bat b/Scripts/NukeBuildOutputs.bat index 77b5b8a373..a8b9c2ee2e 100644 --- a/Scripts/NukeBuildOutputs.bat +++ b/Scripts/NukeBuildOutputs.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 +CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 taskkill /f /im GVFS.Mount.exe 2>&1 verify >nul diff --git a/Scripts/ReinstallGVFS.bat b/Scripts/ReinstallGVFS.bat index b604160041..8d47bea7a7 100644 --- a/Scripts/ReinstallGVFS.bat +++ b/Scripts/ReinstallGVFS.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 +CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") diff --git a/Scripts/RestorePackages.bat b/Scripts/RestorePackages.bat index 5c01dd8ef3..b4c6feeafd 100644 --- a/Scripts/RestorePackages.bat +++ b/Scripts/RestorePackages.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 +CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 SETLOCAL diff --git a/Scripts/RunFunctionalTests.bat b/Scripts/RunFunctionalTests.bat index 51e3d00104..44e175a022 100644 --- a/Scripts/RunFunctionalTests.bat +++ b/Scripts/RunFunctionalTests.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 +CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") diff --git a/Scripts/RunUnitTests.bat b/Scripts/RunUnitTests.bat index ec834d06c8..3e9af434d6 100644 --- a/Scripts/RunUnitTests.bat +++ b/Scripts/RunUnitTests.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 +CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") diff --git a/Scripts/StopAllServices.bat b/Scripts/StopAllServices.bat index 3b45879091..7aed517dbb 100644 --- a/Scripts/StopAllServices.bat +++ b/Scripts/StopAllServices.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 +CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 call %VFS_SCRIPTSDIR%\StopService.bat GVFS.Service call %VFS_SCRIPTSDIR%\StopService.bat Test.GVFS.Service diff --git a/Scripts/UninstallGVFS.bat b/Scripts/UninstallGVFS.bat index b77ae76df7..fbfe281349 100644 --- a/Scripts/UninstallGVFS.bat +++ b/Scripts/UninstallGVFS.bat @@ -1,5 +1,5 @@ @ECHO OFF -CALL %~dp0\EnsureVFSDevShell.bat || EXIT /b 10 +CALL %~dp0\InitializeEnvironment.bat || EXIT /b 10 taskkill /F /T /FI "IMAGENAME eq git.exe" taskkill /F /T /FI "IMAGENAME eq GVFS.exe" From 15fa4f1acd0a4497180ef14ca097bbbfef2646b8 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Fri, 19 Oct 2018 17:17:32 -0700 Subject: [PATCH 195/244] Ignore symlinks when 'prefetch --hydrate'-ing. Update ParseFromLsTreeLine to detect a mode of 120000 which indicates a symlink. --- GVFS/FastFetch/CheckoutJob.cs | 2 +- GVFS/GVFS.Common/Git/DiffTreeResult.cs | 8 ++ GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs | 2 +- GVFS/GVFS.Common/Prefetch/Git/DiffHelper.cs | 18 +++- GVFS/GVFS.UnitTests/Data/forward.txt | 1 + .../Prefetch/DiffHelperTests.cs | 93 ++++++++++++------- .../Prefetch/DiffTreeResultTests.cs | 19 ++++ 7 files changed, 101 insertions(+), 42 deletions(-) diff --git a/GVFS/FastFetch/CheckoutJob.cs b/GVFS/FastFetch/CheckoutJob.cs index a86079f22d..222f0b1177 100644 --- a/GVFS/FastFetch/CheckoutJob.cs +++ b/GVFS/FastFetch/CheckoutJob.cs @@ -39,7 +39,7 @@ public CheckoutJob(int maxParallel, IEnumerable folderList, string targe { this.tracer = tracer.StartActivity(AreaPath, EventLevel.Informational, Keywords.Telemetry, metadata: null); this.enlistment = enlistment; - this.diff = new DiffHelper(tracer, enlistment, new string[0], folderList); + this.diff = new DiffHelper(tracer, enlistment, new string[0], folderList, includeSymLinks: true); this.targetCommitSha = targetCommitSha; this.forceCheckout = forceCheckout; this.AvailableBlobShas = new BlockingCollection(); diff --git a/GVFS/GVFS.Common/Git/DiffTreeResult.cs b/GVFS/GVFS.Common/Git/DiffTreeResult.cs index 6e1167beab..e35b3c00d5 100644 --- a/GVFS/GVFS.Common/Git/DiffTreeResult.cs +++ b/GVFS/GVFS.Common/Git/DiffTreeResult.cs @@ -28,6 +28,7 @@ public enum Operations public Operations Operation { get; set; } public bool SourceIsDirectory { get; set; } public bool TargetIsDirectory { get; set; } + public bool TargetIsSymLink { get; set; } public string TargetPath { get; set; } public string SourceSha { get; set; } public string TargetSha { get; set; } @@ -82,6 +83,12 @@ public static DiffTreeResult ParseFromDiffTreeLine(string line, string repoRoot) DiffTreeResult result = new DiffTreeResult(); result.SourceIsDirectory = ValidTreeModes.Contains(parts[0]); result.TargetIsDirectory = ValidTreeModes.Contains(parts[1]); + + if (!result.TargetIsDirectory) + { + result.TargetIsSymLink = parts[1] == "120000"; + } + result.SourceSha = parts[2]; result.TargetSha = parts[3]; result.Operation = DiffTreeResult.ParseOperation(parts[4]); @@ -146,6 +153,7 @@ public static DiffTreeResult ParseFromLsTreeLine(string line, string repoRoot) if (blobIndex >= 0) { DiffTreeResult blobAdd = new DiffTreeResult(); + blobAdd.TargetIsSymLink = line.StartsWith("120000"); blobAdd.TargetSha = line.Substring(blobIndex + BlobMarker.Length, GVFSConstants.ShaStringLength); blobAdd.TargetPath = ConvertPathToAbsoluteUtf8Path(repoRoot, line.Substring(line.LastIndexOf("\t") + 1)); blobAdd.Operation = DiffTreeResult.Operations.Add; diff --git a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs index 66f1ad0f46..742cc15f67 100644 --- a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs @@ -229,7 +229,7 @@ public void PrefetchWithStats( } } - DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList); + DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList, includeSymLinks: false); ThreadStart performDiff = () => { diff --git a/GVFS/GVFS.Common/Prefetch/Git/DiffHelper.cs b/GVFS/GVFS.Common/Prefetch/Git/DiffHelper.cs index 1e45f42e7e..dedd497380 100644 --- a/GVFS/GVFS.Common/Prefetch/Git/DiffHelper.cs +++ b/GVFS/GVFS.Common/Prefetch/Git/DiffHelper.cs @@ -22,26 +22,29 @@ public class DiffHelper private HashSet stagedFileDeletes = new HashSet(StringComparer.OrdinalIgnoreCase); private Enlistment enlistment; - private GitProcess git; + private GitProcess git; - public DiffHelper(ITracer tracer, Enlistment enlistment, IEnumerable fileList, IEnumerable folderList) - : this(tracer, enlistment, new GitProcess(enlistment), fileList, folderList) + public DiffHelper(ITracer tracer, Enlistment enlistment, IEnumerable fileList, IEnumerable folderList, bool includeSymLinks) + : this(tracer, enlistment, new GitProcess(enlistment), fileList, folderList, includeSymLinks) { } - public DiffHelper(ITracer tracer, Enlistment enlistment, GitProcess git, IEnumerable fileList, IEnumerable folderList) + public DiffHelper(ITracer tracer, Enlistment enlistment, GitProcess git, IEnumerable fileList, IEnumerable folderList, bool includeSymLinks) { this.tracer = tracer; this.fileList = new List(fileList); this.folderList = new List(folderList); this.enlistment = enlistment; this.git = git; + this.ShouldIncludeSymLinks = includeSymLinks; this.DirectoryOperations = new ConcurrentQueue(); this.FileDeleteOperations = new ConcurrentQueue(); this.FileAddOperations = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); this.RequiredBlobs = new BlockingCollection(); - } + } + + public bool ShouldIncludeSymLinks { get; set; } public bool HasFailures { get; private set; } @@ -313,6 +316,11 @@ private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot private bool ShouldIncludeResult(DiffTreeResult blobAdd) { + if (blobAdd.TargetIsSymLink && !this.ShouldIncludeSymLinks) + { + return false; + } + if (blobAdd.TargetPath == null) { return true; diff --git a/GVFS/GVFS.UnitTests/Data/forward.txt b/GVFS/GVFS.UnitTests/Data/forward.txt index e3fb40b616..9a244da99b 100644 --- a/GVFS/GVFS.UnitTests/Data/forward.txt +++ b/GVFS/GVFS.UnitTests/Data/forward.txt @@ -24,3 +24,4 @@ :000000 040000 0000000000000000000000000000000000000000 e30358a677e3a55aa838d0bbe0207f61ce40f401 A folderWasRenamed :000000 100644 0000000000000000000000000000000000000000 35151eac3bb089589836bfece4dfbb84fae502de A folderWasRenamed/existence.txt :000000 100644 0000000000000000000000000000000000000000 9ff3c67e2792e4c17672c6b04a8a6efe732cb07a A newFile.txt +:000000 120000 0000000000000000000000000000000000000000 3bd509d373734a9f9685d6a73ba73324f72931e3 A symLinkToBeCreated.txt \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests/Prefetch/DiffHelperTests.cs b/GVFS/GVFS.UnitTests/Prefetch/DiffHelperTests.cs index e0ebb3eade..e7aded0016 100644 --- a/GVFS/GVFS.UnitTests/Prefetch/DiffHelperTests.cs +++ b/GVFS/GVFS.UnitTests/Prefetch/DiffHelperTests.cs @@ -11,45 +11,68 @@ namespace GVFS.UnitTests.Prefetch { - [TestFixture] + [TestFixtureSource(TestRunners)] public class DiffHelperTests - { - // Make two commits. The first should look like this: - // recursiveDelete - // recursiveDelete/subfolder - // recursiveDelete/subfolder/childFile.txt - // fileToBecomeFolder - // fileToDelete.txt - // fileToEdit.txt - // fileToRename.txt - // fileToRenameEdit.txt - // folderToBeFile - // folderToBeFile/childFile.txt - // folderToDelete - // folderToDelete/childFile.txt - // folderToEdit - // folderToEdit/childFile.txt - // folderToRename - // folderToRename/childFile.txt - // - // The second should follow the action indicated by the file/folder name: - // eg. recursiveDelete should run "rmdir /s/q recursiveDelete" - // eg. folderToBeFile should be deleted and replaced with a file of the same name - // Note that each childFile.txt should have unique contents, but is only a placeholder to force git to add a folder. - // - // Then to generate the diffs, run: - // git diff-tree -r -t Head~1 Head > forward.txt - // git diff-tree -r -t Head Head ~1 > backward.txt + { + public const string TestRunners = "Runners"; + + private static readonly bool[] SymLinkSupport = new bool[] + { + true, + false, + }; + + public DiffHelperTests(bool symLinkSupport) + { + this.IncludeSymLinks = symLinkSupport; + } + + public static bool[] Runners + { + get { return SymLinkSupport; } + } + + public bool IncludeSymLinks { get; set; } + + // Make two commits. The first should look like this: + // recursiveDelete + // recursiveDelete/subfolder + // recursiveDelete/subfolder/childFile.txt + // fileToBecomeFolder + // fileToDelete.txt + // fileToEdit.txt + // fileToRename.txt + // fileToRenameEdit.txt + // folderToBeFile + // folderToBeFile/childFile.txt + // folderToDelete + // folderToDelete/childFile.txt + // folderToEdit + // folderToEdit/childFile.txt + // folderToRename + // folderToRename/childFile.txt + // symLinkToBeCreated.txt + // + // The second should follow the action indicated by the file/folder name: + // eg. recursiveDelete should run "rmdir /s/q recursiveDelete" + // eg. folderToBeFile should be deleted and replaced with a file of the same name + // Note that each childFile.txt should have unique contents, but is only a placeholder to force git to add a folder. + // + // Then to generate the diffs, run: + // git diff-tree -r -t Head~1 Head > forward.txt + // git diff-tree -r -t Head Head ~1 > backward.txt [TestCase] public void CanParseDiffForwards() { MockTracer tracer = new MockTracer(); - DiffHelper diffForwards = new DiffHelper(tracer, new MockGVFSEnlistment(), new List(), new List()); + DiffHelper diffForwards = new DiffHelper(tracer, new MockGVFSEnlistment(), new List(), new List(), includeSymLinks: this.IncludeSymLinks); diffForwards.ParseDiffFile(GetDataPath("forward.txt"), "xx:\\fakeRepo"); - // File added, file edited, file renamed, folder => file, edit-rename file + // File added, file edited, file renamed, folder => file, edit-rename file, SymLink added (if applicable) // Children of: Add folder, Renamed folder, edited folder, file => folder - diffForwards.RequiredBlobs.Count.ShouldEqual(9); + diffForwards.RequiredBlobs.Count.ShouldEqual(diffForwards.ShouldIncludeSymLinks ? 10 : 9); + + diffForwards.FileAddOperations.ContainsKey("3bd509d373734a9f9685d6a73ba73324f72931e3").ShouldEqual(diffForwards.ShouldIncludeSymLinks); // File deleted, folder deleted, file > folder, edit-rename diffForwards.FileDeleteOperations.Count.ShouldEqual(4); @@ -70,7 +93,7 @@ public void CanParseDiffForwards() public void CanParseBackwardsDiff() { MockTracer tracer = new MockTracer(); - DiffHelper diffBackwards = new DiffHelper(tracer, new Mock.Common.MockGVFSEnlistment(), new List(), new List()); + DiffHelper diffBackwards = new DiffHelper(tracer, new Mock.Common.MockGVFSEnlistment(), new List(), new List(), includeSymLinks: this.IncludeSymLinks); diffBackwards.ParseDiffFile(GetDataPath("backward.txt"), "xx:\\fakeRepo"); // File > folder, deleted file, edited file, renamed file, rename-edit file @@ -94,7 +117,7 @@ public void CanParseBackwardsDiff() public void ParsesCaseChangesAsAdds() { MockTracer tracer = new MockTracer(); - DiffHelper diffBackwards = new DiffHelper(tracer, new Mock.Common.MockGVFSEnlistment(), new List(), new List()); + DiffHelper diffBackwards = new DiffHelper(tracer, new Mock.Common.MockGVFSEnlistment(), new List(), new List(), includeSymLinks: this.IncludeSymLinks); diffBackwards.ParseDiffFile(GetDataPath("caseChange.txt"), "xx:\\fakeRepo"); diffBackwards.RequiredBlobs.Count.ShouldEqual(2); @@ -114,7 +137,7 @@ public void DetectsFailuresInDiffTree() MockGitProcess gitProcess = new MockGitProcess(); gitProcess.SetExpectedCommandResult("diff-tree -r -t sha1 sha2", () => new GitProcess.Result(string.Empty, string.Empty, 1)); - DiffHelper diffBackwards = new DiffHelper(tracer, new Mock.Common.MockGVFSEnlistment(), gitProcess, new List(), new List()); + DiffHelper diffBackwards = new DiffHelper(tracer, new Mock.Common.MockGVFSEnlistment(), gitProcess, new List(), new List(), includeSymLinks: this.IncludeSymLinks); diffBackwards.PerformDiff("sha1", "sha2"); diffBackwards.HasFailures.ShouldEqual(true); } @@ -126,7 +149,7 @@ public void DetectsFailuresInLsTree() MockGitProcess gitProcess = new MockGitProcess(); gitProcess.SetExpectedCommandResult("ls-tree -r -t sha1", () => new GitProcess.Result(string.Empty, string.Empty, 1)); - DiffHelper diffBackwards = new DiffHelper(tracer, new Mock.Common.MockGVFSEnlistment(), gitProcess, new List(), new List()); + DiffHelper diffBackwards = new DiffHelper(tracer, new Mock.Common.MockGVFSEnlistment(), gitProcess, new List(), new List(), includeSymLinks: this.IncludeSymLinks); diffBackwards.PerformDiff(null, "sha1"); diffBackwards.HasFailures.ShouldEqual(true); } diff --git a/GVFS/GVFS.UnitTests/Prefetch/DiffTreeResultTests.cs b/GVFS/GVFS.UnitTests/Prefetch/DiffTreeResultTests.cs index 588562f86a..a44606054d 100644 --- a/GVFS/GVFS.UnitTests/Prefetch/DiffTreeResultTests.cs +++ b/GVFS/GVFS.UnitTests/Prefetch/DiffTreeResultTests.cs @@ -33,6 +33,7 @@ public class DiffTreeResultTests private static readonly string BlobLineFromLsTree = $"100644 blob {TestSha1}\t{TestTreePath1}"; private static readonly string TreeLineFromLsTree = $"040000 tree {TestSha1}\t{TestTreePath1}"; private static readonly string InvalidLineFromLsTree = $"040000 bad {TestSha1}\t{TestTreePath1}"; + private static readonly string SymLinkLineFromLsTree = $"120000 blob {TestSha1}\t{TestTreePath1}"; [TestCase] [Category(CategoryConstants.ExceptionExpected)] @@ -300,6 +301,24 @@ public void ParseFromDiffTreeLine_ModifyBlobLine() this.ValidateDiffTreeResult(expected, result); } + [TestCase] + public void ParseFromLsTreeLine_SymLinkLine() + { + DiffTreeResult expected = new DiffTreeResult() + { + Operation = DiffTreeResult.Operations.Add, + SourceIsDirectory = false, + TargetIsDirectory = false, + TargetIsSymLink = true, + TargetPath = Path.Combine(RepoRoot, TestTreePath1.Replace('/', Path.DirectorySeparatorChar)), + SourceSha = null, + TargetSha = TestSha1 + }; + + DiffTreeResult result = DiffTreeResult.ParseFromLsTreeLine(SymLinkLineFromLsTree, RepoRoot); + this.ValidateDiffTreeResult(expected, result); + } + private static string CreateTreePath(string testPath) { return Path.Combine(RepoRoot, testPath.Replace('/', Path.DirectorySeparatorChar)) + Path.DirectorySeparatorChar; From 8bb0fc761320e67098a3acd71cb32a860a08ce10 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 23 Oct 2018 15:01:30 -0700 Subject: [PATCH 196/244] First round of changes for PR feedback --- GVFS/GVFS.Common/GitCommandLineParser.cs | 7 ++++++ GVFS/GVFS.Common/PlaceholderListDatabase.cs | 4 ++-- .../Background/FileSystemTask.cs | 6 ++--- .../FileSystemCallbacks.cs | 10 +++++--- .../GitIndexProjection.GitIndexEntry.cs | 9 ++++++-- .../GitIndexProjection.GitIndexParser.cs | 12 +++++----- .../Projection/GitIndexProjection.cs | 23 +++++++++++-------- 7 files changed, 46 insertions(+), 25 deletions(-) diff --git a/GVFS/GVFS.Common/GitCommandLineParser.cs b/GVFS/GVFS.Common/GitCommandLineParser.cs index e57305a818..c2b207fb12 100644 --- a/GVFS/GVFS.Common/GitCommandLineParser.cs +++ b/GVFS/GVFS.Common/GitCommandLineParser.cs @@ -48,6 +48,13 @@ public bool IsValidGitCommand get { return this.parts != null; } } + public bool IsResetMixed() + { + return + this.IsResetSoftOrMixed() && + !this.HasArgument("--soft"); + } + public bool IsResetSoftOrMixed() { return diff --git a/GVFS/GVFS.Common/PlaceholderListDatabase.cs b/GVFS/GVFS.Common/PlaceholderListDatabase.cs index a4750cf110..667503b8df 100644 --- a/GVFS/GVFS.Common/PlaceholderListDatabase.cs +++ b/GVFS/GVFS.Common/PlaceholderListDatabase.cs @@ -167,7 +167,7 @@ public void GetAllEntries(out List filePlaceholders, out List

filePlaceholdersByPath) + public Dictionary GetAllFileEntries() { try { @@ -190,7 +190,7 @@ public void GetAllFileEntries(out Dictionary x.GetString())); + return this.GetPath(GVFSConstants.GitPathSeparatorString); } public string GetRelativePath() { - return string.Join(PathSeparatorString, this.PathParts.Take(this.NumParts).Select(x => x.GetString())); + return this.GetPath(PathSeparatorString); + } + + private string GetPath(string separator) + { + return string.Join(separator, this.PathParts.Take(this.NumParts).Select(x => x.GetString())); } private unsafe bool RangeContains(byte* bufferPtr, int count, byte value) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index fcf36d012e..6807120eea 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -79,13 +79,13 @@ public FileSystemTaskResult AddMissingModifiedFilesAndRemoveThemFromPlaceholderL throw new InvalidOperationException($"{nameof(this.projection)} cannot be null when calling {nameof(AddMissingModifiedFilesAndRemoveThemFromPlaceholderList)}"); } - Dictionary filePlaceholders; - this.projection.placeholderList.GetAllFileEntries(out filePlaceholders); + Dictionary filePlaceholders = + this.projection.placeholderList.GetAllFileEntries(); FileSystemTaskResult result = this.ParseIndex( tracer, indexStream, - (data) => this.AdjustModifedPathsAndPlaceholdersForIndexEntry(data, filePlaceholders)); + (data) => this.AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfNeeded(data, filePlaceholders)); if (result != FileSystemTaskResult.Success) { @@ -139,12 +139,12 @@ private FileSystemTaskResult AddIndexEntryToProjection(GitIndexEntry data) ///

/// Index entry /// - /// Dictionary of file placeholders. AdjustModifedPathsAndPlaceholdersForIndexEntry will + /// Dictionary of file placeholders. AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfNeeded will /// remove enties from filePlaceholders as they are found in the index. After - /// AdjustModifedPathsAndPlaceholdersForIndexEntry is called for all entries in the index + /// AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfNeeded is called for all entries in the index /// filePlaceholders will contain only those placeholders that are not in the index. /// - private FileSystemTaskResult AdjustModifedPathsAndPlaceholdersForIndexEntry( + private FileSystemTaskResult AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfNeeded( GitIndexEntry gitIndexEntry, Dictionary filePlaceholders) { diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 6b18b89596..1d0b347f9d 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -529,16 +529,21 @@ public FileSystemTaskResult AddMissingModifiedFiles() { if (this.modifiedFilesInvalid) { - FileSystemTaskResult result = this.indexParser.AddMissingModifiedFilesAndRemoveThemFromPlaceholderList( - this.context.Tracer, - this.indexFileStream); - - if (result == FileSystemTaskResult.Success) - { - this.modifiedFilesInvalid = false; + using (ITracer activity = this.context.Tracer.StartActivity( + nameof(this.indexParser.AddMissingModifiedFilesAndRemoveThemFromPlaceholderList), + EventLevel.Informational)) + { + FileSystemTaskResult result = this.indexParser.AddMissingModifiedFilesAndRemoveThemFromPlaceholderList( + activity, + this.indexFileStream); + + if (result == FileSystemTaskResult.Success) + { + this.modifiedFilesInvalid = false; + } + + return result; } - - return result; } } catch (IOException e) From 8705befb15a4d1fc84cf50502be1c4cbc8a75d2c Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 23 Oct 2018 16:55:09 -0700 Subject: [PATCH 197/244] Eliminate LazyPath race condition by not using them on BG task thread --- GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs | 2 +- .../Projection/GitIndexEntryTests.cs | 40 ++++++++-- .../GitIndexProjection.GitIndexEntry.cs | 77 ++++++++++++++++--- .../GitIndexProjection.GitIndexParser.cs | 53 ++++++++----- .../Projection/GitIndexProjection.cs | 10 +-- 5 files changed, 140 insertions(+), 42 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs index 52157be555..5378ffa6db 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs @@ -70,7 +70,7 @@ public override void Dispose() // It's possible that errors from a previous operation (e.g. write(2)) // are only reported in close(). We should *not* retry the close() if // it fails since it may cause a re-used file descriptor from another - // thrad to be closed. + // thread to be closed. int errno = Marshal.GetLastWin32Error(); EventMetadata metadata = this.CreateEventMetadata(errno); diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs index b491cff3ab..9267d2acd7 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs @@ -8,10 +8,30 @@ namespace GVFS.UnitTests.Virtualization.Git { - [TestFixture] + [TestFixtureSource(TestLazyPaths)] public class GitIndexEntryTests { + public const string TestLazyPaths = "ShouldTestLazyPaths"; + private const int DefaultIndexEntryCount = 10; + private bool shouldTestLazyPaths; + + public GitIndexEntryTests(bool shouldTestLazyPaths) + { + this.shouldTestLazyPaths = shouldTestLazyPaths; + } + + public static object[] ShouldTestLazyPaths + { + get + { + return new object[] + { + new object[] { true }, + new object[] { false }, + }; + } + } [OneTimeSetUp] public void Setup() @@ -137,7 +157,7 @@ public void ClearLastParent() private GitIndexEntry SetupIndexEntry(string path) { - GitIndexEntry indexEntry = new GitIndexEntry(); + GitIndexEntry indexEntry = new GitIndexEntry(this.shouldTestLazyPaths); this.ParsePathForIndexEntry(indexEntry, path, replaceIndex: 0); return indexEntry; } @@ -157,11 +177,21 @@ private void TestPathParts(GitIndexEntry indexEntry, string[] pathParts, bool ha indexEntry.NumParts.ShouldEqual(pathParts.Length, nameof(indexEntry.NumParts)); for (int i = 0; i < pathParts.Length; i++) { - indexEntry.PathParts[i].ShouldNotBeNull(); - indexEntry.PathParts[i].GetString().ShouldEqual(pathParts[i]); + if (this.shouldTestLazyPaths) + { + indexEntry.GetLazyPathPart(i).ShouldNotBeNull(); + indexEntry.GetLazyPathPart(i).GetString().ShouldEqual(pathParts[i]); + } + + indexEntry.GetPathPart(i).ShouldNotBeNull(); + indexEntry.GetPathPart(i).ShouldEqual(pathParts[i]); + } + + if (this.shouldTestLazyPaths) + { + indexEntry.GetLazyChildName().GetString().ShouldEqual(pathParts[pathParts.Length - 1]); } - indexEntry.GetChildName().GetString().ShouldEqual(pathParts[pathParts.Length - 1]); indexEntry.GetGitPath().ShouldEqual(string.Join("/", pathParts)); indexEntry.GetRelativePath().ShouldEqual(string.Join(Path.DirectorySeparatorChar.ToString(), pathParts)); } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 791d2d01e2..35c6ca38ea 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -1,7 +1,9 @@ using GVFS.Common; +using System; using System.IO; using System.Linq; - +using System.Text; + namespace GVFS.Virtualization.Projection { public partial class GitIndexProjection @@ -23,9 +25,19 @@ internal class GitIndexEntry private int previousFinalSeparatorIndex = int.MaxValue; - public GitIndexEntry() + private LazyUTF8String[] lazyPathParts; + private string[] utf16PathParts; + + public GitIndexEntry(bool useLazyPaths) { - this.PathParts = new LazyUTF8String[MaxParts]; + if (useLazyPaths) + { + this.lazyPathParts = new LazyUTF8String[MaxParts]; + } + else + { + this.utf16PathParts = new string[MaxParts]; + } } public byte[] Sha { get; } = new byte[20]; @@ -41,19 +53,35 @@ public GitIndexEntry() public byte[] PathBuffer { get; } = new byte[MaxPathBufferSize]; public FolderData LastParent { get; set; } - public LazyUTF8String[] PathParts + public int NumParts { get; private set; } - public int NumParts + public bool HasSameParentAsLastEntry { get; private set; } - public bool HasSameParentAsLastEntry + public string GetPathPart(int index) { - get; private set; + if (this.lazyPathParts != null) + { + return this.lazyPathParts[index].GetString(); + } + + return this.utf16PathParts[index]; + } + + public LazyUTF8String GetLazyPathPart(int index) + { + if (this.lazyPathParts == null) + { + throw new InvalidOperationException( + $"{nameof(GetLazyPathPart)} can only be called when useLazyPaths is set to true when creating {nameof(GitIndexEntry)}"); + } + + return this.lazyPathParts[index]; } public unsafe void ParsePath() @@ -108,11 +136,22 @@ public unsafe void ParsePath() int partIndex = this.NumParts; byte* forLoopPtr = pathPtr + forLoopStartIndex; + byte* bufferPtr; + int bufferLength; for (int i = forLoopStartIndex; i < this.PathLength + 1; i++) { if (*forLoopPtr == PathSeparatorCode) { - this.PathParts[partIndex] = LazyUTF8String.FromByteArray(pathPtr + currentPartStartIndex, i - currentPartStartIndex); + bufferPtr = pathPtr + currentPartStartIndex; + bufferLength = i - currentPartStartIndex; + if (this.lazyPathParts != null) + { + this.lazyPathParts[partIndex] = LazyUTF8String.FromByteArray(bufferPtr, bufferLength); + } + else + { + this.utf16PathParts[partIndex] = Encoding.UTF8.GetString(bufferPtr, bufferLength); + } partIndex++; currentPartStartIndex = i + 1; @@ -125,7 +164,16 @@ public unsafe void ParsePath() } // We unrolled the final part calculation to after the loop, to avoid having to do a 0-byte check inside the for loop - this.PathParts[partIndex] = LazyUTF8String.FromByteArray(pathPtr + currentPartStartIndex, this.PathLength - currentPartStartIndex); + bufferPtr = pathPtr + currentPartStartIndex; + bufferLength = this.PathLength - currentPartStartIndex; + if (this.lazyPathParts != null) + { + this.lazyPathParts[partIndex] = LazyUTF8String.FromByteArray(bufferPtr, bufferLength); + } + else + { + this.utf16PathParts[partIndex] = Encoding.UTF8.GetString(bufferPtr, bufferLength); + } this.NumParts++; } @@ -138,9 +186,9 @@ public void ClearLastParent() this.LastParent = null; } - public LazyUTF8String GetChildName() + public LazyUTF8String GetLazyChildName() { - return this.PathParts[this.NumParts - 1]; + return this.GetLazyPathPart(this.NumParts - 1); } public string GetGitPath() @@ -155,7 +203,12 @@ public string GetRelativePath() private string GetPath(string separator) { - return string.Join(separator, this.PathParts.Take(this.NumParts).Select(x => x.GetString())); + if (this.lazyPathParts != null) + { + return string.Join(separator, this.lazyPathParts.Take(this.NumParts).Select(x => x.GetString())); + } + + return string.Join(separator, this.utf16PathParts.Take(this.NumParts)); } private unsafe bool RangeContains(byte* bufferPtr, int count, byte value) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 6807120eea..238841e22c 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -24,9 +24,14 @@ internal partial class GitIndexParser private GitIndexProjection projection; /// - /// A single GitIndexEntry instance used for parsing all entries in the index + /// A single GitIndexEntry instance used for parsing all entries in the index when building the projection /// - private GitIndexEntry resuableParsedIndexEntry = new GitIndexEntry(); + private GitIndexEntry resuableProjectionBuildingIndexEntry = new GitIndexEntry(useLazyPaths: true); + + /// + /// A single GitIndexEntry instance used by the background task thread for parsing all entries in the index + /// + private GitIndexEntry resuableBackgroundTaskThreadIndexEntry = new GitIndexEntry(useLazyPaths: false); public GitIndexParser(GitIndexProjection projection) { @@ -45,7 +50,7 @@ public enum MergeStage : byte public static void ValidateIndex(ITracer tracer, Stream indexStream) { GitIndexParser indexParser = new GitIndexParser(null); - FileSystemTaskResult result = indexParser.ParseIndex(tracer, indexStream, ValidateIndexEntry); + FileSystemTaskResult result = indexParser.ParseIndex(tracer, indexStream, indexParser.resuableProjectionBuildingIndexEntry, ValidateIndexEntry); if (result != FileSystemTaskResult.Success) { @@ -62,7 +67,12 @@ public void RebuildProjection(ITracer tracer, Stream indexStream) } this.projection.ClearProjectionCaches(); - FileSystemTaskResult result = this.ParseIndex(tracer, indexStream, this.AddIndexEntryToProjection); + FileSystemTaskResult result = this.ParseIndex( + tracer, + indexStream, + this.resuableProjectionBuildingIndexEntry, + this.AddIndexEntryToProjection); + if (result != FileSystemTaskResult.Success) { // RebuildProjection should always result in FileSystemTaskResult.Success (or a thrown exception) @@ -84,9 +94,10 @@ public FileSystemTaskResult AddMissingModifiedFilesAndRemoveThemFromPlaceholderL FileSystemTaskResult result = this.ParseIndex( tracer, - indexStream, + indexStream, + this.resuableBackgroundTaskThreadIndexEntry, (data) => this.AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfNeeded(data, filePlaceholders)); - + if (result != FileSystemTaskResult.Success) { return result; @@ -193,7 +204,11 @@ private FileSystemTaskResult AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfN /// in TryIndexAction returning a FileSystemTaskResult other than Success. All other actions result in success (or an exception in the /// case of a corrupt index) /// - private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func entryAction) + private FileSystemTaskResult ParseIndex( + ITracer tracer, + Stream indexStream, + GitIndexEntry resuableParsedIndexEntry, + Func entryAction) { this.indexStream = indexStream; this.indexStream.Position = 0; @@ -223,7 +238,7 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func SortedFolderEntries.InitializePools(tracer, entryCount); LazyUTF8String.InitializePools(tracer, entryCount); - this.resuableParsedIndexEntry.ClearLastParent(); + resuableParsedIndexEntry.ClearLastParent(); int previousPathLength = 0; bool parseMode = GVFSPlatform.Instance.FileSystem.SupportsFileMode; @@ -268,7 +283,7 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func throw new InvalidDataException($"Invalid file type {typeAndMode.Type:X} found in index"); } - this.resuableParsedIndexEntry.TypeAndMode = typeAndMode; + resuableParsedIndexEntry.TypeAndMode = typeAndMode; this.Skip(12); } @@ -277,7 +292,7 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func this.Skip(40); } - this.ReadSha(this.resuableParsedIndexEntry); + this.ReadSha(resuableParsedIndexEntry); ushort flags = this.ReadUInt16(); if (flags == 0) @@ -285,24 +300,24 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func throw new InvalidDataException("Invalid flags found in index"); } - this.resuableParsedIndexEntry.MergeState = (MergeStage)((flags >> 12) & 3); + resuableParsedIndexEntry.MergeState = (MergeStage)((flags >> 12) & 3); bool isExtended = (flags & ExtendedBit) == ExtendedBit; - this.resuableParsedIndexEntry.PathLength = (ushort)(flags & 0xFFF); + resuableParsedIndexEntry.PathLength = (ushort)(flags & 0xFFF); - this.resuableParsedIndexEntry.SkipWorktree = false; + resuableParsedIndexEntry.SkipWorktree = false; if (isExtended) { ushort extendedFlags = this.ReadUInt16(); - this.resuableParsedIndexEntry.SkipWorktree = (extendedFlags & SkipWorktreeBit) == SkipWorktreeBit; + resuableParsedIndexEntry.SkipWorktree = (extendedFlags & SkipWorktreeBit) == SkipWorktreeBit; } int replaceLength = this.ReadReplaceLength(); - this.resuableParsedIndexEntry.ReplaceIndex = previousPathLength - replaceLength; - int bytesToRead = this.resuableParsedIndexEntry.PathLength - this.resuableParsedIndexEntry.ReplaceIndex + 1; - this.ReadPath(this.resuableParsedIndexEntry, this.resuableParsedIndexEntry.ReplaceIndex, bytesToRead); - previousPathLength = this.resuableParsedIndexEntry.PathLength; + resuableParsedIndexEntry.ReplaceIndex = previousPathLength - replaceLength; + int bytesToRead = resuableParsedIndexEntry.PathLength - resuableParsedIndexEntry.ReplaceIndex + 1; + this.ReadPath(resuableParsedIndexEntry, resuableParsedIndexEntry.ReplaceIndex, bytesToRead); + previousPathLength = resuableParsedIndexEntry.PathLength; - result = entryAction.Invoke(this.resuableParsedIndexEntry); + result = entryAction.Invoke(resuableParsedIndexEntry); if (result != FileSystemTaskResult.Success) { return result; diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 1d0b347f9d..555a75c627 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -669,14 +669,14 @@ private void AddItemFromIndexEntry(GitIndexEntry indexEntry) { if (indexEntry.HasSameParentAsLastEntry) { - indexEntry.LastParent.AddChildFile(indexEntry.GetChildName(), indexEntry.Sha); + indexEntry.LastParent.AddChildFile(indexEntry.GetLazyChildName(), indexEntry.Sha); } else { if (indexEntry.NumParts == 1) { indexEntry.LastParent = this.rootFolderData; - indexEntry.LastParent.AddChildFile(indexEntry.GetChildName(), indexEntry.Sha); + indexEntry.LastParent.AddChildFile(indexEntry.GetLazyChildName(), indexEntry.Sha); } else { @@ -794,7 +794,7 @@ private FolderData AddFileToTree(GitIndexEntry indexEntry) string parentFolderName; if (pathIndex > 0) { - parentFolderName = indexEntry.PathParts[pathIndex - 1].GetString(); + parentFolderName = indexEntry.GetPathPart(pathIndex - 1); } else { @@ -811,10 +811,10 @@ private FolderData AddFileToTree(GitIndexEntry indexEntry) throw new InvalidDataException("Found a file (" + parentFolderName + ") where a folder was expected: " + gitPath); } - parentFolder = parentFolder.ChildEntries.GetOrAddFolder(indexEntry.PathParts[pathIndex]); + parentFolder = parentFolder.ChildEntries.GetOrAddFolder(indexEntry.GetLazyPathPart(pathIndex)); } - parentFolder.AddChildFile(indexEntry.PathParts[indexEntry.NumParts - 1], indexEntry.Sha); + parentFolder.AddChildFile(indexEntry.GetLazyPathPart(indexEntry.NumParts - 1), indexEntry.Sha); return parentFolder; } From 31329e5964d2973e5911e71e8e790209320aa00c Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 24 Oct 2018 09:14:33 -0700 Subject: [PATCH 198/244] Cleanup for PR comments --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 2 +- .../Projection/GitIndexProjection.GitIndexEntry.cs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 6d7f14b155..6aa1e571f1 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -360,7 +360,7 @@ public virtual void OnIndexFileChange() } else if (this.GitCommandLeavesProjectionUnchanged(gitCommand)) { - if (gitCommand.IsResetSoftOrMixed()) + if (gitCommand.IsResetMixed()) { this.GitIndexProjection.InvalidateModifiedFiles(); this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnIndexWriteByResetMixed()); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 35c6ca38ea..5c13294183 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -25,6 +25,9 @@ internal class GitIndexEntry private int previousFinalSeparatorIndex = int.MaxValue; + // lazyPathParts and utf16PathParts are mutually exclusive + // The `useLazyPaths` parameter of the GitIndexEntry constructor determines + // which of these two arrays will be used private LazyUTF8String[] lazyPathParts; private string[] utf16PathParts; From 1e1c41b2a27ada80eae6b3bc3b832d2c79682828 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 24 Oct 2018 09:26:33 -0700 Subject: [PATCH 199/244] Add additional logging --- .../Projection/GitIndexProjection.GitIndexParser.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 238841e22c..23272622d2 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -81,7 +81,7 @@ public void RebuildProjection(ITracer tracer, Stream indexStream) } public FileSystemTaskResult AddMissingModifiedFilesAndRemoveThemFromPlaceholderList( - ITracer tracer, + ITracer tracer, Stream indexStream) { if (this.projection == null) @@ -89,9 +89,17 @@ public FileSystemTaskResult AddMissingModifiedFilesAndRemoveThemFromPlaceholderL throw new InvalidOperationException($"{nameof(this.projection)} cannot be null when calling {nameof(AddMissingModifiedFilesAndRemoveThemFromPlaceholderList)}"); } - Dictionary filePlaceholders = + Dictionary filePlaceholders = this.projection.placeholderList.GetAllFileEntries(); + tracer.RelatedEvent( + EventLevel.Informational, + $"{nameof(this.AddMissingModifiedFilesAndRemoveThemFromPlaceholderList)}_FilePlaceholderCount", + new EventMetadata + { + { "FilePlaceholderCount", filePlaceholders.Count } + }); + FileSystemTaskResult result = this.ParseIndex( tracer, indexStream, From 4d03365bc95e13772ed36235153e50fb46fb17e4 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Wed, 24 Oct 2018 11:41:49 -0400 Subject: [PATCH 200/244] Centralize spcial Mac paths in InitializeEnviroment.sh --- Scripts/Mac/BuildGVFSForMac.sh | 55 +++++++++++---------------- Scripts/Mac/CleanupFunctionalTests.sh | 6 +-- Scripts/Mac/DownloadGVFSGit.sh | 13 +++---- Scripts/Mac/GVFS_Clone.sh | 8 +--- Scripts/Mac/GVFS_Mount.sh | 9 ++--- Scripts/Mac/GVFS_Unmount.sh | 9 ++--- Scripts/Mac/GetGitVersionNumber.sh | 5 ++- Scripts/Mac/InitializeEnvironment.sh | 13 +++++++ Scripts/Mac/NukeBuildOutputs.sh | 15 +++----- Scripts/Mac/PrepFunctionalTests.sh | 7 ++-- Scripts/Mac/RunFunctionalTests.sh | 6 ++- 11 files changed, 69 insertions(+), 77 deletions(-) create mode 100755 Scripts/Mac/InitializeEnvironment.sh diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index 907854e4ad..20190c72be 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -1,57 +1,48 @@ #!/bin/bash +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + CONFIGURATION=$1 if [ -z $CONFIGURATION ]; then CONFIGURATION=Debug fi -SCRIPTDIR="$(dirname ${BASH_SOURCE[0]})" - -# convert to an absolute path because it is required by `dotnet publish` -pushd $SCRIPTDIR -SCRIPTDIR="$(pwd)" -popd - -SRCDIR=$SCRIPTDIR/../.. -ROOTDIR=$SRCDIR/.. -BUILDOUTPUT=$ROOTDIR/BuildOutput -PUBLISHDIR=$ROOTDIR/Publish - -if [ ! -d $BUILDOUTPUT ]; then - mkdir $BUILDOUTPUT +if [ ! -d $VFS_OUTPUTDIR ]; then + mkdir $VFS_OUTPUTDIR fi -PACKAGES=$ROOTDIR/packages - -# Build the ProjFS kext and libraries -$SRCDIR/ProjFS.Mac/Scripts/Build.sh $CONFIGURATION || exit 1 +echo 'Building ProjFS kext and libraries...' +$VFS_SRCDIR/ProjFS.Mac/Scripts/Build.sh $CONFIGURATION || exit 1 # Create the directory where we'll do pre build tasks -BUILDDIR=$BUILDOUTPUT/GVFS.Build +BUILDDIR=$VFS_OUTPUTDIR/GVFS.Build if [ ! -d $BUILDDIR ]; then mkdir $BUILDDIR || exit 1 fi -$SCRIPTDIR/DownloadGVFSGit.sh || exit 1 -GITVERSION="$($SCRIPTDIR/GetGitVersionNumber.sh)" -GITPATH="$(find $PACKAGES/gitformac.gvfs.installer/$GITVERSION -type f -name *.dmg)" || exit 1 +echo 'Downloading a VFS-enabled version of Git...' +$VFS_SCRIPTDIR/DownloadGVFSGit.sh || exit 1 +GITVERSION="$($VFS_SCRIPTDIR/GetGitVersionNumber.sh)" +GITPATH="$(find $VFS_PACKAGESDIR/gitformac.gvfs.installer/$GITVERSION -type f -name *.dmg)" || exit 1 +echo "Downloaded Git $GITVERSION" # Now that we have a path containing the version number, generate GVFSConstants.GitVersion.cs -$SCRIPTDIR/GenerateGitVersionConstants.sh "$GITPATH" $BUILDDIR || exit 1 +$VFS_SCRIPTDIR/GenerateGitVersionConstants.sh "$GITPATH" $BUILDDIR || exit 1 # If we're building the Profiling(Release) configuration, remove Profiling() for building .NET code if [ "$CONFIGURATION" == "Profiling(Release)" ]; then CONFIGURATION=Release fi -dotnet restore $SRCDIR/GVFS.sln /p:Configuration=$CONFIGURATION.Mac --packages $PACKAGES || exit 1 -dotnet build $SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --configuration $CONFIGURATION.Mac /maxcpucount:1 || exit 1 -dotnet publish $SRCDIR/GVFS.sln /p:Configuration=$CONFIGURATION.Mac /p:Platform=x64 --runtime osx-x64 --framework netcoreapp2.1 --self-contained --output $PUBLISHDIR /maxcpucount:1 || exit 1 +echo 'Restoring packages...' +dotnet restore $VFS_SRCDIR/GVFS.sln /p:Configuration=$CONFIGURATION.Mac --packages $VFS_PACKAGESDIR || exit 1 +dotnet build $VFS_SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --configuration $CONFIGURATION.Mac /maxcpucount:1 || exit 1 +dotnet publish $VFS_SRCDIR/GVFS.sln /p:Configuration=$CONFIGURATION.Mac /p:Platform=x64 --runtime osx-x64 --framework netcoreapp2.1 --self-contained --output $VFS_PUBLISHDIR /maxcpucount:1 || exit 1 -NATIVEDIR=$SRCDIR/GVFS/GVFS.Native.Mac -xcodebuild -configuration $CONFIGURATION -workspace $NATIVEDIR/GVFS.Native.Mac.xcworkspace build -scheme GVFS.Native.Mac -derivedDataPath $ROOTDIR/BuildOutput/GVFS.Native.Mac || exit 1 +NATIVEDIR=$VFS_SRCDIR/GVFS/GVFS.Native.Mac +xcodebuild -configuration $CONFIGURATION -workspace $NATIVEDIR/GVFS.Native.Mac.xcworkspace build -scheme GVFS.Native.Mac -derivedDataPath $VFS_OUTPUTDIR/GVFS.Native.Mac || exit 1 -echo 'Copying native binaries to Publish directory' -cp $BUILDOUTPUT/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.ReadObjectHook $PUBLISHDIR || exit 1 -cp $BUILDOUTPUT/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.VirtualFileSystemHook $PUBLISHDIR || exit 1 +echo 'Copying native binaries to Publish directory...' +cp $VFS_OUTPUTDIR/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.ReadObjectHook $VFS_PUBLISHDIR || exit 1 +cp $VFS_OUTPUTDIR/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.VirtualFileSystemHook $VFS_PUBLISHDIR || exit 1 -$PUBLISHDIR/GVFS.UnitTests || exit 1 +$VFS_PUBLISHDIR/GVFS.UnitTests || exit 1 diff --git a/Scripts/Mac/CleanupFunctionalTests.sh b/Scripts/Mac/CleanupFunctionalTests.sh index dc30be675e..c30da56f24 100755 --- a/Scripts/Mac/CleanupFunctionalTests.sh +++ b/Scripts/Mac/CleanupFunctionalTests.sh @@ -1,6 +1,6 @@ -SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) -SRCDIR=$SCRIPTDIR/../.. -$SRCDIR/ProjFS.Mac/Scripts/UnloadPrjFSKext.sh +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + +$VFS_SRCDIR/ProjFS.Mac/Scripts/UnloadPrjFSKext.sh sudo rm -r /GVFS.FT diff --git a/Scripts/Mac/DownloadGVFSGit.sh b/Scripts/Mac/DownloadGVFSGit.sh index 74f8bc5282..99bc86679e 100755 --- a/Scripts/Mac/DownloadGVFSGit.sh +++ b/Scripts/Mac/DownloadGVFSGit.sh @@ -1,8 +1,7 @@ -SCRIPTDIR="$(dirname ${BASH_SOURCE[0]})" -SRCDIR=$SCRIPTDIR/../.. -BUILDDIR=$SRCDIR/../BuildOutput/GVFS.Build -PACKAGESDIR=$SRCDIR/../packages -GITVERSION="$($SCRIPTDIR/GetGitVersionNumber.sh)" -cp $SRCDIR/nuget.config $BUILDDIR +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + +BUILDDIR=$VFS_OUTPUTDIR/GVFS.Build +GITVERSION="$($VFS_SCRIPTDIR/GetGitVersionNumber.sh)" +cp $VFS_SRCDIR/nuget.config $BUILDDIR dotnet new classlib -n GVFS.Restore -o $BUILDDIR --force -dotnet add $BUILDDIR/GVFS.Restore.csproj package --package-directory $PACKAGESDIR GitForMac.GVFS.Installer --version $GITVERSION \ No newline at end of file +dotnet add $BUILDDIR/GVFS.Restore.csproj package --package-directory $VFS_PACKAGESDIR GitForMac.GVFS.Installer --version $GITVERSION \ No newline at end of file diff --git a/Scripts/Mac/GVFS_Clone.sh b/Scripts/Mac/GVFS_Clone.sh index 5aac1cd20f..250d3aa9a1 100755 --- a/Scripts/Mac/GVFS_Clone.sh +++ b/Scripts/Mac/GVFS_Clone.sh @@ -1,4 +1,5 @@ #!/bin/bash +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" REPOURL=$1 @@ -7,9 +8,4 @@ if [ -z $CONFIGURATION ]; then CONFIGURATION=Debug fi -SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) - -ROOTDIR=$SCRIPTDIR/../../.. -PUBLISHDIR=$ROOTDIR/Publish - -$PUBLISHDIR/gvfs clone $REPOURL ~/GVFSTest --local-cache-path ~/GVFSTest/.gvfsCache --no-mount --no-prefetch +$VFS_PUBLISHDIR/gvfs clone $REPOURL ~/GVFSTest --local-cache-path ~/GVFSTest/.gvfsCache --no-mount --no-prefetch diff --git a/Scripts/Mac/GVFS_Mount.sh b/Scripts/Mac/GVFS_Mount.sh index 6a16007511..85a1df7b20 100755 --- a/Scripts/Mac/GVFS_Mount.sh +++ b/Scripts/Mac/GVFS_Mount.sh @@ -1,13 +1,10 @@ #!/bin/bash +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + CONFIGURATION=$1 if [ -z $CONFIGURATION ]; then CONFIGURATION=Debug fi -SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) - -ROOTDIR=$SCRIPTDIR/../../.. -PUBLISHDIR=$ROOTDIR/Publish - -$PUBLISHDIR/gvfs mount ~/GVFSTest \ No newline at end of file +$VFS_PUBLISHDIR/gvfs mount ~/GVFSTest \ No newline at end of file diff --git a/Scripts/Mac/GVFS_Unmount.sh b/Scripts/Mac/GVFS_Unmount.sh index 542bbbca05..162eb64ad7 100755 --- a/Scripts/Mac/GVFS_Unmount.sh +++ b/Scripts/Mac/GVFS_Unmount.sh @@ -1,13 +1,10 @@ #!/bin/bash +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + CONFIGURATION=$1 if [ -z $CONFIGURATION ]; then CONFIGURATION=Debug fi -SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) - -ROOTDIR=$SCRIPTDIR/../../.. -PUBLISHDIR=$ROOTDIR/Publish - -$PUBLISHDIR/gvfs unmount ~/GVFSTest \ No newline at end of file +$VFS_PUBLISHDIR/gvfs unmount ~/GVFSTest \ No newline at end of file diff --git a/Scripts/Mac/GetGitVersionNumber.sh b/Scripts/Mac/GetGitVersionNumber.sh index 6942b3a426..d8604552ab 100755 --- a/Scripts/Mac/GetGitVersionNumber.sh +++ b/Scripts/Mac/GetGitVersionNumber.sh @@ -1,4 +1,5 @@ -SCRIPTDIR="$(dirname ${BASH_SOURCE[0]})" -GVFSPROPS=$SCRIPTDIR/../../GVFS/GVFS.Build/GVFS.props +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + +GVFSPROPS=$VFS_SRCDIR/GVFS/GVFS.Build/GVFS.props GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]+(-\w+)?')" echo $GITVERSION \ No newline at end of file diff --git a/Scripts/Mac/InitializeEnvironment.sh b/Scripts/Mac/InitializeEnvironment.sh new file mode 100755 index 0000000000..bf55c5a1c2 --- /dev/null +++ b/Scripts/Mac/InitializeEnvironment.sh @@ -0,0 +1,13 @@ +SCRIPTDIR="$(dirname ${BASH_SOURCE[0]})" + +# convert to an absolute path because it is required by `dotnet publish` +pushd $SCRIPTDIR &>/dev/null +export VFS_SCRIPTDIR="$(pwd)" +popd &>/dev/null + +export VFS_SRCDIR=$VFS_SCRIPTDIR/../.. + +VFS_ENLISTMENTDIR=$VFS_SRCDIR/.. +export VFS_OUTPUTDIR=$VFS_ENLISTMENTDIR/BuildOutput +export VFS_PUBLISHDIR=$VFS_ENLISTMENTDIR/Publish +export VFS_PACKAGESDIR=$VFS_ENLISTMENTDIR/packages diff --git a/Scripts/Mac/NukeBuildOutputs.sh b/Scripts/Mac/NukeBuildOutputs.sh index 97c57fdfe1..287a657a48 100755 --- a/Scripts/Mac/NukeBuildOutputs.sh +++ b/Scripts/Mac/NukeBuildOutputs.sh @@ -1,13 +1,10 @@ #!/bin/bash -SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" -SRCDIR=$SCRIPTDIR/../.. -ROOTDIR=$SRCDIR/.. +sudo rm -Rf $VFS_OUTPUTDIR +rm -Rf $VFS_PACKAGESDIR +rm -Rf $VFS_PUBLISHDIR -sudo rm -Rf $ROOTDIR/BuildOutput -rm -Rf $ROOTDIR/packages -rm -Rf $ROOTDIR/Publish - -echo git --work-tree=$SRCDIR clean -Xdf -n -git --work-tree=$SRCDIR clean -Xdf -n \ No newline at end of file +echo git --work-tree=$VFS_SRCDIR clean -Xdf -n +git --work-tree=$VFS_SRCDIR clean -Xdf -n diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index d7d9c3cfa1..8921acf588 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -1,14 +1,13 @@ #!/bin/bash -SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" # Ensure the kext isn't loaded before installing Git -$SCRIPTDIR/../../ProjFS.Mac/Scripts/UnloadPrjFSKext.sh +$VFS_SRCDIR/ProjFS.Mac/Scripts/UnloadPrjFSKext.sh # Install GVFS-aware Git (that was downloaded by the build script) GITVERSION="$($SCRIPTDIR/GetGitVersionNumber.sh)" -ROOTDIR=$SCRIPTDIR/../../.. -GITDIR=$ROOTDIR/packages/gitformac.gvfs.installer/$GITVERSION/tools +GITDIR=$VFS_PACKAGESDIR/gitformac.gvfs.installer/$GITVERSION/tools if [[ ! -d $GITDIR ]]; then echo "GVFS-aware Git package not found. Run BuildGVFSForMac.sh and try again" exit 1 diff --git a/Scripts/Mac/RunFunctionalTests.sh b/Scripts/Mac/RunFunctionalTests.sh index 627f8a3c70..48a075faba 100755 --- a/Scripts/Mac/RunFunctionalTests.sh +++ b/Scripts/Mac/RunFunctionalTests.sh @@ -1,5 +1,7 @@ #!/bin/bash +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + CONFIGURATION=$1 if [ -z $CONFIGURATION ]; then CONFIGURATION=Debug @@ -14,5 +16,5 @@ PUBLISHDIR=$ROOTDIR/Publish sudo mkdir /GVFS.FT sudo chown $USER /GVFS.FT -$SRCDIR/ProjFS.Mac/Scripts/LoadPrjFSKext.sh -$PUBLISHDIR/GVFS.FunctionalTests --full-suite $2 +$VFS_SRCDIR/ProjFS.Mac/Scripts/LoadPrjFSKext.sh +$VFS_PUBLISHDIR/GVFS.FunctionalTests --full-suite $2 From ab465bb856821399647f94a3a3b2d7b03660cd56 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 24 Oct 2018 11:20:11 -0700 Subject: [PATCH 201/244] Also update ModifedPaths for update-index command --- .../Background/FileSystemTask.cs | 6 +++--- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs index 35d3a57895..73e3e9feb7 100644 --- a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs +++ b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs @@ -27,7 +27,7 @@ public enum OperationType OnFolderRenamed, OnFolderDeleted, OnFolderFirstWrite, - OnIndexWriteByResetMixed, + OnIndexWriteRequiringModifiedPathsValidation, OnPlaceholderCreationsBlockedForGit, OnFileHardLinkCreated, OnFilePreDelete, @@ -115,9 +115,9 @@ public static FileSystemTask OnFolderPreDelete(string virtualPath) return new FileSystemTask(OperationType.OnFolderPreDelete, virtualPath, oldVirtualPath: null); } - public static FileSystemTask OnIndexWriteByResetMixed() + public static FileSystemTask OnIndexWriteRequiringModifiedPathsValidation() { - return new FileSystemTask(OperationType.OnIndexWriteByResetMixed, virtualPath: null, oldVirtualPath: null); + return new FileSystemTask(OperationType.OnIndexWriteRequiringModifiedPathsValidation, virtualPath: null, oldVirtualPath: null); } public static FileSystemTask OnPlaceholderCreationsBlockedForGit() diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 6aa1e571f1..c807438eb9 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -360,10 +360,10 @@ public virtual void OnIndexFileChange() } else if (this.GitCommandLeavesProjectionUnchanged(gitCommand)) { - if (gitCommand.IsResetMixed()) + if (this.GitCommandRequiresModifiedPathValidationAfterIndexChange(gitCommand)) { this.GitIndexProjection.InvalidateModifiedFiles(); - this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnIndexWriteByResetMixed()); + this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnIndexWriteRequiringModifiedPathsValidation()); } this.InvalidateGitStatusCache(); @@ -627,6 +627,13 @@ private bool GitCommandLeavesProjectionUnchanged(GitCommandLineParser gitCommand gitCommand.IsCheckoutWithFilePaths(); } + private bool GitCommandRequiresModifiedPathValidationAfterIndexChange(GitCommandLineParser gitCommand) + { + return + gitCommand.IsVerb(GitCommandLineParser.Verbs.UpdateIndex) || + gitCommand.IsResetMixed(); + } + private FileSystemTaskResult PreBackgroundOperation() { return this.GitIndexProjection.OpenIndexForRead(); @@ -860,7 +867,7 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate result = FileSystemTaskResult.Success; break; - case FileSystemTask.OperationType.OnIndexWriteByResetMixed: + case FileSystemTask.OperationType.OnIndexWriteRequiringModifiedPathsValidation: result = this.GitIndexProjection.AddMissingModifiedFiles(); break; From 3075aafa336038581c50f44ac35279bf2e12a7b6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 24 Oct 2018 10:02:54 -0400 Subject: [PATCH 202/244] PostFetchJob: Kill running Git process when unmounting The post-fetch job runs up to two Git commands: 'git multi-pack-index write' and 'git commit-graph write'. These can each take up to 30s depending on the amount of work to do. When we unmount, we want to ensure these prcoesses terminate. Before, we were aborting the post-fetch job thread. This is not cross-platform compliant. The new approach is to store a GitProcess object and pass a Kill signal to that process during unmount. There are a few race conditions around this being null, so we guard against some using try/catch blocks. This may be a direction for future improvement. The test suite was previously exercising this code path during the functional tests, as the functional tests frequently ran faster than the background Git processes. That should give some good data on how this works. --- GVFS/GVFS.Common/Git/GitObjects.cs | 21 ------ GVFS/GVFS.Common/Git/GitProcess.cs | 75 ++++++++++++++++--- .../FileSystemCallbacks.cs | 66 ++++++++++++---- 3 files changed, 113 insertions(+), 49 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitObjects.cs b/GVFS/GVFS.Common/Git/GitObjects.cs index dac8f7d72d..f138c31451 100644 --- a/GVFS/GVFS.Common/Git/GitObjects.cs +++ b/GVFS/GVFS.Common/Git/GitObjects.cs @@ -150,27 +150,6 @@ public virtual bool TryDownloadPrefetchPacks(long latestTimestamp, out List 1024 ? result.Output.Substring(1024) : result.Output); - activity.RelatedError(errorMetadata, result.Errors, Keywords.Telemetry); - } - - return !result.HasErrors; - } - } - public virtual string WriteLooseObject(Stream responseStream, string sha, bool overwriteExistingObject, byte[] bufToCopyWith) { try diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 566c5bfa7c..f75f021f3a 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -17,12 +17,24 @@ public class GitProcess private static readonly Encoding UTF8NoBOM = new UTF8Encoding(false); private static bool failedToSetEncoding = false; + /// + /// Lock taken for duration of running executingProcess. + /// private object executionLock = new object(); + /// + /// Lock taken when changing the running state of executingProcess. + /// + /// Can be taken within executionLock. + /// + private object processLock = new object(); + private string gitBinPath; private string workingDirectoryRoot; private string dotGitRoot; private string gvfsHooksRoot; + private Process executingProcess; + private bool stopping; static GitProcess() { @@ -108,6 +120,35 @@ public static bool TryGetVersion(string gitBinPath, out GitVersion gitVersion, o return true; } + public bool TryKillRunningProcess() + { + this.stopping = true; + + try + { + lock (this.processLock) + { + Process process = this.executingProcess; + + if (process != null) + { + process.Kill(); + return true; + } + } + } + catch (Win32Exception) + { + // Thrown when process is already terminating + } + catch (InvalidOperationException) + { + // Process already terminated + } + + return false; + } + public virtual void RevokeCredential(string repoUrl) { this.InvokeGitOutsideEnlistment( @@ -496,19 +537,19 @@ protected virtual Result InvokeGitImpl( // From https://msdn.microsoft.com/en-us/library/system.diagnostics.process.standardoutput.aspx // To avoid deadlocks, use asynchronous read operations on at least one of the streams. // Do not perform a synchronous read to the end of both redirected streams. - using (Process executingProcess = this.GetGitProcess(command, workingDirectory, dotGitDirectory, useReadObjectHook, redirectStandardError: true)) + using (this.executingProcess = this.GetGitProcess(command, workingDirectory, dotGitDirectory, useReadObjectHook, redirectStandardError: true)) { StringBuilder output = new StringBuilder(); StringBuilder errors = new StringBuilder(); - executingProcess.ErrorDataReceived += (sender, args) => + this.executingProcess.ErrorDataReceived += (sender, args) => { if (args.Data != null) { errors.Append(args.Data + "\n"); } }; - executingProcess.OutputDataReceived += (sender, args) => + this.executingProcess.OutputDataReceived += (sender, args) => { if (args.Data != null) { @@ -525,30 +566,40 @@ protected virtual Result InvokeGitImpl( lock (this.executionLock) { - executingProcess.Start(); - - if (writeStdIn != null) + lock (this.processLock) { - writeStdIn(executingProcess.StandardInput); + if (this.stopping) + { + return new Result(string.Empty, nameof(GitProcess) + " is stopping", Result.GenericFailureCode); + } + + this.executingProcess.Start(); } - executingProcess.BeginOutputReadLine(); - executingProcess.BeginErrorReadLine(); + writeStdIn?.Invoke(this.executingProcess.StandardInput); - if (!executingProcess.WaitForExit(timeoutMs)) + this.executingProcess.BeginOutputReadLine(); + this.executingProcess.BeginErrorReadLine(); + + if (!this.executingProcess.WaitForExit(timeoutMs)) { - executingProcess.Kill(); + this.executingProcess.Kill(); + return new Result(output.ToString(), "Operation timed out: " + errors.ToString(), Result.GenericFailureCode); } } - return new Result(output.ToString(), errors.ToString(), executingProcess.ExitCode); + return new Result(output.ToString(), errors.ToString(), this.executingProcess.ExitCode); } } catch (Win32Exception e) { return new Result(string.Empty, e.Message, Result.GenericFailureCode); } + finally + { + this.executingProcess = null; + } } private static string ParseValue(string contents, string prefix) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index e209e3fa2f..6d89c3f6d0 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -39,6 +39,7 @@ public class FileSystemCallbacks : IDisposable, IHeartBeatMetadataProvider private FileSystemVirtualizer fileSystemVirtualizer; private FileProperties logsHeadFileProperties; private Thread postFetchJobThread; + private GitProcess postFetchGitProcess; private object postFetchJobLock; private bool stopping; @@ -197,11 +198,23 @@ public void Stop() this.stopping = true; lock (this.postFetchJobLock) { - // TODO(Mac): System.PlatformNotSupportedException: Thread abort is not supported on this platform - - if (!GVFSPlatform.Instance.IsUnderConstruction) + GitProcess process = this.postFetchGitProcess; + if (process != null) { - this.postFetchJobThread?.Abort(); + if (process.TryKillRunningProcess()) + { + this.context.Tracer.RelatedEvent( + EventLevel.Informational, + PostFetchTelemetryKey + ": killed background Git process during " + nameof(this.Stop), + metadata: null); + } + else + { + this.context.Tracer.RelatedEvent( + EventLevel.Informational, + PostFetchTelemetryKey + ": failed to kill background Git process during " + nameof(this.Stop), + metadata: null); + } } } @@ -568,12 +581,29 @@ private void PostFetchJob(List packIndexes) return; } - if (!this.gitObjects.TryWriteMultiPackIndex(this.context.Tracer, this.context.Enlistment, this.context.FileSystem)) + this.postFetchGitProcess = new GitProcess(this.context.Enlistment); + + using (ITracer activity = this.context.Tracer.StartActivity("TryWriteMultiPackIndex", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { - this.context.Tracer.RelatedWarning( - metadata: null, - message: PostFetchTelemetryKey + ": Failed to generate multi-pack-index for new packfiles", - keywords: Keywords.Telemetry); + if (this.stopping) + { + this.context.Tracer.RelatedWarning( + metadata: null, + message: PostFetchTelemetryKey + ": Not launching 'git multi-pack-index' because the mount is stopping", + keywords: Keywords.Telemetry); + return; + } + + GitProcess.Result result = this.postFetchGitProcess.WriteMultiPackIndex(this.context.Enlistment.GitObjectsRoot); + + if (!this.stopping && result.HasErrors) + { + this.context.Tracer.RelatedWarning( + metadata: null, + message: PostFetchTelemetryKey + ": Failed to generate multi-pack-index for new packfiles:" + result.Errors, + keywords: Keywords.Telemetry); + return; + } } if (packIndexes == null || packIndexes.Count == 0) @@ -584,10 +614,18 @@ private void PostFetchJob(List packIndexes) using (ITracer activity = this.context.Tracer.StartActivity("TryWriteGitCommitGraph", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { - GitProcess process = new GitProcess(this.context.Enlistment); - GitProcess.Result result = process.WriteCommitGraph(this.context.Enlistment.GitObjectsRoot, packIndexes); + if (this.stopping) + { + this.context.Tracer.RelatedWarning( + metadata: null, + message: PostFetchTelemetryKey + ": Not launching 'git commit-graph' because the mount is stopping", + keywords: Keywords.Telemetry); + return; + } - if (result.HasErrors) + GitProcess.Result result = this.postFetchGitProcess.WriteCommitGraph(this.context.Enlistment.GitObjectsRoot, packIndexes); + + if (!this.stopping && result.HasErrors) { this.context.Tracer.RelatedWarning( metadata: null, @@ -598,10 +636,6 @@ private void PostFetchJob(List packIndexes) } } } - catch (ThreadAbortException) - { - this.context.Tracer.RelatedInfo("Aborting post-fetch job due to ThreadAbortException"); - } catch (IOException e) { this.context.Tracer.RelatedWarning( From cf38b0217aa4c5e7ac10d03ed2d9f8ac677558a4 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 24 Oct 2018 11:13:11 -0400 Subject: [PATCH 203/244] PostFetchJob: Delete stale .lock files Before writing commit-graph or multi-pack-index, delete the .lock file that prevents Git from writing to those files. This will unblock users who have a stale .lock file from a previous early termination (BSOD, process kill due to unmount, etc.). If the Git commands are terminating due to legitimate problems, then their logs will be filled with the correct error instead of a message about a .lock file preventing the write. Both of these commands are maintenance commands that users should not be writing on their own, and we do them under our own file- based lock. --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 6d89c3f6d0..3652e41bb4 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -585,6 +585,9 @@ private void PostFetchJob(List packIndexes) using (ITracer activity = this.context.Tracer.StartActivity("TryWriteMultiPackIndex", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { + string midxLockFile = Path.Combine(this.context.Enlistment.GitPackRoot, "multi-pack-index.lock"); + this.context.FileSystem.TryDeleteFile(midxLockFile); + if (this.stopping) { this.context.Tracer.RelatedWarning( @@ -614,6 +617,9 @@ private void PostFetchJob(List packIndexes) using (ITracer activity = this.context.Tracer.StartActivity("TryWriteGitCommitGraph", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { + string graphLockFile = Path.Combine(this.context.Enlistment.GitObjectsRoot, "info", "commit-graph.lock"); + this.context.FileSystem.TryDeleteFile(graphLockFile); + if (this.stopping) { this.context.Tracer.RelatedWarning( From 878c9a9581c2e243568ef9f8ee6df9dd96da4d0e Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 25 Oct 2018 08:12:21 -0400 Subject: [PATCH 204/244] GitProcess: return true when process is null --- GVFS/GVFS.Common/Git/GitProcess.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index f75f021f3a..c4013ef3dd 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -133,8 +133,9 @@ public bool TryKillRunningProcess() if (process != null) { process.Kill(); - return true; } + + return true; } } catch (Win32Exception) From bade0785bf1608fdfca918be6b9d3ab890241e71 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 24 Oct 2018 16:38:24 -0400 Subject: [PATCH 205/244] Prefetch Tests: Add test for .lock files --- .../PrefetchVerbWithoutSharedCacheTests.cs | 27 +++++++++++++++++++ .../FileSystemCallbacks.cs | 6 +++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs index 72771a7d29..822041ecf9 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs @@ -277,6 +277,33 @@ public void PrefetchCleansUpStaleTempPrefetchPacks() otherFilePath.ShouldBeAFile(this.fileSystem).WithContents(otherFileContents); } + [TestCase, Order(9)] + public void PrefetchCleansUpOphanedLockFiles() + { + // Multi-pack-index write happens even if the prefetch downloads nothing, while + // the commit-graph write happens only when the prefetch downloads at least one pack + + string graphPath = Path.Combine(this.Enlistment.GetObjectRoot(this.fileSystem), "info", "commit-graph"); + string graphLockPath = graphPath + ".lock"; + string midxPath = Path.Combine(this.PackRoot, "multi-pack-index"); + string midxLockPath = midxPath + ".lock"; + + this.fileSystem.CreateEmptyFile(graphLockPath); + + // Force deleting the prefetch packs to make the prefetch non-trivial. + this.fileSystem.DeleteDirectory(this.PackRoot); + this.fileSystem.CreateDirectory(this.PackRoot); + this.fileSystem.CreateEmptyFile(midxLockPath); + + this.Enlistment.Prefetch("--commits"); + this.PostFetchJobShouldComplete(); + + this.fileSystem.FileExists(graphLockPath).ShouldBeFalse(); + this.fileSystem.FileExists(midxLockPath).ShouldBeFalse(); + this.fileSystem.FileExists(graphPath).ShouldBeTrue(); + this.fileSystem.FileExists(midxPath).ShouldBeTrue(); + } + private void PackShouldHaveIdxFile(string pathPath) { string idxPath = Path.ChangeExtension(pathPath, ".idx"); diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 3652e41bb4..f2a5f06025 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -20,6 +20,8 @@ public class FileSystemCallbacks : IDisposable, IHeartBeatMetadataProvider { private const string EtwArea = nameof(FileSystemCallbacks); private const string PostFetchLock = "post-fetch.lock"; + private const string CommitGraphLock = "commit-graph.lock"; + private const string MultiPackIndexLock = "multi-pack-index.lock"; private const string PostFetchTelemetryKey = "post-fetch"; private static readonly GitCommandLineParser.Verbs LeavesProjectionUnchangedVerbs = @@ -585,7 +587,7 @@ private void PostFetchJob(List packIndexes) using (ITracer activity = this.context.Tracer.StartActivity("TryWriteMultiPackIndex", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { - string midxLockFile = Path.Combine(this.context.Enlistment.GitPackRoot, "multi-pack-index.lock"); + string midxLockFile = Path.Combine(this.context.Enlistment.GitPackRoot, MultiPackIndexLock); this.context.FileSystem.TryDeleteFile(midxLockFile); if (this.stopping) @@ -617,7 +619,7 @@ private void PostFetchJob(List packIndexes) using (ITracer activity = this.context.Tracer.StartActivity("TryWriteGitCommitGraph", EventLevel.Informational, Keywords.Telemetry, metadata: null)) { - string graphLockFile = Path.Combine(this.context.Enlistment.GitObjectsRoot, "info", "commit-graph.lock"); + string graphLockFile = Path.Combine(this.context.Enlistment.GitObjectsRoot, "info", CommitGraphLock); this.context.FileSystem.TryDeleteFile(graphLockFile); if (this.stopping) From f2c63e8c0d7037af8883aee5aad22c29aa46958a Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Mon, 22 Oct 2018 15:52:22 -0400 Subject: [PATCH 206/244] Mount: Remove tracing on TryAttach Allow the caller to determine how to handle the error. This is because a command line error may fail due to permissions and be retried successfully by the service. --- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 7 ++----- .../Handlers/EnableAndAttachProjFSHandler.cs | 2 +- GVFS/GVFS/CommandLine/MountVerb.cs | 9 ++++++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index d82693c424..2cefd3599f 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -47,7 +47,7 @@ private enum ProjFSInboxStatus public string DriverLogFolderName { get; } = ProjFSFilter.ServiceName; - public static bool TryAttach(ITracer tracer, string enlistmentRoot, out string errorMessage) + public static bool TryAttach(string enlistmentRoot, out string errorMessage) { errorMessage = null; try @@ -56,7 +56,6 @@ public static bool TryAttach(ITracer tracer, string enlistmentRoot, out string e if (!NativeMethods.GetVolumePathName(enlistmentRoot, volumePathName, GVFSConstants.MaxPath)) { errorMessage = "Could not get volume path name"; - tracer.RelatedError($"{nameof(TryAttach)}:{errorMessage}"); return false; } @@ -64,14 +63,12 @@ public static bool TryAttach(ITracer tracer, string enlistmentRoot, out string e if (result != OkResult && result != NameCollisionErrorResult) { errorMessage = string.Format("Attaching the filter driver resulted in: {0}", result); - tracer.RelatedError(errorMessage); return false; } } catch (Exception e) { errorMessage = string.Format("Attaching the filter driver resulted in: {0}", e.Message); - tracer.RelatedError(errorMessage); return false; } @@ -373,7 +370,7 @@ public bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error) return IsServiceRunning(tracer) && IsNativeLibInstalled(tracer, new PhysicalFileSystem()) && - TryAttach(tracer, enlistmentRoot, out error); + TryAttach(enlistmentRoot, out error); } private static bool IsInboxAndEnabled() diff --git a/GVFS/GVFS.Service/Handlers/EnableAndAttachProjFSHandler.cs b/GVFS/GVFS.Service/Handlers/EnableAndAttachProjFSHandler.cs index 38bf6c4915..c255a79b6a 100644 --- a/GVFS/GVFS.Service/Handlers/EnableAndAttachProjFSHandler.cs +++ b/GVFS/GVFS.Service/Handlers/EnableAndAttachProjFSHandler.cs @@ -130,7 +130,7 @@ public void Run() if (!string.IsNullOrEmpty(this.request.EnlistmentRoot)) { - if (!ProjFSFilter.TryAttach(this.tracer, this.request.EnlistmentRoot, out errorMessage)) + if (!ProjFSFilter.TryAttach(this.request.EnlistmentRoot, out errorMessage)) { state = NamedPipeMessages.CompletionState.Failure; this.tracer.RelatedError("Unable to attach filter to volume. Enlistment root: {0} \nError: {1} ", this.request.EnlistmentRoot, errorMessage); diff --git a/GVFS/GVFS/CommandLine/MountVerb.cs b/GVFS/GVFS/CommandLine/MountVerb.cs index ad8e0a2f4b..9295e4b140 100644 --- a/GVFS/GVFS/CommandLine/MountVerb.cs +++ b/GVFS/GVFS/CommandLine/MountVerb.cs @@ -117,7 +117,14 @@ protected override void Execute(GVFSEnlistment enlistment) if (!GVFSPlatform.Instance.KernelDriver.IsReady(tracer, enlistment.EnlistmentRoot, out errorMessage)) { - tracer.RelatedInfo($"{nameof(MountVerb)}.{nameof(this.Execute)}: Enabling and attaching ProjFS through service"); + tracer.RelatedEvent( + EventLevel.Informational, + $"{nameof(MountVerb)}_{nameof(this.Execute)}_EnablingKernelDriverViaService", + new EventMetadata + { + { "KernelDriver.IsReady_Error", errorMessage }, + { TracingConstants.MessageKey.InfoMessage, "Service will retry" } + }); if (!this.ShowStatusWhileRunning( () => { return this.TryEnableAndAttachPrjFltThroughService(enlistment.EnlistmentRoot, out errorMessage); }, From 347e1f013ab73dd382bb567e201b440795567303 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 25 Oct 2018 11:23:41 -0400 Subject: [PATCH 207/244] GitProcessTests: add simple unit test --- GVFS/GVFS.UnitTests/Git/GitProcessTests.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 GVFS/GVFS.UnitTests/Git/GitProcessTests.cs diff --git a/GVFS/GVFS.UnitTests/Git/GitProcessTests.cs b/GVFS/GVFS.UnitTests/Git/GitProcessTests.cs new file mode 100644 index 0000000000..c0d7d0b871 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Git/GitProcessTests.cs @@ -0,0 +1,18 @@ +using GVFS.Common.Git; +using GVFS.Tests.Should; +using GVFS.UnitTests.Mock.Common; +using NUnit.Framework; + +namespace GVFS.UnitTests.Git +{ + [TestFixture] + public class GitProcessTests + { + [TestCase] + public void TryKillRunningProcess_NeverRan() + { + GitProcess process = new GitProcess(new MockGVFSEnlistment()); + process.TryKillRunningProcess().ShouldBeTrue(); + } + } +} From 90b6477d1c15a0370cbc2b1781c93a8d4c1179f1 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Thu, 25 Oct 2018 13:20:23 -0400 Subject: [PATCH 208/244] Modify PrepFunctionalTests.sh to work out of the publish directory --- Scripts/Mac/BuildGVFSForMac.sh | 9 +++++++++ Scripts/Mac/PrepFunctionalTests.sh | 9 ++++----- Scripts/Mac/RunFunctionalTests.sh | 6 ------ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index 20190c72be..758b01834f 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -45,4 +45,13 @@ echo 'Copying native binaries to Publish directory...' cp $VFS_OUTPUTDIR/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.ReadObjectHook $VFS_PUBLISHDIR || exit 1 cp $VFS_OUTPUTDIR/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.VirtualFileSystemHook $VFS_PUBLISHDIR || exit 1 +echo 'Copying Git installer to Publish directory...' +GITPUBLISH=$VFS_PUBLISHDIR/Git +if [[ -d $GITPUBLISH ]] ; then + rm -rf $GITPUBLISH +fi +mkdir $GITPUBLISH +cp $GITPATH $GITPUBLISH + +echo 'Running VFS for Git unit tests...' $VFS_PUBLISHDIR/GVFS.UnitTests || exit 1 diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index 8921acf588..f1cadbab85 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -5,14 +5,13 @@ # Ensure the kext isn't loaded before installing Git $VFS_SRCDIR/ProjFS.Mac/Scripts/UnloadPrjFSKext.sh -# Install GVFS-aware Git (that was downloaded by the build script) -GITVERSION="$($SCRIPTDIR/GetGitVersionNumber.sh)" -GITDIR=$VFS_PACKAGESDIR/gitformac.gvfs.installer/$GITVERSION/tools -if [[ ! -d $GITDIR ]]; then +# Install GVFS-aware Git (that was published by the build script) +GITPUBLISH=$VFS_PUBLISHDIR/Git +if [[ ! -d $GITPUBLISH ]]; then echo "GVFS-aware Git package not found. Run BuildGVFSForMac.sh and try again" exit 1 fi -hdiutil attach $GITDIR/*.dmg || exit 1 +hdiutil attach $GITPUBLISH/*.dmg || exit 1 GITPKG="$(find /Volumes/Git* -type f -name *.pkg)" || exit 1 sudo installer -pkg "$GITPKG" -target / || exit 1 hdiutil detach /Volumes/Git* diff --git a/Scripts/Mac/RunFunctionalTests.sh b/Scripts/Mac/RunFunctionalTests.sh index 48a075faba..b6758ffc80 100755 --- a/Scripts/Mac/RunFunctionalTests.sh +++ b/Scripts/Mac/RunFunctionalTests.sh @@ -7,12 +7,6 @@ if [ -z $CONFIGURATION ]; then CONFIGURATION=Debug fi -SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) - -SRCDIR=$SCRIPTDIR/../.. -ROOTDIR=$SRCDIR/.. -PUBLISHDIR=$ROOTDIR/Publish - sudo mkdir /GVFS.FT sudo chown $USER /GVFS.FT From 75ab20c3c44aebc80afb9a3853b71a44b89b0120 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Thu, 25 Oct 2018 10:36:46 +0200 Subject: [PATCH 209/244] Mac kext: Adds nullability attributes to pointer-typed function parameters. The clang compiler supports _Nonnull and _Nullable annotations as an extension to the C & C++ type systems, and will warn if a NULL value is passed to a function expecting a strictly non-null pointer. This further increases the number of programming errors caught by the type checker. The change adds nullability attributes on all parameters of vfs_context_t type, as there are some instances where a nullptr is used (as it turns out, incorrectly) as VFS context. As soon as there is a nullability specifier on one parameter declaration in a header file, clang warns about non-annotated parameters in the same file, so all pointer parameters have been annotated in headers that use vfs_context_t. --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 24 +++++++++---------- .../PrjFSKext/VirtualizationRoots.cpp | 6 ++--- .../PrjFSKext/PrjFSKext/VnodeUtilities.hpp | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 400c6fd120..abf6de9517 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -34,11 +34,11 @@ static int HandleFileOpOperation( uintptr_t arg2, uintptr_t arg3); -static int GetPid(vfs_context_t context); +static int GetPid(vfs_context_t _Nonnull context); -static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context); +static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t _Nonnull context); static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); -static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t context); +static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t _Nonnull context); static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); static bool IsFileSystemCrawler(char* procname); @@ -59,7 +59,7 @@ static bool ShouldIgnoreVnodeType(vtype vnodeType, vnode_t vnode); static bool ShouldHandleVnodeOpEvent( // In params: - vfs_context_t context, + vfs_context_t _Nonnull context, const vnode_t vnode, kauth_action_t action, ProfileSample& operationSample, @@ -75,7 +75,7 @@ static bool ShouldHandleVnodeOpEvent( static bool ShouldHandleFileOpEvent( // In params: - vfs_context_t context, + vfs_context_t _Nonnull context, const vnode_t vnode, kauth_action_t action, @@ -257,7 +257,7 @@ static int HandleVnodeOperation( ProfileSample functionSample(Probe_VnodeOp); - vfs_context_t context = reinterpret_cast(arg0); + vfs_context_t _Nonnull context = reinterpret_cast(arg0); vnode_t currentVnode = reinterpret_cast(arg1); // arg2 is the (vnode_t) parent vnode int* kauthError = reinterpret_cast(arg3); @@ -427,7 +427,7 @@ static int HandleFileOpOperation( ProfileSample functionSample(Probe_FileOp); - vfs_context_t context = vfs_context_create(NULL); + vfs_context_t _Nonnull context = vfs_context_create(NULL); vnode_t currentVnodeFromPath = NULLVP; if (KAUTH_FILEOP_RENAME == action || @@ -575,7 +575,7 @@ static int HandleFileOpOperation( static bool ShouldHandleVnodeOpEvent( // In params: - vfs_context_t context, + vfs_context_t _Nonnull context, const vnode_t vnode, kauth_action_t action, ProfileSample& operationSample, @@ -861,13 +861,13 @@ static void Sleep(int seconds, void* channel) msleep(channel, nullptr, PUSER, "io.gvfs.PrjFSKext.Sleep", &timeout); } -static int GetPid(vfs_context_t context) +static int GetPid(vfs_context_t _Nonnull context) { proc_t callingProcess = vfs_context_proc(context); return proc_pid(callingProcess); } -static errno_t GetVNodeAttributes(vnode_t vn, vfs_context_t context, struct vnode_attr* attrs) +static errno_t GetVNodeAttributes(vnode_t vn, vfs_context_t _Nonnull context, struct vnode_attr* attrs) { VATTR_INIT(attrs); VATTR_WANTED(attrs, va_flags); @@ -875,7 +875,7 @@ static errno_t GetVNodeAttributes(vnode_t vn, vfs_context_t context, struct vnod return vnode_getattr(vn, attrs, context); } -static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context) +static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t _Nonnull context) { struct vnode_attr attributes = {}; errno_t err = GetVNodeAttributes(vn, context, &attributes); @@ -891,7 +891,7 @@ static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit) return 0 != (fileFlags & bit); } -static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t context) +static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t _Nonnull context) { uint32_t vnodeFileFlags = ReadVNodeFileFlags(vnode, context); return FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index 4949e357bf..ec69c2aefe 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -44,7 +44,7 @@ static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t v // Looks up the vnode and fsid/inode pair among the known roots, and if not found, // detects if there is a hitherto-unknown root at vnode by checking attributes. -static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode); +static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_context_t _Nonnull context, const FsidInode& vnodeFsidInode); static VirtualizationRootHandle FindUnusedIndex_Locked(); static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); @@ -165,7 +165,7 @@ VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const Fs return rootHandle; } -static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode) +static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t _Nonnull vnode, vfs_context_t _Nonnull context, const FsidInode& vnodeFsidInode) { uint32_t vid = vnode_vid(vnode); @@ -328,7 +328,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide assert(nullptr != userClient); vnode_t virtualizationRootVNode = NULLVP; - vfs_context_t vfsContext = vfs_context_create(nullptr); + vfs_context_t _Nonnull vfsContext = vfs_context_create(nullptr); VirtualizationRootHandle rootIndex = RootHandle_None; errno_t err = vnode_lookup(virtualizationRootPath, 0 /* flags */, &virtualizationRootVNode, vfsContext); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp index 068c94efdf..53699c2fb8 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp @@ -10,5 +10,5 @@ struct SizeOrError errno_t error; }; -SizeOrError Vnode_ReadXattr(vnode_t vnode, const char* xattrName, void* buffer, size_t bufferSize, vfs_context_t context); -FsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context); +SizeOrError Vnode_ReadXattr(vnode_t _Nonnull vnode, const char* _Nonnull xattrName, void* _Nullable buffer, size_t bufferSize, vfs_context_t _Nonnull context); +FsidInode Vnode_GetFsidAndInode(vnode_t _Nonnull vnode, vfs_context_t _Nonnull context); From a72a5ab9f428c3b936b5f4be3226c77c000bbe78 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Thu, 25 Oct 2018 10:50:16 +0200 Subject: [PATCH 210/244] Mac kext: Removes unused vfs_context_t parameter The Vnode_ReadXattr() helper function had an unused context parameter, to which we were passing a nullptr in one instance. This would have been incorrect had the parameter been used, but really the parameter should be removed. --- ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp | 10 +++++----- ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp | 8 ++++---- ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp | 2 +- ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index ec69c2aefe..9e349108b7 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -44,7 +44,7 @@ static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t v // Looks up the vnode and fsid/inode pair among the known roots, and if not found, // detects if there is a hitherto-unknown root at vnode by checking attributes. -static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_context_t _Nonnull context, const FsidInode& vnodeFsidInode); +static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, const FsidInode& vnodeFsidInode); static VirtualizationRootHandle FindUnusedIndex_Locked(); static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); @@ -132,7 +132,7 @@ kern_return_t VirtualizationRoots_Cleanup() return KERN_FAILURE; } -VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode) +VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t _Nonnull vnode, const FsidInode& vnodeFsidInode) { ProfileSample functionSample(Probe_VirtualizationRoot_Find); @@ -144,7 +144,7 @@ VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const Fs { ProfileSample iterationSample(Probe_VirtualizationRoot_FindIteration); - rootHandle = FindOrDetectRootAtVnode(vnode, nullptr /* vfs context */, vnodeFsidInode); + rootHandle = FindOrDetectRootAtVnode(vnode, vnodeFsidInode); // Note: if FindOrDetectRootAtVnode returns a "special" handle other // than RootHandle_None, we want to stop the search and return that. if (rootHandle != RootHandle_None) @@ -165,7 +165,7 @@ VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const Fs return rootHandle; } -static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t _Nonnull vnode, vfs_context_t _Nonnull context, const FsidInode& vnodeFsidInode) +static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t _Nonnull vnode, const FsidInode& vnodeFsidInode) { uint32_t vid = vnode_vid(vnode); @@ -180,7 +180,7 @@ static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t _Nonnull vnode, if (rootIndex == RootHandle_None) { PrjFSVirtualizationRootXAttrData rootXattr = {}; - SizeOrError xattrResult = Vnode_ReadXattr(vnode, PrjFSVirtualizationRootXAttrName, &rootXattr, sizeof(rootXattr), context); + SizeOrError xattrResult = Vnode_ReadXattr(vnode, PrjFSVirtualizationRootXAttrName, &rootXattr, sizeof(rootXattr)); if (xattrResult.error == 0) { // TODO: check xattr contents diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp index 2b19b0681d..83cd0f8c0e 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp @@ -21,20 +21,20 @@ enum VirtualizationRootSpecialHandle : VirtualizationRootHandle kern_return_t VirtualizationRoots_Init(void); kern_return_t VirtualizationRoots_Cleanup(void); -VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode); +VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t _Nonnull vnode, const FsidInode& vnodeFsidInode); struct VirtualizationRootResult { errno_t error; VirtualizationRootHandle root; }; -VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProviderUserClient* userClient, pid_t clientPID, const char* virtualizationRootPath); +VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProviderUserClient* _Nonnull userClient, pid_t clientPID, const char* _Nonnull virtualizationRootPath); void ActiveProvider_Disconnect(VirtualizationRootHandle rootHandle); struct Message; errno_t ActiveProvider_SendMessage(VirtualizationRootHandle rootHandle, const Message message); -bool VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode_t vnode); +bool VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode_t _Nonnull vnode); bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootHandle); bool VirtualizationRoot_PIDMatchesProvider(VirtualizationRootHandle rootHandle, pid_t pid); bool VirtualizationRoot_IsValidRootHandle(VirtualizationRootHandle rootHandle); -const char* VirtualizationRoot_GetRootRelativePath(VirtualizationRootHandle rootHandle, const char* path); +const char* _Nonnull VirtualizationRoot_GetRootRelativePath(VirtualizationRootHandle rootHandle, const char* _Nonnull path); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp index c73aa5bfed..bfa021f017 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp @@ -18,7 +18,7 @@ FsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context) return { statfs->f_fsid, attrs.va_fileid }; } -SizeOrError Vnode_ReadXattr(vnode_t vnode, const char* xattrName, void* buffer, size_t bufferSize, vfs_context_t context) +SizeOrError Vnode_ReadXattr(vnode_t vnode, const char* xattrName, void* buffer, size_t bufferSize) { size_t actualSize = bufferSize; errno_t error = mac_vnop_getxattr(vnode, xattrName, static_cast(buffer), bufferSize, &actualSize); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp index 53699c2fb8..323317fb55 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp @@ -10,5 +10,5 @@ struct SizeOrError errno_t error; }; -SizeOrError Vnode_ReadXattr(vnode_t _Nonnull vnode, const char* _Nonnull xattrName, void* _Nullable buffer, size_t bufferSize, vfs_context_t _Nonnull context); +SizeOrError Vnode_ReadXattr(vnode_t _Nonnull vnode, const char* _Nonnull xattrName, void* _Nullable buffer, size_t bufferSize); FsidInode Vnode_GetFsidAndInode(vnode_t _Nonnull vnode, vfs_context_t _Nonnull context); From 583a33bd8e3a3c26af0cd9c8d207acdce7859d7d Mon Sep 17 00:00:00 2001 From: John Briggs Date: Thu, 25 Oct 2018 16:37:25 -0400 Subject: [PATCH 211/244] Relocated Git installer to BuildOuptut --- Scripts/Mac/BuildGVFSForMac.sh | 9 ++------- Scripts/Mac/PrepFunctionalTests.sh | 2 +- Scripts/Mac/PublishGit.sh | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) create mode 100755 Scripts/Mac/PublishGit.sh diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index 758b01834f..7a893784eb 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -45,13 +45,8 @@ echo 'Copying native binaries to Publish directory...' cp $VFS_OUTPUTDIR/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.ReadObjectHook $VFS_PUBLISHDIR || exit 1 cp $VFS_OUTPUTDIR/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.VirtualFileSystemHook $VFS_PUBLISHDIR || exit 1 -echo 'Copying Git installer to Publish directory...' -GITPUBLISH=$VFS_PUBLISHDIR/Git -if [[ -d $GITPUBLISH ]] ; then - rm -rf $GITPUBLISH -fi -mkdir $GITPUBLISH -cp $GITPATH $GITPUBLISH +echo 'Copying Git installer to the output directory...' +$VFS_SCRIPTDIR/PublishGit.sh $GITPATH || exit 1 echo 'Running VFS for Git unit tests...' $VFS_PUBLISHDIR/GVFS.UnitTests || exit 1 diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index f1cadbab85..341ea6fe10 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -6,7 +6,7 @@ $VFS_SRCDIR/ProjFS.Mac/Scripts/UnloadPrjFSKext.sh # Install GVFS-aware Git (that was published by the build script) -GITPUBLISH=$VFS_PUBLISHDIR/Git +GITPUBLISH=$VFS_OUTPUTDIR/Git if [[ ! -d $GITPUBLISH ]]; then echo "GVFS-aware Git package not found. Run BuildGVFSForMac.sh and try again" exit 1 diff --git a/Scripts/Mac/PublishGit.sh b/Scripts/Mac/PublishGit.sh new file mode 100755 index 0000000000..cb048fa1df --- /dev/null +++ b/Scripts/Mac/PublishGit.sh @@ -0,0 +1,15 @@ +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + +GITPATH=$1 +INSTALLER=$(basename $GITPATH) + +GITPUBLISH=$VFS_OUTPUTDIR/Git +if [[ ! -d $GITPUBLISH ]] ; then + mkdir $GITPUBLISH +fi + +find $GITPUBLISH -type f ! -name $INSTALLER -delete + +if [[ ! -e $GITPUBLISH/$INSTALLER ]] ; then + cp $GITPATH $GITPUBLISH +fi From 2753b722d50a19433b0811efe353d19ac273b37e Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 25 Oct 2018 11:46:02 -0700 Subject: [PATCH 212/244] PR Feedback: Use different GitIndexEntry methods when building new projections and running background tasks --- GVFS/GVFS.Common/GVFSPlatform.cs | 2 + GVFS/GVFS.Common/PlaceholderListDatabase.cs | 67 ++++--- .../Tests/GitCommands/UpdateIndexTests.cs | 11 -- ...skLayout12to13Upgrade_FolderPlaceholder.cs | 2 +- ...ut16to17Upgrade_FolderPlaceholderValues.cs | 2 +- .../Common/PlaceholderDatabaseTests.cs | 12 +- .../Projection/GitIndexEntryTests.cs | 48 +++-- .../GitIndexProjection.GitIndexEntry.cs | 169 +++++++++++------- .../GitIndexProjection.GitIndexParser.cs | 10 +- .../Projection/GitIndexProjection.cs | 16 +- 10 files changed, 206 insertions(+), 133 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index 4341b589f8..15ef10e0bb 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -81,6 +81,8 @@ public bool TryGetNormalizedPathRoot(string path, out string pathRoot, out strin public class GVFSPlatformConstants { + public static readonly string PathSeparatorString = Path.DirectorySeparatorChar.ToString(); + public GVFSPlatformConstants(string executableExtension, string installerExtension) { this.ExecutableExtension = executableExtension; diff --git a/GVFS/GVFS.Common/PlaceholderListDatabase.cs b/GVFS/GVFS.Common/PlaceholderListDatabase.cs index 667503b8df..81c8a4d1fe 100644 --- a/GVFS/GVFS.Common/PlaceholderListDatabase.cs +++ b/GVFS/GVFS.Common/PlaceholderListDatabase.cs @@ -16,15 +16,28 @@ public class PlaceholderListDatabase : FileBasedCollection private const char PathTerminator = '\0'; - // This list holds entries that would otherwise be lost because WriteAllEntriesAndFlush has not been called, but a file - // snapshot has been taken using GetAllEntries. - // See the unit test PlaceholderDatabaseTests.HandlesRaceBetweenAddAndWriteAllEntries for example + // This list holds placeholder entries that are created between calls to + // GetAllEntriesAndPrepToWriteAllEntries and WriteAllEntriesAndFlush. // - // With this list, we can no longer call GetAllEntries without a matching WriteAllEntries afterwards. + // Example: + // + // 1) VFS4G parses the updated index (as part of a projection change) + // 2) VFS4G starts the work to update placeholders + // 3) VFS4G calls GetAllEntriesAndPrepToWriteAllEntries + // 4) VFS4G starts updating placeholders + // 5) Some application reads a pure-virtual file (creating a new placeholder) while VFS4G is updating existing placeholders. + // That new placeholder is added to placeholderChangesWhileRebuildingList. + // 6) VFS4G completes updating the placeholders and calls WriteAllEntriesAndFlush. + // Note: this list does *not* include the placeholders created in step 5, as the were not included in GetAllEntries. + // 7) WriteAllEntriesAndFlush writes *both* the entires in placeholderDataEntries and those that were passed in as the parameter. + // + // This scenario is covered in the unit test PlaceholderDatabaseTests.HandlesRaceBetweenAddAndWriteAllEntries + // + // Because of this list, callers must always call WriteAllEntries after calling GetAllEntriesAndPrepToWriteAllEntries. // // This list must always be accessed from inside one of FileBasedCollection's synchronizedAction callbacks because // there is race potential between creating the queue, adding to the queue, and writing to the data file. - private List placeholderDataEntries; + private List placeholderChangesWhileRebuildingList; private PlaceholderListDatabase(ITracer tracer, PhysicalFileSystem fileSystem, string dataFilePath) : base(tracer, fileSystem, dataFilePath, collectionAppendsDirectlyToFile: true) @@ -77,9 +90,9 @@ public void RemoveAndFlush(string path) () => { this.EstimatedCount--; - if (this.placeholderDataEntries != null) + if (this.placeholderChangesWhileRebuildingList != null) { - this.placeholderDataEntries.Add(new PlaceholderDataEntry(path)); + this.placeholderChangesWhileRebuildingList.Add(new PlaceholderDataEntry(path)); } }); } @@ -89,7 +102,14 @@ public void RemoveAndFlush(string path) } } - public List GetAllEntries() + /// + /// Gets all entries and intializes placeholderChangesWhileRebuildingList in preparation fr a call to WriteAllEntriesAndFlush. + /// + /// + /// See placeholderChangesWhileRebuildingList declaration for additional details as to why WriteAllEntriesAndFlush + /// must be called. + /// + public List GetAllEntriesAndPrepToWriteAllEntries() { try { @@ -103,12 +123,12 @@ public List GetAllEntries() out error, () => { - if (this.placeholderDataEntries != null) + if (this.placeholderChangesWhileRebuildingList != null) { - throw new InvalidOperationException("PlaceholderListDatabase should always flush queue placeholders using WriteAllEntriesAndFlush before calling GetAllEntries again."); + throw new InvalidOperationException($"PlaceholderListDatabase should always flush queue placeholders using WriteAllEntriesAndFlush before calling {nameof(this.GetAllEntriesAndPrepToWriteAllEntries)} again."); } - this.placeholderDataEntries = new List(); + this.placeholderChangesWhileRebuildingList = new List(); })) { throw new InvalidDataException(error); @@ -122,7 +142,14 @@ public List GetAllEntries() } } - public void GetAllEntries(out List filePlaceholders, out List folderPlaceholders) + /// + /// Gets all entries and intializes placeholderChangesWhileRebuildingList in preparation fr a call to WriteAllEntriesAndFlush. + /// + /// + /// See placeholderChangesWhileRebuildingList declaration for additional details as to why WriteAllEntriesAndFlush + /// must be called. + /// + public void GetAllEntriesAndPrepToWriteAllEntries(out List filePlaceholders, out List folderPlaceholders) { try { @@ -147,12 +174,12 @@ public void GetAllEntries(out List filePlaceholders, out List

{ - if (this.placeholderDataEntries != null) + if (this.placeholderChangesWhileRebuildingList != null) { - throw new InvalidOperationException("PlaceholderListDatabase should always flush queue placeholders using WriteAllEntriesAndFlush before calling GetAllEntries again."); + throw new InvalidOperationException($"PlaceholderListDatabase should always flush queue placeholders using WriteAllEntriesAndFlush before calling {(nameof(this.GetAllEntriesAndPrepToWriteAllEntries))} again."); } - this.placeholderDataEntries = new List(); + this.placeholderChangesWhileRebuildingList = new List(); })) { throw new InvalidDataException(error); @@ -225,9 +252,9 @@ private IEnumerable GenerateDataLines(IEnumerable updat yield return this.FormatAddLine(updated.Path + PathTerminator + updated.Sha); } - if (this.placeholderDataEntries != null) + if (this.placeholderChangesWhileRebuildingList != null) { - foreach (PlaceholderDataEntry entry in this.placeholderDataEntries) + foreach (PlaceholderDataEntry entry in this.placeholderChangesWhileRebuildingList) { if (entry.DeleteEntry) { @@ -248,7 +275,7 @@ private IEnumerable GenerateDataLines(IEnumerable updat } } - this.placeholderDataEntries = null; + this.placeholderChangesWhileRebuildingList = null; } } @@ -261,9 +288,9 @@ private void AddAndFlush(string path, string sha) () => { this.EstimatedCount++; - if (this.placeholderDataEntries != null) + if (this.placeholderChangesWhileRebuildingList != null) { - this.placeholderDataEntries.Add(new PlaceholderDataEntry(path, sha)); + this.placeholderChangesWhileRebuildingList.Add(new PlaceholderDataEntry(path, sha)); } }); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index feb648353b..1d48aa48c6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -22,7 +22,6 @@ public void UpdateIndexRemoveFileOnDisk() } [TestCase] - [Ignore("TODO 940287: git update-index --add does not work properly if placeholder is not already on disk")] public void UpdateIndexRemoveFileOnDiskDontCheckStatus() { // TODO 940287: Remove this test and re-enable UpdateIndexRemoveFileOnDisk @@ -51,16 +50,6 @@ public void UpdateIndexRemoveAddFileOpenForWrite() GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "update-index --remove Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt"); this.FilesShouldMatchCheckoutOfTargetBranch(); - // Open Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt for write so that it's added to the modified paths database - using ( - FileStream stream = File.Open( - Path.Combine(this.Enlistment.RepoRoot, "Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"), - FileMode.Open, - FileAccess.Write)) - { - // TODO 940287: Remove this File.Open once update-index --add\--remove are working as expected - } - // Add the files back to the index so the git-status that is run during teardown matches GitProcess.InvokeProcess(this.ControlGitRepo.RootPath, "update-index --add Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt"); GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "update-index --add Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt"); diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs index e2d88bf09c..5ad4b0e1c0 100644 --- a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs @@ -48,7 +48,7 @@ public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) .Select(x => x.Substring(workingDirectoryRoot.Length + 1)) .Select(x => new PlaceholderListDatabase.PlaceholderData(x, GVFSConstants.AllZeroSha)); - List placeholderEntries = placeholders.GetAllEntries(); + List placeholderEntries = placeholders.GetAllEntriesAndPrepToWriteAllEntries(); placeholderEntries.AddRange(folderPlaceholderPaths); placeholders.WriteAllEntriesAndFlush(placeholderEntries); diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs index 24893770a0..f335f0d7f1 100644 --- a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs @@ -35,7 +35,7 @@ public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) using (placeholders) { - List oldPlaceholderEntries = placeholders.GetAllEntries(); + List oldPlaceholderEntries = placeholders.GetAllEntriesAndPrepToWriteAllEntries(); List newPlaceholderEntries = new List(); foreach (PlaceholderListDatabase.PlaceholderData entry in oldPlaceholderEntries) diff --git a/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs index 465e4b9075..3940db6abd 100644 --- a/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs @@ -61,7 +61,7 @@ public void WritesPlaceholderAddToFile() } [TestCase] - public void GetAllEntriesReturnsCorrectEntries() + public void GetAllEntriesAndPrepToWriteAllEntriesReturnsCorrectEntries() { ConfigurableFileSystem fs = new ConfigurableFileSystem(); using (PlaceholderListDatabase dut1 = CreatePlaceholderListDatabase(fs, string.Empty)) @@ -75,12 +75,12 @@ public void GetAllEntriesReturnsCorrectEntries() string error; PlaceholderListDatabase dut2; PlaceholderListDatabase.TryCreate(null, MockEntryFileName, fs, out dut2, out error).ShouldEqual(true, error); - List allData = dut2.GetAllEntries(); + List allData = dut2.GetAllEntriesAndPrepToWriteAllEntries(); allData.Count.ShouldEqual(2); } [TestCase] - public void GetAllEntriesSplitsFilesAndFoldersCorrectly() + public void GetAllEntriesAndPrepToWriteAllEntriesSplitsFilesAndFoldersCorrectly() { ConfigurableFileSystem fs = new ConfigurableFileSystem(); using (PlaceholderListDatabase dut1 = CreatePlaceholderListDatabase(fs, string.Empty)) @@ -98,7 +98,7 @@ public void GetAllEntriesSplitsFilesAndFoldersCorrectly() PlaceholderListDatabase.TryCreate(null, MockEntryFileName, fs, out dut2, out error).ShouldEqual(true, error); List fileData; List folderData; - dut2.GetAllEntries(out fileData, out folderData); + dut2.GetAllEntriesAndPrepToWriteAllEntries(out fileData, out folderData); fileData.Count.ShouldEqual(2); folderData.Count.ShouldEqual(2); folderData.ShouldContain( @@ -136,7 +136,7 @@ public void HandlesRaceBetweenAddAndWriteAllEntries() PlaceholderListDatabase dut = CreatePlaceholderListDatabase(fs, ExpectedGitIgnoreEntry); - List existingEntries = dut.GetAllEntries(); + List existingEntries = dut.GetAllEntriesAndPrepToWriteAllEntries(); dut.AddAndFlushFile(InputGitAttributesPath, InputGitAttributesSHA); @@ -154,7 +154,7 @@ public void HandlesRaceBetweenRemoveAndWriteAllEntries() PlaceholderListDatabase dut = CreatePlaceholderListDatabase(fs, ExpectedTwoEntries); - List existingEntries = dut.GetAllEntries(); + List existingEntries = dut.GetAllEntriesAndPrepToWriteAllEntries(); dut.RemoveAndFlush(InputGitAttributesPath); diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs index 9267d2acd7..ebfff8627f 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs @@ -8,20 +8,20 @@ namespace GVFS.UnitTests.Virtualization.Git { - [TestFixtureSource(TestLazyPaths)] + [TestFixtureSource(TestBuildingNewProjection)] public class GitIndexEntryTests { - public const string TestLazyPaths = "ShouldTestLazyPaths"; + public const string TestBuildingNewProjection = "BuildingNewProjection"; private const int DefaultIndexEntryCount = 10; - private bool shouldTestLazyPaths; + private bool buildingNewProjection; public GitIndexEntryTests(bool shouldTestLazyPaths) { - this.shouldTestLazyPaths = shouldTestLazyPaths; + this.buildingNewProjection = shouldTestLazyPaths; } - public static object[] ShouldTestLazyPaths + public static object[] BuildingNewProjection { get { @@ -157,7 +157,7 @@ public void ClearLastParent() private GitIndexEntry SetupIndexEntry(string path) { - GitIndexEntry indexEntry = new GitIndexEntry(this.shouldTestLazyPaths); + GitIndexEntry indexEntry = new GitIndexEntry(this.buildingNewProjection); this.ParsePathForIndexEntry(indexEntry, path, replaceIndex: 0); return indexEntry; } @@ -168,7 +168,15 @@ private void ParsePathForIndexEntry(GitIndexEntry indexEntry, string path, int r Buffer.BlockCopy(pathBuffer, 0, indexEntry.PathBuffer, 0, path.Length); indexEntry.PathLength = path.Length; indexEntry.ReplaceIndex = replaceIndex; - indexEntry.ParsePath(); + + if (this.buildingNewProjection) + { + indexEntry.BuildingProjection_ParsePath(); + } + else + { + indexEntry.BackgroundTask_ParsePath(); + } } private void TestPathParts(GitIndexEntry indexEntry, string[] pathParts, bool hasSameParent) @@ -177,23 +185,27 @@ private void TestPathParts(GitIndexEntry indexEntry, string[] pathParts, bool ha indexEntry.NumParts.ShouldEqual(pathParts.Length, nameof(indexEntry.NumParts)); for (int i = 0; i < pathParts.Length; i++) { - if (this.shouldTestLazyPaths) + if (this.buildingNewProjection) { - indexEntry.GetLazyPathPart(i).ShouldNotBeNull(); - indexEntry.GetLazyPathPart(i).GetString().ShouldEqual(pathParts[i]); + indexEntry.BuildingProjection_PathParts[i].ShouldNotBeNull(); + indexEntry.BuildingProjection_PathParts[i].GetString().ShouldEqual(pathParts[i]); + } + else + { + indexEntry.BackgroundTask_PathParts[i].ShouldNotBeNull(); + indexEntry.BackgroundTask_PathParts[i].ShouldEqual(pathParts[i]); } - - indexEntry.GetPathPart(i).ShouldNotBeNull(); - indexEntry.GetPathPart(i).ShouldEqual(pathParts[i]); } - if (this.shouldTestLazyPaths) + if (this.buildingNewProjection) { - indexEntry.GetLazyChildName().GetString().ShouldEqual(pathParts[pathParts.Length - 1]); + indexEntry.BuildingProjection_GetChildName().GetString().ShouldEqual(pathParts[pathParts.Length - 1]); + indexEntry.BuildingProjection_GetGitRelativePath().ShouldEqual(string.Join("/", pathParts)); + } + else + { + indexEntry.BackgroundTask_GetPlatformRelativePath().ShouldEqual(string.Join(Path.DirectorySeparatorChar.ToString(), pathParts)); } - - indexEntry.GetGitPath().ShouldEqual(string.Join("/", pathParts)); - indexEntry.GetRelativePath().ShouldEqual(string.Join(Path.DirectorySeparatorChar.ToString(), pathParts)); } } } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 5c13294183..4fa30f08cc 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -21,25 +21,17 @@ internal class GitIndexEntry private const int MaxParts = MaxPathBufferSize / 2; private const byte PathSeparatorCode = 0x2F; - private static readonly string PathSeparatorString = Path.DirectorySeparatorChar.ToString(); - private int previousFinalSeparatorIndex = int.MaxValue; - // lazyPathParts and utf16PathParts are mutually exclusive - // The `useLazyPaths` parameter of the GitIndexEntry constructor determines - // which of these two arrays will be used - private LazyUTF8String[] lazyPathParts; - private string[] utf16PathParts; - - public GitIndexEntry(bool useLazyPaths) + public GitIndexEntry(bool buildingNewProjection) { - if (useLazyPaths) + if (buildingNewProjection) { - this.lazyPathParts = new LazyUTF8String[MaxParts]; + this.BuildingProjection_PathParts = new LazyUTF8String[MaxParts]; } else { - this.utf16PathParts = new string[MaxParts]; + this.BackgroundTask_PathParts = new string[MaxParts]; } } @@ -56,38 +48,36 @@ public GitIndexEntry(bool useLazyPaths) public byte[] PathBuffer { get; } = new byte[MaxPathBufferSize]; public FolderData LastParent { get; set; } - public int NumParts + // Only used when buildingNewProjection is true + public LazyUTF8String[] BuildingProjection_PathParts { get; private set; } - public bool HasSameParentAsLastEntry + // Only used when buildingNewProjection is false + public string[] BackgroundTask_PathParts { get; private set; } - public string GetPathPart(int index) + public int NumParts { - if (this.lazyPathParts != null) - { - return this.lazyPathParts[index].GetString(); - } - - return this.utf16PathParts[index]; + get; private set; } - public LazyUTF8String GetLazyPathPart(int index) + public bool HasSameParentAsLastEntry { - if (this.lazyPathParts == null) - { - throw new InvalidOperationException( - $"{nameof(GetLazyPathPart)} can only be called when useLazyPaths is set to true when creating {nameof(GitIndexEntry)}"); - } - - return this.lazyPathParts[index]; + get; private set; } - public unsafe void ParsePath() + ///

+ /// Parses the path using LazyUTF8Strings. It should only be called when building a new projection. + /// + /// + /// Code in this method has been fine-tuned for performance. None of it is shared with + /// BackgroundTask_ParsePath to avoid overhead. + /// + public unsafe void BuildingProjection_ParsePath() { this.PathBuffer[this.PathLength] = 0; @@ -139,22 +129,11 @@ public unsafe void ParsePath() int partIndex = this.NumParts; byte* forLoopPtr = pathPtr + forLoopStartIndex; - byte* bufferPtr; - int bufferLength; for (int i = forLoopStartIndex; i < this.PathLength + 1; i++) { if (*forLoopPtr == PathSeparatorCode) { - bufferPtr = pathPtr + currentPartStartIndex; - bufferLength = i - currentPartStartIndex; - if (this.lazyPathParts != null) - { - this.lazyPathParts[partIndex] = LazyUTF8String.FromByteArray(bufferPtr, bufferLength); - } - else - { - this.utf16PathParts[partIndex] = Encoding.UTF8.GetString(bufferPtr, bufferLength); - } + this.BuildingProjection_PathParts[partIndex] = LazyUTF8String.FromByteArray(pathPtr + currentPartStartIndex, i - currentPartStartIndex); partIndex++; currentPartStartIndex = i + 1; @@ -167,17 +146,91 @@ public unsafe void ParsePath() } // We unrolled the final part calculation to after the loop, to avoid having to do a 0-byte check inside the for loop - bufferPtr = pathPtr + currentPartStartIndex; - bufferLength = this.PathLength - currentPartStartIndex; - if (this.lazyPathParts != null) + this.BuildingProjection_PathParts[partIndex] = LazyUTF8String.FromByteArray(pathPtr + currentPartStartIndex, this.PathLength - currentPartStartIndex); + + this.NumParts++; + } + } + + + /// + /// Parses the path without using LazyUTF8Strings. It should only be called when running a background task. + /// + /// + /// Code in this method has been fine-tuned for performance. None of it is shared with + /// BuildingProjection_ParsePath to avoid overhead. + /// + public unsafe void BackgroundTask_ParsePath() + { + this.PathBuffer[this.PathLength] = 0; + + // The index of that path part that is after the path separator + int currentPartStartIndex = 0; + + // The index to start looking for the next path separator + // Because the previous final separator is stored and we know where the previous path will be replaced + // the code can use the previous final separator to start looking from that point instead of having to + // run through the entire path to break it apart + /* Example: + * Previous path = folder/where/previous/separator/is/used/file.txt + * This path = folder/where/previous/separator/is/used/file2.txt + * ^ ^ + * this.previousFinalSeparatorIndex | + * this.ReplaceIndex + * + * folder/where/previous/separator/is/used/file2.txt + * ^^ + * currentPartStartIndex| + * forLoopStartIndex + */ + int forLoopStartIndex = 0; + + fixed (byte* pathPtr = this.PathBuffer) + { + if (this.previousFinalSeparatorIndex < this.ReplaceIndex && + !this.RangeContains(pathPtr + this.ReplaceIndex, this.PathLength - this.ReplaceIndex, PathSeparatorCode)) { - this.lazyPathParts[partIndex] = LazyUTF8String.FromByteArray(bufferPtr, bufferLength); + // Only need to parse the last part, because the rest of the string is unchanged + + // The logical thing to do would be to start the for loop at previousFinalSeparatorIndex+1, but two + // repeated / characters would make an invalid path, so we'll assume that git would not have stored that path + forLoopStartIndex = this.previousFinalSeparatorIndex + 2; + + // we still do need to start the current part's index at the correct spot, so subtract one for that + currentPartStartIndex = forLoopStartIndex - 1; + + this.NumParts--; + + this.HasSameParentAsLastEntry = true; } else { - this.utf16PathParts[partIndex] = Encoding.UTF8.GetString(bufferPtr, bufferLength); + this.NumParts = 0; + this.ClearLastParent(); } + int partIndex = this.NumParts; + + byte* forLoopPtr = pathPtr + forLoopStartIndex; + for (int i = forLoopStartIndex; i < this.PathLength + 1; i++) + { + if (*forLoopPtr == PathSeparatorCode) + { + this.BackgroundTask_PathParts[partIndex] = Encoding.UTF8.GetString(pathPtr + currentPartStartIndex, i - currentPartStartIndex); + + partIndex++; + currentPartStartIndex = i + 1; + + this.NumParts++; + this.previousFinalSeparatorIndex = i; + } + + ++forLoopPtr; + } + + // We unrolled the final part calculation to after the loop, to avoid having to do a 0-byte check inside the for loop + this.BackgroundTask_PathParts[partIndex] = Encoding.UTF8.GetString(pathPtr + currentPartStartIndex, this.PathLength - currentPartStartIndex); + this.NumParts++; } } @@ -189,29 +242,19 @@ public void ClearLastParent() this.LastParent = null; } - public LazyUTF8String GetLazyChildName() + public LazyUTF8String BuildingProjection_GetChildName() { - return this.GetLazyPathPart(this.NumParts - 1); + return this.BuildingProjection_PathParts[this.NumParts - 1]; } - public string GetGitPath() + public string BuildingProjection_GetGitRelativePath() { - return this.GetPath(GVFSConstants.GitPathSeparatorString); + return string.Join(GVFSConstants.GitPathSeparatorString, this.BuildingProjection_PathParts.Take(this.NumParts).Select(x => x.GetString())); } - public string GetRelativePath() + public string BackgroundTask_GetPlatformRelativePath() { - return this.GetPath(PathSeparatorString); - } - - private string GetPath(string separator) - { - if (this.lazyPathParts != null) - { - return string.Join(separator, this.lazyPathParts.Take(this.NumParts).Select(x => x.GetString())); - } - - return string.Join(separator, this.utf16PathParts.Take(this.NumParts)); + return string.Join(GVFSPlatform.GVFSPlatformConstants.PathSeparatorString, this.BackgroundTask_PathParts.Take(this.NumParts)); } private unsafe bool RangeContains(byte* bufferPtr, int count, byte value) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 23272622d2..3adc800aba 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -26,12 +26,12 @@ internal partial class GitIndexParser /// /// A single GitIndexEntry instance used for parsing all entries in the index when building the projection /// - private GitIndexEntry resuableProjectionBuildingIndexEntry = new GitIndexEntry(useLazyPaths: true); + private GitIndexEntry resuableProjectionBuildingIndexEntry = new GitIndexEntry(buildingNewProjection: true); /// /// A single GitIndexEntry instance used by the background task thread for parsing all entries in the index /// - private GitIndexEntry resuableBackgroundTaskThreadIndexEntry = new GitIndexEntry(useLazyPaths: false); + private GitIndexEntry resuableBackgroundTaskThreadIndexEntry = new GitIndexEntry(buildingNewProjection: false); public GitIndexParser(GitIndexProjection projection) { @@ -142,7 +142,7 @@ private FileSystemTaskResult AddIndexEntryToProjection(GitIndexEntry data) // Never want to project the common ancestor even if the skip worktree bit is on if ((data.MergeState != MergeStage.CommonAncestor && data.SkipWorktree) || data.MergeState == MergeStage.Yours) { - data.ParsePath(); + data.BuildingProjection_ParsePath(); this.projection.AddItemFromIndexEntry(data); } else @@ -167,8 +167,8 @@ private FileSystemTaskResult AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfN GitIndexEntry gitIndexEntry, Dictionary filePlaceholders) { - gitIndexEntry.ParsePath(); - string placeholderRelativePath = gitIndexEntry.GetRelativePath(); + gitIndexEntry.BackgroundTask_ParsePath(); + string placeholderRelativePath = gitIndexEntry.BackgroundTask_GetPlatformRelativePath(); FileSystemTaskResult result = FileSystemTaskResult.Success; diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 555a75c627..663943600d 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -669,14 +669,14 @@ private void AddItemFromIndexEntry(GitIndexEntry indexEntry) { if (indexEntry.HasSameParentAsLastEntry) { - indexEntry.LastParent.AddChildFile(indexEntry.GetLazyChildName(), indexEntry.Sha); + indexEntry.LastParent.AddChildFile(indexEntry.BuildingProjection_GetChildName(), indexEntry.Sha); } else { if (indexEntry.NumParts == 1) { indexEntry.LastParent = this.rootFolderData; - indexEntry.LastParent.AddChildFile(indexEntry.GetLazyChildName(), indexEntry.Sha); + indexEntry.LastParent.AddChildFile(indexEntry.BuildingProjection_GetChildName(), indexEntry.Sha); } else { @@ -694,7 +694,7 @@ private void AddItemFromIndexEntry(GitIndexEntry indexEntry) // TODO(Mac): The line below causes a conversion from LazyUTF8String to .NET string. // Measure the perf and memory overhead of performing this conversion, and determine if we need // a way to keep the path as LazyUTF8String[] - this.nonDefaultFileTypesAndModes.Add(indexEntry.GetGitPath(), indexEntry.TypeAndMode); + this.nonDefaultFileTypesAndModes.Add(indexEntry.BuildingProjection_GetGitRelativePath(), indexEntry.TypeAndMode); } } } @@ -794,14 +794,14 @@ private FolderData AddFileToTree(GitIndexEntry indexEntry) string parentFolderName; if (pathIndex > 0) { - parentFolderName = indexEntry.GetPathPart(pathIndex - 1); + parentFolderName = indexEntry.BuildingProjection_PathParts[pathIndex - 1].GetString(); } else { parentFolderName = this.rootFolderData.Name.GetString(); } - string gitPath = indexEntry.GetGitPath(); + string gitPath = indexEntry.BuildingProjection_GetGitRelativePath(); EventMetadata metadata = CreateEventMetadata(); metadata.Add("gitPath", gitPath); @@ -811,10 +811,10 @@ private FolderData AddFileToTree(GitIndexEntry indexEntry) throw new InvalidDataException("Found a file (" + parentFolderName + ") where a folder was expected: " + gitPath); } - parentFolder = parentFolder.ChildEntries.GetOrAddFolder(indexEntry.GetLazyPathPart(pathIndex)); + parentFolder = parentFolder.ChildEntries.GetOrAddFolder(indexEntry.BuildingProjection_PathParts[pathIndex]); } - parentFolder.AddChildFile(indexEntry.GetLazyPathPart(indexEntry.NumParts - 1), indexEntry.Sha); + parentFolder.AddChildFile(indexEntry.BuildingProjection_PathParts[indexEntry.NumParts - 1], indexEntry.Sha); return parentFolder; } @@ -1111,7 +1111,7 @@ private void UpdatePlaceholders() List placeholderFilesListCopy; List placeholderFoldersListCopy; - this.placeholderList.GetAllEntries(out placeholderFilesListCopy, out placeholderFoldersListCopy); + this.placeholderList.GetAllEntriesAndPrepToWriteAllEntries(out placeholderFilesListCopy, out placeholderFoldersListCopy); EventMetadata metadata = new EventMetadata(); metadata.Add("File placeholder count", placeholderFilesListCopy.Count); From 97dfe315b49b03d1cbc10a2a64eb35068f18277c Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 25 Oct 2018 14:13:14 -0700 Subject: [PATCH 213/244] StyleCop fixes --- .../Projection/GitIndexProjection.GitIndexEntry.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 4fa30f08cc..7261257647 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -152,7 +152,6 @@ public unsafe void BuildingProjection_ParsePath() } } - /// /// Parses the path without using LazyUTF8Strings. It should only be called when running a background task. /// From 8a6b1bd2f065d346370c89988486846d03a9b1db Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Thu, 25 Oct 2018 13:00:24 -0700 Subject: [PATCH 214/244] Implement a generic bool array for use with parameterized TestFixtures. --- GVFS/GVFS.Tests/DataSources.cs | 18 ++++++++++++++++++ .../GVFS.UnitTests/Prefetch/DiffHelperTests.cs | 18 +++--------------- 2 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 GVFS/GVFS.Tests/DataSources.cs diff --git a/GVFS/GVFS.Tests/DataSources.cs b/GVFS/GVFS.Tests/DataSources.cs new file mode 100644 index 0000000000..c333ed1e03 --- /dev/null +++ b/GVFS/GVFS.Tests/DataSources.cs @@ -0,0 +1,18 @@ +using System; +namespace GVFS.Tests +{ + public class DataSources + { + public static object[] AllBools + { + get + { + return new object[] + { + new object[] { true }, + new object[] { false }, + }; + } + } + } +} diff --git a/GVFS/GVFS.UnitTests/Prefetch/DiffHelperTests.cs b/GVFS/GVFS.UnitTests/Prefetch/DiffHelperTests.cs index e7aded0016..b0daa4ed2c 100644 --- a/GVFS/GVFS.UnitTests/Prefetch/DiffHelperTests.cs +++ b/GVFS/GVFS.UnitTests/Prefetch/DiffHelperTests.cs @@ -1,5 +1,6 @@ using GVFS.Common.Git; using GVFS.Common.Prefetch.Git; +using GVFS.Tests; using GVFS.Tests.Should; using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.Git; @@ -11,29 +12,16 @@ namespace GVFS.UnitTests.Prefetch { - [TestFixtureSource(TestRunners)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] public class DiffHelperTests { - public const string TestRunners = "Runners"; - - private static readonly bool[] SymLinkSupport = new bool[] - { - true, - false, - }; - public DiffHelperTests(bool symLinkSupport) { this.IncludeSymLinks = symLinkSupport; } - - public static bool[] Runners - { - get { return SymLinkSupport; } - } public bool IncludeSymLinks { get; set; } - + // Make two commits. The first should look like this: // recursiveDelete // recursiveDelete/subfolder From f032c944a19251ec2b103094b3ffbbae3213d679 Mon Sep 17 00:00:00 2001 From: Ravindra saini <43451826+ravi-saini35@users.noreply.github.com> Date: Fri, 26 Oct 2018 19:52:50 +0530 Subject: [PATCH 215/244] Update Readme.md make the readme.md more attractive. --- Readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Readme.md b/Readme.md index b67630cd0c..f8656d60ab 100644 --- a/Readme.md +++ b/Readme.md @@ -55,13 +55,13 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer Note that VFS for Git on Mac is under active development. -* Ensure you have Xcode installed, have accepted the terms of use, and have launched Xcode at least once. +* Ensure you have `Xcode` installed, have accepted the terms of use, and have launched `Xcode` at least once. * Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). * If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it]. You can check what version you have with `dotnet --version`.(https://www.microsoft.com/net/download/dotnet-core/2.1) -* If you're using Xcode for the first time, you may have to login to Xcode with your Apple ID to generate a codesigning certificate. You can do this by launching Xcode.app, opening the PrjFS.xcworkspace and trying to build. You can find the signing options in the General tab of the project's settings. +* If you're using `Xcode` for the first time, you may have to login to `Xcode` with your Apple ID to generate a codesigning certificate. You can do this by launching `Xcode.app`, opening the `PrjFS.xcworkspace` and trying to build. You can find the signing options in the General tab of the project's settings. * Create a `VFSForGit` directory and Clone VFSForGit into a directory called `src` inside it: ``` @@ -93,7 +93,7 @@ Note that VFS for Git on Mac is under active development. ``` xcodebuild: error: SDK "macosx10.13" cannot be located. ``` - You may have the "XCode Command Line Tools" installed (helpfully by Mac OS) instead of full XCode. + You may have the "XCode Command Line Tools" installed (helpfully by Mac OS) instead of full `XCode`. Make sure ``` xcode-select -p From d9edf15a14da962dc6b9dfa16ca422e73a7716d5 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Fri, 26 Oct 2018 13:46:00 -0400 Subject: [PATCH 216/244] Migrate nuget feeds --- nuget.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nuget.config b/nuget.config index dcdfdfb432..4f42ab44d0 100644 --- a/nuget.config +++ b/nuget.config @@ -5,11 +5,11 @@ - + - + From efd971a13c0fcdbb0eeaedb9aba5dd5a8c3195f4 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 26 Oct 2018 11:25:01 -0700 Subject: [PATCH 217/244] PerfProfiling should not initialize GVFSPlatform twice --- GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs b/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs index 8f4d6a777f..3bca15df74 100644 --- a/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs +++ b/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs @@ -23,7 +23,6 @@ public ProfilingEnvironment(string enlistmentRootPath) private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) { - GVFSPlatform.Register(new WindowsPlatform()); string gitBinPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); string hooksPath = ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSHooksExecutableName); From 9d84295d5d626a50f745bfef378f58809a138a51 Mon Sep 17 00:00:00 2001 From: Ravindra saini <43451826+ravi-saini35@users.noreply.github.com> Date: Sat, 27 Oct 2018 01:07:09 +0530 Subject: [PATCH 218/244] Update Readme.md --- Readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index f8656d60ab..4f903221c7 100644 --- a/Readme.md +++ b/Readme.md @@ -93,13 +93,13 @@ Note that VFS for Git on Mac is under active development. ``` xcodebuild: error: SDK "macosx10.13" cannot be located. ``` - You may have the "XCode Command Line Tools" installed (helpfully by Mac OS) instead of full `XCode`. + You may have the "Xcode Command Line Tools" installed (helpfully by Mac OS) instead of full `Xcode`. Make sure ``` xcode-select -p ``` - shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install Xcode and then launch it (you can close it afterwards.) + shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install `Xcode` and then launch it (you can close it afterwards.) * In order to build VFS for Git on Mac (and PrjFSKext) you will have to disable the SIP (System Integrity Protection) in order to load the kext). From 77f84ae6581ea358a7b8b75a0d85ef5438414b9a Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 26 Oct 2018 12:24:23 -0700 Subject: [PATCH 219/244] Use StringBuilder in BG task and update unit tests --- GVFS/GVFS.Common/GVFSPlatform.cs | 2 +- .../Projection/GitIndexEntryTests.cs | 68 +++++--- .../GitIndexProjection.GitIndexEntry.cs | 157 ++++++++---------- .../GitIndexProjection.GitIndexParser.cs | 2 - .../Projection/GitIndexProjection.cs | 16 +- 5 files changed, 123 insertions(+), 122 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index 15ef10e0bb..6f1c09bbf1 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -81,7 +81,7 @@ public bool TryGetNormalizedPathRoot(string path, out string pathRoot, out strin public class GVFSPlatformConstants { - public static readonly string PathSeparatorString = Path.DirectorySeparatorChar.ToString(); + public static readonly char PathSeparator = Path.DirectorySeparatorChar; public GVFSPlatformConstants(string executableExtension, string installerExtension) { diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs index ebfff8627f..afc65895bb 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs @@ -16,9 +16,9 @@ public class GitIndexEntryTests private const int DefaultIndexEntryCount = 10; private bool buildingNewProjection; - public GitIndexEntryTests(bool shouldTestLazyPaths) + public GitIndexEntryTests(bool buildingNewProjection) { - this.buildingNewProjection = shouldTestLazyPaths; + this.buildingNewProjection = buildingNewProjection; } public static object[] BuildingNewProjection @@ -67,6 +67,19 @@ public void ReplaceFileName() this.TestPathParts(indexEntry, pathParts2, hasSameParent: true); } + [TestCase] + public void ReplaceNonASCIIFileName() + { + string[] pathParts = new[] { "توبر", "مارسأغ", "FCIBBinaries.kml" }; + string path = string.Join("/", pathParts); + GitIndexEntry indexEntry = this.SetupIndexEntry(path); + this.TestPathParts(indexEntry, pathParts, hasSameParent: false); + + string[] pathParts2 = new[] { "توبر", "مارسأغ", "FCIBBinaries.txt" }; + this.ParsePathForIndexEntry(indexEntry, string.Join("/", pathParts2), replaceIndex: Encoding.UTF8.GetByteCount(path) - 3); + this.TestPathParts(indexEntry, pathParts2, hasSameParent: true); + } + [TestCase] public void ReplaceFileNameShorter() { @@ -83,11 +96,25 @@ public void ReplaceFileNameShorter() public void TestComponentsWithSimilarNames() { string[] pathParts = new[] { "MergedComponents", "SDK", "FCIBBinaries.kml" }; - GitIndexEntry indexEntry = this.SetupIndexEntry(string.Join("/", pathParts)); + string path = string.Join("/", pathParts); + GitIndexEntry indexEntry = this.SetupIndexEntry(path); this.TestPathParts(indexEntry, pathParts, hasSameParent: false); string[] pathParts2 = new[] { "MergedComponents", "SDK", "FCIBBinaries", "TH2Legacy", "amd64", "mdmerge.exe" }; - this.ParsePathForIndexEntry(indexEntry, string.Join("/", pathParts2), replaceIndex: 17); + this.ParsePathForIndexEntry(indexEntry, string.Join("/", pathParts2), replaceIndex: path.Length - 4); + this.TestPathParts(indexEntry, pathParts2, hasSameParent: false); + } + + [TestCase] + public void TestComponentsWithSimilarNonASCIINames() + { + string[] pathParts = new[] { "توبر", "مارسأغ", "FCIBBinaries.kml" }; + string path = string.Join("/", pathParts); + GitIndexEntry indexEntry = this.SetupIndexEntry(path); + this.TestPathParts(indexEntry, pathParts, hasSameParent: false); + + string[] pathParts2 = new[] { "توبر", "مارسأغ", "FCIBBinaries", "TH2Legacy", "amd64", "mdmerge.exe" }; + this.ParsePathForIndexEntry(indexEntry, string.Join("/", pathParts2), replaceIndex: Encoding.UTF8.GetByteCount(path) - 4); this.TestPathParts(indexEntry, pathParts2, hasSameParent: false); } @@ -147,12 +174,16 @@ public void ClearLastParent() this.TestPathParts(indexEntry, pathParts, hasSameParent: false); string[] pathParts2 = new[] { "folder", "one", "newfile.txt" }; - this.ParsePathForIndexEntry(indexEntry, string.Join("/", pathParts2), replaceIndex: 12); + this.ParsePathForIndexEntry(indexEntry, string.Join("/", pathParts2), replaceIndex: 11); this.TestPathParts(indexEntry, pathParts2, hasSameParent: true); - indexEntry.LastParent = new FolderData(); - indexEntry.ClearLastParent(); - indexEntry.HasSameParentAsLastEntry.ShouldBeFalse(); - indexEntry.LastParent.ShouldBeNull(); + + if (this.buildingNewProjection) + { + indexEntry.BuildingProjection_LastParent = new FolderData(); + indexEntry.ClearLastParent(); + indexEntry.BuildingProjection_HasSameParentAsLastEntry.ShouldBeFalse(); + indexEntry.BuildingProjection_LastParent.ShouldBeNull(); + } } private GitIndexEntry SetupIndexEntry(string path) @@ -164,9 +195,9 @@ private GitIndexEntry SetupIndexEntry(string path) private void ParsePathForIndexEntry(GitIndexEntry indexEntry, string path, int replaceIndex) { - byte[] pathBuffer = Encoding.ASCII.GetBytes(path); - Buffer.BlockCopy(pathBuffer, 0, indexEntry.PathBuffer, 0, path.Length); - indexEntry.PathLength = path.Length; + byte[] pathBuffer = Encoding.UTF8.GetBytes(path); + Buffer.BlockCopy(pathBuffer, 0, indexEntry.PathBuffer, 0, pathBuffer.Length); + indexEntry.PathLength = pathBuffer.Length; indexEntry.ReplaceIndex = replaceIndex; if (this.buildingNewProjection) @@ -181,8 +212,12 @@ private void ParsePathForIndexEntry(GitIndexEntry indexEntry, string path, int r private void TestPathParts(GitIndexEntry indexEntry, string[] pathParts, bool hasSameParent) { - indexEntry.HasSameParentAsLastEntry.ShouldEqual(hasSameParent, nameof(indexEntry.HasSameParentAsLastEntry)); - indexEntry.NumParts.ShouldEqual(pathParts.Length, nameof(indexEntry.NumParts)); + if (this.buildingNewProjection) + { + indexEntry.BuildingProjection_HasSameParentAsLastEntry.ShouldEqual(hasSameParent, nameof(indexEntry.BuildingProjection_HasSameParentAsLastEntry)); + indexEntry.BuildingProjection_NumParts.ShouldEqual(pathParts.Length, nameof(indexEntry.BuildingProjection_NumParts)); + } + for (int i = 0; i < pathParts.Length; i++) { if (this.buildingNewProjection) @@ -190,11 +225,6 @@ private void TestPathParts(GitIndexEntry indexEntry, string[] pathParts, bool ha indexEntry.BuildingProjection_PathParts[i].ShouldNotBeNull(); indexEntry.BuildingProjection_PathParts[i].GetString().ShouldEqual(pathParts[i]); } - else - { - indexEntry.BackgroundTask_PathParts[i].ShouldNotBeNull(); - indexEntry.BackgroundTask_PathParts[i].ShouldEqual(pathParts[i]); - } } if (this.buildingNewProjection) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 7261257647..a00881562e 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -1,6 +1,5 @@ using GVFS.Common; using System; -using System.IO; using System.Linq; using System.Text; @@ -21,7 +20,11 @@ internal class GitIndexEntry private const int MaxParts = MaxPathBufferSize / 2; private const byte PathSeparatorCode = 0x2F; - private int previousFinalSeparatorIndex = int.MaxValue; + private int buildingProjectionPreviousFinalSeparatorIndex = int.MaxValue; + + // Only used when buildingNewProjection is false + private string backgroundTaskRelativePath; + private StringBuilder backgroundTaskRelativePathBuilder; public GitIndexEntry(bool buildingNewProjection) { @@ -31,7 +34,7 @@ public GitIndexEntry(bool buildingNewProjection) } else { - this.BackgroundTask_PathParts = new string[MaxParts]; + this.backgroundTaskRelativePathBuilder = new StringBuilder(MaxPathBufferSize); } } @@ -46,7 +49,7 @@ public GitIndexEntry(bool buildingNewProjection) /// public int PathLength { get; set; } public byte[] PathBuffer { get; } = new byte[MaxPathBufferSize]; - public FolderData LastParent { get; set; } + public FolderData BuildingProjection_LastParent { get; set; } // Only used when buildingNewProjection is true public LazyUTF8String[] BuildingProjection_PathParts @@ -54,18 +57,12 @@ public LazyUTF8String[] BuildingProjection_PathParts get; private set; } - // Only used when buildingNewProjection is false - public string[] BackgroundTask_PathParts - { - get; private set; - } - - public int NumParts + public int BuildingProjection_NumParts { get; private set; } - public bool HasSameParentAsLastEntry + public bool BuildingProjection_HasSameParentAsLastEntry { get; private set; } @@ -73,10 +70,6 @@ public bool HasSameParentAsLastEntry /// /// Parses the path using LazyUTF8Strings. It should only be called when building a new projection. /// - /// - /// Code in this method has been fine-tuned for performance. None of it is shared with - /// BackgroundTask_ParsePath to avoid overhead. - /// public unsafe void BuildingProjection_ParsePath() { this.PathBuffer[this.PathLength] = 0; @@ -104,29 +97,29 @@ public unsafe void BuildingProjection_ParsePath() fixed (byte* pathPtr = this.PathBuffer) { - if (this.previousFinalSeparatorIndex < this.ReplaceIndex && + if (this.buildingProjectionPreviousFinalSeparatorIndex < this.ReplaceIndex && !this.RangeContains(pathPtr + this.ReplaceIndex, this.PathLength - this.ReplaceIndex, PathSeparatorCode)) { // Only need to parse the last part, because the rest of the string is unchanged // The logical thing to do would be to start the for loop at previousFinalSeparatorIndex+1, but two // repeated / characters would make an invalid path, so we'll assume that git would not have stored that path - forLoopStartIndex = this.previousFinalSeparatorIndex + 2; + forLoopStartIndex = this.buildingProjectionPreviousFinalSeparatorIndex + 2; // we still do need to start the current part's index at the correct spot, so subtract one for that currentPartStartIndex = forLoopStartIndex - 1; - this.NumParts--; + this.BuildingProjection_NumParts--; - this.HasSameParentAsLastEntry = true; + this.BuildingProjection_HasSameParentAsLastEntry = true; } else { - this.NumParts = 0; + this.BuildingProjection_NumParts = 0; this.ClearLastParent(); } - int partIndex = this.NumParts; + int partIndex = this.BuildingProjection_NumParts; byte* forLoopPtr = pathPtr + forLoopStartIndex; for (int i = forLoopStartIndex; i < this.PathLength + 1; i++) @@ -138,8 +131,8 @@ public unsafe void BuildingProjection_ParsePath() partIndex++; currentPartStartIndex = i + 1; - this.NumParts++; - this.previousFinalSeparatorIndex = i; + this.BuildingProjection_NumParts++; + this.buildingProjectionPreviousFinalSeparatorIndex = i; } ++forLoopPtr; @@ -148,112 +141,92 @@ public unsafe void BuildingProjection_ParsePath() // We unrolled the final part calculation to after the loop, to avoid having to do a 0-byte check inside the for loop this.BuildingProjection_PathParts[partIndex] = LazyUTF8String.FromByteArray(pathPtr + currentPartStartIndex, this.PathLength - currentPartStartIndex); - this.NumParts++; + this.BuildingProjection_NumParts++; } } /// - /// Parses the path without using LazyUTF8Strings. It should only be called when running a background task. + /// Parses the path from the index as platform-specific relative path. + /// It should only be called when running a background task. /// - /// - /// Code in this method has been fine-tuned for performance. None of it is shared with - /// BuildingProjection_ParsePath to avoid overhead. - /// public unsafe void BackgroundTask_ParsePath() { this.PathBuffer[this.PathLength] = 0; - // The index of that path part that is after the path separator - int currentPartStartIndex = 0; + // The index in the buffer at which to start parsing + int loopStartIndex = 0; - // The index to start looking for the next path separator - // Because the previous final separator is stored and we know where the previous path will be replaced - // the code can use the previous final separator to start looking from that point instead of having to - // run through the entire path to break it apart - /* Example: - * Previous path = folder/where/previous/separator/is/used/file.txt - * This path = folder/where/previous/separator/is/used/file2.txt - * ^ ^ - * this.previousFinalSeparatorIndex | - * this.ReplaceIndex - * - * folder/where/previous/separator/is/used/file2.txt - * ^^ - * currentPartStartIndex| - * forLoopStartIndex - */ - int forLoopStartIndex = 0; + // backgroundTaskRelativePathBuilder is reset to empty if the last index entry contained non-ASCII character(s) + // Only start from ReplaceIndex if the last entry contained *only* ASCII characters + if (this.backgroundTaskRelativePathBuilder.Length > 0) + { + loopStartIndex = this.ReplaceIndex; + this.backgroundTaskRelativePathBuilder.Length = this.ReplaceIndex; + } fixed (byte* pathPtr = this.PathBuffer) { - if (this.previousFinalSeparatorIndex < this.ReplaceIndex && - !this.RangeContains(pathPtr + this.ReplaceIndex, this.PathLength - this.ReplaceIndex, PathSeparatorCode)) + byte* bufferPtrForLoop = pathPtr + loopStartIndex; + while (loopStartIndex < this.PathLength) { - // Only need to parse the last part, because the rest of the string is unchanged - - // The logical thing to do would be to start the for loop at previousFinalSeparatorIndex+1, but two - // repeated / characters would make an invalid path, so we'll assume that git would not have stored that path - forLoopStartIndex = this.previousFinalSeparatorIndex + 2; - - // we still do need to start the current part's index at the correct spot, so subtract one for that - currentPartStartIndex = forLoopStartIndex - 1; - - this.NumParts--; - - this.HasSameParentAsLastEntry = true; - } - else - { - this.NumParts = 0; - this.ClearLastParent(); - } - - int partIndex = this.NumParts; - - byte* forLoopPtr = pathPtr + forLoopStartIndex; - for (int i = forLoopStartIndex; i < this.PathLength + 1; i++) - { - if (*forLoopPtr == PathSeparatorCode) + if (*bufferPtrForLoop <= 127) + { + if (*bufferPtrForLoop == PathSeparatorCode) + { + this.backgroundTaskRelativePathBuilder.Append(GVFSPlatform.GVFSPlatformConstants.PathSeparator); + } + else + { + this.backgroundTaskRelativePathBuilder.Append(Convert.ToChar(*bufferPtrForLoop)); + } + } + else { - this.BackgroundTask_PathParts[partIndex] = Encoding.UTF8.GetString(pathPtr + currentPartStartIndex, i - currentPartStartIndex); + // The string has non-ASCII characters in it, fall back to full parsing + this.backgroundTaskRelativePath = Encoding.UTF8.GetString(pathPtr, this.PathLength); + if (GVFSConstants.GitPathSeparator != GVFSPlatform.GVFSPlatformConstants.PathSeparator) + { + this.backgroundTaskRelativePath = this.backgroundTaskRelativePath.Replace(GVFSConstants.GitPathSeparator, GVFSPlatform.GVFSPlatformConstants.PathSeparator); + } - partIndex++; - currentPartStartIndex = i + 1; + // Record that this entry had non-ASCII characters by clearing backgroundTaskRelativePathBuilder + this.backgroundTaskRelativePathBuilder.Clear(); - this.NumParts++; - this.previousFinalSeparatorIndex = i; + return; } - ++forLoopPtr; + ++bufferPtrForLoop; + ++loopStartIndex; } - // We unrolled the final part calculation to after the loop, to avoid having to do a 0-byte check inside the for loop - this.BackgroundTask_PathParts[partIndex] = Encoding.UTF8.GetString(pathPtr + currentPartStartIndex, this.PathLength - currentPartStartIndex); - - this.NumParts++; + this.backgroundTaskRelativePath = this.backgroundTaskRelativePathBuilder.ToString(); } } + /// + /// Clear last parent data that's tracked when GitIndexEntry is used for building a new projection. + /// No-op when GitIndexEntry is used for background tasks. + /// public void ClearLastParent() { - this.previousFinalSeparatorIndex = int.MaxValue; - this.HasSameParentAsLastEntry = false; - this.LastParent = null; + this.buildingProjectionPreviousFinalSeparatorIndex = int.MaxValue; + this.BuildingProjection_HasSameParentAsLastEntry = false; + this.BuildingProjection_LastParent = null; } public LazyUTF8String BuildingProjection_GetChildName() { - return this.BuildingProjection_PathParts[this.NumParts - 1]; + return this.BuildingProjection_PathParts[this.BuildingProjection_NumParts - 1]; } public string BuildingProjection_GetGitRelativePath() { - return string.Join(GVFSConstants.GitPathSeparatorString, this.BuildingProjection_PathParts.Take(this.NumParts).Select(x => x.GetString())); + return string.Join(GVFSConstants.GitPathSeparatorString, this.BuildingProjection_PathParts.Take(this.BuildingProjection_NumParts).Select(x => x.GetString())); } public string BackgroundTask_GetPlatformRelativePath() { - return string.Join(GVFSPlatform.GVFSPlatformConstants.PathSeparatorString, this.BackgroundTask_PathParts.Take(this.NumParts)); + return this.backgroundTaskRelativePath; } private unsafe bool RangeContains(byte* bufferPtr, int count, byte value) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 3adc800aba..ff80078d4d 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -191,8 +191,6 @@ private FileSystemTaskResult AddEntryToModifiedPathsAndRemoveFromPlaceholdersIfN } else { - gitIndexEntry.ClearLastParent(); - filePlaceholders.Remove(placeholderRelativePath); } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 663943600d..e05a849aba 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -667,20 +667,20 @@ private static List ConvertToProjectedFileInfos(SortedFolderE private void AddItemFromIndexEntry(GitIndexEntry indexEntry) { - if (indexEntry.HasSameParentAsLastEntry) + if (indexEntry.BuildingProjection_HasSameParentAsLastEntry) { - indexEntry.LastParent.AddChildFile(indexEntry.BuildingProjection_GetChildName(), indexEntry.Sha); + indexEntry.BuildingProjection_LastParent.AddChildFile(indexEntry.BuildingProjection_GetChildName(), indexEntry.Sha); } else { - if (indexEntry.NumParts == 1) + if (indexEntry.BuildingProjection_NumParts == 1) { - indexEntry.LastParent = this.rootFolderData; - indexEntry.LastParent.AddChildFile(indexEntry.BuildingProjection_GetChildName(), indexEntry.Sha); + indexEntry.BuildingProjection_LastParent = this.rootFolderData; + indexEntry.BuildingProjection_LastParent.AddChildFile(indexEntry.BuildingProjection_GetChildName(), indexEntry.Sha); } else { - indexEntry.LastParent = this.AddFileToTree(indexEntry); + indexEntry.BuildingProjection_LastParent = this.AddFileToTree(indexEntry); } } @@ -787,7 +787,7 @@ private string GetChildName(string gitPath, int pathSeparatorIndex) private FolderData AddFileToTree(GitIndexEntry indexEntry) { FolderData parentFolder = this.rootFolderData; - for (int pathIndex = 0; pathIndex < indexEntry.NumParts - 1; ++pathIndex) + for (int pathIndex = 0; pathIndex < indexEntry.BuildingProjection_NumParts - 1; ++pathIndex) { if (parentFolder == null) { @@ -814,7 +814,7 @@ private FolderData AddFileToTree(GitIndexEntry indexEntry) parentFolder = parentFolder.ChildEntries.GetOrAddFolder(indexEntry.BuildingProjection_PathParts[pathIndex]); } - parentFolder.AddChildFile(indexEntry.BuildingProjection_PathParts[indexEntry.NumParts - 1], indexEntry.Sha); + parentFolder.AddChildFile(indexEntry.BuildingProjection_PathParts[indexEntry.BuildingProjection_NumParts - 1], indexEntry.Sha); return parentFolder; } From 7cde6c2cd76066a87e692658c7394e775fe7c9fd Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 26 Oct 2018 15:39:24 -0700 Subject: [PATCH 220/244] Additional cleanup --- GVFS/GVFS.Common/PlaceholderListDatabase.cs | 22 ++++++++++++++----- .../GitIndexProjection.GitIndexEntry.cs | 2 +- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.Common/PlaceholderListDatabase.cs b/GVFS/GVFS.Common/PlaceholderListDatabase.cs index 81c8a4d1fe..dd13b37ef0 100644 --- a/GVFS/GVFS.Common/PlaceholderListDatabase.cs +++ b/GVFS/GVFS.Common/PlaceholderListDatabase.cs @@ -103,11 +103,16 @@ public void RemoveAndFlush(string path) } /// - /// Gets all entries and intializes placeholderChangesWhileRebuildingList in preparation fr a call to WriteAllEntriesAndFlush. + /// Gets all entries and prepares the PlaceholderListDatabase for a call to WriteAllEntriesAndFlush. /// + /// + /// GetAllEntriesAndPrepToWriteAllEntries was called (a second time) without first calling WriteAllEntriesAndFlush. + /// /// - /// See placeholderChangesWhileRebuildingList declaration for additional details as to why WriteAllEntriesAndFlush - /// must be called. + /// Usage notes: + /// - All calls to GetAllEntriesAndPrepToWriteAllEntries must be paired with a subsequent call to WriteAllEntriesAndFlush + /// - If WriteAllEntriesAndFlush is *not* called entries that were added to the PlaceholderListDatabase after + /// calling GetAllEntriesAndPrepToWriteAllEntries will be lost /// public List GetAllEntriesAndPrepToWriteAllEntries() { @@ -143,11 +148,16 @@ public List GetAllEntriesAndPrepToWriteAllEntries() } /// - /// Gets all entries and intializes placeholderChangesWhileRebuildingList in preparation fr a call to WriteAllEntriesAndFlush. + /// Gets all entries and prepares the PlaceholderListDatabase for a call to WriteAllEntriesAndFlush. /// + /// + /// GetAllEntriesAndPrepToWriteAllEntries was called (a second time) without first calling WriteAllEntriesAndFlush. + /// /// - /// See placeholderChangesWhileRebuildingList declaration for additional details as to why WriteAllEntriesAndFlush - /// must be called. + /// Usage notes: + /// - All calls to GetAllEntriesAndPrepToWriteAllEntries must be paired with a subsequent call to WriteAllEntriesAndFlush + /// - If WriteAllEntriesAndFlush is *not* called entries that were added to the PlaceholderListDatabase after + /// calling GetAllEntriesAndPrepToWriteAllEntries will be lost /// public void GetAllEntriesAndPrepToWriteAllEntries(out List filePlaceholders, out List folderPlaceholders) { diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index a00881562e..85d7c23860 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -177,7 +177,7 @@ public unsafe void BackgroundTask_ParsePath() } else { - this.backgroundTaskRelativePathBuilder.Append(Convert.ToChar(*bufferPtrForLoop)); + this.backgroundTaskRelativePathBuilder.Append((char)(*bufferPtrForLoop)); } } else From 5be95117670591f2ddc78cca911677437189e3df Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 26 Oct 2018 15:49:11 -0700 Subject: [PATCH 221/244] Use DataSources.AllBools in unit tests --- .../Projection/GitIndexEntryTests.cs | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs index afc65895bb..97052c8dd6 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/GitIndexEntryTests.cs @@ -1,4 +1,5 @@ -using GVFS.Tests.Should; +using GVFS.Tests; +using GVFS.Tests.Should; using GVFS.UnitTests.Mock.Common; using NUnit.Framework; using System; @@ -8,11 +9,9 @@ namespace GVFS.UnitTests.Virtualization.Git { - [TestFixtureSource(TestBuildingNewProjection)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] public class GitIndexEntryTests { - public const string TestBuildingNewProjection = "BuildingNewProjection"; - private const int DefaultIndexEntryCount = 10; private bool buildingNewProjection; @@ -21,18 +20,6 @@ public GitIndexEntryTests(bool buildingNewProjection) this.buildingNewProjection = buildingNewProjection; } - public static object[] BuildingNewProjection - { - get - { - return new object[] - { - new object[] { true }, - new object[] { false }, - }; - } - } - [OneTimeSetUp] public void Setup() { From 0f79ab08bb1842ba45708ccbe8ddb1a47e96f948 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 23 Oct 2018 09:52:49 -0700 Subject: [PATCH 222/244] Mac: Skip events when file attributes cannot be read --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index abf6de9517..c67e2162ca 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -879,8 +879,17 @@ static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t _Nonnull context) { struct vnode_attr attributes = {}; errno_t err = GetVNodeAttributes(vn, context, &attributes); - // TODO: May fail on some file system types? Perhaps we should early-out depending on mount point anyway. - assert(0 == err); + if (0 != err) + { + // TODO(Mac): May fail on some file system types? Perhaps we should early-out depending on mount point anyway. + // We should also consider: + // - Logging this error + // - Falling back on vnode lookup (or custom cache) to determine if file is in the root + // - Assuming files are empty if we can't read the flags + + return 0; + } + assert(VATTR_IS_SUPPORTED(&attributes, va_flags)); return attributes.va_flags; } From 12c17a3877717a3b6d0a99b75e8eec126a135976 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 25 Oct 2018 14:56:32 -0700 Subject: [PATCH 223/244] Assert that file attribute read does not fail in FileOp events --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index c67e2162ca..de4a82ae9a 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -36,9 +36,9 @@ static int HandleFileOpOperation( static int GetPid(vfs_context_t _Nonnull context); -static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t _Nonnull context); +static bool TryReadVNodeFileFlags(vnode_t vn, vfs_context_t _Nonnull context, uint32_t* flags); static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); -static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t _Nonnull context); +static inline bool TryGetFileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t _Nonnull context, bool* flaggedInRoot); static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); static bool IsFileSystemCrawler(char* procname); @@ -498,7 +498,9 @@ static int HandleFileOpOperation( goto CleanupAndReturn; } - bool fileFlaggedInRoot = FileIsFlaggedAsInRoot(currentVnode, context); + bool fileFlaggedInRoot; + assert(TryGetFileIsFlaggedAsInRoot(currentVnode, context, &fileFlaggedInRoot)); + if (fileFlaggedInRoot && KAUTH_FILEOP_CLOSE_MODIFIED != closeFlags) { goto CleanupAndReturn; @@ -609,7 +611,9 @@ static bool ShouldHandleVnodeOpEvent( { ProfileSample readflags(Probe_ReadFileFlags); - *vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + + // TODO(Mac): Don't ignore failures of TryReadVNodeFileFlags + TryReadVNodeFileFlags(vnode, context, vnodeFileFlags); } if (!FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) @@ -875,9 +879,10 @@ static errno_t GetVNodeAttributes(vnode_t vn, vfs_context_t _Nonnull context, st return vnode_getattr(vn, attrs, context); } -static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t _Nonnull context) +static bool TryReadVNodeFileFlags(vnode_t vn, vfs_context_t _Nonnull context, uint32_t* flags) { struct vnode_attr attributes = {}; + *flags = 0; errno_t err = GetVNodeAttributes(vn, context, &attributes); if (0 != err) { @@ -887,11 +892,12 @@ static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t _Nonnull context) // - Falling back on vnode lookup (or custom cache) to determine if file is in the root // - Assuming files are empty if we can't read the flags - return 0; + return false; } assert(VATTR_IS_SUPPORTED(&attributes, va_flags)); - return attributes.va_flags; + *flags = attributes.va_flags; + return true; } static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit) @@ -900,10 +906,17 @@ static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit) return 0 != (fileFlags & bit); } -static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t _Nonnull context) +static inline bool TryGetFileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t _Nonnull context, bool* flaggedInRoot) { - uint32_t vnodeFileFlags = ReadVNodeFileFlags(vnode, context); - return FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot); + uint32_t vnodeFileFlags; + *flaggedInRoot = false; + if (!TryReadVNodeFileFlags(vnode, context, &vnodeFileFlags)) + { + return false; + } + + *flaggedInRoot = FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot); + return true; } static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask) { From 43cf47a6f6e3ba6190b398774ed2bc08c126683f Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 25 Oct 2018 15:41:25 -0700 Subject: [PATCH 224/244] Deny I/O when file attribute read fails --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index de4a82ae9a..442bf10120 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -71,7 +71,8 @@ static bool ShouldHandleVnodeOpEvent( FsidInode* vnodeFsidInode, int* pid, char procname[MAXCOMLEN + 1], - int* kauthResult); + int* kauthResult, + int* kauthError); static bool ShouldHandleFileOpEvent( // In params: @@ -307,7 +308,8 @@ static int HandleVnodeOperation( &vnodeFsidInode, &pid, procname, - &kauthResult)) + &kauthResult, + kauthError)) { goto CleanupAndReturn; } @@ -589,7 +591,8 @@ static bool ShouldHandleVnodeOpEvent( FsidInode* vnodeFsidInode, int* pid, char procname[MAXCOMLEN + 1], - int* kauthResult) + int* kauthResult, + int* kauthError) { *kauthResult = KAUTH_RESULT_DEFER; *root = RootHandle_None; @@ -611,9 +614,12 @@ static bool ShouldHandleVnodeOpEvent( { ProfileSample readflags(Probe_ReadFileFlags); - - // TODO(Mac): Don't ignore failures of TryReadVNodeFileFlags - TryReadVNodeFileFlags(vnode, context, vnodeFileFlags); + if (!TryReadVNodeFileFlags(vnode, context, vnodeFileFlags)) + { + *kauthError = EBADF; + *kauthResult = KAUTH_RESULT_DENY; + return false; + } } if (!FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) From c7e221b5ed193cef368c48961a545233cbc911a6 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Thu, 25 Oct 2018 14:36:15 +0200 Subject: [PATCH 225/244] Mac PrjFSLib: Use IOSharedDataQueue workaround library if available This allows PrjFSLib and the prjfs-log tool to use alternate versions of the IODataQueueDequeue() and IODataQueuePeek() functions from a dylib on OS versions where the built-in functions are buggy. On affected versions, a library "libSharedDataQueue.dylib" is loaded if available, and its implementations of the aforementioned functions are used for the message and kext logging queues instead of the ones supplied by the OS's IOKit.framework. --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 6 +- ProjFS.Mac/PrjFSLib/PrjFSUser.cpp | 109 ++++++++++++++++++++ ProjFS.Mac/PrjFSLib/PrjFSUser.hpp | 3 + ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp | 4 +- 4 files changed, 117 insertions(+), 5 deletions(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 3b0012971d..a77d0842c9 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -215,7 +215,7 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( while (1) { - IODataQueueEntry* entry = IODataQueuePeek(dataQueue.queueMemory); + IODataQueueEntry* entry = DataQueue_Peek(dataQueue.queueMemory); if (nullptr == entry) { // No more items in queue @@ -226,13 +226,13 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( if (messageSize < sizeof(Message)) { cerr << "Bad message size: got " << messageSize << " bytes, expected minimum of " << sizeof(Message) << ", skipping. Kernel/user version mismatch?\n"; - IODataQueueDequeue(dataQueue.queueMemory, nullptr, nullptr); + DataQueue_Dequeue(dataQueue.queueMemory, nullptr, nullptr); continue; } void* messageMemory = malloc(messageSize); uint32_t dequeuedSize = messageSize; - IOReturn result = IODataQueueDequeue(dataQueue.queueMemory, messageMemory, &dequeuedSize); + IOReturn result = DataQueue_Dequeue(dataQueue.queueMemory, messageMemory, &dequeuedSize); if (kIOReturnSuccess != result || dequeuedSize != messageSize) { cerr << "Unexpected result dequeueing message - result 0x" << hex << result << " dequeued " << dequeuedSize << "/" << messageSize << " bytes\n"; diff --git a/ProjFS.Mac/PrjFSLib/PrjFSUser.cpp b/ProjFS.Mac/PrjFSLib/PrjFSUser.cpp index a120131189..00d94671e8 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSUser.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSUser.cpp @@ -3,6 +3,20 @@ #include #include #include +#include +#include + +struct DarwinVersion +{ + unsigned long major, minor, revision; +}; + +typedef decltype(IODataQueueDequeue)* ioDataQueueDequeueFunctionPtr; +static ioDataQueueDequeueFunctionPtr ioDataQueueDequeueFunction = nullptr; +typedef decltype(IODataQueuePeek)* ioDataQueuePeekFunctionPtr; +static ioDataQueuePeekFunctionPtr ioDataQueuePeekFunction = nullptr; + +static void InitDataQueueFunctions(); io_connect_t PrjFSService_ConnectToDriver(enum PrjFSServiceUserClientType clientType) @@ -122,3 +136,98 @@ bool PrjFSService_DataQueueInit( return false; } +IOReturn DataQueue_Dequeue(IODataQueueMemory* dataQueue, void* data, uint32_t* dataSize) +{ + if (nullptr == ioDataQueueDequeueFunction) + { + InitDataQueueFunctions(); + } + return ioDataQueueDequeueFunction(dataQueue, data, dataSize); +} + +IODataQueueEntry* DataQueue_Peek(IODataQueueMemory* dataQueue) +{ + if (nullptr == ioDataQueuePeekFunction) + { + InitDataQueueFunctions(); + } + return ioDataQueuePeekFunction(dataQueue); +} + + +static bool GetDarwinVersion(DarwinVersion& outVersion) +{ + utsname unameInfo = {}; + if (0 != uname(&unameInfo)) + { + return false; + } + + char* fieldEnd = nullptr; + unsigned long majorVersion = strtoul(unameInfo.release, &fieldEnd, 10); + if (nullptr == fieldEnd || *fieldEnd != '.') + { + return false; + } + + unsigned long minorVersion = strtoul(fieldEnd + 1, &fieldEnd, 10); + if (nullptr == fieldEnd || (*fieldEnd != '.' && *fieldEnd != '\0')) + { + return false; + } + + outVersion.major = majorVersion; + outVersion.minor = minorVersion; + outVersion.revision = 0; + + if (*fieldEnd != '\0') + { + unsigned long revision = strtoul(fieldEnd + 1, &fieldEnd, 10); + if (nullptr == fieldEnd || (*fieldEnd != '.' && *fieldEnd != '\0')) + { + return false; + } + outVersion.revision = revision; + } + + return true; +} + +static void InitDataQueueFunctions() +{ + ioDataQueueDequeueFunction = &IODataQueueDequeue; + ioDataQueuePeekFunction = &IODataQueuePeek; + + DarwinVersion osVersion = {}; + if (!GetDarwinVersion(osVersion)) + { + return; + } + + if ((osVersion.major == 17 && osVersion.minor >= 7) // macOS 10.13.6+ + || (osVersion.major == 18 && osVersion.minor == 0)) // macOS 10.14(.0) exactly + { + void* dataQueueLibrary = dlopen("libSharedDataQueue.dylib", RTLD_LAZY); + if (nullptr == dataQueueLibrary) + { + fprintf(stderr, "Error opening data queue client library: %s\n", dlerror()); + } + else + { + void* sym = dlsym(dataQueueLibrary, "IODataQueueDequeue"); + if (nullptr != sym) + { + ioDataQueueDequeueFunction = reinterpret_cast(sym); + } + + sym = dlsym(dataQueueLibrary, "IODataQueuePeek"); + if (nullptr != sym) + { + ioDataQueuePeekFunction = reinterpret_cast(sym); + } + + // Allow the dataQueueLibrary handle to leak; if we called dlclose(), + // the library would be unloaded, breaking our function pointers. + } + } +} diff --git a/ProjFS.Mac/PrjFSLib/PrjFSUser.hpp b/ProjFS.Mac/PrjFSLib/PrjFSUser.hpp index 3da3e2c2e0..494dcc5da6 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSUser.hpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSUser.hpp @@ -23,3 +23,6 @@ bool PrjFSService_DataQueueInit( uint32_t clientPortType, uint32_t clientMemoryType, dispatch_queue_t eventHandlingQueue); + +IODataQueueEntry* DataQueue_Peek(IODataQueueMemory* dataQueue); +IOReturn DataQueue_Dequeue(IODataQueueMemory* dataQueue, void* data, uint32_t* dataSize); diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp index aaa32b1c4e..813f7ed04c 100644 --- a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp @@ -44,7 +44,7 @@ int main(int argc, const char * argv[]) while(true) { - IODataQueueEntry* entry = IODataQueuePeek(dataQueue.queueMemory); + IODataQueueEntry* entry = DataQueue_Peek(dataQueue.queueMemory); if(entry == nullptr) { break; @@ -65,7 +65,7 @@ int main(int argc, const char * argv[]) lineCount++; } - IODataQueueDequeue(dataQueue.queueMemory, nullptr, nullptr); + DataQueue_Dequeue(dataQueue.queueMemory, nullptr, nullptr); } }); dispatch_resume(dataQueue.dispatchSource); From 4809082d40f1b196a000966d713122a4f89de413 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Mon, 29 Oct 2018 16:10:00 -0700 Subject: [PATCH 226/244] Consume libSharedDataQueue.dylib by downloading it as a NuGet package. Add this new script to PrepFunctionalTests.sh to ensure everyone is on a healthy version of the library. --- Scripts/Mac/InstallSharedDataQueueDylib.sh | 9 +++++++++ Scripts/Mac/PrepFunctionalTests.sh | 2 ++ 2 files changed, 11 insertions(+) create mode 100755 Scripts/Mac/InstallSharedDataQueueDylib.sh diff --git a/Scripts/Mac/InstallSharedDataQueueDylib.sh b/Scripts/Mac/InstallSharedDataQueueDylib.sh new file mode 100755 index 0000000000..2943d82c36 --- /dev/null +++ b/Scripts/Mac/InstallSharedDataQueueDylib.sh @@ -0,0 +1,9 @@ +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + +BUILDDIR=$VFS_OUTPUTDIR/GVFS.Build +cp $VFS_SRCDIR/nuget.config $BUILDDIR +dotnet new classlib -n GVFS.Restore -o $BUILDDIR --force +dotnet add $BUILDDIR/GVFS.Restore.csproj package --package-directory $VFS_PACKAGESDIR SharedDataQueueDylib --version '1.0.0' + +# DYLD_LIBRARY_PATH contains /usr/local/lib by default, so we'll copy this library there. +cp $VFS_PACKAGESDIR/shareddataqueuedylib/1.0.0/libSharedDataQueue.dylib /usr/local/lib \ No newline at end of file diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index 341ea6fe10..38eb724a43 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -37,6 +37,8 @@ fi git-credential-manager install +$VFS_SCRIPTDIR/InstallSharedDataQueueDylib.sh || exit 1 + # If we're running on an agent where the PAT environment variable is set and a URL is passed into the script, add it to the keychain. PATURL=$1 PAT=$2 From 835e950cd986a9794c11d0b176877d50dba685d3 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 25 Oct 2018 16:08:11 -0700 Subject: [PATCH 227/244] Mac: Run GitCommandsTests with and without validating the working directory during setup --- GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs | 2 + GVFS/GVFS.FunctionalTests/Program.cs | 14 ++++++ .../Tests/GitCommands/AddStageTests.cs | 5 ++- .../Tests/GitCommands/CheckoutTests.cs | 5 ++- .../GitCommands/CherryPickConflictTests.cs | 5 ++- .../GitCommands/CreatePlaceholderTests.cs | 5 ++- .../GitCommands/DeleteEmptyFolderTests.cs | 5 ++- .../Tests/GitCommands/EnumerationMergeTest.cs | 5 ++- .../Tests/GitCommands/GitCommandsTests.cs | 5 ++- .../Tests/GitCommands/GitRepoTests.cs | 45 ++++++++++++++++--- .../Tests/GitCommands/HashObjectTests.cs | 5 ++- .../Tests/GitCommands/MergeConflictTests.cs | 5 ++- .../Tests/GitCommands/RebaseConflictTests.cs | 5 ++- .../Tests/GitCommands/RebaseTests.cs | 5 ++- .../Tests/GitCommands/ResetHardTests.cs | 7 +-- .../Tests/GitCommands/ResetMixedTests.cs | 5 ++- .../Tests/GitCommands/ResetSoftTests.cs | 5 ++- .../Tests/GitCommands/RmTests.cs | 5 ++- .../Tests/GitCommands/StatusTests.cs | 5 ++- .../Tests/GitCommands/UpdateIndexTests.cs | 5 ++- .../Tests/GitCommands/UpdateRefTests.cs | 5 ++- 21 files changed, 109 insertions(+), 44 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs b/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs index 7f65687538..ecf6d2fcb6 100644 --- a/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs +++ b/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs @@ -12,6 +12,8 @@ public static class GVFSTestConfig public static object[] FileSystemRunners { get; set; } + public static object[] GitCommandTestWorkTreeValidation { get; set; } + public static bool TestGVFSOnPath { get; set; } public static bool ReplaceInboxProjFS { get; set; } diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index f7db4b9698..7eed002a12 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -1,5 +1,6 @@ using GVFS.FunctionalTests.Tests; using GVFS.FunctionalTests.Tools; +using GVFS.FunctionalTests.Tests.GitCommands; using GVFS.Tests; using System; using System.Collections.Generic; @@ -43,6 +44,13 @@ public static void Main(string[] args) { Console.WriteLine("Running the full suite of tests"); + GVFSTestConfig.GitCommandTestWorkTreeValidation = + new object[] + { + new object[] { GitRepoTests.ValidateWorkingTreeOptions.ValidateWorkingTree }, + new object[] { GitRepoTests.ValidateWorkingTreeOptions.DoNotValidateWorkingTree } + }; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { GVFSTestConfig.FileSystemRunners = FileSystemRunners.FileSystemRunner.AllWindowsRunners; @@ -51,11 +59,17 @@ public static void Main(string[] args) { GVFSTestConfig.FileSystemRunners = FileSystemRunners.FileSystemRunner.AllMacRunners; } + } else { excludeCategories.Add(Categories.FullSuiteOnly); GVFSTestConfig.FileSystemRunners = FileSystemRunners.FileSystemRunner.DefaultRunners; + GVFSTestConfig.GitCommandTestWorkTreeValidation = + new object[] + { + new object[] { GitRepoTests.ValidateWorkingTreeOptions.ValidateWorkingTree } + }; } if (runner.HasCustomArg("--windows-only")) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index d4e834dad4..76a801c32a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -5,11 +5,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class AddStageTests : GitRepoTests { - public AddStageTests() : base(enlistmentPerTest: false) + public AddStageTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: false, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 9a5383f1dd..c4ec6f478c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -11,11 +11,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class CheckoutTests : GitRepoTests { - public CheckoutTests() : base(enlistmentPerTest: true) + public CheckoutTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs index 9009c4519a..aafd213d02 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs @@ -2,11 +2,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class CherryPickConflictTests : GitRepoTests { - public CherryPickConflictTests() : base(enlistmentPerTest: true) + public CherryPickConflictTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs index 3704e5e286..a6e457c9f6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs @@ -7,13 +7,14 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class CreatePlaceholderTests : GitRepoTests { private static readonly string FileToRead = Path.Combine("GVFS", "GVFS", "Program.cs"); - public CreatePlaceholderTests() : base(enlistmentPerTest: true) + public CreatePlaceholderTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs index 9fd1343136..c4ab9d7dad 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs @@ -4,11 +4,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class DeleteEmptyFolderTests : GitRepoTests { - public DeleteEmptyFolderTests() : base(enlistmentPerTest: true) + public DeleteEmptyFolderTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs index 4306529ab1..b33554d8a4 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs @@ -2,7 +2,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class EnumerationMergeTest : GitRepoTests { @@ -10,7 +10,8 @@ public class EnumerationMergeTest : GitRepoTests // enumeration when they don't fit in a user's buffer private const string EnumerationReproCommitish = "FunctionalTests/20170602"; - public EnumerationMergeTest() : base(enlistmentPerTest: true) + public EnumerationMergeTest(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 99da0d9dce..324d193f45 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -8,7 +8,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class GitCommandsTests : GitRepoTests { @@ -26,7 +26,8 @@ public class GitCommandsTests : GitRepoTests private static readonly string RenameFolderPathFrom = Path.Combine("GVFS", "GVFS.Common", "PrefetchPacks"); private static readonly string RenameFolderPathTo = Path.Combine("GVFS", "GVFS.Common", "PrefetchPacksRenamed"); - public GitCommandsTests() : base(enlistmentPerTest: false) + public GitCommandsTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: false, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index d68e380959..3cf36dfe5a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -13,6 +13,8 @@ namespace GVFS.FunctionalTests.Tests.GitCommands [TestFixture] public abstract class GitRepoTests { + public const string ValidateWorkingTree = "WorkTreeValidation"; + protected const string ConflictSourceBranch = "FunctionalTests/20170206_Conflict_Source"; protected const string ConflictTargetBranch = "FunctionalTests/20170206_Conflict_Target"; protected const string NoConflictSourceBranch = "FunctionalTests/20170209_NoConflict_Source"; @@ -23,13 +25,27 @@ public abstract class GitRepoTests protected const string DeepDirectoryWithOneDifferentFile = "FunctionalTests/20181010_DeepFolderOneDifferentFile"; private bool enlistmentPerTest; + private bool validateWorkingTree; - public GitRepoTests(bool enlistmentPerTest) + public GitRepoTests(bool enlistmentPerTest, ValidateWorkingTreeOptions validateWorkingTree) { this.enlistmentPerTest = enlistmentPerTest; + this.validateWorkingTree = validateWorkingTree == ValidateWorkingTreeOptions.ValidateWorkingTree; this.FileSystem = new SystemIORunner(); } + public enum ValidateWorkingTreeOptions + { + Invalid = 0, + DoNotValidateWorkingTree, + ValidateWorkingTree, + } + + public static object[] WorkTreeValidation + { + get { return GVFSTestConfig.GitCommandTestWorkTreeValidation; } + } + public ControlGitRepo ControlGitRepo { get; private set; @@ -74,8 +90,13 @@ public virtual void SetupForTest() this.ValidateGitCommand("checkout " + this.ControlGitRepo.Commitish); this.CheckHeadCommitTree(); - this.Enlistment.RepoRoot.ShouldBeADirectory(this.FileSystem) - .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath); + + if (this.validateWorkingTree) + { + this.Enlistment.RepoRoot.ShouldBeADirectory(this.FileSystem) + .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath); + } + this.ValidateGitCommand("status"); } @@ -90,16 +111,26 @@ protected void TestValidationAndCleanup(bool ignoreCase = false) try { this.CheckHeadCommitTree(); - this.Enlistment.RepoRoot.ShouldBeADirectory(this.FileSystem) - .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, ignoreCase: ignoreCase); + + if (this.validateWorkingTree) + { + this.Enlistment.RepoRoot.ShouldBeADirectory(this.FileSystem) + .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, ignoreCase: ignoreCase); + } this.RunGitCommand("reset --hard -q HEAD"); this.RunGitCommand("clean -d -f -x"); this.ValidateGitCommand("checkout " + this.ControlGitRepo.Commitish); this.CheckHeadCommitTree(); - this.Enlistment.RepoRoot.ShouldBeADirectory(this.FileSystem) - .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, ignoreCase: ignoreCase); + + // If enlistmentPerTest is true we can always validate the working tree because + // this is the last place we'll use it + if (this.validateWorkingTree || this.enlistmentPerTest) + { + this.Enlistment.RepoRoot.ShouldBeADirectory(this.FileSystem) + .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, ignoreCase: ignoreCase); + } } finally { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs index aca8358171..a68c9e49dc 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs @@ -4,12 +4,13 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] [Category(Categories.MacTODO.M3)] public class HashObjectTests : GitRepoTests { - public HashObjectTests() : base(enlistmentPerTest: false) + public HashObjectTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: false, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs index 594d8aeb45..01dc74f8e5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs @@ -4,11 +4,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class MergeConflictTests : GitRepoTests { - public MergeConflictTests() : base(enlistmentPerTest: true) + public MergeConflictTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs index a6beeabcc3..004160f76b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs @@ -2,11 +2,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class RebaseConflictTests : GitRepoTests { - public RebaseConflictTests() : base(enlistmentPerTest: true) + public RebaseConflictTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs index 4f7a62926d..ddfc3229a1 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs @@ -2,11 +2,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class RebaseTests : GitRepoTests { - public RebaseTests() : base(enlistmentPerTest: true) + public RebaseTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs index 17ccf772fe..d8bf897329 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs @@ -1,15 +1,16 @@ -using GVFS.FunctionalTests.Should; +using GVFS.FunctionalTests.Should; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class ResetHardTests : GitRepoTests { private const string ResetHardCommand = "reset --hard"; - public ResetHardTests() : base(enlistmentPerTest: true) + public ResetHardTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs index 1b8f9cc0a6..751617ffd3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs @@ -3,11 +3,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class ResetMixedTests : GitRepoTests { - public ResetMixedTests() : base(enlistmentPerTest: true) + public ResetMixedTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs index 8aaed0884e..8906a8d467 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs @@ -2,11 +2,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class ResetSoftTests : GitRepoTests { - public ResetSoftTests() : base(enlistmentPerTest: true) + public ResetSoftTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs index 14443a24d0..1a7a5bb152 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs @@ -3,10 +3,11 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] public class RmTests : GitRepoTests { - public RmTests() : base(enlistmentPerTest: false) + public RmTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: false, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs index 116ebd7000..6f51e14d74 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs @@ -5,11 +5,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class StatusTests : GitRepoTests { - public StatusTests() : base(enlistmentPerTest: true) + public StatusTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index 1d48aa48c6..f14e273651 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -4,11 +4,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class UpdateIndexTests : GitRepoTests { - public UpdateIndexTests() : base(enlistmentPerTest: true) + public UpdateIndexTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs index 20281ecc85..39a0a8634d 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs @@ -2,11 +2,12 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixture] + [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] [Category(Categories.GitCommands)] public class UpdateRefTests : GitRepoTests { - public UpdateRefTests() : base(enlistmentPerTest: true) + public UpdateRefTests(ValidateWorkingTreeOptions validateWorkingTree) + : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } From 6bb8059eba475c2533be4bcd19d13417dc39b81f Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Tue, 30 Oct 2018 11:36:38 -0700 Subject: [PATCH 228/244] Address PR feedback on naming --- Scripts/Mac/DownloadGVFSGit.sh | 4 ++-- Scripts/Mac/InstallSharedDataQueueDylib.sh | 9 --------- Scripts/Mac/InstallSharedDataQueueStallWorkaround.sh | 9 +++++++++ Scripts/Mac/PrepFunctionalTests.sh | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) delete mode 100755 Scripts/Mac/InstallSharedDataQueueDylib.sh create mode 100755 Scripts/Mac/InstallSharedDataQueueStallWorkaround.sh diff --git a/Scripts/Mac/DownloadGVFSGit.sh b/Scripts/Mac/DownloadGVFSGit.sh index 99bc86679e..54a742b906 100755 --- a/Scripts/Mac/DownloadGVFSGit.sh +++ b/Scripts/Mac/DownloadGVFSGit.sh @@ -3,5 +3,5 @@ BUILDDIR=$VFS_OUTPUTDIR/GVFS.Build GITVERSION="$($VFS_SCRIPTDIR/GetGitVersionNumber.sh)" cp $VFS_SRCDIR/nuget.config $BUILDDIR -dotnet new classlib -n GVFS.Restore -o $BUILDDIR --force -dotnet add $BUILDDIR/GVFS.Restore.csproj package --package-directory $VFS_PACKAGESDIR GitForMac.GVFS.Installer --version $GITVERSION \ No newline at end of file +dotnet new classlib -n Restore.GitInstaller -o $BUILDDIR --force +dotnet add $BUILDDIR/Restore.GitInstaller.csproj package --package-directory $VFS_PACKAGESDIR GitForMac.GVFS.Installer --version $GITVERSION \ No newline at end of file diff --git a/Scripts/Mac/InstallSharedDataQueueDylib.sh b/Scripts/Mac/InstallSharedDataQueueDylib.sh deleted file mode 100755 index 2943d82c36..0000000000 --- a/Scripts/Mac/InstallSharedDataQueueDylib.sh +++ /dev/null @@ -1,9 +0,0 @@ -. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" - -BUILDDIR=$VFS_OUTPUTDIR/GVFS.Build -cp $VFS_SRCDIR/nuget.config $BUILDDIR -dotnet new classlib -n GVFS.Restore -o $BUILDDIR --force -dotnet add $BUILDDIR/GVFS.Restore.csproj package --package-directory $VFS_PACKAGESDIR SharedDataQueueDylib --version '1.0.0' - -# DYLD_LIBRARY_PATH contains /usr/local/lib by default, so we'll copy this library there. -cp $VFS_PACKAGESDIR/shareddataqueuedylib/1.0.0/libSharedDataQueue.dylib /usr/local/lib \ No newline at end of file diff --git a/Scripts/Mac/InstallSharedDataQueueStallWorkaround.sh b/Scripts/Mac/InstallSharedDataQueueStallWorkaround.sh new file mode 100755 index 0000000000..e764522986 --- /dev/null +++ b/Scripts/Mac/InstallSharedDataQueueStallWorkaround.sh @@ -0,0 +1,9 @@ +. "$(dirname ${BASH_SOURCE[0]})/InitializeEnvironment.sh" + +BUILDDIR=$VFS_OUTPUTDIR/GVFS.Build +cp $VFS_SRCDIR/nuget.config $BUILDDIR +dotnet new classlib -n Restore.SharedDataQueueStallWorkaround -o $BUILDDIR --force +dotnet add $BUILDDIR/Restore.SharedDataQueueStallWorkaround.csproj package --package-directory $VFS_PACKAGESDIR SharedDataQueueStallWorkaround --version '1.0.0' + +# DYLD_LIBRARY_PATH contains /usr/local/lib by default, so we'll copy this library there. +cp $VFS_PACKAGESDIR/shareddataqueuestallworkaround/1.0.0/libSharedDataQueue.dylib /usr/local/lib diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index 38eb724a43..7650f5e2b2 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -37,7 +37,7 @@ fi git-credential-manager install -$VFS_SCRIPTDIR/InstallSharedDataQueueDylib.sh || exit 1 +$VFS_SCRIPTDIR/InstallSharedDataQueueStallWorkaround.sh || exit 1 # If we're running on an agent where the PAT environment variable is set and a URL is passed into the script, add it to the keychain. PATURL=$1 From a84150da1e2397acc2a821ec99fb439f82e619e0 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 30 Oct 2018 11:44:46 -0700 Subject: [PATCH 229/244] Fix StyleCop issues --- GVFS/GVFS.FunctionalTests/Program.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index 7eed002a12..c90cd33a6d 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -1,6 +1,6 @@ using GVFS.FunctionalTests.Tests; -using GVFS.FunctionalTests.Tools; using GVFS.FunctionalTests.Tests.GitCommands; +using GVFS.FunctionalTests.Tools; using GVFS.Tests; using System; using System.Collections.Generic; @@ -59,7 +59,6 @@ public static void Main(string[] args) { GVFSTestConfig.FileSystemRunners = FileSystemRunners.FileSystemRunner.AllMacRunners; } - } else { From a89a45539750213bcf388c2de8046d7317246c5e Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 30 Oct 2018 14:33:24 -0700 Subject: [PATCH 230/244] Enable HashObjectTests and RmTests on Mac --- .../Tests/GitCommands/HashObjectTests.cs | 14 +++++++------- .../Tests/GitCommands/RmTests.cs | 16 +++++++--------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs index a68c9e49dc..31091cd5d4 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs @@ -1,16 +1,16 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; using NUnit.Framework; +using System.IO; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class HashObjectTests : GitRepoTests { - public HashObjectTests(ValidateWorkingTreeOptions validateWorkingTree) - : base(enlistmentPerTest: false, validateWorkingTree: validateWorkingTree) + public HashObjectTests() + : base(enlistmentPerTest: false, validateWorkingTree: ValidateWorkingTreeOptions.DoNotValidateWorkingTree) { } @@ -19,8 +19,8 @@ public void CanReadFileAfterHashObject() { this.ValidateGitCommand("status"); - // Validate that Readme.md is not on disk at all - string fileName = "Readme.md"; + // Validate that Scripts\RunUnitTests.bad is not on disk at all + string fileName = Path.Combine("Scripts", "RunUnitTests.bat"); this.Enlistment.UnmountGVFS(); this.Enlistment.GetVirtualPathTo(fileName).ShouldNotExistOnDisk(this.FileSystem); @@ -29,7 +29,7 @@ public void CanReadFileAfterHashObject() // TODO 1087312: Fix 'git hash-oject' so that it works for files that aren't on disk yet GitHelpers.InvokeGitAgainstGVFSRepo( this.Enlistment.RepoRoot, - "hash-object " + fileName); + "hash-object " + fileName.Replace("\\", "/")); this.FileContentsShouldMatch(fileName); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs index 1a7a5bb152..a5cc9ff8f5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs @@ -1,32 +1,30 @@ using GVFS.FunctionalTests.Should; using NUnit.Framework; +using System.IO; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixture] public class RmTests : GitRepoTests { - public RmTests(ValidateWorkingTreeOptions validateWorkingTree) - : base(enlistmentPerTest: false, validateWorkingTree: validateWorkingTree) + public RmTests() + : base(enlistmentPerTest: false, validateWorkingTree: ValidateWorkingTreeOptions.DoNotValidateWorkingTree) { } - // Mac(TODO): Something is triggering Readme.md to get created on disk before this - // test validates that it's not present [TestCase] - [Category(Categories.MacTODO.M3)] public void CanReadFileAfterGitRmDryRun() { this.ValidateGitCommand("status"); - // Validate that Readme.md is not on disk at all - string fileName = "Readme.md"; + // Validate that Scripts\RunUnitTests.bad is not on disk at all + string fileName = Path.Combine("Scripts", "RunUnitTests.bat"); this.Enlistment.UnmountGVFS(); this.Enlistment.GetVirtualPathTo(fileName).ShouldNotExistOnDisk(this.FileSystem); this.Enlistment.MountGVFS(); - this.ValidateGitCommand("rm --dry-run " + fileName); + this.ValidateGitCommand("rm --dry-run " + fileName.Replace("\\", "/")); this.FileContentsShouldMatch(fileName); } } From f2deecd90e0623f852225cf6edde82965c928633 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 30 Oct 2018 15:02:48 -0700 Subject: [PATCH 231/244] Clean up NUnit source parameterization --- .../Windows/Tests/WindowsFileSystemTests.cs | 32 +++++----- .../FileSystemRunners/FileSystemRunner.cs | 5 -- GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs | 2 - GVFS/GVFS.FunctionalTests/Program.cs | 13 ---- .../BasicFileSystemTests.cs | 61 +++++++++---------- .../EnlistmentPerFixture/GitFilesTests.cs | 2 +- .../GitMoveRenameTests.cs | 2 +- .../MoveRenameFileTests.cs | 2 +- .../MoveRenameFileTests_2.cs | 2 +- .../MoveRenameFolderTests.cs | 2 +- .../MultithreadedReadWriteTests.cs | 4 +- .../WorkingDirectoryTests.cs | 2 +- .../ModifiedPathsTests.cs | 10 +-- .../PersistedWorkingDirectoryTests.cs | 4 +- .../Tests/GitCommands/AddStageTests.cs | 5 +- .../Tests/GitCommands/CheckoutTests.cs | 5 +- .../GitCommands/CherryPickConflictTests.cs | 7 ++- .../GitCommands/CreatePlaceholderTests.cs | 5 +- .../GitCommands/DeleteEmptyFolderTests.cs | 5 +- .../Tests/GitCommands/EnumerationMergeTest.cs | 7 ++- .../Tests/GitCommands/GitCommandsTests.cs | 5 +- .../Tests/GitCommands/GitRepoTests.cs | 18 +----- .../Tests/GitCommands/HashObjectTests.cs | 2 +- .../Tests/GitCommands/MergeConflictTests.cs | 5 +- .../Tests/GitCommands/RebaseConflictTests.cs | 7 ++- .../Tests/GitCommands/RebaseTests.cs | 7 ++- .../Tests/GitCommands/ResetHardTests.cs | 5 +- .../Tests/GitCommands/ResetMixedTests.cs | 5 +- .../Tests/GitCommands/ResetSoftTests.cs | 7 ++- .../Tests/GitCommands/RmTests.cs | 2 +- .../Tests/GitCommands/StatusTests.cs | 7 ++- .../Tests/GitCommands/UpdateIndexTests.cs | 5 +- .../Tests/GitCommands/UpdateRefTests.cs | 7 ++- 33 files changed, 120 insertions(+), 139 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs index 1cd0c45a68..235a111994 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs @@ -28,7 +28,7 @@ private enum CreationDisposition TruncateExisting = 5 // TRUNCATE_EXISTING } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void CaseOnlyRenameEmptyVirtualNTFSFolder(FileSystemRunner fileSystem, string parentFolder) { string testFolderName = Path.Combine(parentFolder, "test_folder"); @@ -50,7 +50,7 @@ public void CaseOnlyRenameEmptyVirtualNTFSFolder(FileSystemRunner fileSystem, st newFolderVirtualPath.ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void CaseOnlyRenameToAllCapsEmptyVirtualNTFSFolder(FileSystemRunner fileSystem) { string testFolderName = Path.Combine("test_folder"); @@ -72,7 +72,7 @@ public void CaseOnlyRenameToAllCapsEmptyVirtualNTFSFolder(FileSystemRunner fileS newFolderVirtualPath.ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void CaseOnlyRenameTopOfVirtualNTFSFolderTree(FileSystemRunner fileSystem) { string testFolderParent = "test_folder_parent"; @@ -117,7 +117,7 @@ public void CaseOnlyRenameTopOfVirtualNTFSFolderTree(FileSystemRunner fileSystem this.Enlistment.GetVirtualPathTo(relativeTestFilePath).ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void CaseOnlyRenameFullDotGitFolder(FileSystemRunner fileSystem) { string testFolderName = ".git\\test_folder"; @@ -139,7 +139,7 @@ public void CaseOnlyRenameFullDotGitFolder(FileSystemRunner fileSystem) newFolderVirtualPath.ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void CaseOnlyRenameTopOfDotGitFullFolderTree(FileSystemRunner fileSystem) { string testFolderParent = ".git\\test_folder_parent"; @@ -181,7 +181,7 @@ public void CaseOnlyRenameTopOfDotGitFullFolderTree(FileSystemRunner fileSystem) this.Enlistment.GetVirtualPathTo(Path.Combine(".git", newFolderParentName)).ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void StreamAccessReadFromMemoryMappedVirtualNTFSFile(string parentFolder) { // Use SystemIORunner as the text we are writing is too long to pass to the command line @@ -221,7 +221,7 @@ public void StreamAccessReadFromMemoryMappedVirtualNTFSFile(string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void RandomAccessReadFromMemoryMappedVirtualNTFSFile(string parentFolder) { // Use SystemIORunner as the text we are writing is too long to pass to the command line @@ -267,7 +267,7 @@ public void RandomAccessReadFromMemoryMappedVirtualNTFSFile(string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void StreamAccessReadWriteFromMemoryMappedVirtualNTFSFile(string parentFolder) { // Use SystemIORunner as the text we are writing is too long to pass to the command line @@ -335,7 +335,7 @@ public void StreamAccessReadWriteFromMemoryMappedVirtualNTFSFile(string parentFo FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void RandomAccessReadWriteFromMemoryMappedVirtualNTFSFile(string parentFolder) { // Use SystemIORunner as the text we are writing is too long to pass to the command line @@ -405,7 +405,7 @@ public void RandomAccessReadWriteFromMemoryMappedVirtualNTFSFile(string parentFo FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void StreamAccessToExistingMemoryMappedFile(string parentFolder) { // Use SystemIORunner as the text we are writing is too long to pass to the command line @@ -474,7 +474,7 @@ public void StreamAccessToExistingMemoryMappedFile(string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void RandomAccessToExistingMemoryMappedFile(string parentFolder) { // Use SystemIORunner as the text we are writing is too long to pass to the command line @@ -543,7 +543,7 @@ public void RandomAccessToExistingMemoryMappedFile(string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void NativeReadAndWriteSeparateHandles(string parentFolder) { FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner; @@ -557,7 +557,7 @@ public void NativeReadAndWriteSeparateHandles(string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void NativeReadAndWriteSameHandle(string parentFolder) { FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner; @@ -575,7 +575,7 @@ public void NativeReadAndWriteSameHandle(string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void NativeReadAndWriteRepeatedly(string parentFolder) { FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner; @@ -593,7 +593,7 @@ public void NativeReadAndWriteRepeatedly(string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void NativeRemoveReadOnlyAttribute(string parentFolder) { FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner; @@ -607,7 +607,7 @@ public void NativeRemoveReadOnlyAttribute(string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] public void NativeCannotWriteToReadOnlyFile(string parentFolder) { FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner; diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs index e064be50ab..4d2b9dba60 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs @@ -5,11 +5,6 @@ namespace GVFS.FunctionalTests.FileSystemRunners { public abstract class FileSystemRunner { - /// - /// String that identifies which list to use when running tests - /// - public const string TestRunners = "Runners"; - private static FileSystemRunner defaultRunner = new SystemIORunner(); public static object[] AllWindowsRunners { get; } = diff --git a/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs b/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs index ecf6d2fcb6..7f65687538 100644 --- a/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs +++ b/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs @@ -12,8 +12,6 @@ public static class GVFSTestConfig public static object[] FileSystemRunners { get; set; } - public static object[] GitCommandTestWorkTreeValidation { get; set; } - public static bool TestGVFSOnPath { get; set; } public static bool ReplaceInboxProjFS { get; set; } diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index c90cd33a6d..f7db4b9698 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -1,5 +1,4 @@ using GVFS.FunctionalTests.Tests; -using GVFS.FunctionalTests.Tests.GitCommands; using GVFS.FunctionalTests.Tools; using GVFS.Tests; using System; @@ -44,13 +43,6 @@ public static void Main(string[] args) { Console.WriteLine("Running the full suite of tests"); - GVFSTestConfig.GitCommandTestWorkTreeValidation = - new object[] - { - new object[] { GitRepoTests.ValidateWorkingTreeOptions.ValidateWorkingTree }, - new object[] { GitRepoTests.ValidateWorkingTreeOptions.DoNotValidateWorkingTree } - }; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { GVFSTestConfig.FileSystemRunners = FileSystemRunners.FileSystemRunner.AllWindowsRunners; @@ -64,11 +56,6 @@ public static void Main(string[] args) { excludeCategories.Add(Categories.FullSuiteOnly); GVFSTestConfig.FileSystemRunners = FileSystemRunners.FileSystemRunner.DefaultRunners; - GVFSTestConfig.GitCommandTestWorkTreeValidation = - new object[] - { - new object[] { GitRepoTests.ValidateWorkingTreeOptions.ValidateWorkingTree } - }; } if (runner.HasCustomArg("--windows-only")) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index 982e02decf..6079a42242 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -1,4 +1,4 @@ -using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.FileSystemRunners; using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tests.EnlistmentPerFixture; using GVFS.FunctionalTests.Tools; @@ -19,7 +19,7 @@ public class BasicFileSystemTests : TestsWithEnlistmentPerFixture private const int FileAttributeReparsePoint = 0x00000400; private const int FileAttributeRecallOnDataAccess = 0x00400000; - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void ShrinkFileContents(FileSystemRunner fileSystem, string parentFolder) { string filename = Path.Combine(parentFolder, "ShrinkFileContents"); @@ -35,7 +35,7 @@ public void ShrinkFileContents(FileSystemRunner fileSystem, string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void GrowFileContents(FileSystemRunner fileSystem, string parentFolder) { string filename = Path.Combine(parentFolder, "GrowFileContents"); @@ -51,7 +51,7 @@ public void GrowFileContents(FileSystemRunner fileSystem, string parentFolder) FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void FilesAreBufferedAndCanBeFlushed(FileSystemRunner fileSystem, string parentFolder) { string filename = Path.Combine(parentFolder, "FilesAreBufferedAndCanBeFlushed"); @@ -78,7 +78,7 @@ public void FilesAreBufferedAndCanBeFlushed(FileSystemRunner fileSystem, string fileSystem.DeleteFile(filePath); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] [Category(Categories.WindowsOnly)] public void NewFileAttributesAreUpdated(string parentFolder) { @@ -106,7 +106,7 @@ public void NewFileAttributesAreUpdated(string parentFolder) virtualFile.ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] [Category(Categories.WindowsOnly)] public void NewFolderAttributesAreUpdated(string parentFolder) { @@ -201,7 +201,7 @@ public void UnhydratedFolderAttributesAreUpdated() .WithInfo(testValue, testValue, testValue, FileAttributes.Hidden | FileAttributes.Directory, ignoreRecallAttributes: true); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void CannotWriteToReadOnlyFile(FileSystemRunner fileSystem, string parentFolder) { string filename = Path.Combine(parentFolder, "CannotWriteToReadOnlyFile"); @@ -228,7 +228,7 @@ public void CannotWriteToReadOnlyFile(FileSystemRunner fileSystem, string parent FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void ReadonlyCanBeSetAndUnset(FileSystemRunner fileSystem, string parentFolder) { string filename = Path.Combine(parentFolder, "ReadonlyCanBeSetAndUnset"); @@ -252,7 +252,7 @@ public void ReadonlyCanBeSetAndUnset(FileSystemRunner fileSystem, string parentF FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, filename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void ChangeVirtualNTFSFileNameCase(FileSystemRunner fileSystem, string parentFolder) { string oldFilename = Path.Combine(parentFolder, "ChangePhysicalFileNameCase.txt"); @@ -273,7 +273,7 @@ public void ChangeVirtualNTFSFileNameCase(FileSystemRunner fileSystem, string pa FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, newFilename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void ChangeVirtualNTFSFileName(FileSystemRunner fileSystem, string parentFolder) { string oldFilename = Path.Combine(parentFolder, "ChangePhysicalFileName.txt"); @@ -294,7 +294,7 @@ public void ChangeVirtualNTFSFileName(FileSystemRunner fileSystem, string parent FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, newFilename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void MoveVirtualNTFSFileToVirtualNTFSFolder(FileSystemRunner fileSystem, string parentFolder) { string testFolderName = Path.Combine(parentFolder, "test_folder"); @@ -324,7 +324,7 @@ public void MoveVirtualNTFSFileToVirtualNTFSFolder(FileSystemRunner fileSystem, FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, testFolderName, parentFolder); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void MoveWorkingDirectoryFileToDotGitFolder(FileSystemRunner fileSystem) { string testFolderName = ".git"; @@ -346,7 +346,7 @@ public void MoveWorkingDirectoryFileToDotGitFolder(FileSystemRunner fileSystem) newTestFileVirtualPath.ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void MoveDotGitFileToWorkingDirectoryFolder(FileSystemRunner fileSystem) { string testFolderName = "test_folder"; @@ -375,7 +375,7 @@ public void MoveDotGitFileToWorkingDirectoryFolder(FileSystemRunner fileSystem) this.Enlistment.GetVirtualPathTo(testFolderName).ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void MoveVirtualNTFSFileToOverwriteVirtualNTFSFile(FileSystemRunner fileSystem, string parentFolder) { string targetFilename = Path.Combine(parentFolder, "TargetFile.txt"); @@ -401,7 +401,7 @@ public void MoveVirtualNTFSFileToOverwriteVirtualNTFSFile(FileSystemRunner fileS FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, targetFilename, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void MoveVirtualNTFSFileToInvalidFolder(FileSystemRunner fileSystem, string parentFolder) { string testFolderName = Path.Combine(parentFolder, "test_folder"); @@ -425,7 +425,7 @@ public void MoveVirtualNTFSFileToInvalidFolder(FileSystemRunner fileSystem, stri FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, testFileName, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void DeletedFilesCanBeImmediatelyRecreated(FileSystemRunner fileSystem, string parentFolder) { string filename = Path.Combine(parentFolder, "DeletedFilesCanBeImmediatelyRecreated"); @@ -447,7 +447,7 @@ public void DeletedFilesCanBeImmediatelyRecreated(FileSystemRunner fileSystem, s } // WindowsOnly due to differences between POSIX and Windows delete - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestCanDeleteFilesWhileTheyAreOpenRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.CanDeleteFilesWhileTheyAreOpenRunners))] [Category(Categories.WindowsOnly)] public void CanDeleteFilesWhileTheyAreOpen(FileSystemRunner fileSystem, string parentFolder) { @@ -514,6 +514,8 @@ public void CanDeleteHydratedFilesWhileTheyAreOpenForWrite() virtualPath.ShouldNotExistOnDisk(fileSystem); } + // WindowsOnly because file timestamps on Mac are set to the time at which + // placeholders are written [TestCase] [Category(Categories.WindowsOnly)] public void ProjectedBlobFileTimesMatchHead() @@ -562,7 +564,7 @@ public void ProjectedBlobFolderTimesMatchHead() folderInfo.LastWriteTime.ShouldBeAtMost(DateTime.Now); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void NonExistentItemBehaviorIsCorrect(FileSystemRunner fileSystem, string parentFolder) { string nonExistentItem = Path.Combine(parentFolder, "BadFolderName"); @@ -584,7 +586,7 @@ public void NonExistentItemBehaviorIsCorrect(FileSystemRunner fileSystem, string // fileSystem.ReplaceDirectoryShouldNotBeFound(nonExistentItem, true) } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void RenameEmptyVirtualNTFSFolder(FileSystemRunner fileSystem, string parentFolder) { string testFolderName = Path.Combine(parentFolder, "test_folder"); @@ -606,7 +608,7 @@ public void RenameEmptyVirtualNTFSFolder(FileSystemRunner fileSystem, string par newFolderVirtualPath.ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void MoveVirtualNTFSFolderIntoVirtualNTFSFolder(FileSystemRunner fileSystem, string parentFolder) { string testFolderName = Path.Combine(parentFolder, "test_folder"); @@ -642,7 +644,7 @@ public void MoveVirtualNTFSFolderIntoVirtualNTFSFolder(FileSystemRunner fileSyst FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, targetFolderName, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void RenameAndMoveVirtualNTFSFolderIntoVirtualNTFSFolder(FileSystemRunner fileSystem, string parentFolder) { string testFolderName = Path.Combine(parentFolder, "test_folder"); @@ -679,7 +681,7 @@ public void RenameAndMoveVirtualNTFSFolderIntoVirtualNTFSFolder(FileSystemRunner FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, targetFolderName, parentFolder); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void MoveVirtualNTFSFolderTreeIntoVirtualNTFSFolder(FileSystemRunner fileSystem) { string testFolderParent = "test_folder_parent"; @@ -743,7 +745,7 @@ public void MoveVirtualNTFSFolderTreeIntoVirtualNTFSFolder(FileSystemRunner file this.Enlistment.GetVirtualPathTo(relativeTestFilePath).ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void MoveDotGitFullFolderTreeToDotGitFullFolder(FileSystemRunner fileSystem) { string testFolderRoot = ".git"; @@ -808,7 +810,7 @@ public void MoveDotGitFullFolderTreeToDotGitFullFolder(FileSystemRunner fileSyst this.Enlistment.GetVirtualPathTo(relativeTestFilePath).ShouldNotExistOnDisk(fileSystem); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void DeleteIndexFileFails(FileSystemRunner fileSystem) { string indexFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(".git", "index")); @@ -817,7 +819,7 @@ public void DeleteIndexFileFails(FileSystemRunner fileSystem) indexFilePath.ShouldBeAFile(fileSystem); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Runners))] public void MoveVirtualNTFSFolderIntoInvalidFolder(FileSystemRunner fileSystem, string parentFolder) { string testFolderParent = Path.Combine(parentFolder, "test_folder_parent"); @@ -865,7 +867,7 @@ public void MoveVirtualNTFSFolderIntoInvalidFolder(FileSystemRunner fileSystem, FileRunnersAndFolders.ShouldNotExistOnDisk(this.Enlistment, fileSystem, relativeTestFilePath, parentFolder); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] [Category(Categories.WindowsOnly)] public void CreateFileInheritsParentDirectoryAttributes(string parentFolder) { @@ -882,7 +884,7 @@ public void CreateFileInheritsParentDirectoryAttributes(string parentFolder) FileSystemRunner.DefaultRunner.DeleteDirectory(parentDirectoryPath); } - [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] + [TestCaseSource(typeof(FileRunnersAndFolders), nameof(FileRunnersAndFolders.Folders))] [Category(Categories.WindowsOnly)] public void CreateDirectoryInheritsParentDirectoryAttributes(string parentFolder) { @@ -901,10 +903,7 @@ public void CreateDirectoryInheritsParentDirectoryAttributes(string parentFolder private class FileRunnersAndFolders { - public const string TestFolders = "Folders"; - public const string TestRunners = "Runners"; - public const string TestCanDeleteFilesWhileTheyAreOpenRunners = "CanDeleteFilesWhileTheyAreOpenRunners"; - public const string DotGitFolder = ".git"; + private const string DotGitFolder = ".git"; private static object[] allFolders = { diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index ad4bc8087c..6f020c1a27 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -11,7 +11,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { - [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestFixtureSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public class GitFilesTests : TestsWithEnlistmentPerFixture { private FileSystemRunner fileSystem; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs index b26a3e3b74..900870b7f0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs @@ -10,7 +10,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { - [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestFixtureSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] [Category(Categories.GitCommands)] public class GitMoveRenameTests : TestsWithEnlistmentPerFixture { diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs index e83547cd49..e5c7d1068e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs @@ -9,7 +9,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { // TODO 452590 - Combine all of the MoveRenameTests into a single fixture, and have each use different // well known files - [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestFixtureSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public class MoveRenameFileTests : TestsWithEnlistmentPerFixture { public const string TestFileContents = diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs index b38d460aca..87b55e2757 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs @@ -7,7 +7,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { // TODO 452590 - Combine all of the MoveRenameTests into a single fixture, and have each use different // well known files - [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestFixtureSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public class MoveRenameFileTests_2 : TestsWithEnlistmentPerFixture { private const string TestFileFolder = "Test_EPF_MoveRenameFileTests_2"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs index 2f0071a048..0e0d53ea72 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs @@ -6,7 +6,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { - [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestFixtureSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public class MoveRenameFolderTests : TestsWithEnlistmentPerFixture { private const string TestFileContents = diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs index df916dc3cc..f003167509 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs @@ -1,4 +1,4 @@ -using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.FileSystemRunners; using GVFS.FunctionalTests.Should; using GVFS.Tests.Should; using NUnit.Framework; @@ -109,7 +109,7 @@ public void CanReadHydratedPlaceholderInParallel() } } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] [Order(3)] public void CanReadWriteAFileInParallel(FileSystemRunner fileSystem) { diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index e33de7b6e1..559e48561b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -13,7 +13,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { - [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestFixtureSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public class WorkingDirectoryTests : TestsWithEnlistmentPerFixture { public const string TestFileContents = diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 88d440a1d1..62a8970c75 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -40,7 +40,7 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase $"A {FolderToDelete}/", }; - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void DeletedTempFileIsRemovedFromModifiedFiles(FileSystemRunner fileSystem) { string tempFile = this.CreateFile(fileSystem, "temp.txt"); @@ -50,7 +50,7 @@ public void DeletedTempFileIsRemovedFromModifiedFiles(FileSystemRunner fileSyste GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, fileSystem, "temp.txt"); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void DeletedTempFolderIsRemovedFromModifiedFiles(FileSystemRunner fileSystem) { string tempFolder = this.CreateDirectory(fileSystem, "Temp"); @@ -60,7 +60,7 @@ public void DeletedTempFolderIsRemovedFromModifiedFiles(FileSystemRunner fileSys GVFSHelpers.ModifiedPathsShouldNotContain(this.Enlistment, fileSystem, "Temp/"); } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void DeletedTempFolderDeletesFilesFromModifiedFiles(FileSystemRunner fileSystem) { string tempFolder = this.CreateDirectory(fileSystem, "Temp"); @@ -75,7 +75,7 @@ public void DeletedTempFolderDeletesFilesFromModifiedFiles(FileSystemRunner file } [Category(Categories.MacTODO.NeedsRenameOldPath)] - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) { string fileToAdd = this.Enlistment.GetVirtualPathTo(FileToAdd); @@ -156,7 +156,7 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) } } - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void ModifiedPathsCorrectAfterHardLinking(FileSystemRunner fileSystem) { string[] expectedModifiedFilesContentsAfterHardlinks = diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedWorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedWorkingDirectoryTests.cs index 02b44fced5..da471c8fb3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedWorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedWorkingDirectoryTests.cs @@ -11,7 +11,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase [Category(Categories.FullSuiteOnly)] public class PersistedWorkingDirectoryTests : TestsWithEnlistmentPerTestCase { - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void PersistedDirectoryLazyLoad(FileSystemRunner fileSystem) { string enumerateDirectoryName = Path.Combine("GVFS", "GVFS"); @@ -68,7 +68,7 @@ public void PersistedDirectoryLazyLoad(FileSystemRunner fileSystem) /// test persistence, we want to save as much time in tests runs as possible by only /// remounting once. /// - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), nameof(FileSystemRunner.Runners))] public void PersistedDirectoryTests(FileSystemRunner fileSystem) { // Delete File Setup diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index 76a801c32a..ad1886fe51 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -1,15 +1,16 @@ using GVFS.FunctionalTests.Tools; +using GVFS.Tests; using NUnit.Framework; using System.IO; using System.Threading; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class AddStageTests : GitRepoTests { - public AddStageTests(ValidateWorkingTreeOptions validateWorkingTree) + public AddStageTests(bool validateWorkingTree) : base(enlistmentPerTest: false, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index c4ec6f478c..41873ec85e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -1,5 +1,6 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; +using GVFS.Tests; using GVFS.Tests.Should; using Microsoft.Win32.SafeHandles; using NUnit.Framework; @@ -11,11 +12,11 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class CheckoutTests : GitRepoTests { - public CheckoutTests(ValidateWorkingTreeOptions validateWorkingTree) + public CheckoutTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs index aafd213d02..a131d49814 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs @@ -1,12 +1,13 @@ -using NUnit.Framework; +using GVFS.Tests; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class CherryPickConflictTests : GitRepoTests { - public CherryPickConflictTests(ValidateWorkingTreeOptions validateWorkingTree) + public CherryPickConflictTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs index a6e457c9f6..ae3f20a8d4 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs @@ -1,5 +1,6 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; +using GVFS.Tests; using NUnit.Framework; using System.IO; using System.Runtime.InteropServices; @@ -7,13 +8,13 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class CreatePlaceholderTests : GitRepoTests { private static readonly string FileToRead = Path.Combine("GVFS", "GVFS", "Program.cs"); - public CreatePlaceholderTests(ValidateWorkingTreeOptions validateWorkingTree) + public CreatePlaceholderTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs index c4ab9d7dad..b2a6a3f9cb 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs @@ -1,14 +1,15 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; +using GVFS.Tests; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class DeleteEmptyFolderTests : GitRepoTests { - public DeleteEmptyFolderTests(ValidateWorkingTreeOptions validateWorkingTree) + public DeleteEmptyFolderTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs index b33554d8a4..beea21a288 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs @@ -1,8 +1,9 @@ -using NUnit.Framework; +using GVFS.Tests; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class EnumerationMergeTest : GitRepoTests { @@ -10,7 +11,7 @@ public class EnumerationMergeTest : GitRepoTests // enumeration when they don't fit in a user's buffer private const string EnumerationReproCommitish = "FunctionalTests/20170602"; - public EnumerationMergeTest(ValidateWorkingTreeOptions validateWorkingTree) + public EnumerationMergeTest(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 324d193f45..54894a8ad3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -1,5 +1,6 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; +using GVFS.Tests; using GVFS.Tests.Should; using NUnit.Framework; using System; @@ -8,7 +9,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class GitCommandsTests : GitRepoTests { @@ -26,7 +27,7 @@ public class GitCommandsTests : GitRepoTests private static readonly string RenameFolderPathFrom = Path.Combine("GVFS", "GVFS.Common", "PrefetchPacks"); private static readonly string RenameFolderPathTo = Path.Combine("GVFS", "GVFS.Common", "PrefetchPacksRenamed"); - public GitCommandsTests(ValidateWorkingTreeOptions validateWorkingTree) + public GitCommandsTests(bool validateWorkingTree) : base(enlistmentPerTest: false, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index 3cf36dfe5a..b7f0e0a71e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -13,8 +13,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands [TestFixture] public abstract class GitRepoTests { - public const string ValidateWorkingTree = "WorkTreeValidation"; - protected const string ConflictSourceBranch = "FunctionalTests/20170206_Conflict_Source"; protected const string ConflictTargetBranch = "FunctionalTests/20170206_Conflict_Target"; protected const string NoConflictSourceBranch = "FunctionalTests/20170209_NoConflict_Source"; @@ -27,25 +25,13 @@ public abstract class GitRepoTests private bool enlistmentPerTest; private bool validateWorkingTree; - public GitRepoTests(bool enlistmentPerTest, ValidateWorkingTreeOptions validateWorkingTree) + public GitRepoTests(bool enlistmentPerTest, bool validateWorkingTree) { this.enlistmentPerTest = enlistmentPerTest; - this.validateWorkingTree = validateWorkingTree == ValidateWorkingTreeOptions.ValidateWorkingTree; + this.validateWorkingTree = validateWorkingTree; this.FileSystem = new SystemIORunner(); } - public enum ValidateWorkingTreeOptions - { - Invalid = 0, - DoNotValidateWorkingTree, - ValidateWorkingTree, - } - - public static object[] WorkTreeValidation - { - get { return GVFSTestConfig.GitCommandTestWorkTreeValidation; } - } - public ControlGitRepo ControlGitRepo { get; private set; diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs index 31091cd5d4..e79edec465 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs @@ -10,7 +10,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands public class HashObjectTests : GitRepoTests { public HashObjectTests() - : base(enlistmentPerTest: false, validateWorkingTree: ValidateWorkingTreeOptions.DoNotValidateWorkingTree) + : base(enlistmentPerTest: false, validateWorkingTree: false) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs index 01dc74f8e5..99d8a84543 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs @@ -1,14 +1,15 @@ using GVFS.FunctionalTests.Tools; +using GVFS.Tests; using GVFS.Tests.Should; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class MergeConflictTests : GitRepoTests { - public MergeConflictTests(ValidateWorkingTreeOptions validateWorkingTree) + public MergeConflictTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs index 004160f76b..2be832a418 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs @@ -1,12 +1,13 @@ -using NUnit.Framework; +using GVFS.Tests; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class RebaseConflictTests : GitRepoTests { - public RebaseConflictTests(ValidateWorkingTreeOptions validateWorkingTree) + public RebaseConflictTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs index ddfc3229a1..959640a04b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs @@ -1,12 +1,13 @@ -using NUnit.Framework; +using GVFS.Tests; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class RebaseTests : GitRepoTests { - public RebaseTests(ValidateWorkingTreeOptions validateWorkingTree) + public RebaseTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs index d8bf897329..3b24aff1e7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs @@ -1,15 +1,16 @@ using GVFS.FunctionalTests.Should; +using GVFS.Tests; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class ResetHardTests : GitRepoTests { private const string ResetHardCommand = "reset --hard"; - public ResetHardTests(ValidateWorkingTreeOptions validateWorkingTree) + public ResetHardTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs index 751617ffd3..51b9ac8512 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs @@ -1,13 +1,14 @@ using GVFS.FunctionalTests.Should; +using GVFS.Tests; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class ResetMixedTests : GitRepoTests { - public ResetMixedTests(ValidateWorkingTreeOptions validateWorkingTree) + public ResetMixedTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs index 8906a8d467..148b720f0b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs @@ -1,12 +1,13 @@ -using NUnit.Framework; +using GVFS.Tests; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class ResetSoftTests : GitRepoTests { - public ResetSoftTests(ValidateWorkingTreeOptions validateWorkingTree) + public ResetSoftTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs index a5cc9ff8f5..81a31a3e87 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs @@ -8,7 +8,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands public class RmTests : GitRepoTests { public RmTests() - : base(enlistmentPerTest: false, validateWorkingTree: ValidateWorkingTreeOptions.DoNotValidateWorkingTree) + : base(enlistmentPerTest: false, validateWorkingTree: false) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs index 6f51e14d74..30c5d9ddad 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs @@ -1,15 +1,16 @@ -using GVFS.Tests.Should; +using GVFS.Tests; +using GVFS.Tests.Should; using NUnit.Framework; using System.IO; using System.Threading; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class StatusTests : GitRepoTests { - public StatusTests(ValidateWorkingTreeOptions validateWorkingTree) + public StatusTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index f14e273651..55def3d24d 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -1,14 +1,15 @@ using GVFS.FunctionalTests.Tools; +using GVFS.Tests; using NUnit.Framework; using System.IO; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class UpdateIndexTests : GitRepoTests { - public UpdateIndexTests(ValidateWorkingTreeOptions validateWorkingTree) + public UpdateIndexTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs index 39a0a8634d..4aa5ced43f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs @@ -1,12 +1,13 @@ -using NUnit.Framework; +using GVFS.Tests; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(GitRepoTests), GitRepoTests.ValidateWorkingTree)] + [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] [Category(Categories.GitCommands)] public class UpdateRefTests : GitRepoTests { - public UpdateRefTests(ValidateWorkingTreeOptions validateWorkingTree) + public UpdateRefTests(bool validateWorkingTree) : base(enlistmentPerTest: true, validateWorkingTree: validateWorkingTree) { } From 4478d095e4a00b2c6bc17d95c6bddee7cf8ec9e5 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 30 Oct 2018 16:16:35 -0600 Subject: [PATCH 232/244] Add ability to specify an iKey for the ETW listener --- GVFS/GVFS.Common/GVFSConstants.cs | 1 + .../ETWTelemetryEventListener.cs | 90 +++++++++---------- 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 8d2bd49891..89a89d7166 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -32,6 +32,7 @@ public static class GitConfig public const string DeprecatedCacheEndpointSuffix = ".cache-server-url"; public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; + public const string IKey = GitConfig.GVFSPrefix + "ikey"; public const string HooksExtension = ".hooks"; } diff --git a/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs b/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs index 3ca1adcd52..4e34e33d91 100644 --- a/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs +++ b/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs @@ -34,29 +34,27 @@ public class ETWTelemetryEventListener : InProcEventListener private EventSource eventSource; private string enlistmentId; private string mountId; + private string ikey; - private ETWTelemetryEventListener(string providerName, string[] traitsList, string enlistmentId, string mountId) + private ETWTelemetryEventListener(string providerName, string[] traitsList, string enlistmentId, string mountId, string ikey) : base(EventLevel.Verbose, Keywords.Telemetry) { this.eventSource = new EventSource(providerName, EventSourceSettings.EtwSelfDescribingEventFormat, traitsList); this.enlistmentId = enlistmentId; this.mountId = mountId; + this.ikey = ikey; } public static ETWTelemetryEventListener CreateTelemetryListenerIfEnabled(string gitBinRoot, string providerName, string enlistmentId, string mountId) { // This listener is disabled unless the user specifies the proper git config setting. - GitProcess.Result result = GitProcess.GetFromSystemConfig(gitBinRoot, GVFSConstants.GitConfig.GVFSTelemetryId); - if (result.HasErrors || string.IsNullOrEmpty(result.Output.TrimEnd('\r', '\n'))) - { - result = GitProcess.GetFromGlobalConfig(gitBinRoot, GVFSConstants.GitConfig.GVFSTelemetryId); - } - - if (!result.HasErrors && !string.IsNullOrEmpty(result.Output.TrimEnd('\r', '\n'))) + string traits = GetConfigValue(gitBinRoot, GVFSConstants.GitConfig.GVFSTelemetryId); + if (!string.IsNullOrEmpty(traits)) { - string[] traitsList = result.Output.TrimEnd('\r', '\n').Split('|'); - return new ETWTelemetryEventListener(providerName, traitsList, enlistmentId, mountId); + string[] traitsList = traits.Split('|'); + string ikey = GetConfigValue(gitBinRoot, GVFSConstants.GitConfig.IKey); + return new ETWTelemetryEventListener(providerName, traitsList, enlistmentId, mountId, ikey); } else { @@ -85,62 +83,60 @@ protected override void RecordMessageInternal( EventSourceOptions options = this.CreateOptions(level, keywords, opcode); EventSource.SetCurrentThreadActivityId(activityId); - if (jsonPayload != null) + if (string.IsNullOrEmpty(jsonPayload)) { - JsonPayload payload = new JsonPayload(jsonPayload, this.enlistmentId, this.mountId); + jsonPayload = "{}"; + } + + if (string.IsNullOrEmpty(this.ikey)) + { + var payload = new + { + Json = jsonPayload, + EnlistmentId = this.enlistmentId, + MountId = this.mountId, + }; this.eventSource.Write(eventName, ref options, ref activityId, ref parentActivityId, ref payload); } else { - Payload payload = new Payload(this.enlistmentId, this.mountId); + var payload = new + { + PartA_iKey = this.ikey, + Json = jsonPayload, + EnlistmentId = this.enlistmentId, + MountId = this.mountId, + }; this.eventSource.Write(eventName, ref options, ref activityId, ref parentActivityId, ref payload); } } - private EventSourceOptions CreateOptions(EventLevel level, Keywords keywords, EventOpcode opcode) + private static string GetConfigValue(string gitBinRoot, string configKey) { - EventSourceOptions options = new EventSourceOptions(); - options.Keywords = (EventKeywords)keywords; - options.Keywords |= (EventKeywords)MeasureKeyword; - - options.Level = (Microsoft.Diagnostics.Tracing.EventLevel)level; - options.Opcode = (Microsoft.Diagnostics.Tracing.EventOpcode)opcode; + GitProcess.Result result = GitProcess.GetFromSystemConfig(gitBinRoot, configKey); + if (result.HasErrors || string.IsNullOrEmpty(result.Output.TrimEnd('\r', '\n'))) + { + result = GitProcess.GetFromGlobalConfig(gitBinRoot, configKey); + } - return options; - } - - [EventData] - public struct Payload - { - public Payload(string enlistmentId, string mountId) + if (result.HasErrors || string.IsNullOrEmpty(result.Output.TrimEnd('\r', '\n'))) { - this.EnlistmentId = enlistmentId; - this.MountId = mountId; + return string.Empty; } - [EventField] - public string EnlistmentId { get; } - [EventField] - public string MountId { get; } + return result.Output.TrimEnd('\r', '\n'); } - [EventData] - public struct JsonPayload + private EventSourceOptions CreateOptions(EventLevel level, Keywords keywords, EventOpcode opcode) { - public JsonPayload(string payload, string enlistmentId, string mountId) - { - this.Json = payload; - this.EnlistmentId = enlistmentId; - this.MountId = mountId; - } + EventSourceOptions options = new EventSourceOptions(); + options.Keywords = (EventKeywords)keywords; + options.Keywords |= (EventKeywords)MeasureKeyword; - [EventField] - public string Json { get; } + options.Level = (Microsoft.Diagnostics.Tracing.EventLevel)level; + options.Opcode = (Microsoft.Diagnostics.Tracing.EventOpcode)opcode; - [EventField] - public string EnlistmentId { get; } - [EventField] - public string MountId { get; } + return options; } } } From 1ce6738196799b89ec8a1862aa793df7bdb6a011 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 31 Oct 2018 09:52:13 -0700 Subject: [PATCH 233/244] Add path helper method, only run with both validation options when running the full suite of tests --- GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs | 2 ++ GVFS/GVFS.FunctionalTests/Program.cs | 8 ++++++++ .../Tests/EnlistmentPerFixture/MountTests.cs | 2 +- .../Tests/GitCommands/AddStageTests.cs | 3 +-- .../Tests/GitCommands/CheckoutTests.cs | 3 +-- .../Tests/GitCommands/CherryPickConflictTests.cs | 5 ++--- .../Tests/GitCommands/CreatePlaceholderTests.cs | 3 +-- .../Tests/GitCommands/DeleteEmptyFolderTests.cs | 3 +-- .../Tests/GitCommands/EnumerationMergeTest.cs | 5 ++--- .../Tests/GitCommands/GitCommandsTests.cs | 3 +-- .../Tests/GitCommands/GitRepoTests.cs | 8 ++++++++ .../Tests/GitCommands/HashObjectTests.cs | 8 ++++---- .../Tests/GitCommands/MergeConflictTests.cs | 3 +-- .../Tests/GitCommands/RebaseConflictTests.cs | 5 ++--- .../Tests/GitCommands/RebaseTests.cs | 5 ++--- .../Tests/GitCommands/ResetHardTests.cs | 3 +-- .../Tests/GitCommands/ResetMixedTests.cs | 3 +-- .../Tests/GitCommands/ResetSoftTests.cs | 5 ++--- GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs | 9 +++++---- .../Tests/GitCommands/StatusTests.cs | 5 ++--- .../Tests/GitCommands/UpdateIndexTests.cs | 3 +-- .../Tests/GitCommands/UpdateRefTests.cs | 5 ++--- GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs | 8 ++++++++ 23 files changed, 59 insertions(+), 48 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs b/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs index 7f65687538..45080d4754 100644 --- a/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs +++ b/GVFS/GVFS.FunctionalTests/GVFSTestConfig.cs @@ -12,6 +12,8 @@ public static class GVFSTestConfig public static object[] FileSystemRunners { get; set; } + public static object[] GitRepoTestsValidateWorkTree { get; set; } + public static bool TestGVFSOnPath { get; set; } public static bool ReplaceInboxProjFS { get; set; } diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index f7db4b9698..0de9e79ee6 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -43,6 +43,8 @@ public static void Main(string[] args) { Console.WriteLine("Running the full suite of tests"); + GVFSTestConfig.GitRepoTestsValidateWorkTree = DataSources.AllBools; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { GVFSTestConfig.FileSystemRunners = FileSystemRunners.FileSystemRunner.AllWindowsRunners; @@ -54,6 +56,12 @@ public static void Main(string[] args) } else { + GVFSTestConfig.GitRepoTestsValidateWorkTree = + new object[] + { + new object[] { true } + }; + excludeCategories.Add(Categories.FullSuiteOnly); GVFSTestConfig.FileSystemRunners = FileSystemRunners.FileSystemRunner.DefaultRunners; } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MountTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MountTests.cs index d1d53ff5c8..b597a2b193 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MountTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MountTests.cs @@ -64,7 +64,7 @@ public void MountSetsCoreHooksPath() this.Enlistment.MountGVFS(); string expectedHooksPath = Path.Combine(this.Enlistment.RepoRoot, ".git", "hooks"); - expectedHooksPath = expectedHooksPath.Replace("\\", "/"); + expectedHooksPath = GitHelpers.ConvertPathToGitFormat(expectedHooksPath); GitProcess.Invoke( this.Enlistment.RepoRoot, "config core.hookspath") diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index ad1886fe51..fc784a1f83 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -1,12 +1,11 @@ using GVFS.FunctionalTests.Tools; -using GVFS.Tests; using NUnit.Framework; using System.IO; using System.Threading; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class AddStageTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 41873ec85e..b54ef2df45 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -1,6 +1,5 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; -using GVFS.Tests; using GVFS.Tests.Should; using Microsoft.Win32.SafeHandles; using NUnit.Framework; @@ -12,7 +11,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class CheckoutTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs index a131d49814..e0c302e9b1 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs @@ -1,9 +1,8 @@ -using GVFS.Tests; -using NUnit.Framework; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class CherryPickConflictTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs index ae3f20a8d4..2eafb216b3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs @@ -1,6 +1,5 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; -using GVFS.Tests; using NUnit.Framework; using System.IO; using System.Runtime.InteropServices; @@ -8,7 +7,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class CreatePlaceholderTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs index b2a6a3f9cb..00c29138af 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs @@ -1,11 +1,10 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; -using GVFS.Tests; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class DeleteEmptyFolderTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs index beea21a288..0a30a99ed0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs @@ -1,9 +1,8 @@ -using GVFS.Tests; -using NUnit.Framework; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class EnumerationMergeTest : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 54894a8ad3..d31ec1f6a1 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -1,6 +1,5 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; -using GVFS.Tests; using GVFS.Tests.Should; using NUnit.Framework; using System; @@ -9,7 +8,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class GitCommandsTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index b7f0e0a71e..88e47dd5e5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -32,6 +32,14 @@ public GitRepoTests(bool enlistmentPerTest, bool validateWorkingTree) this.FileSystem = new SystemIORunner(); } + public static object[] ValidateWorkingTree + { + get + { + return GVFSTestConfig.GitRepoTestsValidateWorkTree; + } + } + public ControlGitRepo ControlGitRepo { get; private set; diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs index e79edec465..ed0586ac84 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs @@ -20,18 +20,18 @@ public void CanReadFileAfterHashObject() this.ValidateGitCommand("status"); // Validate that Scripts\RunUnitTests.bad is not on disk at all - string fileName = Path.Combine("Scripts", "RunUnitTests.bat"); + string filePath = Path.Combine("Scripts", "RunUnitTests.bat"); this.Enlistment.UnmountGVFS(); - this.Enlistment.GetVirtualPathTo(fileName).ShouldNotExistOnDisk(this.FileSystem); + this.Enlistment.GetVirtualPathTo(filePath).ShouldNotExistOnDisk(this.FileSystem); this.Enlistment.MountGVFS(); // TODO 1087312: Fix 'git hash-oject' so that it works for files that aren't on disk yet GitHelpers.InvokeGitAgainstGVFSRepo( this.Enlistment.RepoRoot, - "hash-object " + fileName.Replace("\\", "/")); + "hash-object " + GitHelpers.ConvertPathToGitFormat(filePath)); - this.FileContentsShouldMatch(fileName); + this.FileContentsShouldMatch(filePath); } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs index 99d8a84543..c3853d8204 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs @@ -1,11 +1,10 @@ using GVFS.FunctionalTests.Tools; -using GVFS.Tests; using GVFS.Tests.Should; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class MergeConflictTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs index 2be832a418..094d7ef846 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs @@ -1,9 +1,8 @@ -using GVFS.Tests; -using NUnit.Framework; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class RebaseConflictTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs index 959640a04b..64f1d23532 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs @@ -1,9 +1,8 @@ -using GVFS.Tests; -using NUnit.Framework; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class RebaseTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs index 3b24aff1e7..6009e128cc 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs @@ -1,10 +1,9 @@ using GVFS.FunctionalTests.Should; -using GVFS.Tests; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class ResetHardTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs index 51b9ac8512..69a0d9cc18 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs @@ -1,10 +1,9 @@ using GVFS.FunctionalTests.Should; -using GVFS.Tests; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class ResetMixedTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs index 148b720f0b..fccf86de3c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs @@ -1,9 +1,8 @@ -using GVFS.Tests; -using NUnit.Framework; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class ResetSoftTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs index 81a31a3e87..3c9836443b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs @@ -1,4 +1,5 @@ using GVFS.FunctionalTests.Should; +using GVFS.FunctionalTests.Tools; using NUnit.Framework; using System.IO; @@ -18,14 +19,14 @@ public void CanReadFileAfterGitRmDryRun() this.ValidateGitCommand("status"); // Validate that Scripts\RunUnitTests.bad is not on disk at all - string fileName = Path.Combine("Scripts", "RunUnitTests.bat"); + string filePath = Path.Combine("Scripts", "RunUnitTests.bat"); this.Enlistment.UnmountGVFS(); - this.Enlistment.GetVirtualPathTo(fileName).ShouldNotExistOnDisk(this.FileSystem); + this.Enlistment.GetVirtualPathTo(filePath).ShouldNotExistOnDisk(this.FileSystem); this.Enlistment.MountGVFS(); - this.ValidateGitCommand("rm --dry-run " + fileName.Replace("\\", "/")); - this.FileContentsShouldMatch(fileName); + this.ValidateGitCommand("rm --dry-run " + GitHelpers.ConvertPathToGitFormat(filePath)); + this.FileContentsShouldMatch(filePath); } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs index 30c5d9ddad..137cd0b80f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs @@ -1,12 +1,11 @@ -using GVFS.Tests; -using GVFS.Tests.Should; +using GVFS.Tests.Should; using NUnit.Framework; using System.IO; using System.Threading; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class StatusTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index 55def3d24d..3424103259 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -1,11 +1,10 @@ using GVFS.FunctionalTests.Tools; -using GVFS.Tests; using NUnit.Framework; using System.IO; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class UpdateIndexTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs index 4aa5ced43f..04fdab6c2b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs @@ -1,9 +1,8 @@ -using GVFS.Tests; -using NUnit.Framework; +using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands { - [TestFixtureSource(typeof(DataSources), nameof(DataSources.AllBools))] + [TestFixtureSource(typeof(GitRepoTests), nameof(GitRepoTests.ValidateWorkingTree))] [Category(Categories.GitCommands)] public class UpdateRefTests : GitRepoTests { diff --git a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs index bc1d8dd54c..2ef5cee533 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs @@ -20,6 +20,9 @@ public static class GitHelpers private const string LockHolderCommandName = @"GVFS.FunctionalTests.LockHolder"; private const string LockHolderCommand = @"GVFS.FunctionalTests.LockHolder.exe"; + private const string WindowsPathSeparator = "\\"; + private const string GitPathSeparator = "/"; + private static string LockHolderCommandPath { get @@ -28,6 +31,11 @@ private static string LockHolderCommandPath } } + public static string ConvertPathToGitFormat(string relativePath) + { + return relativePath.Replace(WindowsPathSeparator, GitPathSeparator); + } + public static void CheckGitCommand(string virtualRepoRoot, string command, params string[] expectedLinesInResult) { ProcessResult result = GitProcess.InvokeProcess(virtualRepoRoot, command); From 76ee853821429bd6c27d0c910058e6b6fd075227 Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Tue, 23 Oct 2018 13:34:14 -0400 Subject: [PATCH 234/244] Telemetry: Record when a mount succesfully starts --- GVFS/GVFS.Mount/InProcessMount.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index 038d31c790..03706ba79e 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -134,14 +134,15 @@ public void Mount(EventLevel verbosity, Keywords keywords) Console.Title = "GVFS " + ProcessHelper.GetCurrentProcessVersion() + " - " + this.enlistment.EnlistmentRoot; this.tracer.RelatedEvent( - EventLevel.Critical, + EventLevel.Informational, "Mount", new EventMetadata { // Use TracingConstants.MessageKey.InfoMessage rather than TracingConstants.MessageKey.CriticalMessage // as this message should not appear as an error { TracingConstants.MessageKey.InfoMessage, "Virtual repo is ready" }, - }); + }, + Keywords.Telemetry); this.currentState = MountState.Ready; From accaacb9fb5279d3c314c5efb629bbf79d4aafe7 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Wed, 31 Oct 2018 16:20:41 -0700 Subject: [PATCH 235/244] Disable panic in fileop handler --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 442bf10120..a817cc85db 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -500,8 +500,14 @@ static int HandleFileOpOperation( goto CleanupAndReturn; } + // We used to assert if we couldn't read the attributes of a file here. + // In practice, we always found that these panics were on accesses outside of a + // virtualization root, so we'll log as opposed to panicing. bool fileFlaggedInRoot; - assert(TryGetFileIsFlaggedAsInRoot(currentVnode, context, &fileFlaggedInRoot)); + if (!TryGetFileIsFlaggedAsInRoot(currentVnode, context, &fileFlaggedInRoot)) + { + KextLog_Info("Failed to read attributes when handling FileOp operation. Path is %s", path); + } if (fileFlaggedInRoot && KAUTH_FILEOP_CLOSE_MODIFIED != closeFlags) { From 676ab687d8151b5c0739b435c953f304ae7699c8 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 1 Nov 2018 11:04:36 -0700 Subject: [PATCH 236/244] Disable Native_ProjFS_MoveFile_PartialToOutside functional test --- .../Windows/Tests/WindowsFileSystemTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs index 235a111994..a768233c0e 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs @@ -1142,6 +1142,7 @@ public void Native_ProjFS_MoveFile_VirtualToOutside() ProjFS_MoveFileTest.ProjFS_MoveFile_VirtualToOutside(Path.GetDirectoryName(this.Enlistment.RepoRoot), this.Enlistment.RepoRoot).ShouldEqual(true); } + [Ignore("Disable this test until we can surface native test errors, see #454")] [TestCase] public void Native_ProjFS_MoveFile_PartialToOutside() { From d66c5b07006f98258d44859714c3bb1adaf04c45 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Thu, 1 Nov 2018 11:07:46 -0700 Subject: [PATCH 237/244] Address PR feedback --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index a817cc85db..3330a45881 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -500,13 +500,11 @@ static int HandleFileOpOperation( goto CleanupAndReturn; } - // We used to assert if we couldn't read the attributes of a file here. - // In practice, we always found that these panics were on accesses outside of a - // virtualization root, so we'll log as opposed to panicing. bool fileFlaggedInRoot; if (!TryGetFileIsFlaggedAsInRoot(currentVnode, context, &fileFlaggedInRoot)) { KextLog_Info("Failed to read attributes when handling FileOp operation. Path is %s", path); + goto CleanupAndReturn; } if (fileFlaggedInRoot && KAUTH_FILEOP_CLOSE_MODIFIED != closeFlags) From 8a0b9101a7a9addc1a6b447baa8b3c9c26a32cbb Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 1 Nov 2018 11:52:44 -0700 Subject: [PATCH 238/244] Mac: Disable StatusTests.MoveFileIntoDotGitDirectory --- GVFS/GVFS.FunctionalTests/Categories.cs | 4 ++++ GVFS/GVFS.FunctionalTests/Program.cs | 1 + GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs | 1 + 3 files changed, 6 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests/Categories.cs b/GVFS/GVFS.FunctionalTests/Categories.cs index 3ad24ce7d8..31e19eb5cd 100644 --- a/GVFS/GVFS.FunctionalTests/Categories.cs +++ b/GVFS/GVFS.FunctionalTests/Categories.cs @@ -26,6 +26,10 @@ public static class MacTODO // Tests for GVFS features that are not required for correct git functionality public const string M4 = "M4_GVFSFeatures"; + + // Tests that have been flaky on build servers and need additional logging and\or + // investigation + public const string FlakyTest = "MacFlakyTest"; } } } diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index 0de9e79ee6..2e12e4753b 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -78,6 +78,7 @@ public static void Main(string[] args) excludeCategories.Add(Categories.MacTODO.NeedsRenameOldPath); excludeCategories.Add(Categories.MacTODO.M3); excludeCategories.Add(Categories.MacTODO.M4); + excludeCategories.Add(Categories.MacTODO.FlakyTest); excludeCategories.Add(Categories.WindowsOnly); } else diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs index 137cd0b80f..bd9b10f5bd 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs @@ -15,6 +15,7 @@ public StatusTests(bool validateWorkingTree) } [TestCase] + [Category(Categories.MacTODO.FlakyTest)] public void MoveFileIntoDotGitDirectory() { string srcPath = @"Readme.md"; From f1d1c7aeefed24fe557b7c85d8e2bc2a5642d3d0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 1 Nov 2018 13:59:02 -0600 Subject: [PATCH 239/244] Use explicit class declarations instead of an anonymous class for payload --- .../ETWTelemetryEventListener.cs | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs b/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs index 4e34e33d91..b897f00a9d 100644 --- a/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs +++ b/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs @@ -90,23 +90,12 @@ protected override void RecordMessageInternal( if (string.IsNullOrEmpty(this.ikey)) { - var payload = new - { - Json = jsonPayload, - EnlistmentId = this.enlistmentId, - MountId = this.mountId, - }; + Payload payload = new Payload(jsonPayload, this.enlistmentId, this.mountId); this.eventSource.Write(eventName, ref options, ref activityId, ref parentActivityId, ref payload); } else { - var payload = new - { - PartA_iKey = this.ikey, - Json = jsonPayload, - EnlistmentId = this.enlistmentId, - MountId = this.mountId, - }; + PayloadWithIKey payload = new PayloadWithIKey(jsonPayload, this.enlistmentId, this.mountId, this.ikey); this.eventSource.Write(eventName, ref options, ref activityId, ref parentActivityId, ref payload); } } @@ -138,5 +127,36 @@ private EventSourceOptions CreateOptions(EventLevel level, Keywords keywords, Ev return options; } + + [EventData] + private class Payload + { + public Payload(string jsonPayload, string enlistmentId, string mountId) + { + this.Json = jsonPayload; + this.EnlistmentId = enlistmentId; + this.MountId = mountId; + } + + [EventField] + public string Json { get; } + [EventField] + public string EnlistmentId { get; } + [EventField] + public string MountId { get; } + } + + [EventData] + private class PayloadWithIKey : Payload + { + public PayloadWithIKey(string jsonPayload, string enlistmentId, string mountId, string ikey) + : base(jsonPayload, enlistmentId, mountId) + { + this.PartA_iKey = ikey; + } + + [EventField] + public string PartA_iKey { get; } + } } } From 5b19140f7fd3df3de57b6ec5fe0bbd57b863d549 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Thu, 1 Nov 2018 13:43:26 -0700 Subject: [PATCH 240/244] Before putting a PAT on the keychain in PrepFunctionalTests.sh, try removing the old one first if it wasn't properly cleaned up (likely in the case of a panic) --- Scripts/Mac/PrepFunctionalTests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index 7650f5e2b2..a26cd36dcf 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -43,5 +43,6 @@ $VFS_SCRIPTDIR/InstallSharedDataQueueStallWorkaround.sh || exit 1 PATURL=$1 PAT=$2 if [[ ! -z $PAT && ! -z $PATURL ]] ; then + security delete-generic-password -s "gcm4ml:git:$PATURL" security add-generic-password -a "Personal Access Token" -s "gcm4ml:git:$PATURL" -D Credential -w $PAT || exit 1 fi From 2c4dcf8ebd0f8987d135925779e77f60a6f50d80 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 2 Nov 2018 08:32:11 -0400 Subject: [PATCH 241/244] PrefetchVerbTests: Remove commit-graph check In the EnlistmentPerFixture.PrefetchVerbTests class, we have the same PostFetchJobShouldComplete() method as we do in the version without a shared cache, except that none of our tests in the class actually trigger a commit-graph write. The commit-graph write requires a prefetch that downloads a new prefetch packfile, while the multi-pack-index is rewritten on every prefetch (in case a non-prefetch packfile was added). We don't write the commit-graph on clone because we need a full enlistment to guarantee missing object downloads. Somehow, these test succeed in the full test suite, but do not succeed when only the one class is run. This caused some pain for someone stepping through the tests in a debugger. Instead of removing the commit-graph check, instead see if the commit-graph file exists before calling `git commit-graph read`. This allows us to still check that the commit-graph file exists and is in good condition (when it exists). The multi-pack-index is guaranteed to exist after every post-fetch job. --- .../Tests/EnlistmentPerFixture/PrefetchVerbTests.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs index 1703560e40..a236a08ff2 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs @@ -140,9 +140,13 @@ private void PostFetchJobShouldComplete() ProcessResult midxResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "multi-pack-index verify --object-dir=\"" + objectDir + "\""); midxResult.ExitCode.ShouldEqual(0); - ProcessResult graphResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "commit-graph read --object-dir=\"" + objectDir + "\""); - graphResult.ExitCode.ShouldEqual(0); - graphResult.Output.ShouldContain("43475048"); // Header from commit-graph file. + // A commit graph is not always generated, but if it is, then we want to ensure it is in a good state + if (this.fileSystem.FileExists(Path.Combine(objectDir, "info", "commit-graph"))) + { + ProcessResult graphResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "commit-graph read --object-dir=\"" + objectDir + "\""); + graphResult.ExitCode.ShouldEqual(0); + graphResult.Output.ShouldContain("43475048"); // Header from commit-graph file. + } } } } From 11ab1fbd66dbaab14f60faa1126029924b44c3ef Mon Sep 17 00:00:00 2001 From: John Briggs Date: Tue, 30 Oct 2018 17:16:57 -0700 Subject: [PATCH 242/244] Add a script used by the CI to create a build drop for running functional tests on other machines --- Scripts/Mac/CI/CreateBuildDrop.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 Scripts/Mac/CI/CreateBuildDrop.sh diff --git a/Scripts/Mac/CI/CreateBuildDrop.sh b/Scripts/Mac/CI/CreateBuildDrop.sh new file mode 100755 index 0000000000..1aa95eae4b --- /dev/null +++ b/Scripts/Mac/CI/CreateBuildDrop.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +. "$(dirname ${BASH_SOURCE[0]})/../InitializeEnvironment.sh" + +CONFIGURATION=$1 +BUILDDROP_ROOT=$2 +if [ -z $BUILDDROP_ROOT ] || [ -z $CONFIGURATION ]; then + echo 'ERROR: Usage: CreateBuildDrop.sh [configuration] [build drop root directory]' + exit 1 +fi + +# Set up some paths +BUILDDROP_BUILDOUTPUT=$BUILDDROP_ROOT/BuildOutput +BUILDDROP_SRC=$BUILDDROP_ROOT/src +BUILDDROP_PROJFS=$BUILDDROP_SRC/ProjFS.Mac +BUILDDROP_KEXT=$BUILDDROP_BUILDOUTPUT/ProjFS.Mac/Native/Build/Products/$CONFIGURATION + +# Set up the build drop directory structure +rm -rf $BUILDDROP_ROOT +mkdir -p $BUILDDROP_BUILDOUTPUT +mkdir -p $BUILDDROP_SRC +mkdir -p $BUILDDROP_PROJFS +mkdir -p $BUILDDROP_KEXT + +# Copy to the build drop, retaining directory structure. +rsync -avm $VFS_OUTPUTDIR/Git $BUILDDROP_BUILDOUTPUT +rsync -avm $VFS_PUBLISHDIR $BUILDDROP_ROOT +rsync -avm $VFS_SCRIPTDIR $BUILDDROP_SRC/Scripts +rsync -avm $VFS_SRCDIR/ProjFS.Mac/Scripts $BUILDDROP_PROJFS +rsync -avm $VFS_OUTPUTDIR/ProjFS.Mac/Native/Build/Products/$CONFIGURATION/PrjFSKext.kext $BUILDDROP_KEXT From f0c3a60d9aa6215043102b5ce595197191e7ca16 Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Mon, 5 Nov 2018 10:43:21 -0500 Subject: [PATCH 243/244] TryDeleteStatusCacheFile as warning This is an expected condition that happens when status is being read to and removed at the same time. This will self heal but we want to keep a warning to know how often it happens. --- GVFS/GVFS.Common/GitStatusCache.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Common/GitStatusCache.cs b/GVFS/GVFS.Common/GitStatusCache.cs index ae7d2e18f6..e5861c4220 100644 --- a/GVFS/GVFS.Common/GitStatusCache.cs +++ b/GVFS/GVFS.Common/GitStatusCache.cs @@ -439,9 +439,10 @@ private bool TryDeleteStatusCacheFile() metadata.Add("Area", EtwArea); metadata.Add("Exception", ex.ToString()); - this.context.Tracer.RelatedError( + this.context.Tracer.RelatedWarning( metadata, - string.Format("GitStatusCache encountered exception attempting to delete cache file at {0}.", this.serializedGitStatusFilePath)); + string.Format("GitStatusCache encountered exception attempting to delete cache file at {0}.", this.serializedGitStatusFilePath), + Keywords.Telemetry); return false; } From 700a9beb17d9b8f2aec0ba10bec583272c03cc8c Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Mon, 23 May 2022 11:47:53 -0400 Subject: [PATCH 244/244] Create dd --- dd | 1 + 1 file changed, 1 insertion(+) create mode 100644 dd diff --git a/dd b/dd new file mode 100644 index 0000000000..f03f6945fb --- /dev/null +++ b/dd @@ -0,0 +1 @@ +dd