From a6f95195173810382c4572f9e1a1e4fe6f064fbe Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Fri, 27 Aug 2021 16:56:48 +0200 Subject: [PATCH 1/3] Add support for resuming file upload/download --- src/Renci.SshNet/IServiceFactory.cs | 5 +- src/Renci.SshNet/ServiceFactory.cs | 7 +- src/Renci.SshNet/Sftp/ISftpSession.cs | 3 +- src/Renci.SshNet/Sftp/SftpFileReader.cs | 5 +- src/Renci.SshNet/Sftp/SftpSession.cs | 350 +++++++++--------- src/Renci.SshNet/SftpClient.cs | 113 +++--- ...tpFileReader_EndLStatThrowsSshException.cs | 2 +- ...izeIsAlmostSixTimesGreaterThanChunkSize.cs | 2 +- ...tpFileReader_FileSizeIsEqualToChunkSize.cs | 2 +- ...eIsExactlyFiveTimesGreaterThanChunkSize.cs | 2 +- ...pFileReader_FileSizeIsLessThanChunkSize.cs | 2 +- ...leMoreThanFiveTimesGreaterThanChunkSize.cs | 2 +- ...eIsMoreThanTenTimesGreaterThanChunkSize.cs | 2 +- ...est_CreateSftpFileReader_FileSizeIsZero.cs | 2 +- 14 files changed, 260 insertions(+), 239 deletions(-) diff --git a/src/Renci.SshNet/IServiceFactory.cs b/src/Renci.SshNet/IServiceFactory.cs index 8b2c8650b..ebeb9cbe3 100644 --- a/src/Renci.SshNet/IServiceFactory.cs +++ b/src/Renci.SshNet/IServiceFactory.cs @@ -83,17 +83,18 @@ internal partial interface IServiceFactory /// No key exchange algorithm is supported by both client and server. IKeyExchange CreateKeyExchange(IDictionary clientAlgorithms, string[] serverAlgorithms); - /// + /// /// Creates an for the specified file and with the specified /// buffer size. /// /// The file to read. /// The SFTP session to use. /// The size of buffer. + /// The offset to resume from. /// /// An . /// - ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize); + ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize, ulong offset = 0); /// /// Creates a new instance. diff --git a/src/Renci.SshNet/ServiceFactory.cs b/src/Renci.SshNet/ServiceFactory.cs index c41c3679e..1eaccf83e 100644 --- a/src/Renci.SshNet/ServiceFactory.cs +++ b/src/Renci.SshNet/ServiceFactory.cs @@ -138,10 +138,11 @@ public INetConfSession CreateNetConfSession(ISession session, int operationTimeo /// The file to read. /// The SFTP session to use. /// The size of buffer. + /// The offset to resume from. /// /// An . /// - public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize) + public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize, ulong offset = 0) { const int defaultMaxPendingReads = 3; @@ -163,7 +164,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe { var fileAttributes = sftpSession.EndLStat(statAsyncResult); fileSize = fileAttributes.Size; - maxPendingReads = Math.Min(10, (int) Math.Ceiling((double) fileAttributes.Size / chunkSize) + 1); + maxPendingReads = Math.Min(10, (int) Math.Ceiling((double)(fileSize - (long)offset) / chunkSize) + 1); } catch (SshException ex) { @@ -173,7 +174,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe DiagnosticAbstraction.Log(string.Format("Failed to obtain size of file. Allowing maximum {0} pending reads: {1}", maxPendingReads, ex)); } - return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize); + return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize, offset); } /// diff --git a/src/Renci.SshNet/Sftp/ISftpSession.cs b/src/Renci.SshNet/Sftp/ISftpSession.cs index ed9b72369..f64145273 100644 --- a/src/Renci.SshNet/Sftp/ISftpSession.cs +++ b/src/Renci.SshNet/Sftp/ISftpSession.cs @@ -494,10 +494,11 @@ void RequestWrite(byte[] handle, /// The maximum number of bytes to read with each chunk. /// The maximum number of pending reads. /// The size of the file or when the size could not be determined. + /// The offset to resume from. /// /// An for reading the content of the file represented by the /// specified . /// - ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize); + ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0); } } diff --git a/src/Renci.SshNet/Sftp/SftpFileReader.cs b/src/Renci.SshNet/Sftp/SftpFileReader.cs index af32f87a7..f288e6c0a 100644 --- a/src/Renci.SshNet/Sftp/SftpFileReader.cs +++ b/src/Renci.SshNet/Sftp/SftpFileReader.cs @@ -56,12 +56,15 @@ internal sealed class SftpFileReader : ISftpFileReader /// The size of a individual read-ahead chunk. /// The maximum number of pending reads. /// The size of the file, if known; otherwise, . - public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize) + /// The offset to resume from. + public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0) { _handle = handle; _sftpSession = sftpSession; _chunkSize = chunkSize; _fileSize = fileSize; + _offset = offset; + _readAheadOffset = offset; _semaphore = new SemaphoreLight(maxPendingReads); _queue = new Dictionary(maxPendingReads); _readLock = new object(); diff --git a/src/Renci.SshNet/Sftp/SftpSession.cs b/src/Renci.SshNet/Sftp/SftpSession.cs index 0fec92230..749d32b7e 100644 --- a/src/Renci.SshNet/Sftp/SftpSession.cs +++ b/src/Renci.SshNet/Sftp/SftpSession.cs @@ -9,6 +9,9 @@ using Renci.SshNet.Sftp.Requests; using Renci.SshNet.Sftp.Responses; +#pragma warning disable SA1005 // Single line comment should begin with a space +#pragma warning disable SA1137 // Elements should have the same indentation + namespace Renci.SshNet.Sftp { /// @@ -113,7 +116,7 @@ public string GetCanonicalPath(string path) return canonizedPath; } - // Check for special cases + // Check for special cases if (fullPath.EndsWith("/.", StringComparison.OrdinalIgnoreCase) || fullPath.EndsWith("/..", StringComparison.OrdinalIgnoreCase) || fullPath.Equals("/", StringComparison.OrdinalIgnoreCase) || @@ -239,13 +242,14 @@ public async Task GetCanonicalPathAsync(string path, CancellationToken c /// The maximum number of bytes to read with each chunk. /// The maximum number of pending reads. /// The size of the file or when the size could not be determined. + /// The offset to resume from. /// /// An for reading the content of the file represented by the /// specified . /// - public ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize) + public ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0) { - return new SftpFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize); + return new SftpFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize, offset); } internal string GetFullRemotePath(string path) @@ -278,7 +282,7 @@ protected override void OnChannelOpen() throw new NotSupportedException(string.Format(CultureInfo.CurrentCulture, "Server SFTP version {0} is not supported.", ProtocolVersion)); } - // Resolve current directory + // Resolve current directory WorkingDirectory = RequestRealPath(".")[0].Key; } @@ -471,16 +475,16 @@ public byte[] RequestOpen(string path, Flags flags, bool nullOnError = false) path, _encoding, flags, - response => - { - handle = response.Handle; + response => + { + handle = response.Handle; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -551,14 +555,14 @@ public SftpOpenAsyncResult BeginOpen(string path, Flags flags, AsyncCallback cal path, _encoding, flags, - response => - { + response => + { asyncResult.SetAsCompleted(response.Handle, completedSynchronously: false); - }, - response => - { + }, + response => + { asyncResult.SetAsCompleted(GetSftpException(response), completedSynchronously: false); - }); + }); SendRequest(request); @@ -614,11 +618,11 @@ public void RequestClose(byte[] handle) var request = new SftpCloseRequest(ProtocolVersion, NextRequestId, handle, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -689,10 +693,10 @@ public SftpCloseAsyncResult BeginClose(byte[] handle, AsyncCallback callback, ob var request = new SftpCloseRequest(ProtocolVersion, NextRequestId, handle, - response => - { + response => + { asyncResult.SetAsCompleted(GetSftpException(response), completedSynchronously: false); - }); + }); SendRequest(request); return asyncResult; @@ -749,21 +753,21 @@ public SftpReadAsyncResult BeginRead(byte[] handle, ulong offset, uint length, A handle, offset, length, - response => - { + response => + { asyncResult.SetAsCompleted(response.Data, completedSynchronously: false); - }, - response => - { - if (response.StatusCode != StatusCodes.Eof) - { + }, + response => + { + if (response.StatusCode != StatusCodes.Eof) + { asyncResult.SetAsCompleted(GetSftpException(response), completedSynchronously: false); - } - else - { + } + else + { asyncResult.SetAsCompleted(Array.Empty(), completedSynchronously: false); - } - }); + } + }); SendRequest(request); return asyncResult; @@ -827,24 +831,24 @@ public byte[] RequestRead(byte[] handle, ulong offset, uint length) handle, offset, length, - response => - { - data = response.Data; + response => + { + data = response.Data; _ = wait.Set(); - }, - response => - { - if (response.StatusCode != StatusCodes.Eof) - { - exception = GetSftpException(response); - } - else - { + }, + response => + { + if (response.StatusCode != StatusCodes.Eof) + { + exception = GetSftpException(response); + } + else + { data = Array.Empty(); - } + } _ = wait.Set(); - }); + }); SendRequest(request); @@ -935,15 +939,15 @@ public void RequestWrite(byte[] handle, offset, length, response => - { + { writeCompleted?.Invoke(response); - exception = GetSftpException(response); - if (wait != null) + exception = GetSftpException(response); + if (wait != null) { _ = wait.Set(); } - }); + }); SendRequest(request); @@ -1025,16 +1029,16 @@ public SftpFileAttributes RequestLStat(string path) NextRequestId, path, _encoding, - response => - { - attributes = response.Attributes; + response => + { + attributes = response.Attributes; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1068,14 +1072,14 @@ public SFtpStatAsyncResult BeginLStat(string path, AsyncCallback callback, objec NextRequestId, path, _encoding, - response => - { + response => + { asyncResult.SetAsCompleted(response.Attributes, completedSynchronously: false); - }, - response => - { + }, + response => + { asyncResult.SetAsCompleted(GetSftpException(response), completedSynchronously: false); - }); + }); SendRequest(request); return asyncResult; @@ -1131,16 +1135,16 @@ public SftpFileAttributes RequestFStat(byte[] handle, bool nullOnError) var request = new SftpFStatRequest(ProtocolVersion, NextRequestId, handle, - response => - { - attributes = response.Attributes; + response => + { + attributes = response.Attributes; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1204,11 +1208,11 @@ public void RequestSetStat(string path, SftpFileAttributes attributes) path, _encoding, attributes, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1238,11 +1242,11 @@ public void RequestFSetStat(byte[] handle, SftpFileAttributes attributes) NextRequestId, handle, attributes, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1275,16 +1279,16 @@ public byte[] RequestOpenDir(string path, bool nullOnError = false) NextRequestId, path, _encoding, - response => - { - handle = response.Handle; + response => + { + handle = response.Handle; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1352,20 +1356,20 @@ public KeyValuePair[] RequestReadDir(byte[] handle) var request = new SftpReadDirRequest(ProtocolVersion, NextRequestId, handle, - response => - { - result = response.Files; + response => + { + result = response.Files; _ = wait.Set(); - }, - response => - { - if (response.StatusCode != StatusCodes.Eof) - { - exception = GetSftpException(response); - } + }, + response => + { + if (response.StatusCode != StatusCodes.Eof) + { + exception = GetSftpException(response); + } _ = wait.Set(); - }); + }); SendRequest(request); @@ -1439,11 +1443,11 @@ public void RequestRemove(string path) NextRequestId, path, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1512,11 +1516,11 @@ public void RequestMkDir(string path) NextRequestId, path, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1545,11 +1549,11 @@ public void RequestRmDir(string path) NextRequestId, path, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1584,16 +1588,16 @@ internal KeyValuePair[] RequestRealPath(string path, NextRequestId, path, _encoding, - response => - { - result = response.Files; + response => + { + result = response.Files; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1719,16 +1723,16 @@ public SftpFileAttributes RequestStat(string path, bool nullOnError = false) NextRequestId, path, _encoding, - response => - { - attributes = response.Attributes; + response => + { + attributes = response.Attributes; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1822,11 +1826,11 @@ public void RequestRename(string oldPath, string newPath) oldPath, newPath, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1909,16 +1913,16 @@ internal KeyValuePair[] RequestReadLink(string path, NextRequestId, path, _encoding, - response => - { - result = response.Files; + response => + { + result = response.Files; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1956,11 +1960,11 @@ public void RequestSymLink(string linkpath, string targetpath) linkpath, targetpath, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1996,11 +2000,11 @@ public void RequestPosixRename(string oldPath, string newPath) oldPath, newPath, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); if (!_supportedExtensions.ContainsKey(request.Name)) { @@ -2045,16 +2049,16 @@ public SftpFileSytemInformation RequestStatVfs(string path, bool nullOnError = f NextRequestId, path, _encoding, - response => - { - information = response.GetReply().Information; + response => + { + information = response.GetReply().Information; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); if (!_supportedExtensions.ContainsKey(request.Name)) { @@ -2139,16 +2143,16 @@ internal SftpFileSytemInformation RequestFStatVfs(byte[] handle, bool nullOnErro var request = new FStatVfsRequest(ProtocolVersion, NextRequestId, handle, - response => - { - information = response.GetReply().Information; + response => + { + information = response.GetReply().Information; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); if (!_supportedExtensions.ContainsKey(request.Name)) { @@ -2190,11 +2194,11 @@ internal void HardLink(string oldPath, string newPath) NextRequestId, oldPath, newPath, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); if (!_supportedExtensions.ContainsKey(request.Name)) { @@ -2268,7 +2272,7 @@ public uint CalculateOptimalWriteLength(uint bufferSize, byte[] handle) * WinSCP uses data length of 32739 bytes (total 32768 bytes; 32739 + 25 + 4 bytes for handle) */ - var lengthOfNonDataProtocolFields = 25u + (uint) handle.Length; + var lengthOfNonDataProtocolFields = 25u + (uint)handle.Length; var maximumPacketSize = Channel.RemotePacketSize; return Math.Min(bufferSize, maximumPacketSize) - lengthOfNonDataProtocolFields; } diff --git a/src/Renci.SshNet/SftpClient.cs b/src/Renci.SshNet/SftpClient.cs index aa30efadb..763aa6558 100644 --- a/src/Renci.SshNet/SftpClient.cs +++ b/src/Renci.SshNet/SftpClient.cs @@ -13,6 +13,9 @@ using Renci.SshNet.Common; using Renci.SshNet.Sftp; +#pragma warning disable SA1005 // Single line comment should begin with a space +#pragma warning disable SA1137 // Elements should have the same indentation + namespace Renci.SshNet { /// @@ -695,7 +698,7 @@ public IAsyncResult BeginListDirectory(string path, AsyncCallback asyncCallback, public IEnumerable EndListDirectory(IAsyncResult asyncResult) { if (asyncResult is not SftpListDirectoryAsyncResult ar || ar.EndInvokeCalled) - { + { throw new ArgumentException("Either the IAsyncResult object did not come from the corresponding async method on this type, or EndExecute was called multiple times with the same IAsyncResult."); } @@ -803,7 +806,7 @@ public bool Exists(string path) /// is or contains only whitespace characters. /// Client is not connected. /// Permission to perform the operation was denied by the remote host. -or- A SSH command was denied by the server. - /// was not found on the remote host./// + /// was not found on the remote host. /// A SSH error where is the message from the remote host. /// The method was called after the client was disposed. /// @@ -978,7 +981,11 @@ public void UploadFile(Stream input, string path, bool canOverride, Action 0) + { + flags = Flags.Write | Flags.Append; + } + else if (canOverride) { flags |= Flags.CreateNewOrOpen; } @@ -1116,7 +1123,11 @@ public IAsyncResult BeginUploadFile(Stream input, string path, bool canOverride, var flags = Flags.Write | Flags.Truncate; - if (canOverride) + if (input.Position > 0) + { + flags = Flags.Write | Flags.Append; + } + else if (canOverride) { flags |= Flags.CreateNewOrOpen; } @@ -1132,10 +1143,10 @@ public IAsyncResult BeginUploadFile(Stream input, string path, bool canOverride, try { InternalUploadFile(input, path, flags, asyncResult, offset => - { - asyncResult.Update(offset); + { + asyncResult.Update(offset); uploadCallback?.Invoke(offset); - }); + }); asyncResult.SetAsCompleted(exception: null, completedSynchronously: false); } @@ -2144,18 +2155,18 @@ public IAsyncResult BeginSynchronizeDirectories(string sourcePath, string destin var asyncResult = new SftpSynchronizeDirectoriesAsyncResult(asyncCallback, state); ThreadAbstraction.ExecuteThread(() => + { + try { - try - { - var result = InternalSynchronizeDirectories(sourcePath, destinationPath, searchPattern, asyncResult); + var result = InternalSynchronizeDirectories(sourcePath, destinationPath, searchPattern, asyncResult); asyncResult.SetAsCompleted(result, completedSynchronously: false); - } - catch (Exception exp) - { + } + catch (Exception exp) + { asyncResult.SetAsCompleted(exp, completedSynchronously: false); - } - }); + } + }); return asyncResult; } @@ -2195,61 +2206,61 @@ private List InternalSynchronizeDirectories(string sourcePath, string { if (!sourceFiles.MoveNext()) { - return uploadedFiles; + return uploadedFiles; } - #region Existing Files at The Destination + #region Existing Files at The Destination var destFiles = InternalListDirectory(destinationPath, listCallback: null); var destDict = new Dictionary(); - foreach (var destFile in destFiles) - { - if (destFile.IsDirectory) + foreach (var destFile in destFiles) + { + if (destFile.IsDirectory) { - continue; + continue; } - destDict.Add(destFile.Name, destFile); - } + destDict.Add(destFile.Name, destFile); + } - #endregion + #endregion - #region Upload the difference + #region Upload the difference - const Flags uploadFlag = Flags.Write | Flags.Truncate | Flags.CreateNewOrOpen; + const Flags uploadFlag = Flags.Write | Flags.Truncate | Flags.CreateNewOrOpen; do { var localFile = sourceFiles.Current; if (localFile is null) - { + { continue; } var isDifferent = true; if (destDict.TryGetValue(localFile.Name, out var remoteFile)) - { + { // File exists at the destination, use filesize to detect if there's a difference isDifferent = localFile.Length != remoteFile.Length; - } + } - if (isDifferent) + if (isDifferent) + { + var remoteFileName = string.Format(CultureInfo.InvariantCulture, @"{0}/{1}", destinationPath, localFile.Name); + try { - var remoteFileName = string.Format(CultureInfo.InvariantCulture, @"{0}/{1}", destinationPath, localFile.Name); - try - { #pragma warning disable CA2000 // Dispose objects before losing scope; false positive - using (var file = File.OpenRead(localFile.FullName)) + using (var file = File.OpenRead(localFile.FullName)) #pragma warning restore CA2000 // Dispose objects before losing scope; false positive - { + { InternalUploadFile(file, remoteFileName, uploadFlag, asyncResult: null, uploadCallback: null); - } + } - uploadedFiles.Add(localFile); + uploadedFiles.Add(localFile); asynchResult?.Update(uploadedFiles.Count); - } - catch (Exception ex) - { + } + catch (Exception ex) + { throw new SshException($"Failed to upload {localFile.FullName} to {remoteFileName}", ex); } } @@ -2308,16 +2319,16 @@ private List InternalListDirectory(string path, Action listCallb while (files is not null) { foreach (var f in files) - { + { result.Add(new SftpFile(_sftpSession, string.Format(CultureInfo.InvariantCulture, "{0}{1}", basePath, f.Key), f.Value)); } - // Call callback to report number of files read + // Call callback to report number of files read if (listCallback is not null) { - // Execute callback on different thread + // Execute callback on different thread ThreadAbstraction.ExecuteThread(() => listCallback(result.Count)); } @@ -2358,13 +2369,13 @@ private void InternalDownloadFile(string path, Stream output, SftpDownloadAsyncR var fullPath = _sftpSession.GetCanonicalPath(path); - using (var fileReader = ServiceFactory.CreateSftpFileReader(fullPath, _sftpSession, _bufferSize)) + using (var fileReader = ServiceFactory.CreateSftpFileReader(fullPath, _sftpSession, _bufferSize, (ulong)output.Position)) { var totalBytesRead = 0UL; while (true) { - // Cancel download + // Cancel download if (asyncResult is not null && asyncResult.IsDownloadCanceled) { break; @@ -2385,7 +2396,7 @@ private void InternalDownloadFile(string path, Stream output, SftpDownloadAsyncR // Copy offset to ensure it's not modified between now and execution of callback var downloadOffset = totalBytesRead; - // Execute callback on different thread + // Execute callback on different thread ThreadAbstraction.ExecuteThread(() => { downloadCallback(downloadOffset); }); } } @@ -2424,7 +2435,7 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo var handle = _sftpSession.RequestOpen(fullPath, flags); - ulong offset = 0; + var offset = (ulong)input.Position; // create buffer of optimal length var buffer = new byte[_sftpSession.CalculateOptimalWriteLength(_bufferSize, handle)]; @@ -2435,7 +2446,7 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo do { - // Cancel upload + // Cancel upload if (asyncResult is not null && asyncResult.IsUploadCanceled) { break; @@ -2452,10 +2463,10 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo _ = Interlocked.Decrement(ref expectedResponses); _ = responseReceivedWaitHandle.Set(); - // Call callback to report number of bytes written + // Call callback to report number of bytes written if (uploadCallback is not null) { - // Execute callback on different thread + // Execute callback on different thread ThreadAbstraction.ExecuteThread(() => uploadCallback(writtenBytes)); } } @@ -2469,7 +2480,7 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo } else if (expectedResponses > 0) { - // Wait for expectedResponses to change + // Wait for expectedResponses to change _sftpSession.WaitOnHandle(responseReceivedWaitHandle, _operationTimeout); } } diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs index ee3db7cba..4671e5112 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs @@ -59,7 +59,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Throws(new SshException()); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 3, null)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 3, null, 0)) .Returns(_sftpFileReaderMock.Object); } diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsAlmostSixTimesGreaterThanChunkSize.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsAlmostSixTimesGreaterThanChunkSize.cs index c89bfb0f2..23f66870f 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsAlmostSixTimesGreaterThanChunkSize.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsAlmostSixTimesGreaterThanChunkSize.cs @@ -63,7 +63,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Returns(_fileAttributes); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize, 0)) .Returns(_sftpFileReaderMock.Object); } diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsEqualToChunkSize.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsEqualToChunkSize.cs index dcab293cc..684f158f2 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsEqualToChunkSize.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsEqualToChunkSize.cs @@ -63,7 +63,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Returns(_fileAttributes); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize, 0)) .Returns(_sftpFileReaderMock.Object); } diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsExactlyFiveTimesGreaterThanChunkSize.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsExactlyFiveTimesGreaterThanChunkSize.cs index affac00ab..f93013336 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsExactlyFiveTimesGreaterThanChunkSize.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsExactlyFiveTimesGreaterThanChunkSize.cs @@ -63,7 +63,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Returns(_fileAttributes); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 6, _fileSize)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 6, _fileSize, 0)) .Returns(_sftpFileReaderMock.Object); } diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLessThanChunkSize.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLessThanChunkSize.cs index 2c822a9c8..96f745153 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLessThanChunkSize.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLessThanChunkSize.cs @@ -63,7 +63,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Returns(_fileAttributes); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize, 0)) .Returns(_sftpFileReaderMock.Object); } diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLittleMoreThanFiveTimesGreaterThanChunkSize.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLittleMoreThanFiveTimesGreaterThanChunkSize.cs index e1229298c..40fc7ed10 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLittleMoreThanFiveTimesGreaterThanChunkSize.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLittleMoreThanFiveTimesGreaterThanChunkSize.cs @@ -63,7 +63,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Returns(_fileAttributes); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize, 0)) .Returns(_sftpFileReaderMock.Object); } diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs index dad634472..48a5f497e 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs @@ -63,7 +63,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Returns(_fileAttributes); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, _fileSize)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, _fileSize, 0)) .Returns(_sftpFileReaderMock.Object); } diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsZero.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsZero.cs index c34b9cccf..df13e8ff0 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsZero.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsZero.cs @@ -63,7 +63,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Returns(_fileAttributes); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 1, _fileSize)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 1, _fileSize, 0)) .Returns(_sftpFileReaderMock.Object); } From f9a3b018b874193f7be75e27edbcda8ae720fe54 Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Wed, 1 Nov 2023 19:26:10 +0100 Subject: [PATCH 2/3] Fix rebase issues (whitespace) --- src/Renci.SshNet/IServiceFactory.cs | 2 +- src/Renci.SshNet/Sftp/SftpSession.cs | 345 +++++++++++++-------------- src/Renci.SshNet/SftpClient.cs | 99 ++++---- 3 files changed, 220 insertions(+), 226 deletions(-) diff --git a/src/Renci.SshNet/IServiceFactory.cs b/src/Renci.SshNet/IServiceFactory.cs index ebeb9cbe3..b03cc4f13 100644 --- a/src/Renci.SshNet/IServiceFactory.cs +++ b/src/Renci.SshNet/IServiceFactory.cs @@ -83,7 +83,7 @@ internal partial interface IServiceFactory /// No key exchange algorithm is supported by both client and server. IKeyExchange CreateKeyExchange(IDictionary clientAlgorithms, string[] serverAlgorithms); - /// + /// /// Creates an for the specified file and with the specified /// buffer size. /// diff --git a/src/Renci.SshNet/Sftp/SftpSession.cs b/src/Renci.SshNet/Sftp/SftpSession.cs index 749d32b7e..604362532 100644 --- a/src/Renci.SshNet/Sftp/SftpSession.cs +++ b/src/Renci.SshNet/Sftp/SftpSession.cs @@ -9,9 +9,6 @@ using Renci.SshNet.Sftp.Requests; using Renci.SshNet.Sftp.Responses; -#pragma warning disable SA1005 // Single line comment should begin with a space -#pragma warning disable SA1137 // Elements should have the same indentation - namespace Renci.SshNet.Sftp { /// @@ -116,7 +113,7 @@ public string GetCanonicalPath(string path) return canonizedPath; } - // Check for special cases + // Check for special cases if (fullPath.EndsWith("/.", StringComparison.OrdinalIgnoreCase) || fullPath.EndsWith("/..", StringComparison.OrdinalIgnoreCase) || fullPath.Equals("/", StringComparison.OrdinalIgnoreCase) || @@ -282,7 +279,7 @@ protected override void OnChannelOpen() throw new NotSupportedException(string.Format(CultureInfo.CurrentCulture, "Server SFTP version {0} is not supported.", ProtocolVersion)); } - // Resolve current directory + // Resolve current directory WorkingDirectory = RequestRealPath(".")[0].Key; } @@ -475,16 +472,16 @@ public byte[] RequestOpen(string path, Flags flags, bool nullOnError = false) path, _encoding, flags, - response => - { - handle = response.Handle; + response => + { + handle = response.Handle; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -555,14 +552,14 @@ public SftpOpenAsyncResult BeginOpen(string path, Flags flags, AsyncCallback cal path, _encoding, flags, - response => - { + response => + { asyncResult.SetAsCompleted(response.Handle, completedSynchronously: false); - }, - response => - { + }, + response => + { asyncResult.SetAsCompleted(GetSftpException(response), completedSynchronously: false); - }); + }); SendRequest(request); @@ -618,11 +615,11 @@ public void RequestClose(byte[] handle) var request = new SftpCloseRequest(ProtocolVersion, NextRequestId, handle, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -693,10 +690,10 @@ public SftpCloseAsyncResult BeginClose(byte[] handle, AsyncCallback callback, ob var request = new SftpCloseRequest(ProtocolVersion, NextRequestId, handle, - response => - { + response => + { asyncResult.SetAsCompleted(GetSftpException(response), completedSynchronously: false); - }); + }); SendRequest(request); return asyncResult; @@ -753,21 +750,21 @@ public SftpReadAsyncResult BeginRead(byte[] handle, ulong offset, uint length, A handle, offset, length, - response => - { + response => + { asyncResult.SetAsCompleted(response.Data, completedSynchronously: false); - }, - response => - { - if (response.StatusCode != StatusCodes.Eof) - { + }, + response => + { + if (response.StatusCode != StatusCodes.Eof) + { asyncResult.SetAsCompleted(GetSftpException(response), completedSynchronously: false); - } - else - { + } + else + { asyncResult.SetAsCompleted(Array.Empty(), completedSynchronously: false); - } - }); + } + }); SendRequest(request); return asyncResult; @@ -831,24 +828,24 @@ public byte[] RequestRead(byte[] handle, ulong offset, uint length) handle, offset, length, - response => - { - data = response.Data; + response => + { + data = response.Data; _ = wait.Set(); - }, - response => - { - if (response.StatusCode != StatusCodes.Eof) - { - exception = GetSftpException(response); - } - else - { + }, + response => + { + if (response.StatusCode != StatusCodes.Eof) + { + exception = GetSftpException(response); + } + else + { data = Array.Empty(); - } + } _ = wait.Set(); - }); + }); SendRequest(request); @@ -939,15 +936,15 @@ public void RequestWrite(byte[] handle, offset, length, response => - { + { writeCompleted?.Invoke(response); - exception = GetSftpException(response); - if (wait != null) + exception = GetSftpException(response); + if (wait != null) { _ = wait.Set(); } - }); + }); SendRequest(request); @@ -1029,16 +1026,16 @@ public SftpFileAttributes RequestLStat(string path) NextRequestId, path, _encoding, - response => - { - attributes = response.Attributes; + response => + { + attributes = response.Attributes; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1072,14 +1069,14 @@ public SFtpStatAsyncResult BeginLStat(string path, AsyncCallback callback, objec NextRequestId, path, _encoding, - response => - { + response => + { asyncResult.SetAsCompleted(response.Attributes, completedSynchronously: false); - }, - response => - { + }, + response => + { asyncResult.SetAsCompleted(GetSftpException(response), completedSynchronously: false); - }); + }); SendRequest(request); return asyncResult; @@ -1135,16 +1132,16 @@ public SftpFileAttributes RequestFStat(byte[] handle, bool nullOnError) var request = new SftpFStatRequest(ProtocolVersion, NextRequestId, handle, - response => - { - attributes = response.Attributes; + response => + { + attributes = response.Attributes; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1208,11 +1205,11 @@ public void RequestSetStat(string path, SftpFileAttributes attributes) path, _encoding, attributes, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1242,11 +1239,11 @@ public void RequestFSetStat(byte[] handle, SftpFileAttributes attributes) NextRequestId, handle, attributes, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1279,16 +1276,16 @@ public byte[] RequestOpenDir(string path, bool nullOnError = false) NextRequestId, path, _encoding, - response => - { - handle = response.Handle; + response => + { + handle = response.Handle; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1356,20 +1353,20 @@ public KeyValuePair[] RequestReadDir(byte[] handle) var request = new SftpReadDirRequest(ProtocolVersion, NextRequestId, handle, - response => - { - result = response.Files; + response => + { + result = response.Files; _ = wait.Set(); - }, - response => - { - if (response.StatusCode != StatusCodes.Eof) - { - exception = GetSftpException(response); - } + }, + response => + { + if (response.StatusCode != StatusCodes.Eof) + { + exception = GetSftpException(response); + } _ = wait.Set(); - }); + }); SendRequest(request); @@ -1443,11 +1440,11 @@ public void RequestRemove(string path) NextRequestId, path, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1516,11 +1513,11 @@ public void RequestMkDir(string path) NextRequestId, path, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1549,11 +1546,11 @@ public void RequestRmDir(string path) NextRequestId, path, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1588,16 +1585,16 @@ internal KeyValuePair[] RequestRealPath(string path, NextRequestId, path, _encoding, - response => - { - result = response.Files; + response => + { + result = response.Files; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1723,16 +1720,16 @@ public SftpFileAttributes RequestStat(string path, bool nullOnError = false) NextRequestId, path, _encoding, - response => - { - attributes = response.Attributes; + response => + { + attributes = response.Attributes; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1826,11 +1823,11 @@ public void RequestRename(string oldPath, string newPath) oldPath, newPath, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1913,16 +1910,16 @@ internal KeyValuePair[] RequestReadLink(string path, NextRequestId, path, _encoding, - response => - { - result = response.Files; + response => + { + result = response.Files; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -1960,11 +1957,11 @@ public void RequestSymLink(string linkpath, string targetpath) linkpath, targetpath, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); SendRequest(request); @@ -2000,11 +1997,11 @@ public void RequestPosixRename(string oldPath, string newPath) oldPath, newPath, _encoding, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); if (!_supportedExtensions.ContainsKey(request.Name)) { @@ -2049,16 +2046,16 @@ public SftpFileSytemInformation RequestStatVfs(string path, bool nullOnError = f NextRequestId, path, _encoding, - response => - { - information = response.GetReply().Information; + response => + { + information = response.GetReply().Information; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); if (!_supportedExtensions.ContainsKey(request.Name)) { @@ -2143,16 +2140,16 @@ internal SftpFileSytemInformation RequestFStatVfs(byte[] handle, bool nullOnErro var request = new FStatVfsRequest(ProtocolVersion, NextRequestId, handle, - response => - { - information = response.GetReply().Information; + response => + { + information = response.GetReply().Information; _ = wait.Set(); - }, - response => - { - exception = GetSftpException(response); + }, + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); if (!_supportedExtensions.ContainsKey(request.Name)) { @@ -2194,11 +2191,11 @@ internal void HardLink(string oldPath, string newPath) NextRequestId, oldPath, newPath, - response => - { - exception = GetSftpException(response); + response => + { + exception = GetSftpException(response); _ = wait.Set(); - }); + }); if (!_supportedExtensions.ContainsKey(request.Name)) { @@ -2272,7 +2269,7 @@ public uint CalculateOptimalWriteLength(uint bufferSize, byte[] handle) * WinSCP uses data length of 32739 bytes (total 32768 bytes; 32739 + 25 + 4 bytes for handle) */ - var lengthOfNonDataProtocolFields = 25u + (uint)handle.Length; + var lengthOfNonDataProtocolFields = 25u + (uint) handle.Length; var maximumPacketSize = Channel.RemotePacketSize; return Math.Min(bufferSize, maximumPacketSize) - lengthOfNonDataProtocolFields; } diff --git a/src/Renci.SshNet/SftpClient.cs b/src/Renci.SshNet/SftpClient.cs index 763aa6558..01134b854 100644 --- a/src/Renci.SshNet/SftpClient.cs +++ b/src/Renci.SshNet/SftpClient.cs @@ -13,9 +13,6 @@ using Renci.SshNet.Common; using Renci.SshNet.Sftp; -#pragma warning disable SA1005 // Single line comment should begin with a space -#pragma warning disable SA1137 // Elements should have the same indentation - namespace Renci.SshNet { /// @@ -698,7 +695,7 @@ public IAsyncResult BeginListDirectory(string path, AsyncCallback asyncCallback, public IEnumerable EndListDirectory(IAsyncResult asyncResult) { if (asyncResult is not SftpListDirectoryAsyncResult ar || ar.EndInvokeCalled) - { + { throw new ArgumentException("Either the IAsyncResult object did not come from the corresponding async method on this type, or EndExecute was called multiple times with the same IAsyncResult."); } @@ -806,7 +803,7 @@ public bool Exists(string path) /// is or contains only whitespace characters. /// Client is not connected. /// Permission to perform the operation was denied by the remote host. -or- A SSH command was denied by the server. - /// was not found on the remote host. + /// was not found on the remote host./// /// A SSH error where is the message from the remote host. /// The method was called after the client was disposed. /// @@ -1143,10 +1140,10 @@ public IAsyncResult BeginUploadFile(Stream input, string path, bool canOverride, try { InternalUploadFile(input, path, flags, asyncResult, offset => - { - asyncResult.Update(offset); + { + asyncResult.Update(offset); uploadCallback?.Invoke(offset); - }); + }); asyncResult.SetAsCompleted(exception: null, completedSynchronously: false); } @@ -2155,18 +2152,18 @@ public IAsyncResult BeginSynchronizeDirectories(string sourcePath, string destin var asyncResult = new SftpSynchronizeDirectoriesAsyncResult(asyncCallback, state); ThreadAbstraction.ExecuteThread(() => - { - try { - var result = InternalSynchronizeDirectories(sourcePath, destinationPath, searchPattern, asyncResult); + try + { + var result = InternalSynchronizeDirectories(sourcePath, destinationPath, searchPattern, asyncResult); asyncResult.SetAsCompleted(result, completedSynchronously: false); - } - catch (Exception exp) - { + } + catch (Exception exp) + { asyncResult.SetAsCompleted(exp, completedSynchronously: false); - } - }); + } + }); return asyncResult; } @@ -2206,61 +2203,61 @@ private List InternalSynchronizeDirectories(string sourcePath, string { if (!sourceFiles.MoveNext()) { - return uploadedFiles; + return uploadedFiles; } - #region Existing Files at The Destination + #region Existing Files at The Destination var destFiles = InternalListDirectory(destinationPath, listCallback: null); var destDict = new Dictionary(); - foreach (var destFile in destFiles) - { - if (destFile.IsDirectory) + foreach (var destFile in destFiles) + { + if (destFile.IsDirectory) { - continue; + continue; } - destDict.Add(destFile.Name, destFile); - } + destDict.Add(destFile.Name, destFile); + } - #endregion + #endregion - #region Upload the difference + #region Upload the difference - const Flags uploadFlag = Flags.Write | Flags.Truncate | Flags.CreateNewOrOpen; + const Flags uploadFlag = Flags.Write | Flags.Truncate | Flags.CreateNewOrOpen; do { var localFile = sourceFiles.Current; if (localFile is null) - { + { continue; } var isDifferent = true; if (destDict.TryGetValue(localFile.Name, out var remoteFile)) - { + { // File exists at the destination, use filesize to detect if there's a difference isDifferent = localFile.Length != remoteFile.Length; - } + } - if (isDifferent) - { - var remoteFileName = string.Format(CultureInfo.InvariantCulture, @"{0}/{1}", destinationPath, localFile.Name); - try + if (isDifferent) { + var remoteFileName = string.Format(CultureInfo.InvariantCulture, @"{0}/{1}", destinationPath, localFile.Name); + try + { #pragma warning disable CA2000 // Dispose objects before losing scope; false positive - using (var file = File.OpenRead(localFile.FullName)) + using (var file = File.OpenRead(localFile.FullName)) #pragma warning restore CA2000 // Dispose objects before losing scope; false positive - { + { InternalUploadFile(file, remoteFileName, uploadFlag, asyncResult: null, uploadCallback: null); - } + } - uploadedFiles.Add(localFile); + uploadedFiles.Add(localFile); asynchResult?.Update(uploadedFiles.Count); - } - catch (Exception ex) - { + } + catch (Exception ex) + { throw new SshException($"Failed to upload {localFile.FullName} to {remoteFileName}", ex); } } @@ -2319,16 +2316,16 @@ private List InternalListDirectory(string path, Action listCallb while (files is not null) { foreach (var f in files) - { + { result.Add(new SftpFile(_sftpSession, string.Format(CultureInfo.InvariantCulture, "{0}{1}", basePath, f.Key), f.Value)); } - // Call callback to report number of files read + // Call callback to report number of files read if (listCallback is not null) { - // Execute callback on different thread + // Execute callback on different thread ThreadAbstraction.ExecuteThread(() => listCallback(result.Count)); } @@ -2369,13 +2366,13 @@ private void InternalDownloadFile(string path, Stream output, SftpDownloadAsyncR var fullPath = _sftpSession.GetCanonicalPath(path); - using (var fileReader = ServiceFactory.CreateSftpFileReader(fullPath, _sftpSession, _bufferSize, (ulong)output.Position)) + using (var fileReader = ServiceFactory.CreateSftpFileReader(fullPath, _sftpSession, _bufferSize)) { var totalBytesRead = 0UL; while (true) { - // Cancel download + // Cancel download if (asyncResult is not null && asyncResult.IsDownloadCanceled) { break; @@ -2396,7 +2393,7 @@ private void InternalDownloadFile(string path, Stream output, SftpDownloadAsyncR // Copy offset to ensure it's not modified between now and execution of callback var downloadOffset = totalBytesRead; - // Execute callback on different thread + // Execute callback on different thread ThreadAbstraction.ExecuteThread(() => { downloadCallback(downloadOffset); }); } } @@ -2446,7 +2443,7 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo do { - // Cancel upload + // Cancel upload if (asyncResult is not null && asyncResult.IsUploadCanceled) { break; @@ -2463,10 +2460,10 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo _ = Interlocked.Decrement(ref expectedResponses); _ = responseReceivedWaitHandle.Set(); - // Call callback to report number of bytes written + // Call callback to report number of bytes written if (uploadCallback is not null) { - // Execute callback on different thread + // Execute callback on different thread ThreadAbstraction.ExecuteThread(() => uploadCallback(writtenBytes)); } } @@ -2480,7 +2477,7 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo } else if (expectedResponses > 0) { - // Wait for expectedResponses to change + // Wait for expectedResponses to change _sftpSession.WaitOnHandle(responseReceivedWaitHandle, _operationTimeout); } } From b739243550f40615f0653228e0740a8e6873d070 Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Wed, 1 Nov 2023 20:07:16 +0100 Subject: [PATCH 3/3] add comments --- src/Renci.SshNet/ServiceFactory.cs | 2 ++ src/Renci.SshNet/Sftp/SftpFileReader.cs | 7 +++++-- src/Renci.SshNet/SftpClient.cs | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Renci.SshNet/ServiceFactory.cs b/src/Renci.SshNet/ServiceFactory.cs index 1eaccf83e..c3697650a 100644 --- a/src/Renci.SshNet/ServiceFactory.cs +++ b/src/Renci.SshNet/ServiceFactory.cs @@ -164,6 +164,8 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe { var fileAttributes = sftpSession.EndLStat(statAsyncResult); fileSize = fileAttributes.Size; + + // calculate maxPendingReads based on remaining size, not total filesize (needed for resume support) maxPendingReads = Math.Min(10, (int) Math.Ceiling((double)(fileSize - (long)offset) / chunkSize) + 1); } catch (SshException ex) diff --git a/src/Renci.SshNet/Sftp/SftpFileReader.cs b/src/Renci.SshNet/Sftp/SftpFileReader.cs index f288e6c0a..2c535613e 100644 --- a/src/Renci.SshNet/Sftp/SftpFileReader.cs +++ b/src/Renci.SshNet/Sftp/SftpFileReader.cs @@ -63,8 +63,6 @@ public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, i _sftpSession = sftpSession; _chunkSize = chunkSize; _fileSize = fileSize; - _offset = offset; - _readAheadOffset = offset; _semaphore = new SemaphoreLight(maxPendingReads); _queue = new Dictionary(maxPendingReads); _readLock = new object(); @@ -72,6 +70,11 @@ public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, i _disposingWaitHandle = new ManualResetEvent(initialState: false); _waitHandles = _sftpSession.CreateWaitHandleArray(_disposingWaitHandle, _semaphore.AvailableWaitHandle); + // When resuming a download (offset > 0), set the initial offset of the remote file to + // the same offset as the local output file. Read-ahead also starts at the same offset. + _offset = offset; + _readAheadOffset = offset; + StartReadAhead(); } diff --git a/src/Renci.SshNet/SftpClient.cs b/src/Renci.SshNet/SftpClient.cs index 01134b854..887b07fde 100644 --- a/src/Renci.SshNet/SftpClient.cs +++ b/src/Renci.SshNet/SftpClient.cs @@ -980,6 +980,7 @@ public void UploadFile(Stream input, string path, bool canOverride, Action 0) { + // if the local stream position is not zero, open the remote file in APPEND mode to resume upload flags = Flags.Write | Flags.Append; } else if (canOverride) @@ -1122,6 +1123,7 @@ public IAsyncResult BeginUploadFile(Stream input, string path, bool canOverride, if (input.Position > 0) { + // if the local stream position is not zero, open the remote file in APPEND mode to resume upload flags = Flags.Write | Flags.Append; } else if (canOverride) @@ -2432,6 +2434,7 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo var handle = _sftpSession.RequestOpen(fullPath, flags); + // Set the initial offset of the remote file to the same as the local file to allow resuming var offset = (ulong)input.Position; // create buffer of optimal length