From 9cfe6e7e20ee6203492af92423d570990d18ebd7 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 21 May 2016 01:00:23 -0400 Subject: [PATCH 01/29] build number fix --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 7265b38f3..1bb0b5fd2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,7 +7,7 @@ # - Section names should be unique on each level. # version format -version: 2.2{build} +version: 2.2.{build} shallow_clone: true From 2b1ad86fc43a67e9e79c4cb81a472dd8c5d199bb Mon Sep 17 00:00:00 2001 From: titanium007 Date: Mon, 23 May 2016 18:14:27 -0400 Subject: [PATCH 02/29] Use Version instead of string object --- .../EventArguments/SessionEventArgs.cs | 6 +- Titanium.Web.Proxy/Http/HttpWebClient.cs | 14 ++++- Titanium.Web.Proxy/Http/Request.cs | 2 +- Titanium.Web.Proxy/Http/Response.cs | 3 +- .../Network/TcpConnectionManager.cs | 47 ++++++++------ Titanium.Web.Proxy/RequestHandler.cs | 63 +++++++++++-------- Titanium.Web.Proxy/ResponseHandler.cs | 7 ++- 7 files changed, 88 insertions(+), 54 deletions(-) diff --git a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs index af59ed70a..4b533128f 100644 --- a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs @@ -36,7 +36,7 @@ internal SessionEventArgs() /// /// Does this session uses SSL /// - public bool IsHttps { get; internal set; } + public bool IsHttps => WebSession.Request.RequestUri.Scheme == Uri.UriSchemeHttps; /// /// A web session corresponding to a single request/response sequence @@ -87,7 +87,7 @@ private async Task ReadRequestBody() await this.Client.ClientStreamReader.CopyBytesToStream(requestBodyStream, WebSession.Request.ContentLength).ConfigureAwait(false); } - else if (WebSession.Request.HttpVersion.ToLower().Trim().Equals("http/1.0")) + else if(WebSession.Response.HttpVersion.Major == 1 && WebSession.Response.HttpVersion.Minor == 0) await WebSession.ServerConnection.StreamReader.CopyBytesToStream(requestBodyStream, long.MaxValue).ConfigureAwait(false); } WebSession.Request.RequestBody = await GetDecompressedResponseBody(WebSession.Request.ContentEncoding, requestBodyStream.ToArray()).ConfigureAwait(false); @@ -123,7 +123,7 @@ private async Task ReadResponseBody() await WebSession.ServerConnection.StreamReader.CopyBytesToStream(responseBodyStream, WebSession.Response.ContentLength).ConfigureAwait(false); } - else if(WebSession.Response.HttpVersion.ToLower().Trim().Equals("http/1.0")) + else if(WebSession.Response.HttpVersion.Major == 1 && WebSession.Response.HttpVersion.Minor == 0) await WebSession.ServerConnection.StreamReader.CopyBytesToStream(responseBodyStream, long.MaxValue).ConfigureAwait(false); } diff --git a/Titanium.Web.Proxy/Http/HttpWebClient.cs b/Titanium.Web.Proxy/Http/HttpWebClient.cs index 11c681467..d1f04cb3b 100644 --- a/Titanium.Web.Proxy/Http/HttpWebClient.cs +++ b/Titanium.Web.Proxy/Http/HttpWebClient.cs @@ -46,7 +46,7 @@ internal async Task SendRequest() { this.Request.Method, this.Request.RequestUri.PathAndQuery, - this.Request.HttpVersion + string.Format("HTTP/{0}.{1}",this.Request.HttpVersion.Major, this.Request.HttpVersion.Minor) })); foreach (HttpHeader httpHeader in this.Request.RequestHeaders) @@ -95,8 +95,18 @@ internal async Task ReceiveResponse() { await ServerConnection.StreamReader.ReadLineAsync().ConfigureAwait(false); } + var httpVersion = httpResult[0].Trim().ToLower(); + Version version; + if (httpVersion == "http/1.1") + { + version = new Version(1, 1); + } + else + { + version = new Version(1, 0); + } - this.Response.HttpVersion = httpResult[0].Trim(); + this.Response.HttpVersion = version; this.Response.ResponseStatusCode = httpResult[1].Trim(); this.Response.ResponseStatusDescription = httpResult[2].Trim(); diff --git a/Titanium.Web.Proxy/Http/Request.cs b/Titanium.Web.Proxy/Http/Request.cs index 7953b8664..4bfecffb1 100644 --- a/Titanium.Web.Proxy/Http/Request.cs +++ b/Titanium.Web.Proxy/Http/Request.cs @@ -11,7 +11,7 @@ public class Request { public string Method { get; set; } public Uri RequestUri { get; set; } - public string HttpVersion { get; set; } + public Version HttpVersion { get; set; } internal string Host { diff --git a/Titanium.Web.Proxy/Http/Response.cs b/Titanium.Web.Proxy/Http/Response.cs index 92347fdea..d08c8ec9c 100644 --- a/Titanium.Web.Proxy/Http/Response.cs +++ b/Titanium.Web.Proxy/Http/Response.cs @@ -4,6 +4,7 @@ using System.Text; using Titanium.Web.Proxy.Models; using Titanium.Web.Proxy.Extensions; +using System; namespace Titanium.Web.Proxy.Http { @@ -31,7 +32,7 @@ internal string ContentEncoding } } - internal string HttpVersion { get; set; } + internal Version HttpVersion { get; set; } internal bool ResponseKeepAlive { get diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index ab06b5148..f206fae46 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -37,21 +37,31 @@ internal TcpConnection() internal class TcpConnectionManager { - static List ConnectionCache = new List(); - + static List connectionCache = new List(); + static SemaphoreSlim connectionAccessLock = new SemaphoreSlim(1); internal static async Task GetClient(SessionEventArgs sessionArgs, string hostname, int port, bool isSecure, Version version) { TcpConnection cached = null; while (true) { - lock (ConnectionCache) + await connectionAccessLock.WaitAsync(); + try { - cached = ConnectionCache.FirstOrDefault(x => x.HostName == hostname && x.port == port && + cached = connectionCache.FirstOrDefault(x => x.HostName == hostname && x.port == port && x.IsSecure == isSecure && x.TcpClient.Connected && x.Version.Equals(version)); + //just create one more preemptively + if (connectionCache.Where(x => x.HostName == hostname && x.port == port && + x.IsSecure == isSecure && x.TcpClient.Connected && x.Version.Equals(version)).Count() < 2) + { + var task = CreateClient(sessionArgs, hostname, port, isSecure, version) + .ContinueWith(async (x) => { if (x.Status == TaskStatus.RanToCompletion) await ReleaseClient(x.Result); }); + } + if (cached != null) - ConnectionCache.Remove(cached); + connectionCache.Remove(cached); } + finally { connectionAccessLock.Release(); } if (cached != null && !cached.TcpClient.Client.IsConnected()) continue; @@ -62,15 +72,7 @@ internal static async Task GetClient(SessionEventArgs sessionArgs if (cached == null) cached = await CreateClient(sessionArgs, hostname, port, isSecure, version).ConfigureAwait(false); - - //just create one more preemptively - if (ConnectionCache.Where(x => x.HostName == hostname && x.port == port && - x.IsSecure == isSecure && x.TcpClient.Connected && x.Version.Equals(version)).Count() < 2) - { - var task = CreateClient(sessionArgs, hostname, port, isSecure, version) - .ContinueWith(x => { if (x.Status == TaskStatus.RanToCompletion) ReleaseClient(x.Result); }); - } - + return cached; } @@ -155,27 +157,34 @@ private static async Task CreateClient(SessionEventArgs sessionAr } - internal static void ReleaseClient(TcpConnection Connection) + internal static async Task ReleaseClient(TcpConnection Connection) { Connection.LastAccess = DateTime.Now; - ConnectionCache.Add(Connection); + await connectionAccessLock.WaitAsync(); + try + { + connectionCache.Add(Connection); + } + finally { connectionAccessLock.Release(); } } internal async static void ClearIdleConnections() { while (true) { - lock (ConnectionCache) + await connectionAccessLock.WaitAsync(); + try { var cutOff = DateTime.Now.AddSeconds(-60); - ConnectionCache + connectionCache .Where(x => x.LastAccess < cutOff) .ToList() .ForEach(x => x.TcpClient.Close()); - ConnectionCache.RemoveAll(x => x.LastAccess < cutOff); + connectionCache.RemoveAll(x => x.LastAccess < cutOff); } + finally { connectionAccessLock.Release(); } await Task.Delay(1000 * 60 * 3).ConfigureAwait(false); } diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 7f5e9555b..a27b87b67 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -52,10 +52,21 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient else httpRemoteUri = new Uri(httpCmdSplit[1]); - string httpVersion = "HTTP/1.1"; - + Version version = new Version(1, 1); if (httpCmdSplit.Length == 3) - httpVersion = httpCmdSplit[2]; + { + string httpVersion = httpCmdSplit[1].Trim(); + + if (httpVersion == "http/1.1") + { + version = new Version(1, 1); + } + else + { + version = new Version(1, 0); + } + + } var excluded = endPoint.ExcludedHttpsHostNameRegex != null ? endPoint.ExcludedHttpsHostNameRegex.Any(x => Regex.IsMatch(httpRemoteUri.Host, x)) : false; @@ -65,7 +76,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient httpRemoteUri = new Uri("https://" + httpCmdSplit[1]); await clientStreamReader.ReadAllLinesAsync().ConfigureAwait(false); - await WriteConnectResponse(clientStreamWriter, httpVersion).ConfigureAwait(false); + await WriteConnectResponse(clientStreamWriter, version).ConfigureAwait(false); var certificate = await CertManager.CreateCertificate(httpRemoteUri.Host); @@ -101,7 +112,7 @@ await sslStream.AuthenticateAsServerAsync(certificate, false, else if (httpVerb.ToUpper() == "CONNECT") { await clientStreamReader.ReadAllLinesAsync().ConfigureAwait(false); - await WriteConnectResponse(clientStreamWriter, httpVersion).ConfigureAwait(false); + await WriteConnectResponse(clientStreamWriter, version).ConfigureAwait(false); await TcpHelper.SendRaw(clientStream, null, null, httpRemoteUri.Host, httpRemoteUri.Port, false).ConfigureAwait(false); @@ -178,7 +189,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http CustomBinaryReader clientStreamReader, StreamWriter clientStreamWriter, bool IsHttps) { TcpConnection connection = null; - string lastRequestHostName = null; + string lastRequest = null; while (true) { @@ -197,10 +208,10 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http var httpCmdSplit = httpCmd.Split(Constants.SpaceSplit, 3); var httpMethod = httpCmdSplit[0]; - var httpVersion = httpCmdSplit[2]; + var httpVersion = httpCmdSplit[2].ToLower().Trim(); Version version; - if (httpVersion == "HTTP/1.1") + if (httpVersion == "http/1.1") { version = new Version(1, 1); } @@ -219,26 +230,18 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http } var httpRemoteUri = new Uri(!IsHttps ? httpCmdSplit[1] : (string.Concat("https://", args.WebSession.Request.Host, httpCmdSplit[1]))); - args.IsHttps = IsHttps; args.WebSession.Request.RequestUri = httpRemoteUri; args.WebSession.Request.Method = httpMethod; - args.WebSession.Request.HttpVersion = httpVersion; + args.WebSession.Request.HttpVersion = version; args.Client.ClientStream = clientStream; args.Client.ClientStreamReader = clientStreamReader; args.Client.ClientStreamWriter = clientStreamWriter; - if (args.WebSession.Request.UpgradeToWebSocket) - { - await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHeaders, - httpRemoteUri.Host, httpRemoteUri.Port, args.IsHttps).ConfigureAwait(false); - Dispose(client, clientStream, clientStreamReader, clientStreamWriter, args); - return; - } - + PrepareRequestHeaders(args.WebSession.Request.RequestHeaders, args.WebSession); - args.WebSession.Request.Host = args.WebSession.Request.RequestUri.Host; + args.WebSession.Request.Host = args.WebSession.Request.RequestUri.Authority; //If requested interception if (BeforeRequest != null) @@ -251,16 +254,26 @@ await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHe handlerTasks[i] = ((Func)invocationList[i])(null, args); } - await Task.WhenAll(handlerTasks).ConfigureAwait(false); + await Task.WhenAll(handlerTasks).ConfigureAwait(false); + } + + if (args.WebSession.Request.UpgradeToWebSocket) + { + await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHeaders, + httpRemoteUri.Host, httpRemoteUri.Port, args.IsHttps).ConfigureAwait(false); + Dispose(client, clientStream, clientStreamReader, clientStreamWriter, args); + return; } //construct the web request that we are going to issue on behalf of the client. connection = connection == null ? await TcpConnectionManager.GetClient(args, args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) - : lastRequestHostName != args.WebSession.Request.RequestUri.Host ? await TcpConnectionManager.GetClient(args, args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) + : lastRequest != args.WebSession.Request.RequestUri.Host ? await TcpConnectionManager.GetClient(args, args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) : connection; - lastRequestHostName = args.WebSession.Request.RequestUri.Host; + lastRequest = string.Concat(args.WebSession.Request.RequestUri.Host,":", + args.WebSession.Request.RequestUri.Port,":", + args.IsHttps,":", version.ToString()); args.WebSession.Request.RequestLocked = true; @@ -347,12 +360,12 @@ await TcpConnectionManager.GetClient(args, args.WebSession.Request.RequestUri.Ho } if (connection != null) - TcpConnectionManager.ReleaseClient(connection); + await TcpConnectionManager.ReleaseClient(connection); } - private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, string httpVersion) + private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, Version httpVersion) { - await clientStreamWriter.WriteLineAsync(httpVersion + " 200 Connection established").ConfigureAwait(false); + await clientStreamWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2}", httpVersion.Major, httpVersion.Minor,"200 Connection established")).ConfigureAwait(false); await clientStreamWriter.WriteLineAsync(string.Format("Timestamp: {0}", DateTime.Now)).ConfigureAwait(false); await clientStreamWriter.WriteLineAsync().ConfigureAwait(false); await clientStreamWriter.FlushAsync().ConfigureAwait(false); diff --git a/Titanium.Web.Proxy/ResponseHandler.cs b/Titanium.Web.Proxy/ResponseHandler.cs index 4577feef3..3cc4626c4 100644 --- a/Titanium.Web.Proxy/ResponseHandler.cs +++ b/Titanium.Web.Proxy/ResponseHandler.cs @@ -80,7 +80,8 @@ public static async Task HandleHttpSessionResponse(SessionEventArgs args) { await WriteResponseHeaders(args.Client.ClientStreamWriter, args.WebSession.Response.ResponseHeaders); - if (args.WebSession.Response.IsChunked || args.WebSession.Response.ContentLength > 0 || args.WebSession.Response.HttpVersion.ToLower().Trim() == "http/1.0") + if (args.WebSession.Response.IsChunked || args.WebSession.Response.ContentLength > 0 || + (args.WebSession.Response.HttpVersion.Major == 1 && args.WebSession.Response.HttpVersion.Minor == 0)) await WriteResponseBody(args.WebSession.ServerConnection.StreamReader, args.Client.ClientStream, args.WebSession.Response.IsChunked, args.WebSession.Response.ContentLength).ConfigureAwait(false); } @@ -105,10 +106,10 @@ private static async Task GetCompressedResponseBody(string encodingType, } - private static void WriteResponseStatus(string version, string code, string description, + private static void WriteResponseStatus(Version version, string code, string description, StreamWriter responseWriter) { - responseWriter.WriteLineAsync(string.Format("{0} {1} {2}", version, code, description)); + responseWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2} {3}", version.Major, version.Minor, code, description)); } private static async Task WriteResponseHeaders(StreamWriter responseWriter, List headers) From 6ee02250c0a16b0796b309cd8476207ac02065c2 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Mon, 23 May 2016 18:15:23 -0400 Subject: [PATCH 03/29] update version --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 1bb0b5fd2..7e00f509f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,7 +7,7 @@ # - Section names should be unique on each level. # version format -version: 2.2.{build} +version: 2.3.{build} shallow_clone: true From f38199d0affac3a54f8df3c5dbe823567e8c7171 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Mon, 23 May 2016 18:50:31 -0400 Subject: [PATCH 04/29] fix build number --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 7e00f509f..1f250dc36 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,7 +7,7 @@ # - Section names should be unique on each level. # version format -version: 2.3.{build} +version: 2.3000.{build} shallow_clone: true From 1849d31e8e50d9fbc660d9cfc28630ea6a193fb9 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Thu, 26 May 2016 20:30:58 -0400 Subject: [PATCH 05/29] optimize connection manager; fix asyn/await --- Titanium.Web.Proxy/Helpers/SystemProxy.cs | 20 +++--- Titanium.Web.Proxy/Http/HttpWebClient.cs | 11 ++- .../Network/TcpConnectionManager.cs | 69 +++++++++++++------ Titanium.Web.Proxy/RequestHandler.cs | 18 ++--- Titanium.Web.Proxy/ResponseHandler.cs | 12 ++-- 5 files changed, 73 insertions(+), 57 deletions(-) diff --git a/Titanium.Web.Proxy/Helpers/SystemProxy.cs b/Titanium.Web.Proxy/Helpers/SystemProxy.cs index a1113955f..cf2202084 100644 --- a/Titanium.Web.Proxy/Helpers/SystemProxy.cs +++ b/Titanium.Web.Proxy/Helpers/SystemProxy.cs @@ -18,11 +18,11 @@ internal class HttpSystemProxyValue { public string HostName { get; set; } public int Port { get; set; } - public bool IsSecure { get; set; } + public bool IsHttps { get; set; } public override string ToString() { - if (!IsSecure) + if (!IsHttps) return "http=" + HostName + ":" + Port; else return "https=" + HostName + ":" + Port; @@ -44,11 +44,11 @@ public static void SetHttpProxy(string hostname, int port) var exisitingContent = reg.GetValue("ProxyServer") as string; var existingSystemProxyValues = GetSystemProxyValues(exisitingContent); - existingSystemProxyValues.RemoveAll(x => !x.IsSecure); + existingSystemProxyValues.RemoveAll(x => !x.IsHttps); existingSystemProxyValues.Add(new HttpSystemProxyValue() { HostName = hostname, - IsSecure = false, + IsHttps = false, Port = port }); @@ -71,7 +71,7 @@ public static void RemoveHttpProxy() var exisitingContent = reg.GetValue("ProxyServer") as string; var existingSystemProxyValues = GetSystemProxyValues(exisitingContent); - existingSystemProxyValues.RemoveAll(x => !x.IsSecure); + existingSystemProxyValues.RemoveAll(x => !x.IsHttps); if (!(existingSystemProxyValues.Count() == 0)) { @@ -101,11 +101,11 @@ public static void SetHttpsProxy(string hostname, int port) var exisitingContent = reg.GetValue("ProxyServer") as string; var existingSystemProxyValues = GetSystemProxyValues(exisitingContent); - existingSystemProxyValues.RemoveAll(x => x.IsSecure); + existingSystemProxyValues.RemoveAll(x => x.IsHttps); existingSystemProxyValues.Add(new HttpSystemProxyValue() { HostName = hostname, - IsSecure = true, + IsHttps = true, Port = port }); @@ -127,7 +127,7 @@ public static void RemoveHttpsProxy() var exisitingContent = reg.GetValue("ProxyServer") as string; var existingSystemProxyValues = GetSystemProxyValues(exisitingContent); - existingSystemProxyValues.RemoveAll(x => x.IsSecure); + existingSystemProxyValues.RemoveAll(x => x.IsHttps); if (!(existingSystemProxyValues.Count() == 0)) { @@ -197,7 +197,7 @@ private static HttpSystemProxyValue parseProxyValue(string value) { HostName = endPoint.Split(':')[0], Port = int.Parse(endPoint.Split(':')[1]), - IsSecure = false + IsHttps = false }; } else if (tmp.StartsWith("https=")) @@ -207,7 +207,7 @@ private static HttpSystemProxyValue parseProxyValue(string value) { HostName = endPoint.Split(':')[0], Port = int.Parse(endPoint.Split(':')[1]), - IsSecure = true + IsHttps = true }; } return null; diff --git a/Titanium.Web.Proxy/Http/HttpWebClient.cs b/Titanium.Web.Proxy/Http/HttpWebClient.cs index d1f04cb3b..c605ca843 100644 --- a/Titanium.Web.Proxy/Http/HttpWebClient.cs +++ b/Titanium.Web.Proxy/Http/HttpWebClient.cs @@ -16,7 +16,7 @@ public class HttpWebSession public Request Request { get; set; } public Response Response { get; set; } - public bool IsSecure + public bool IsHttps { get { @@ -96,12 +96,9 @@ internal async Task ReceiveResponse() await ServerConnection.StreamReader.ReadLineAsync().ConfigureAwait(false); } var httpVersion = httpResult[0].Trim().ToLower(); - Version version; - if (httpVersion == "http/1.1") - { - version = new Version(1, 1); - } - else + + var version = new Version(1,1); + if (httpVersion == "http/1.0") { version = new Version(1, 0); } diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index f206fae46..8396f4912 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -19,7 +19,7 @@ public class TcpConnection { internal string HostName { get; set; } internal int port { get; set; } - internal bool IsSecure { get; set; } + internal bool IsHttps { get; set; } internal Version Version { get; set; } internal TcpClient TcpClient { get; set; } @@ -37,29 +37,31 @@ internal TcpConnection() internal class TcpConnectionManager { - static List connectionCache = new List(); + static Dictionary> connectionCache = new Dictionary>(); static SemaphoreSlim connectionAccessLock = new SemaphoreSlim(1); - internal static async Task GetClient(SessionEventArgs sessionArgs, string hostname, int port, bool isSecure, Version version) + internal static async Task GetClient(SessionEventArgs sessionArgs, string hostname, int port, bool isHttps, Version version) { + List cachedConnections = null; TcpConnection cached = null; + + var key = GetConnectionKey(hostname, port, isHttps, version); + while (true) { await connectionAccessLock.WaitAsync(); try { - cached = connectionCache.FirstOrDefault(x => x.HostName == hostname && x.port == port && - x.IsSecure == isSecure && x.TcpClient.Connected && x.Version.Equals(version)); + connectionCache.TryGetValue(key, out cachedConnections); - //just create one more preemptively - if (connectionCache.Where(x => x.HostName == hostname && x.port == port && - x.IsSecure == isSecure && x.TcpClient.Connected && x.Version.Equals(version)).Count() < 2) + if (cachedConnections != null && cachedConnections.Count > 0) { - var task = CreateClient(sessionArgs, hostname, port, isSecure, version) - .ContinueWith(async (x) => { if (x.Status == TaskStatus.RanToCompletion) await ReleaseClient(x.Result); }); + cached = cachedConnections.First(); + cachedConnections.Remove(cached); + } + else + { + cached = null; } - - if (cached != null) - connectionCache.Remove(cached); } finally { connectionAccessLock.Release(); } @@ -71,17 +73,30 @@ internal static async Task GetClient(SessionEventArgs sessionArgs } if (cached == null) - cached = await CreateClient(sessionArgs, hostname, port, isSecure, version).ConfigureAwait(false); - + cached = await CreateClient(sessionArgs, hostname, port, isHttps, version).ConfigureAwait(false); + + + //just create one more preemptively + if (cachedConnections == null || cachedConnections.Count() < 2) + { + var task = CreateClient(sessionArgs, hostname, port, isHttps, version) + .ContinueWith(async (x) => { if (x.Status == TaskStatus.RanToCompletion) await ReleaseClient(x.Result); }); + } + return cached; } - private static async Task CreateClient(SessionEventArgs sessionArgs, string hostname, int port, bool isSecure, Version version) + internal static string GetConnectionKey(string hostname, int port, bool isHttps, Version version) + { + return string.Format("{0}:{1}:{2}:{3}:{4}", hostname.ToLower(), port, isHttps, version.Major, version.Minor); + } + + private static async Task CreateClient(SessionEventArgs sessionArgs, string hostname, int port, bool isHttps, Version version) { TcpClient client; Stream stream; - if (isSecure) + if (isHttps) { CustomSslStream sslStream = null; @@ -148,7 +163,7 @@ private static async Task CreateClient(SessionEventArgs sessionAr { HostName = hostname, port = port, - IsSecure = isSecure, + IsHttps = isHttps, TcpClient = client, StreamReader = new CustomBinaryReader(stream), Stream = stream, @@ -157,14 +172,23 @@ private static async Task CreateClient(SessionEventArgs sessionAr } - internal static async Task ReleaseClient(TcpConnection Connection) + internal static async Task ReleaseClient(TcpConnection connection) { - Connection.LastAccess = DateTime.Now; + connection.LastAccess = DateTime.Now; + var key = GetConnectionKey(connection.HostName, connection.port, connection.IsHttps, connection.Version); await connectionAccessLock.WaitAsync(); try { - connectionCache.Add(Connection); + List cachedConnections; + connectionCache.TryGetValue(key, out cachedConnections); + + if (cachedConnections != null) + cachedConnections.Add(connection); + else + + connectionCache.Add(key, new List() { connection }); } + finally { connectionAccessLock.Release(); } } @@ -178,11 +202,12 @@ internal async static void ClearIdleConnections() var cutOff = DateTime.Now.AddSeconds(-60); connectionCache + .SelectMany(x => x.Value) .Where(x => x.LastAccess < cutOff) .ToList() .ForEach(x => x.TcpClient.Close()); - connectionCache.RemoveAll(x => x.LastAccess < cutOff); + connectionCache.ToList().ForEach(x => x.Value.RemoveAll(y => y.LastAccess < cutOff)); } finally { connectionAccessLock.Release(); } diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index a27b87b67..e948ceb8c 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -57,11 +57,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient { string httpVersion = httpCmdSplit[1].Trim(); - if (httpVersion == "http/1.1") - { - version = new Version(1, 1); - } - else + if (httpVersion == "http/1.0") { version = new Version(1, 0); } @@ -186,7 +182,7 @@ await HandleHttpSessionRequest(tcpClient, httpCmd, clientStream, clientStreamRea } private static async Task HandleHttpSessionRequest(TcpClient client, string httpCmd, Stream clientStream, - CustomBinaryReader clientStreamReader, StreamWriter clientStreamWriter, bool IsHttps) + CustomBinaryReader clientStreamReader, StreamWriter clientStreamWriter, bool isHttps) { TcpConnection connection = null; string lastRequest = null; @@ -229,7 +225,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http args.WebSession.Request.RequestHeaders.Add(new HttpHeader(header[0], header[1])); } - var httpRemoteUri = new Uri(!IsHttps ? httpCmdSplit[1] : (string.Concat("https://", args.WebSession.Request.Host, httpCmdSplit[1]))); + var httpRemoteUri = new Uri(!isHttps ? httpCmdSplit[1] : (string.Concat("https://", args.WebSession.Request.Host, httpCmdSplit[1]))); args.WebSession.Request.RequestUri = httpRemoteUri; @@ -271,9 +267,7 @@ await TcpConnectionManager.GetClient(args, args.WebSession.Request.RequestUri.Ho : lastRequest != args.WebSession.Request.RequestUri.Host ? await TcpConnectionManager.GetClient(args, args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) : connection; - lastRequest = string.Concat(args.WebSession.Request.RequestUri.Host,":", - args.WebSession.Request.RequestUri.Port,":", - args.IsHttps,":", version.ToString()); + lastRequest = TcpConnectionManager.GetConnectionKey(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version); args.WebSession.Request.RequestLocked = true; @@ -292,13 +286,13 @@ await TcpConnectionManager.GetClient(args, args.WebSession.Request.RequestUri.Ho if (Enable100ContinueBehaviour) if (args.WebSession.Request.Is100Continue) { - WriteResponseStatus(args.WebSession.Response.HttpVersion, "100", + await WriteResponseStatus(args.WebSession.Response.HttpVersion, "100", "Continue", args.Client.ClientStreamWriter); await args.Client.ClientStreamWriter.WriteLineAsync(); } else if (args.WebSession.Request.ExpectationFailed) { - WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", + await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", "Expectation Failed", args.Client.ClientStreamWriter); await args.Client.ClientStreamWriter.WriteLineAsync(); } diff --git a/Titanium.Web.Proxy/ResponseHandler.cs b/Titanium.Web.Proxy/ResponseHandler.cs index 3cc4626c4..95aa7e717 100644 --- a/Titanium.Web.Proxy/ResponseHandler.cs +++ b/Titanium.Web.Proxy/ResponseHandler.cs @@ -44,19 +44,19 @@ public static async Task HandleHttpSessionResponse(SessionEventArgs args) if (args.WebSession.Response.Is100Continue) { - WriteResponseStatus(args.WebSession.Response.HttpVersion, "100", + await WriteResponseStatus(args.WebSession.Response.HttpVersion, "100", "Continue", args.Client.ClientStreamWriter); await args.Client.ClientStreamWriter.WriteLineAsync(); } else if (args.WebSession.Response.ExpectationFailed) { - WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", + await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", "Expectation Failed", args.Client.ClientStreamWriter); await args.Client.ClientStreamWriter.WriteLineAsync(); } - WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession.Response.ResponseStatusCode, - args.WebSession.Response.ResponseStatusDescription, args.Client.ClientStreamWriter); + await WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession.Response.ResponseStatusCode, + args.WebSession.Response.ResponseStatusDescription, args.Client.ClientStreamWriter); if (args.WebSession.Response.ResponseBodyRead) { @@ -106,10 +106,10 @@ private static async Task GetCompressedResponseBody(string encodingType, } - private static void WriteResponseStatus(Version version, string code, string description, + private static async Task WriteResponseStatus(Version version, string code, string description, StreamWriter responseWriter) { - responseWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2} {3}", version.Major, version.Minor, code, description)); + await responseWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2} {3}", version.Major, version.Minor, code, description)); } private static async Task WriteResponseHeaders(StreamWriter responseWriter, List headers) From 167bb2f7d8d9dcb521a3221e9a87cf90de84918f Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Fri, 27 May 2016 08:27:44 -0400 Subject: [PATCH 06/29] Fix version parse --- Titanium.Web.Proxy/RequestHandler.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index e948ceb8c..f240a52d4 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -55,7 +55,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient Version version = new Version(1, 1); if (httpCmdSplit.Length == 3) { - string httpVersion = httpCmdSplit[1].Trim(); + string httpVersion = httpCmdSplit[2].Trim(); if (httpVersion == "http/1.0") { @@ -204,16 +204,16 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http var httpCmdSplit = httpCmd.Split(Constants.SpaceSplit, 3); var httpMethod = httpCmdSplit[0]; - var httpVersion = httpCmdSplit[2].ToLower().Trim(); - Version version; - if (httpVersion == "http/1.1") + Version version = new Version(1, 1); + if (httpCmdSplit.Length == 3) { - version = new Version(1, 1); - } - else - { - version = new Version(1, 0); + var httpVersion = httpCmdSplit[2].ToLower().Trim(); + + if (httpVersion == "http/1.0") + { + version = new Version(1, 0); + } } args.WebSession.Request.RequestHeaders = new List(); @@ -221,7 +221,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http string tmpLine; while (!string.IsNullOrEmpty(tmpLine = await clientStreamReader.ReadLineAsync().ConfigureAwait(false))) { - var header = tmpLine.Split(new char[] { ':' }, 2); + var header = tmpLine.Split(Constants.ColonSplit, 2); args.WebSession.Request.RequestHeaders.Add(new HttpHeader(header[0], header[1])); } From 554069ba6cf5f4fbe1e325e4760d4488b241bd39 Mon Sep 17 00:00:00 2001 From: titanium007 Date: Wed, 1 Jun 2016 09:57:18 -0400 Subject: [PATCH 07/29] Fix bug caused by using wrong variable in SessionEventArgs --- Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs | 6 +++--- Titanium.Web.Proxy/Http/HttpWebClient.cs | 4 ++-- Titanium.Web.Proxy/ProxyServer.cs | 4 ++++ Titanium.Web.Proxy/RequestHandler.cs | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs index 4b533128f..de1ce7523 100644 --- a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs @@ -25,7 +25,7 @@ public class SessionEventArgs : EventArgs, IDisposable internal SessionEventArgs() { Client = new ProxyClient(); - WebSession = new HttpWebSession(); + WebSession = new HttpWebClient(); } /// @@ -42,7 +42,7 @@ internal SessionEventArgs() /// A web session corresponding to a single request/response sequence /// within a proxy connection /// - public HttpWebSession WebSession { get; set; } + public HttpWebClient WebSession { get; set; } /// @@ -87,7 +87,7 @@ private async Task ReadRequestBody() await this.Client.ClientStreamReader.CopyBytesToStream(requestBodyStream, WebSession.Request.ContentLength).ConfigureAwait(false); } - else if(WebSession.Response.HttpVersion.Major == 1 && WebSession.Response.HttpVersion.Minor == 0) + else if(WebSession.Request.HttpVersion.Major == 1 && WebSession.Request.HttpVersion.Minor == 0) await WebSession.ServerConnection.StreamReader.CopyBytesToStream(requestBodyStream, long.MaxValue).ConfigureAwait(false); } WebSession.Request.RequestBody = await GetDecompressedResponseBody(WebSession.Request.ContentEncoding, requestBodyStream.ToArray()).ConfigureAwait(false); diff --git a/Titanium.Web.Proxy/Http/HttpWebClient.cs b/Titanium.Web.Proxy/Http/HttpWebClient.cs index c605ca843..a7611f7b4 100644 --- a/Titanium.Web.Proxy/Http/HttpWebClient.cs +++ b/Titanium.Web.Proxy/Http/HttpWebClient.cs @@ -9,7 +9,7 @@ namespace Titanium.Web.Proxy.Http { - public class HttpWebSession + public class HttpWebClient { internal TcpConnection ServerConnection { get; set; } @@ -30,7 +30,7 @@ internal void SetConnection(TcpConnection Connection) ServerConnection = Connection; } - internal HttpWebSession() + internal HttpWebClient() { this.Request = new Request(); this.Response = new Response(); diff --git a/Titanium.Web.Proxy/ProxyServer.cs b/Titanium.Web.Proxy/ProxyServer.cs index fef2895dd..2e29b5d50 100644 --- a/Titanium.Web.Proxy/ProxyServer.cs +++ b/Titanium.Web.Proxy/ProxyServer.cs @@ -236,6 +236,10 @@ private static void OnAcceptConnection(IAsyncResult asyn) // so just return. return; } + catch + { + + } } diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index f240a52d4..62de3d323 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -365,7 +365,7 @@ private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, await clientStreamWriter.FlushAsync().ConfigureAwait(false); } - private static void PrepareRequestHeaders(List requestHeaders, HttpWebSession webRequest) + private static void PrepareRequestHeaders(List requestHeaders, HttpWebClient webRequest) { for (var i = 0; i < requestHeaders.Count; i++) { From dfc6d9a68edf4cba7c6980e140e2d3a65340e31a Mon Sep 17 00:00:00 2001 From: titanium007 Date: Wed, 1 Jun 2016 10:04:29 -0400 Subject: [PATCH 08/29] config await --- Titanium.Web.Proxy/Network/TcpConnectionManager.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index 8396f4912..896a6b27c 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -107,11 +107,11 @@ private static async Task CreateClient(SessionEventArgs sessionAr using (var writer = new StreamWriter(stream, Encoding.ASCII, Constants.BUFFER_SIZE, true)) { - await writer.WriteLineAsync(string.Format("CONNECT {0}:{1} {2}", sessionArgs.WebSession.Request.RequestUri.Host, sessionArgs.WebSession.Request.RequestUri.Port, sessionArgs.WebSession.Request.HttpVersion)); - await writer.WriteLineAsync(string.Format("Host: {0}:{1}", sessionArgs.WebSession.Request.RequestUri.Host, sessionArgs.WebSession.Request.RequestUri.Port)); - await writer.WriteLineAsync("Connection: Keep-Alive"); - await writer.WriteLineAsync(); - await writer.FlushAsync(); + await writer.WriteLineAsync(string.Format("CONNECT {0}:{1} {2}", sessionArgs.WebSession.Request.RequestUri.Host, sessionArgs.WebSession.Request.RequestUri.Port, sessionArgs.WebSession.Request.HttpVersion)).ConfigureAwait(false); + await writer.WriteLineAsync(string.Format("Host: {0}:{1}", sessionArgs.WebSession.Request.RequestUri.Host, sessionArgs.WebSession.Request.RequestUri.Port)).ConfigureAwait(false); + await writer.WriteLineAsync("Connection: Keep-Alive").ConfigureAwait(false); + await writer.WriteLineAsync().ConfigureAwait(false); + await writer.FlushAsync().ConfigureAwait(false); writer.Close(); } From e4dfa442daf80c2120c1441bccf6546669bf5efb Mon Sep 17 00:00:00 2001 From: titanium007 Date: Fri, 3 Jun 2016 17:33:33 -0400 Subject: [PATCH 09/29] issue #13 Fix Mark items internal; keep only MRU certificates in cache --- .../Decompression/DecompressionFactory.cs | 4 +- .../Decompression/DefaultDecompression.cs | 2 +- .../Decompression/DeflateDecompression.cs | 2 +- .../Decompression/GZipDecompression.cs | 2 +- .../Decompression/IDecompression.cs | 4 +- .../Decompression/ZlibDecompression.cs | 2 +- .../Extensions/HttpWebRequestExtensions.cs | 4 +- .../Extensions/HttpWebResponseExtensions.cs | 4 +- .../Extensions/StreamExtensions.cs | 4 +- .../Extensions/TcpExtensions.cs | 4 +- .../Helpers/CertificateManager.cs | 159 ++++++++++++------ .../Helpers/CustomBinaryReader.cs | 16 +- Titanium.Web.Proxy/Helpers/SystemProxy.cs | 22 +-- Titanium.Web.Proxy/Helpers/Tcp.cs | 4 +- Titanium.Web.Proxy/Network/CustomSslStream.cs | 2 +- .../Network/TcpConnectionManager.cs | 11 +- Titanium.Web.Proxy/ProxyServer.cs | 16 +- Titanium.Web.Proxy/RequestHandler.cs | 4 +- Titanium.Web.Proxy/Shared/Constants.cs | 2 +- 19 files changed, 164 insertions(+), 104 deletions(-) diff --git a/Titanium.Web.Proxy/Decompression/DecompressionFactory.cs b/Titanium.Web.Proxy/Decompression/DecompressionFactory.cs index 7f4e61d51..4548fc147 100644 --- a/Titanium.Web.Proxy/Decompression/DecompressionFactory.cs +++ b/Titanium.Web.Proxy/Decompression/DecompressionFactory.cs @@ -1,8 +1,8 @@ namespace Titanium.Web.Proxy.Decompression { - class DecompressionFactory + internal class DecompressionFactory { - public IDecompression Create(string type) + internal IDecompression Create(string type) { switch(type) { diff --git a/Titanium.Web.Proxy/Decompression/DefaultDecompression.cs b/Titanium.Web.Proxy/Decompression/DefaultDecompression.cs index a754f9252..e8fa483c0 100644 --- a/Titanium.Web.Proxy/Decompression/DefaultDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/DefaultDecompression.cs @@ -2,7 +2,7 @@ namespace Titanium.Web.Proxy.Decompression { - class DefaultDecompression : IDecompression + internal class DefaultDecompression : IDecompression { public Task Decompress(byte[] compressedArray) { diff --git a/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs b/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs index 429a98473..02b184b82 100644 --- a/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs @@ -5,7 +5,7 @@ namespace Titanium.Web.Proxy.Decompression { - class DeflateDecompression : IDecompression + internal class DeflateDecompression : IDecompression { public async Task Decompress(byte[] compressedArray) { diff --git a/Titanium.Web.Proxy/Decompression/GZipDecompression.cs b/Titanium.Web.Proxy/Decompression/GZipDecompression.cs index bfe0820c4..8df6c7be9 100644 --- a/Titanium.Web.Proxy/Decompression/GZipDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/GZipDecompression.cs @@ -5,7 +5,7 @@ namespace Titanium.Web.Proxy.Decompression { - class GZipDecompression : IDecompression + internal class GZipDecompression : IDecompression { public async Task Decompress(byte[] compressedArray) { diff --git a/Titanium.Web.Proxy/Decompression/IDecompression.cs b/Titanium.Web.Proxy/Decompression/IDecompression.cs index 3e01352d0..f215bd1ad 100644 --- a/Titanium.Web.Proxy/Decompression/IDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/IDecompression.cs @@ -3,8 +3,8 @@ namespace Titanium.Web.Proxy.Decompression { - interface IDecompression + internal interface IDecompression { - Task Decompress(byte[] compressedArray); + Task Decompress(byte[] compressedArray); } } diff --git a/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs b/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs index a4aee4f90..9cede3d54 100644 --- a/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs @@ -5,7 +5,7 @@ namespace Titanium.Web.Proxy.Decompression { - class ZlibDecompression : IDecompression + internal class ZlibDecompression : IDecompression { public async Task Decompress(byte[] compressedArray) { diff --git a/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs b/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs index c172192bf..bcc3ff9a8 100644 --- a/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs @@ -7,10 +7,10 @@ namespace Titanium.Web.Proxy.Extensions /// /// Extensions on HttpWebSession object /// - public static class HttpWebRequestExtensions + internal static class HttpWebRequestExtensions { //Get encoding of the HTTP request - public static Encoding GetEncoding(this Request request) + internal static Encoding GetEncoding(this Request request) { try { diff --git a/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs b/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs index c3ff3ef30..029eb107a 100644 --- a/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs @@ -4,9 +4,9 @@ namespace Titanium.Web.Proxy.Extensions { - public static class HttpWebResponseExtensions + internal static class HttpWebResponseExtensions { - public static Encoding GetResponseCharacterEncoding(this Response response) + internal static Encoding GetResponseCharacterEncoding(this Response response) { try { diff --git a/Titanium.Web.Proxy/Extensions/StreamExtensions.cs b/Titanium.Web.Proxy/Extensions/StreamExtensions.cs index 69152a9a6..c86bcb264 100644 --- a/Titanium.Web.Proxy/Extensions/StreamExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/StreamExtensions.cs @@ -8,9 +8,9 @@ namespace Titanium.Web.Proxy.Extensions { - public static class StreamHelper + internal static class StreamHelper { - public static async Task CopyToAsync(this Stream input, string initialData, Stream output) + internal static async Task CopyToAsync(this Stream input, string initialData, Stream output) { if (!string.IsNullOrEmpty(initialData)) { diff --git a/Titanium.Web.Proxy/Extensions/TcpExtensions.cs b/Titanium.Web.Proxy/Extensions/TcpExtensions.cs index 085c5b15e..4d1ce7adb 100644 --- a/Titanium.Web.Proxy/Extensions/TcpExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/TcpExtensions.cs @@ -1,12 +1,10 @@ using System.Net.Sockets; - namespace Titanium.Web.Proxy.Extensions { - internal static class TcpExtensions { - public static bool IsConnected(this Socket client) + internal static bool IsConnected(this Socket client) { // This is how you can determine whether a socket is still connected. bool blockingState = client.Blocking; diff --git a/Titanium.Web.Proxy/Helpers/CertificateManager.cs b/Titanium.Web.Proxy/Helpers/CertificateManager.cs index 6d5cbcb12..3a2306c02 100644 --- a/Titanium.Web.Proxy/Helpers/CertificateManager.cs +++ b/Titanium.Web.Proxy/Helpers/CertificateManager.cs @@ -6,24 +6,37 @@ using System.Reflection; using System.Threading.Tasks; using System.Threading; +using System.Linq; namespace Titanium.Web.Proxy.Helpers { - public class CertificateManager : IDisposable + internal class CachedCertificate + { + internal X509Certificate2 Certificate { get; set; } + + internal DateTime LastAccess { get; set; } + + internal CachedCertificate() + { + LastAccess = DateTime.Now; + } + + } + internal class CertificateManager : IDisposable { private const string CertCreateFormat = "-ss {0} -n \"CN={1}, O={2}\" -sky {3} -cy {4} -m 120 -a sha256 -eku 1.3.6.1.5.5.7.3.1 {5}"; - private readonly IDictionary _certificateCache; + private readonly IDictionary certificateCache; private static SemaphoreSlim semaphoreLock = new SemaphoreSlim(1); - public string Issuer { get; private set; } - public string RootCertificateName { get; private set; } + internal string Issuer { get; private set; } + internal string RootCertificateName { get; private set; } - public X509Store MyStore { get; private set; } - public X509Store RootStore { get; private set; } + internal X509Store MyStore { get; private set; } + internal X509Store RootStore { get; private set; } - public CertificateManager(string issuer, string rootCertificateName) + internal CertificateManager(string issuer, string rootCertificateName) { Issuer = issuer; RootCertificateName = rootCertificateName; @@ -31,17 +44,17 @@ public CertificateManager(string issuer, string rootCertificateName) MyStore = new X509Store(StoreName.My, StoreLocation.CurrentUser); RootStore = new X509Store(StoreName.Root, StoreLocation.CurrentUser); - _certificateCache = new Dictionary(); + certificateCache = new Dictionary(); } /// /// Attempts to move a self-signed certificate to the root store. /// /// true if succeeded, else false - public async Task CreateTrustedRootCertificate() + internal async Task CreateTrustedRootCertificate() { X509Certificate2 rootCertificate = - await CreateCertificate(RootStore, RootCertificateName); + await CreateCertificate(RootStore, RootCertificateName, true); return rootCertificate != null; } @@ -49,12 +62,12 @@ public async Task CreateTrustedRootCertificate() /// Attempts to remove the self-signed certificate from the root store. /// /// true if succeeded, else false - public async Task DestroyTrustedRootCertificate() + internal bool DestroyTrustedRootCertificate() { - return await DestroyCertificate(RootStore, RootCertificateName); + return DestroyCertificate(RootStore, RootCertificateName, false); } - public X509Certificate2Collection FindCertificates(string certificateSubject) + internal X509Certificate2Collection FindCertificates(string certificateSubject) { return FindCertificates(MyStore, certificateSubject); } @@ -67,29 +80,36 @@ protected virtual X509Certificate2Collection FindCertificates(X509Store store, s discoveredCertificates : null; } - public async Task CreateCertificate(string certificateName) + internal async Task CreateCertificate(string certificateName, bool isRootCertificate) { - return await CreateCertificate(MyStore, certificateName); + return await CreateCertificate(MyStore, certificateName, isRootCertificate); } - protected async virtual Task CreateCertificate(X509Store store, string certificateName) - { - - if (_certificateCache.ContainsKey(certificateName)) - return _certificateCache[certificateName]; + protected async virtual Task CreateCertificate(X509Store store, string certificateName, bool isRootCertificate) + { await semaphoreLock.WaitAsync(); - X509Certificate2 certificate = null; try { + if (certificateCache.ContainsKey(certificateName)) + { + var cached = certificateCache[certificateName]; + cached.LastAccess = DateTime.Now; + return cached.Certificate; + } + X509Certificate2 certificate = null; store.Open(OpenFlags.ReadWrite); string certificateSubject = string.Format("CN={0}, O={1}", certificateName, Issuer); - var certificates = - FindCertificates(store, certificateSubject); + X509Certificate2Collection certificates; + + if (isRootCertificate) + { + certificates = FindCertificates(store, certificateSubject); - if (certificates != null) - certificate = certificates[0]; + if (certificates != null) + certificate = certificates[0]; + } if (certificate == null) { @@ -99,22 +119,26 @@ protected async virtual Task CreateCertificate(X509Store store await CreateCertificate(args); certificates = FindCertificates(store, certificateSubject); - return certificates != null ? - certificates[0] : null; + //remove it from store + if (!isRootCertificate) + DestroyCertificate(certificateName); + + if (certificates != null) + certificate = certificates[0]; } store.Close(); - if (certificate != null && !_certificateCache.ContainsKey(certificateName)) - _certificateCache.Add(certificateName, certificate); + if (certificate != null && !certificateCache.ContainsKey(certificateName)) + certificateCache.Add(certificateName, new CachedCertificate() { Certificate = certificate }); return certificate; } finally { - semaphoreLock.Release(); + semaphoreLock.Release(); } - } + protected virtual Task CreateCertificate(string[] args) { @@ -153,40 +177,32 @@ protected virtual Task CreateCertificate(string[] args) } - public async Task DestroyCertificate(string certificateName) + internal bool DestroyCertificate(string certificateName) { - return await DestroyCertificate(MyStore, certificateName); + return DestroyCertificate(MyStore, certificateName, false); } - protected virtual async Task DestroyCertificate(X509Store store, string certificateName) - { - await semaphoreLock.WaitAsync(); + protected virtual bool DestroyCertificate(X509Store store, string certificateName, bool removeFromCache) + { X509Certificate2Collection certificates = null; - try - { - store.Open(OpenFlags.ReadWrite); - string certificateSubject = string.Format("CN={0}, O={1}", certificateName, Issuer); + store.Open(OpenFlags.ReadWrite); + string certificateSubject = string.Format("CN={0}, O={1}", certificateName, Issuer); + + certificates = FindCertificates(store, certificateSubject); + if (certificates != null) + { + store.RemoveRange(certificates); certificates = FindCertificates(store, certificateSubject); - if (certificates != null) - { - store.RemoveRange(certificates); - certificates = FindCertificates(store, certificateSubject); - } - - store.Close(); - if (certificates == null && - _certificateCache.ContainsKey(certificateName)) - { - _certificateCache.Remove(certificateName); - } - return certificates == null; } - finally + + store.Close(); + if (removeFromCache && + certificateCache.ContainsKey(certificateName)) { - semaphoreLock.Release(); + certificateCache.Remove(certificateName); } - + return certificates == null; } protected virtual string GetCertificateCreateArgs(X509Store store, string certificateName) @@ -203,6 +219,37 @@ protected virtual string GetCertificateCreateArgs(X509Store store, string certif return certCreatArgs; } + private static bool clearCertificates { get; set; } + + internal void StopClearIdleCertificates() + { + clearCertificates = false; + } + + internal async void ClearIdleCertificates() + { + clearCertificates = true; + while (clearCertificates) + { + await semaphoreLock.WaitAsync(); + + try + { + var cutOff = DateTime.Now.AddSeconds(-60); + + var outdated = certificateCache + .Where(x => x.Value.LastAccess < cutOff) + .ToList(); + + foreach (var cache in outdated) + certificateCache.Remove(cache.Key); + } + finally { semaphoreLock.Release(); } + + await Task.Delay(1000 * 60 * 3).ConfigureAwait(false); + } + } + public void Dispose() { if (MyStore != null) diff --git a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs index eb4553c9e..7fc4437e1 100644 --- a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs +++ b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs @@ -15,18 +15,16 @@ namespace Titanium.Web.Proxy.Helpers /// using the specified encoding /// as well as to read bytes as required /// - public class CustomBinaryReader : IDisposable + internal class CustomBinaryReader : IDisposable { private Stream stream; - internal CustomBinaryReader(Stream stream) { this.stream = stream; } - - public Stream BaseStream => stream; + internal Stream BaseStream => stream; /// /// Read a line from the byte stream @@ -41,25 +39,25 @@ internal async Task ReadLineAsync() var lastChar = default(char); var buffer = new byte[1]; - while (await this.stream.ReadAsync(buffer, 0, 1).ConfigureAwait(false) > 0) + while (this.stream.Read(buffer, 0, 1) > 0) { if (lastChar == '\r' && buffer[0] == '\n') { - return readBuffer.Remove(readBuffer.Length - 1, 1).ToString(); + return await Task.FromResult(readBuffer.Remove(readBuffer.Length - 1, 1).ToString()); } if (buffer[0] == '\0') { - return readBuffer.ToString(); + return await Task.FromResult(readBuffer.ToString()); } readBuffer.Append((char)buffer[0]); lastChar = (char)buffer[0]; } - return readBuffer.ToString(); + return await Task.FromResult(readBuffer.ToString()); } catch (IOException) { - return readBuffer.ToString(); + return await Task.FromResult(readBuffer.ToString()); } } diff --git a/Titanium.Web.Proxy/Helpers/SystemProxy.cs b/Titanium.Web.Proxy/Helpers/SystemProxy.cs index cf2202084..f46426dcd 100644 --- a/Titanium.Web.Proxy/Helpers/SystemProxy.cs +++ b/Titanium.Web.Proxy/Helpers/SystemProxy.cs @@ -16,9 +16,9 @@ internal static extern bool InternetSetOption(IntPtr hInternet, int dwOption, In internal class HttpSystemProxyValue { - public string HostName { get; set; } - public int Port { get; set; } - public bool IsHttps { get; set; } + internal string HostName { get; set; } + internal int Port { get; set; } + internal bool IsHttps { get; set; } public override string ToString() { @@ -29,12 +29,12 @@ public override string ToString() } } - public static class SystemProxyHelper + internal static class SystemProxyHelper { - public const int InternetOptionSettingsChanged = 39; - public const int InternetOptionRefresh = 37; + internal const int InternetOptionSettingsChanged = 39; + internal const int InternetOptionRefresh = 37; - public static void SetHttpProxy(string hostname, int port) + internal static void SetHttpProxy(string hostname, int port) { var reg = Registry.CurrentUser.OpenSubKey( "Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings", true); @@ -60,7 +60,7 @@ public static void SetHttpProxy(string hostname, int port) } - public static void RemoveHttpProxy() + internal static void RemoveHttpProxy() { var reg = Registry.CurrentUser.OpenSubKey( "Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings", true); @@ -89,7 +89,7 @@ public static void RemoveHttpProxy() Refresh(); } - public static void SetHttpsProxy(string hostname, int port) + internal static void SetHttpsProxy(string hostname, int port) { var reg = Registry.CurrentUser.OpenSubKey( "Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings", true); @@ -116,7 +116,7 @@ public static void SetHttpsProxy(string hostname, int port) Refresh(); } - public static void RemoveHttpsProxy() + internal static void RemoveHttpsProxy() { var reg = Registry.CurrentUser.OpenSubKey( "Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings", true); @@ -146,7 +146,7 @@ public static void RemoveHttpsProxy() Refresh(); } - public static void DisableAllProxy() + internal static void DisableAllProxy() { var reg = Registry.CurrentUser.OpenSubKey( "Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings", true); diff --git a/Titanium.Web.Proxy/Helpers/Tcp.cs b/Titanium.Web.Proxy/Helpers/Tcp.cs index 93d7ce129..cd754aa27 100644 --- a/Titanium.Web.Proxy/Helpers/Tcp.cs +++ b/Titanium.Web.Proxy/Helpers/Tcp.cs @@ -12,9 +12,9 @@ namespace Titanium.Web.Proxy.Helpers { - public class TcpHelper + internal class TcpHelper { - public async static Task SendRaw(Stream clientStream, string httpCmd, List requestHeaders, string hostName, + internal async static Task SendRaw(Stream clientStream, string httpCmd, List requestHeaders, string hostName, int tunnelPort, bool isHttps) { StringBuilder sb = null; diff --git a/Titanium.Web.Proxy/Network/CustomSslStream.cs b/Titanium.Web.Proxy/Network/CustomSslStream.cs index 39100623c..27173b0d6 100644 --- a/Titanium.Web.Proxy/Network/CustomSslStream.cs +++ b/Titanium.Web.Proxy/Network/CustomSslStream.cs @@ -19,7 +19,7 @@ internal class CustomSslStream : SslStream /// internal SessionEventArgs Session { get; set; } - public CustomSslStream(Stream innerStream, bool leaveInnerStreamOpen, RemoteCertificateValidationCallback userCertificateValidationCallback) + internal CustomSslStream(Stream innerStream, bool leaveInnerStreamOpen, RemoteCertificateValidationCallback userCertificateValidationCallback) :base(innerStream, leaveInnerStreamOpen, userCertificateValidationCallback) { diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index 896a6b27c..08d154caa 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -28,7 +28,6 @@ public class TcpConnection internal DateTime LastAccess { get; set; } - internal TcpConnection() { LastAccess = DateTime.Now; @@ -192,9 +191,17 @@ internal static async Task ReleaseClient(TcpConnection connection) finally { connectionAccessLock.Release(); } } + private static bool clearConenctions { get; set; } + + internal static void StopClearIdleConnections() + { + clearConenctions = false; + } + internal async static void ClearIdleConnections() { - while (true) + clearConenctions = true; + while (clearConenctions) { await connectionAccessLock.WaitAsync(); try diff --git a/Titanium.Web.Proxy/ProxyServer.cs b/Titanium.Web.Proxy/ProxyServer.cs index 2e29b5d50..65022673c 100644 --- a/Titanium.Web.Proxy/ProxyServer.cs +++ b/Titanium.Web.Proxy/ProxyServer.cs @@ -21,8 +21,7 @@ public partial class ProxyServer static ProxyServer() { - ProxyEndPoints = new List(); - Initialize(); + ProxyEndPoints = new List(); } private static CertificateManager CertManager { get; set; } @@ -58,6 +57,13 @@ static ProxyServer() public static void Initialize() { TcpConnectionManager.ClearIdleConnections(); + CertManager.ClearIdleCertificates(); + } + + public static void Quit() + { + TcpConnectionManager.StopClearIdleConnections(); + CertManager.StopClearIdleCertificates(); } public static void AddEndPoint(ProxyEndPoint endPoint) @@ -153,7 +159,7 @@ public static void Start() CertManager = new CertificateManager(RootCertificateIssuerName, RootCertificateName); - + certTrusted = CertManager.CreateTrustedRootCertificate().Result; foreach (var endPoint in ProxyEndPoints) @@ -161,6 +167,8 @@ public static void Start() Listen(endPoint); } + Initialize(); + proxyRunning = true; } @@ -186,6 +194,8 @@ public static void Stop() CertManager.Dispose(); + Quit(); + proxyRunning = false; } diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 62de3d323..a9ddec2c6 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -74,7 +74,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient await WriteConnectResponse(clientStreamWriter, version).ConfigureAwait(false); - var certificate = await CertManager.CreateCertificate(httpRemoteUri.Host); + var certificate = await CertManager.CreateCertificate(httpRemoteUri.Host, false); SslStream sslStream = null; @@ -146,7 +146,7 @@ private static async void HandleClient(TransparentProxyEndPoint endPoint, TcpCli // certificate = CertManager.CreateCertificate(hostName); //} //else - certificate = await CertManager.CreateCertificate(endPoint.GenericCertificateName); + certificate = await CertManager.CreateCertificate(endPoint.GenericCertificateName, false); try { diff --git a/Titanium.Web.Proxy/Shared/Constants.cs b/Titanium.Web.Proxy/Shared/Constants.cs index 7a6bc21ed..5b9934c01 100644 --- a/Titanium.Web.Proxy/Shared/Constants.cs +++ b/Titanium.Web.Proxy/Shared/Constants.cs @@ -10,7 +10,7 @@ namespace Titanium.Web.Proxy.Shared /// internal class Constants { - public static readonly int BUFFER_SIZE = 8192; + internal static readonly int BUFFER_SIZE = 8192; internal static readonly char[] SpaceSplit = { ' ' }; internal static readonly char[] ColonSplit = { ':' }; From c525dda79166f597283936b1a3ab47795bbd8095 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Fri, 3 Jun 2016 17:38:51 -0400 Subject: [PATCH 10/29] let users change timeout minutes --- Titanium.Web.Proxy/Helpers/CertificateManager.cs | 2 +- Titanium.Web.Proxy/Network/TcpConnectionManager.cs | 2 +- Titanium.Web.Proxy/ProxyServer.cs | 14 +++++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Titanium.Web.Proxy/Helpers/CertificateManager.cs b/Titanium.Web.Proxy/Helpers/CertificateManager.cs index 3a2306c02..f3b92504f 100644 --- a/Titanium.Web.Proxy/Helpers/CertificateManager.cs +++ b/Titanium.Web.Proxy/Helpers/CertificateManager.cs @@ -235,7 +235,7 @@ internal async void ClearIdleCertificates() try { - var cutOff = DateTime.Now.AddSeconds(-60); + var cutOff = DateTime.Now.AddMinutes(-1 * ProxyServer.CertificateCacheTimeOutMinutes); var outdated = certificateCache .Where(x => x.Value.LastAccess < cutOff) diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index 08d154caa..a913d83c0 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -206,7 +206,7 @@ internal async static void ClearIdleConnections() await connectionAccessLock.WaitAsync(); try { - var cutOff = DateTime.Now.AddSeconds(-60); + var cutOff = DateTime.Now.AddMinutes(-1 * ProxyServer.ConnectionCacheTimeOutMinutes); connectionCache .SelectMany(x => x.Value) diff --git a/Titanium.Web.Proxy/ProxyServer.cs b/Titanium.Web.Proxy/ProxyServer.cs index 65022673c..cd3e48172 100644 --- a/Titanium.Web.Proxy/ProxyServer.cs +++ b/Titanium.Web.Proxy/ProxyServer.cs @@ -21,7 +21,9 @@ public partial class ProxyServer static ProxyServer() { - ProxyEndPoints = new List(); + ProxyEndPoints = new List(); + ConnectionCacheTimeOutMinutes = 3; + CertificateCacheTimeOutMinutes = 60; } private static CertificateManager CertManager { get; set; } @@ -32,6 +34,16 @@ static ProxyServer() public static string RootCertificateIssuerName { get; set; } public static string RootCertificateName { get; set; } public static bool Enable100ContinueBehaviour { get; set; } + + /// + /// Minutes TCP connection cache to servers to be kept alive when in idle state + /// + public static int ConnectionCacheTimeOutMinutes { get; set; } + + /// + /// Minutes certificates should be kept in cache when not used + /// + public static int CertificateCacheTimeOutMinutes { get; set; } public static event Func BeforeRequest; public static event Func BeforeResponse; From 65b923fb300c93d37b471b5f909d15ab9df6b05b Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 4 Jun 2016 00:41:31 -0400 Subject: [PATCH 11/29] Add mutual authentication capability --- .../ProxyTestController.cs | 19 ++- README.md | 32 ++-- .../CertificateSelectionEventArgs.cs | 21 +++ .../CertificateValidationEventArgs.cs | 9 -- .../Helpers/CertificateManager.cs | 1 + Titanium.Web.Proxy/Models/ConnectRequest.cs | 11 ++ Titanium.Web.Proxy/Network/CustomSslStream.cs | 6 +- .../Network/TcpConnectionManager.cs | 25 +-- Titanium.Web.Proxy/ProxyServer.cs | 15 +- Titanium.Web.Proxy/RequestHandler.cs | 143 ++++++++++++++---- Titanium.Web.Proxy/Titanium.Web.Proxy.csproj | 2 + 11 files changed, 221 insertions(+), 63 deletions(-) create mode 100644 Titanium.Web.Proxy/EventArguments/CertificateSelectionEventArgs.cs create mode 100644 Titanium.Web.Proxy/Models/ConnectRequest.cs diff --git a/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs b/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs index 5b23814ca..dc2ba74af 100644 --- a/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs +++ b/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs @@ -15,6 +15,7 @@ public void StartProxy() ProxyServer.BeforeRequest += OnRequest; ProxyServer.BeforeResponse += OnResponse; ProxyServer.ServerCertificateValidationCallback += OnCertificateValidation; + ProxyServer.ClientCertificateSelectionCallback += OnCertificateSelection; //Exclude Https addresses you don't want to proxy //Usefull for clients that use certificate pinning @@ -129,13 +130,25 @@ public async Task OnResponse(object sender, SessionEventArgs e) /// /// /// - public async Task OnCertificateValidation(object sender, CertificateValidationEventArgs e) + public Task OnCertificateValidation(object sender, CertificateValidationEventArgs e) { //set IsValid to true/false based on Certificate Errors if (e.SslPolicyErrors == System.Net.Security.SslPolicyErrors.None) e.IsValid = true; - else - await e.Session.Ok("Cannot validate server certificate! Not safe to proceed."); + + return Task.FromResult(0); + } + + /// + /// Allows overriding default client certificate selection logic during mutual authentication + /// + /// + /// + public Task OnCertificateSelection(object sender, CertificateSelectionEventArgs e) + { + //set e.clientCertificate to override + + return Task.FromResult(0); } } } \ No newline at end of file diff --git a/README.md b/README.md index bb9a9fd3e..ce1ad3703 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,11 @@ Features ======== * Supports Http(s) and most features of HTTP 1.1 -* Supports relaying of WebSockets -* Supports script injection +* Support redirect/block/update requests +* Supports updating response +* Safely relays WebSocket requests over Http +* Support mutual SSL authentication +* Fully asynchronous proxy Usage ===== @@ -35,6 +38,8 @@ Setup HTTP proxy: ProxyServer.BeforeRequest += OnRequest; ProxyServer.BeforeResponse += OnResponse; ProxyServer.ServerCertificateValidationCallback += OnCertificateValidation; + ProxyServer.ClientCertificateSelectionCallback += OnCertificateSelection; + //Exclude Https addresses you don't want to proxy //Usefull for clients that use certificate pinning @@ -84,9 +89,7 @@ Setup HTTP proxy: ``` Sample request and response event handlers -```csharp - - //intecept & cancel, redirect or update requests +```csharp public async Task OnRequest(object sender, SessionEventArgs e) { Console.WriteLine(e.WebSession.Request.Url); @@ -148,20 +151,27 @@ Sample request and response event handlers } } - - /// Allows overriding default certificate validation logic - public async Task OnCertificateValidation(object sender, CertificateValidationEventArgs e) + /// Allows overriding default certificate validation logic + public Task OnCertificateValidation(object sender, CertificateValidationEventArgs e) { //set IsValid to true/false based on Certificate Errors if (e.SslPolicyErrors == System.Net.Security.SslPolicyErrors.None) e.IsValid = true; - else - await e.Session.Ok("Cannot validate server certificate! Not safe to proceed."); + + return Task.FromResult(0); + } + + /// Allows overriding default client certificate selection logic during mutual authentication + public Task OnCertificateSelection(object sender, CertificateSelectionEventArgs e) + { + //set e.clientCertificate to override + + return Task.FromResult(0); } ``` Future roadmap ============ -* Support mutual authentication +* Implement Kerberos/NTLM authentication over HTTP protocols for windows domain * Support Server Name Indication (SNI) for transparent endpoints * Support HTTP 2.0 diff --git a/Titanium.Web.Proxy/EventArguments/CertificateSelectionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/CertificateSelectionEventArgs.cs new file mode 100644 index 000000000..eed8cc774 --- /dev/null +++ b/Titanium.Web.Proxy/EventArguments/CertificateSelectionEventArgs.cs @@ -0,0 +1,21 @@ +using System; +using System.Security.Cryptography.X509Certificates; + +namespace Titanium.Web.Proxy.EventArguments +{ + public class CertificateSelectionEventArgs : EventArgs, IDisposable + { + public object sender { get; internal set; } + public string targetHost { get; internal set; } + public X509CertificateCollection localCertificates { get; internal set; } + public X509Certificate remoteCertificate { get; internal set; } + public string[] acceptableIssuers { get; internal set; } + + public X509Certificate clientCertificate { get; set; } + + public void Dispose() + { + throw new NotImplementedException(); + } + } +} diff --git a/Titanium.Web.Proxy/EventArguments/CertificateValidationEventArgs.cs b/Titanium.Web.Proxy/EventArguments/CertificateValidationEventArgs.cs index 90512978c..6704cc33c 100644 --- a/Titanium.Web.Proxy/EventArguments/CertificateValidationEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/CertificateValidationEventArgs.cs @@ -1,20 +1,11 @@ using System; -using System.Collections.Generic; -using System.Linq; using System.Net.Security; using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading.Tasks; -using Titanium.Web.Proxy.Http; namespace Titanium.Web.Proxy.EventArguments { public class CertificateValidationEventArgs : EventArgs, IDisposable { - public string HostName => Session.WebSession.Request.Host; - - public SessionEventArgs Session { get; internal set; } - public X509Certificate Certificate { get; internal set; } public X509Chain Chain { get; internal set; } public SslPolicyErrors SslPolicyErrors { get; internal set; } diff --git a/Titanium.Web.Proxy/Helpers/CertificateManager.cs b/Titanium.Web.Proxy/Helpers/CertificateManager.cs index f3b92504f..46604f733 100644 --- a/Titanium.Web.Proxy/Helpers/CertificateManager.cs +++ b/Titanium.Web.Proxy/Helpers/CertificateManager.cs @@ -97,6 +97,7 @@ protected async virtual Task CreateCertificate(X509Store store cached.LastAccess = DateTime.Now; return cached.Certificate; } + X509Certificate2 certificate = null; store.Open(OpenFlags.ReadWrite); string certificateSubject = string.Format("CN={0}, O={1}", certificateName, Issuer); diff --git a/Titanium.Web.Proxy/Models/ConnectRequest.cs b/Titanium.Web.Proxy/Models/ConnectRequest.cs new file mode 100644 index 000000000..d7e2e080e --- /dev/null +++ b/Titanium.Web.Proxy/Models/ConnectRequest.cs @@ -0,0 +1,11 @@ +using System; +using System.IO; + +namespace Titanium.Web.Proxy.Models +{ + internal class ConnectRequest + { + internal Stream Stream { get; set; } + internal Uri Uri { get; set; } + } +} diff --git a/Titanium.Web.Proxy/Network/CustomSslStream.cs b/Titanium.Web.Proxy/Network/CustomSslStream.cs index 27173b0d6..0762f8478 100644 --- a/Titanium.Web.Proxy/Network/CustomSslStream.cs +++ b/Titanium.Web.Proxy/Network/CustomSslStream.cs @@ -17,10 +17,10 @@ internal class CustomSslStream : SslStream /// /// Holds the current session /// - internal SessionEventArgs Session { get; set; } + internal object Param { get; set; } - internal CustomSslStream(Stream innerStream, bool leaveInnerStreamOpen, RemoteCertificateValidationCallback userCertificateValidationCallback) - :base(innerStream, leaveInnerStreamOpen, userCertificateValidationCallback) + internal CustomSslStream(Stream innerStream, bool leaveInnerStreamOpen, RemoteCertificateValidationCallback userCertificateValidationCallback, LocalCertificateSelectionCallback clientCertificateSelectionCallback) + :base(innerStream, leaveInnerStreamOpen, userCertificateValidationCallback, clientCertificateSelectionCallback) { } diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index a913d83c0..ed3518be7 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -12,6 +12,7 @@ using Titanium.Web.Proxy.Shared; using System.Security.Cryptography.X509Certificates; using Titanium.Web.Proxy.EventArguments; +using Titanium.Web.Proxy.Models; namespace Titanium.Web.Proxy.Network { @@ -38,7 +39,12 @@ internal class TcpConnectionManager { static Dictionary> connectionCache = new Dictionary>(); static SemaphoreSlim connectionAccessLock = new SemaphoreSlim(1); - internal static async Task GetClient(SessionEventArgs sessionArgs, string hostname, int port, bool isHttps, Version version) + + internal static async Task GetClient(string hostname, int port, bool isHttps, Version version) + { + return await GetClient(null, hostname, port, isHttps, version); + } + internal static async Task GetClient(ConnectRequest connectRequest, string hostname, int port, bool isHttps, Version version) { List cachedConnections = null; TcpConnection cached = null; @@ -72,14 +78,14 @@ internal static async Task GetClient(SessionEventArgs sessionArgs } if (cached == null) - cached = await CreateClient(sessionArgs, hostname, port, isHttps, version).ConfigureAwait(false); + cached = await CreateClient(connectRequest, hostname, port, isHttps, version).ConfigureAwait(false); //just create one more preemptively if (cachedConnections == null || cachedConnections.Count() < 2) { - var task = CreateClient(sessionArgs, hostname, port, isHttps, version) - .ContinueWith(async (x) => { if (x.Status == TaskStatus.RanToCompletion) await ReleaseClient(x.Result); }); + var task = CreateClient(connectRequest, hostname, port, isHttps, version) + .ContinueWith(async (x) => { if (x.Status == TaskStatus.RanToCompletion) await ReleaseClient(x.Result); }); } return cached; @@ -90,7 +96,7 @@ internal static string GetConnectionKey(string hostname, int port, bool isHttps, return string.Format("{0}:{1}:{2}:{3}:{4}", hostname.ToLower(), port, isHttps, version.Major, version.Minor); } - private static async Task CreateClient(SessionEventArgs sessionArgs, string hostname, int port, bool isHttps, Version version) + private static async Task CreateClient(ConnectRequest connectRequest, string hostname, int port, bool isHttps, Version version) { TcpClient client; Stream stream; @@ -106,8 +112,8 @@ private static async Task CreateClient(SessionEventArgs sessionAr using (var writer = new StreamWriter(stream, Encoding.ASCII, Constants.BUFFER_SIZE, true)) { - await writer.WriteLineAsync(string.Format("CONNECT {0}:{1} {2}", sessionArgs.WebSession.Request.RequestUri.Host, sessionArgs.WebSession.Request.RequestUri.Port, sessionArgs.WebSession.Request.HttpVersion)).ConfigureAwait(false); - await writer.WriteLineAsync(string.Format("Host: {0}:{1}", sessionArgs.WebSession.Request.RequestUri.Host, sessionArgs.WebSession.Request.RequestUri.Port)).ConfigureAwait(false); + await writer.WriteLineAsync(string.Format("CONNECT {0}:{1} {2}", hostname, port, version)).ConfigureAwait(false); + await writer.WriteLineAsync(string.Format("Host: {0}:{1}", hostname, port)).ConfigureAwait(false); await writer.WriteLineAsync("Connection: Keep-Alive").ConfigureAwait(false); await writer.WriteLineAsync().ConfigureAwait(false); await writer.FlushAsync().ConfigureAwait(false); @@ -132,8 +138,9 @@ private static async Task CreateClient(SessionEventArgs sessionAr try { - sslStream = new CustomSslStream(stream, true, new RemoteCertificateValidationCallback(ProxyServer.ValidateServerCertificate)); - sslStream.Session = sessionArgs; + sslStream = new CustomSslStream(stream, true, new RemoteCertificateValidationCallback(ProxyServer.ValidateServerCertificate), + new LocalCertificateSelectionCallback(ProxyServer.SelectClientCertificate)); + sslStream.Param = connectRequest; await sslStream.AuthenticateAsClientAsync(hostname, null, Constants.SupportedProtocols, false).ConfigureAwait(false); stream = (Stream)sslStream; } diff --git a/Titanium.Web.Proxy/ProxyServer.cs b/Titanium.Web.Proxy/ProxyServer.cs index cd3e48172..52adbd12e 100644 --- a/Titanium.Web.Proxy/ProxyServer.cs +++ b/Titanium.Web.Proxy/ProxyServer.cs @@ -45,7 +45,16 @@ static ProxyServer() /// public static int CertificateCacheTimeOutMinutes { get; set; } + + /// + /// Intercept request to server + /// public static event Func BeforeRequest; + + + /// + /// Intercept response from server + /// public static event Func BeforeResponse; /// @@ -62,7 +71,11 @@ static ProxyServer() /// Verifies the remote Secure Sockets Layer (SSL) certificate used for authentication /// public static event Func ServerCertificateValidationCallback; - + + /// + /// Callback tooverride client certificate during SSL mutual authentication + /// + public static event Func ClientCertificateSelectionCallback; public static List ProxyEndPoints { get; set; } diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index a9ddec2c6..f0363721b 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -1,12 +1,10 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Net.Security; using System.Net.Sockets; using System.Security.Authentication; -using System.Text; using System.Text.RegularExpressions; using Titanium.Web.Proxy.EventArguments; using Titanium.Web.Proxy.Helpers; @@ -20,6 +18,7 @@ namespace Titanium.Web.Proxy { + partial class ProxyServer { //This is called when client is aware of proxy @@ -33,7 +32,6 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient try { //read the first line HTTP command - var httpCmd = await clientStreamReader.ReadLineAsync().ConfigureAwait(false); if (string.IsNullOrEmpty(httpCmd)) @@ -56,7 +54,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient if (httpCmdSplit.Length == 3) { string httpVersion = httpCmdSplit[2].Trim(); - + if (httpVersion == "http/1.0") { version = new Version(1, 0); @@ -80,18 +78,28 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient try { - sslStream = new SslStream(clientStream, true); + var connectRequest = new ConnectRequest() { Stream = clientStream, Uri = httpRemoteUri }; + + await TcpConnectionManager.GetClient(connectRequest, httpRemoteUri.Host, httpRemoteUri.Port, true, version).ConfigureAwait(false); - //Successfully managed to authenticate the client using the fake certificate - await sslStream.AuthenticateAsServerAsync(certificate, false, - Constants.SupportedProtocols, false).ConfigureAwait(false); + if (clientStream is SslStream) + { + sslStream = clientStream as SslStream; + } + else + { + sslStream = new SslStream(clientStream, true); + //Successfully managed to authenticate the client using the fake certificate + await sslStream.AuthenticateAsServerAsync(certificate, false, + Constants.SupportedProtocols, false).ConfigureAwait(false); + //HTTPS server created - we can now decrypt the client's traffic + clientStream = sslStream; + } clientStreamReader = new CustomBinaryReader(sslStream); clientStreamWriter = new StreamWriter(sslStream); - //HTTPS server created - we can now decrypt the client's traffic - clientStream = sslStream; - } + } catch { if (sslStream != null) @@ -226,7 +234,13 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http } var httpRemoteUri = new Uri(!isHttps ? httpCmdSplit[1] : (string.Concat("https://", args.WebSession.Request.Host, httpCmdSplit[1]))); - +#if DEBUG + if (httpRemoteUri.Host.Contains("localhost")) + { + Dispose(client, clientStream, clientStreamReader, clientStreamWriter, null); + break; + } +#endif args.WebSession.Request.RequestUri = httpRemoteUri; args.WebSession.Request.Method = httpMethod; @@ -235,7 +249,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http args.Client.ClientStreamReader = clientStreamReader; args.Client.ClientStreamWriter = clientStreamWriter; - + PrepareRequestHeaders(args.WebSession.Request.RequestHeaders, args.WebSession); args.WebSession.Request.Host = args.WebSession.Request.RequestUri.Authority; @@ -250,7 +264,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http handlerTasks[i] = ((Func)invocationList[i])(null, args); } - await Task.WhenAll(handlerTasks).ConfigureAwait(false); + await Task.WhenAll(handlerTasks).ConfigureAwait(false); } if (args.WebSession.Request.UpgradeToWebSocket) @@ -263,8 +277,8 @@ await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHe //construct the web request that we are going to issue on behalf of the client. connection = connection == null ? - await TcpConnectionManager.GetClient(args, args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) - : lastRequest != args.WebSession.Request.RequestUri.Host ? await TcpConnectionManager.GetClient(args, args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) + await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) + : lastRequest != args.WebSession.Request.RequestUri.Host ? await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) : connection; lastRequest = TcpConnectionManager.GetConnectionKey(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version); @@ -294,7 +308,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "100", { await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", "Expectation Failed", args.Client.ClientStreamWriter); - await args.Client.ClientStreamWriter.WriteLineAsync(); + await args.Client.ClientStreamWriter.WriteLineAsync(); } if (!args.WebSession.Request.ExpectContinue) @@ -306,7 +320,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", //If request was modified by user if (args.WebSession.Request.RequestBodyRead) { - if(args.WebSession.Request.ContentEncoding!=null) + if (args.WebSession.Request.ContentEncoding != null) { args.WebSession.Request.RequestBody = await GetCompressedResponseBody(args.WebSession.Request.ContentEncoding, args.WebSession.Request.RequestBody); } @@ -354,12 +368,12 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", } if (connection != null) - await TcpConnectionManager.ReleaseClient(connection); + await TcpConnectionManager.ReleaseClient(connection); } private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, Version httpVersion) { - await clientStreamWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2}", httpVersion.Major, httpVersion.Minor,"200 Connection established")).ConfigureAwait(false); + await clientStreamWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2}", httpVersion.Major, httpVersion.Minor, "200 Connection established")).ConfigureAwait(false); await clientStreamWriter.WriteLineAsync(string.Format("Timestamp: {0}", DateTime.Now)).ConfigureAwait(false); await clientStreamWriter.WriteLineAsync().ConfigureAwait(false); await clientStreamWriter.FlushAsync().ConfigureAwait(false); @@ -446,13 +460,12 @@ internal static bool ValidateServerCertificate( X509Chain chain, SslPolicyErrors sslPolicyErrors) { - var param = sender as CustomSslStream; + var customSslStream = sender as CustomSslStream; if (ServerCertificateValidationCallback != null) { var args = new CertificateValidationEventArgs(); - args.Session = param.Session; args.Certificate = certificate; args.Chain = chain; args.SslPolicyErrors = sslPolicyErrors; @@ -468,11 +481,6 @@ internal static bool ValidateServerCertificate( Task.WhenAll(handlerTasks).Wait(); - if (!args.IsValid) - { - param.Session.WebSession.Request.CancelRequest = true; - } - return args.IsValid; } @@ -484,6 +492,87 @@ internal static bool ValidateServerCertificate( return false; } + /// + /// Call back to select client certificate used for mutual authentication + /// + /// + /// + /// + /// + /// + internal static X509Certificate SelectClientCertificate( + object sender, + string targetHost, + X509CertificateCollection localCertificates, + X509Certificate remoteCertificate, + string[] acceptableIssuers) + { + X509Certificate clientCertificate = null; + var customSslStream = sender as CustomSslStream; + + if (customSslStream.Param is ConnectRequest && remoteCertificate != null) + { + var connectRequest = customSslStream.Param as ConnectRequest; + + var sslStream = new SslStream(connectRequest.Stream, true); + + var certificate = CertManager.CreateCertificate(connectRequest.Uri.Host, false).Result; + //Successfully managed to authenticate the client using the fake certificate + sslStream.AuthenticateAsServerAsync(certificate, true, + Constants.SupportedProtocols, false).Wait(); + + connectRequest.Stream = sslStream; + + clientCertificate = sslStream.RemoteCertificate; + + } + else if (customSslStream.Param is ConnectRequest) + { + if (acceptableIssuers != null && + acceptableIssuers.Length > 0 && + localCertificates != null && + localCertificates.Count > 0) + { + // Use the first certificate that is from an acceptable issuer. + foreach (X509Certificate certificate in localCertificates) + { + string issuer = certificate.Issuer; + if (Array.IndexOf(acceptableIssuers, issuer) != -1) + clientCertificate = certificate; + } + } + + if (localCertificates != null && + localCertificates.Count > 0) + clientCertificate = localCertificates[0]; + } + + if (ClientCertificateSelectionCallback != null) + { + var args = new CertificateSelectionEventArgs(); + + args.targetHost = targetHost; + args.localCertificates = localCertificates; + args.remoteCertificate = remoteCertificate; + args.acceptableIssuers = acceptableIssuers; + args.clientCertificate = clientCertificate; + + Delegate[] invocationList = ClientCertificateSelectionCallback.GetInvocationList(); + Task[] handlerTasks = new Task[invocationList.Length]; + + for (int i = 0; i < invocationList.Length; i++) + { + handlerTasks[i] = ((Func)invocationList[i])(null, args); + } + + Task.WhenAll(handlerTasks).Wait(); + + return args.clientCertificate; + } + + return clientCertificate; + + } } } \ No newline at end of file diff --git a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj index 596166e7e..8b05565b7 100644 --- a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj +++ b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj @@ -60,7 +60,9 @@ + + From ed3ce01cc9fb33baa30dbc1621b6ff2c69f8f17d Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 4 Jun 2016 01:34:17 -0400 Subject: [PATCH 12/29] remove caching which can cause errors --- Titanium.Web.Proxy/Helpers/CertificateManager.cs | 1 + Titanium.Web.Proxy/Network/TcpConnectionManager.cs | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Titanium.Web.Proxy/Helpers/CertificateManager.cs b/Titanium.Web.Proxy/Helpers/CertificateManager.cs index 46604f733..8bf52beb9 100644 --- a/Titanium.Web.Proxy/Helpers/CertificateManager.cs +++ b/Titanium.Web.Proxy/Helpers/CertificateManager.cs @@ -129,6 +129,7 @@ protected async virtual Task CreateCertificate(X509Store store } store.Close(); + if (certificate != null && !certificateCache.ContainsKey(certificateName)) certificateCache.Add(certificateName, new CachedCertificate() { Certificate = certificate }); diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index ed3518be7..d8e4c3c3f 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -80,14 +80,6 @@ internal static async Task GetClient(ConnectRequest connectReques if (cached == null) cached = await CreateClient(connectRequest, hostname, port, isHttps, version).ConfigureAwait(false); - - //just create one more preemptively - if (cachedConnections == null || cachedConnections.Count() < 2) - { - var task = CreateClient(connectRequest, hostname, port, isHttps, version) - .ContinueWith(async (x) => { if (x.Status == TaskStatus.RanToCompletion) await ReleaseClient(x.Result); }); - } - return cached; } From c3e407866ce21f0bede8a5a4c51d7f8290d84ca8 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 4 Jun 2016 02:54:24 -0400 Subject: [PATCH 13/29] fix performance issues --- Titanium.Web.Proxy/Models/ConnectRequest.cs | 11 --- Titanium.Web.Proxy/Network/CustomSslStream.cs | 4 - .../Network/TcpConnectionManager.cs | 21 +++-- Titanium.Web.Proxy/RequestHandler.cs | 87 ++++++------------- Titanium.Web.Proxy/Titanium.Web.Proxy.csproj | 1 - 5 files changed, 41 insertions(+), 83 deletions(-) delete mode 100644 Titanium.Web.Proxy/Models/ConnectRequest.cs diff --git a/Titanium.Web.Proxy/Models/ConnectRequest.cs b/Titanium.Web.Proxy/Models/ConnectRequest.cs deleted file mode 100644 index d7e2e080e..000000000 --- a/Titanium.Web.Proxy/Models/ConnectRequest.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; -using System.IO; - -namespace Titanium.Web.Proxy.Models -{ - internal class ConnectRequest - { - internal Stream Stream { get; set; } - internal Uri Uri { get; set; } - } -} diff --git a/Titanium.Web.Proxy/Network/CustomSslStream.cs b/Titanium.Web.Proxy/Network/CustomSslStream.cs index 0762f8478..0bf5496bd 100644 --- a/Titanium.Web.Proxy/Network/CustomSslStream.cs +++ b/Titanium.Web.Proxy/Network/CustomSslStream.cs @@ -14,10 +14,6 @@ namespace Titanium.Web.Proxy.Network /// internal class CustomSslStream : SslStream { - /// - /// Holds the current session - /// - internal object Param { get; set; } internal CustomSslStream(Stream innerStream, bool leaveInnerStreamOpen, RemoteCertificateValidationCallback userCertificateValidationCallback, LocalCertificateSelectionCallback clientCertificateSelectionCallback) :base(innerStream, leaveInnerStreamOpen, userCertificateValidationCallback, clientCertificateSelectionCallback) diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index d8e4c3c3f..645692f19 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -40,11 +40,8 @@ internal class TcpConnectionManager static Dictionary> connectionCache = new Dictionary>(); static SemaphoreSlim connectionAccessLock = new SemaphoreSlim(1); + internal static async Task GetClient(string hostname, int port, bool isHttps, Version version) - { - return await GetClient(null, hostname, port, isHttps, version); - } - internal static async Task GetClient(ConnectRequest connectRequest, string hostname, int port, bool isHttps, Version version) { List cachedConnections = null; TcpConnection cached = null; @@ -71,14 +68,24 @@ internal static async Task GetClient(ConnectRequest connectReques finally { connectionAccessLock.Release(); } if (cached != null && !cached.TcpClient.Client.IsConnected()) + { + cached.TcpClient.Client.Dispose(); + cached.TcpClient.Close(); continue; + } if (cached == null) break; } if (cached == null) - cached = await CreateClient(connectRequest, hostname, port, isHttps, version).ConfigureAwait(false); + cached = await CreateClient(hostname, port, isHttps, version).ConfigureAwait(false); + + if (cachedConnections == null || cachedConnections.Count() <= 2) + { + await CreateClient(hostname, port, isHttps, version) + .ContinueWith(async (x) => { if (x.Status == TaskStatus.RanToCompletion) await ReleaseClient(x.Result); }); + } return cached; } @@ -88,7 +95,7 @@ internal static string GetConnectionKey(string hostname, int port, bool isHttps, return string.Format("{0}:{1}:{2}:{3}:{4}", hostname.ToLower(), port, isHttps, version.Major, version.Minor); } - private static async Task CreateClient(ConnectRequest connectRequest, string hostname, int port, bool isHttps, Version version) + private static async Task CreateClient(string hostname, int port, bool isHttps, Version version) { TcpClient client; Stream stream; @@ -132,7 +139,6 @@ private static async Task CreateClient(ConnectRequest connectRequ { sslStream = new CustomSslStream(stream, true, new RemoteCertificateValidationCallback(ProxyServer.ValidateServerCertificate), new LocalCertificateSelectionCallback(ProxyServer.SelectClientCertificate)); - sslStream.Param = connectRequest; await sslStream.AuthenticateAsClientAsync(hostname, null, Constants.SupportedProtocols, false).ConfigureAwait(false); stream = (Stream)sslStream; } @@ -172,6 +178,7 @@ private static async Task CreateClient(ConnectRequest connectRequ internal static async Task ReleaseClient(TcpConnection connection) { + connection.LastAccess = DateTime.Now; var key = GetConnectionKey(connection.HostName, connection.port, connection.IsHttps, connection.Version); await connectionAccessLock.WaitAsync(); diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index f0363721b..8b7fe7143 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -72,29 +72,23 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient await WriteConnectResponse(clientStreamWriter, version).ConfigureAwait(false); - var certificate = await CertManager.CreateCertificate(httpRemoteUri.Host, false); + SslStream sslStream = null; try { - var connectRequest = new ConnectRequest() { Stream = clientStream, Uri = httpRemoteUri }; - await TcpConnectionManager.GetClient(connectRequest, httpRemoteUri.Host, httpRemoteUri.Port, true, version).ConfigureAwait(false); + await TcpConnectionManager.GetClient(httpRemoteUri.Host, httpRemoteUri.Port, true, version).ConfigureAwait(false); + + sslStream = new SslStream(clientStream, true); + var certificate = await CertManager.CreateCertificate(httpRemoteUri.Host, false); + //Successfully managed to authenticate the client using the fake certificate + await sslStream.AuthenticateAsServerAsync(certificate, false, + Constants.SupportedProtocols, false).ConfigureAwait(false); + //HTTPS server created - we can now decrypt the client's traffic + clientStream = sslStream; - if (clientStream is SslStream) - { - sslStream = clientStream as SslStream; - } - else - { - sslStream = new SslStream(clientStream, true); - //Successfully managed to authenticate the client using the fake certificate - await sslStream.AuthenticateAsServerAsync(certificate, false, - Constants.SupportedProtocols, false).ConfigureAwait(false); - //HTTPS server created - we can now decrypt the client's traffic - clientStream = sslStream; - } clientStreamReader = new CustomBinaryReader(sslStream); clientStreamWriter = new StreamWriter(sslStream); @@ -192,8 +186,6 @@ await HandleHttpSessionRequest(tcpClient, httpCmd, clientStream, clientStreamRea private static async Task HandleHttpSessionRequest(TcpClient client, string httpCmd, Stream clientStream, CustomBinaryReader clientStreamReader, StreamWriter clientStreamWriter, bool isHttps) { - TcpConnection connection = null; - string lastRequest = null; while (true) { @@ -276,12 +268,8 @@ await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHe } //construct the web request that we are going to issue on behalf of the client. - connection = connection == null ? - await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) - : lastRequest != args.WebSession.Request.RequestUri.Host ? await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false) - : connection; + var connection = await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false); - lastRequest = TcpConnectionManager.GetConnectionKey(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version); args.WebSession.Request.RequestLocked = true; @@ -350,11 +338,12 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", //if connection is closing exit if (args.WebSession.Response.ResponseKeepAlive == false) { - connection.TcpClient.Close(); Dispose(client, clientStream, clientStreamReader, clientStreamWriter, args); return; } + await TcpConnectionManager.ReleaseClient(connection); + // read the next request httpCmd = await clientStreamReader.ReadLineAsync().ConfigureAwait(false); @@ -367,8 +356,6 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", } - if (connection != null) - await TcpConnectionManager.ReleaseClient(connection); } private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, Version httpVersion) @@ -460,8 +447,6 @@ internal static bool ValidateServerCertificate( X509Chain chain, SslPolicyErrors sslPolicyErrors) { - var customSslStream = sender as CustomSslStream; - if (ServerCertificateValidationCallback != null) { var args = new CertificateValidationEventArgs(); @@ -510,43 +495,25 @@ internal static X509Certificate SelectClientCertificate( X509Certificate clientCertificate = null; var customSslStream = sender as CustomSslStream; - if (customSslStream.Param is ConnectRequest && remoteCertificate != null) - { - var connectRequest = customSslStream.Param as ConnectRequest; - - var sslStream = new SslStream(connectRequest.Stream, true); - - var certificate = CertManager.CreateCertificate(connectRequest.Uri.Host, false).Result; - //Successfully managed to authenticate the client using the fake certificate - sslStream.AuthenticateAsServerAsync(certificate, true, - Constants.SupportedProtocols, false).Wait(); - - connectRequest.Stream = sslStream; - - clientCertificate = sslStream.RemoteCertificate; - - } - else if (customSslStream.Param is ConnectRequest) + if (acceptableIssuers != null && + acceptableIssuers.Length > 0 && + localCertificates != null && + localCertificates.Count > 0) { - if (acceptableIssuers != null && - acceptableIssuers.Length > 0 && - localCertificates != null && - localCertificates.Count > 0) + // Use the first certificate that is from an acceptable issuer. + foreach (X509Certificate certificate in localCertificates) { - // Use the first certificate that is from an acceptable issuer. - foreach (X509Certificate certificate in localCertificates) - { - string issuer = certificate.Issuer; - if (Array.IndexOf(acceptableIssuers, issuer) != -1) - clientCertificate = certificate; - } + string issuer = certificate.Issuer; + if (Array.IndexOf(acceptableIssuers, issuer) != -1) + clientCertificate = certificate; } - - if (localCertificates != null && - localCertificates.Count > 0) - clientCertificate = localCertificates[0]; } + if (localCertificates != null && + localCertificates.Count > 0) + clientCertificate = localCertificates[0]; + + if (ClientCertificateSelectionCallback != null) { var args = new CertificateSelectionEventArgs(); diff --git a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj index 8b05565b7..4327e0a95 100644 --- a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj +++ b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj @@ -62,7 +62,6 @@ - From a8e16d49da65ff852bafa489e5fa6f5f821a4b63 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 4 Jun 2016 02:59:22 -0400 Subject: [PATCH 14/29] remove async --- Titanium.Web.Proxy/Network/TcpConnectionManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index 645692f19..06e0c24e1 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -83,7 +83,7 @@ internal static async Task GetClient(string hostname, int port, b if (cachedConnections == null || cachedConnections.Count() <= 2) { - await CreateClient(hostname, port, isHttps, version) + var task = CreateClient(hostname, port, isHttps, version) .ContinueWith(async (x) => { if (x.Status == TaskStatus.RanToCompletion) await ReleaseClient(x.Result); }); } From 8211b5467f65eb9dc1fd1a1a8193380a07a48d88 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 4 Jun 2016 03:13:11 -0400 Subject: [PATCH 15/29] connection perf --- Titanium.Web.Proxy/RequestHandler.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 8b7fe7143..93eb0dce0 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -186,6 +186,7 @@ await HandleHttpSessionRequest(tcpClient, httpCmd, clientStream, clientStreamRea private static async Task HandleHttpSessionRequest(TcpClient client, string httpCmd, Stream clientStream, CustomBinaryReader clientStreamReader, StreamWriter clientStreamWriter, bool isHttps) { + TcpConnection connection = null; while (true) { @@ -268,7 +269,7 @@ await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHe } //construct the web request that we are going to issue on behalf of the client. - var connection = await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false); + connection = await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false); args.WebSession.Request.RequestLocked = true; @@ -342,8 +343,6 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", return; } - await TcpConnectionManager.ReleaseClient(connection); - // read the next request httpCmd = await clientStreamReader.ReadLineAsync().ConfigureAwait(false); @@ -356,6 +355,8 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", } + if (connection!=null) + await TcpConnectionManager.ReleaseClient(connection); } private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, Version httpVersion) From 3a41b1d36b18dc85f59bbe118d71d5af1e679d14 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 4 Jun 2016 13:13:58 -0400 Subject: [PATCH 16/29] fix proxy-connection --- Titanium.Web.Proxy/ResponseHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Titanium.Web.Proxy/ResponseHandler.cs b/Titanium.Web.Proxy/ResponseHandler.cs index 95aa7e717..233ddfd21 100644 --- a/Titanium.Web.Proxy/ResponseHandler.cs +++ b/Titanium.Web.Proxy/ResponseHandler.cs @@ -140,7 +140,7 @@ private static void FixResponseProxyHeaders(List headers) } else { - connectionHeader.Value = "close"; + connectionHeader.Value = proxyHeader.Value; } headers.RemoveAll(x => x.Name.ToLower() == "proxy-connection"); From 5d2536eab6e80fff2419eabdc2a8321cfcf162c8 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 4 Jun 2016 13:38:37 -0400 Subject: [PATCH 17/29] Fix async causing performance issues --- Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs | 2 +- Titanium.Web.Proxy/RequestHandler.cs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs index 7fc4437e1..26976fa14 100644 --- a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs +++ b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs @@ -39,7 +39,7 @@ internal async Task ReadLineAsync() var lastChar = default(char); var buffer = new byte[1]; - while (this.stream.Read(buffer, 0, 1) > 0) + while ((await this.stream.ReadAsync(buffer, 0, 1)) > 0) { if (lastChar == '\r' && buffer[0] == '\n') { diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 93eb0dce0..5ce4631b0 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -233,6 +233,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http Dispose(client, clientStream, clientStreamReader, clientStreamWriter, null); break; } + #endif args.WebSession.Request.RequestUri = httpRemoteUri; @@ -269,7 +270,7 @@ await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHe } //construct the web request that we are going to issue on behalf of the client. - connection = await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false); + connection = await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false); args.WebSession.Request.RequestLocked = true; @@ -343,6 +344,8 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", return; } + await TcpConnectionManager.ReleaseClient(connection); + // read the next request httpCmd = await clientStreamReader.ReadLineAsync().ConfigureAwait(false); @@ -355,8 +358,6 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", } - if (connection!=null) - await TcpConnectionManager.ReleaseClient(connection); } private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, Version httpVersion) From f6aa5914c452234333967b277893b001d9cd30e6 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 4 Jun 2016 17:05:35 -0400 Subject: [PATCH 18/29] release client --- Titanium.Web.Proxy/RequestHandler.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 5ce4631b0..b08f5ea9d 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -79,7 +79,8 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient try { - await TcpConnectionManager.GetClient(httpRemoteUri.Host, httpRemoteUri.Port, true, version).ConfigureAwait(false); + var tunnelClient = await TcpConnectionManager.GetClient(httpRemoteUri.Host, httpRemoteUri.Port, true, version).ConfigureAwait(false); + await TcpConnectionManager.ReleaseClient(tunnelClient); sslStream = new SslStream(clientStream, true); var certificate = await CertManager.CreateCertificate(httpRemoteUri.Host, false); From 5051fd5ee27dc2720a6d1442f71490d2c76fa439 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 11 Jun 2016 15:02:17 -0400 Subject: [PATCH 19/29] Expose client IP Address #77 --- .../Decompression/DeflateDecompression.cs | 2 +- .../Decompression/GZipDecompression.cs | 2 +- .../Decompression/ZlibDecompression.cs | 2 +- .../EventArguments/SessionEventArgs.cs | 18 ++++++-- .../Extensions/HttpWebRequestExtensions.cs | 2 +- .../Extensions/HttpWebResponseExtensions.cs | 2 +- .../Extensions/StreamExtensions.cs | 4 +- .../Helpers/CustomBinaryReader.cs | 8 ++-- Titanium.Web.Proxy/Helpers/Tcp.cs | 2 +- Titanium.Web.Proxy/Http/HttpWebClient.cs | 6 +-- Titanium.Web.Proxy/Network/CustomSslStream.cs | 24 ----------- .../Network/TcpConnectionManager.cs | 8 ++-- Titanium.Web.Proxy/RequestHandler.cs | 30 ++++++------- Titanium.Web.Proxy/ResponseHandler.cs | 42 +++++++++---------- .../{Constants.cs => ProxyConstants.cs} | 11 +++-- Titanium.Web.Proxy/Titanium.Web.Proxy.csproj | 3 +- 16 files changed, 75 insertions(+), 91 deletions(-) delete mode 100644 Titanium.Web.Proxy/Network/CustomSslStream.cs rename Titanium.Web.Proxy/Shared/{Constants.cs => ProxyConstants.cs} (70%) diff --git a/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs b/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs index 02b184b82..e7fddd3b0 100644 --- a/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs @@ -13,7 +13,7 @@ public async Task Decompress(byte[] compressedArray) using (var decompressor = new DeflateStream(stream, CompressionMode.Decompress)) { - var buffer = new byte[Constants.BUFFER_SIZE]; + var buffer = new byte[ProxyConstants.BUFFER_SIZE]; using (var output = new MemoryStream()) { diff --git a/Titanium.Web.Proxy/Decompression/GZipDecompression.cs b/Titanium.Web.Proxy/Decompression/GZipDecompression.cs index 8df6c7be9..e18414c53 100644 --- a/Titanium.Web.Proxy/Decompression/GZipDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/GZipDecompression.cs @@ -11,7 +11,7 @@ public async Task Decompress(byte[] compressedArray) { using (var decompressor = new GZipStream(new MemoryStream(compressedArray), CompressionMode.Decompress)) { - var buffer = new byte[Constants.BUFFER_SIZE]; + var buffer = new byte[ProxyConstants.BUFFER_SIZE]; using (var output = new MemoryStream()) { int read; diff --git a/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs b/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs index 9cede3d54..24d797149 100644 --- a/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs @@ -12,7 +12,7 @@ public async Task Decompress(byte[] compressedArray) var memoryStream = new MemoryStream(compressedArray); using (var decompressor = new ZlibStream(memoryStream, CompressionMode.Decompress)) { - var buffer = new byte[Constants.BUFFER_SIZE]; + var buffer = new byte[ProxyConstants.BUFFER_SIZE]; using (var output = new MemoryStream()) { diff --git a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs index de1ce7523..9158ef8e0 100644 --- a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs @@ -8,6 +8,8 @@ using Titanium.Web.Proxy.Extensions; using System.Threading.Tasks; using Titanium.Web.Proxy.Network; +using System.Net; +using System.Net.Sockets; namespace Titanium.Web.Proxy.EventArguments { @@ -24,26 +26,34 @@ public class SessionEventArgs : EventArgs, IDisposable /// internal SessionEventArgs() { - Client = new ProxyClient(); + ProxyClient = new ProxyClient(); WebSession = new HttpWebClient(); } /// /// Holds a reference to server connection /// - internal ProxyClient Client { get; set; } + internal ProxyClient ProxyClient { get; set; } /// /// Does this session uses SSL /// public bool IsHttps => WebSession.Request.RequestUri.Scheme == Uri.UriSchemeHttps; + + public IPAddress ClientIpAddress => ((IPEndPoint)Client.Client.RemoteEndPoint).Address; + /// /// A web session corresponding to a single request/response sequence /// within a proxy connection /// public HttpWebClient WebSession { get; set; } + /// + /// Reference to client connection + /// + internal TcpClient Client { get; set; } + /// /// implement any cleanup here @@ -76,7 +86,7 @@ private async Task ReadRequestBody() //For chunked request we need to read data as they arrive, until we reach a chunk end symbol if (WebSession.Request.IsChunked) { - await this.Client.ClientStreamReader.CopyBytesToStreamChunked(requestBodyStream).ConfigureAwait(false); + await this.ProxyClient.ClientStreamReader.CopyBytesToStreamChunked(requestBodyStream).ConfigureAwait(false); } else { @@ -84,7 +94,7 @@ private async Task ReadRequestBody() if (WebSession.Request.ContentLength > 0) { //If not chunked then its easy just read the amount of bytes mentioned in content length header of response - await this.Client.ClientStreamReader.CopyBytesToStream(requestBodyStream, WebSession.Request.ContentLength).ConfigureAwait(false); + await this.ProxyClient.ClientStreamReader.CopyBytesToStream(requestBodyStream, WebSession.Request.ContentLength).ConfigureAwait(false); } else if(WebSession.Request.HttpVersion.Major == 1 && WebSession.Request.HttpVersion.Minor == 0) diff --git a/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs b/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs index bcc3ff9a8..1e096a416 100644 --- a/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs @@ -19,7 +19,7 @@ internal static Encoding GetEncoding(this Request request) return Encoding.GetEncoding("ISO-8859-1"); //extract the encoding by finding the charset - var contentTypes = request.ContentType.Split(Constants.SemiColonSplit); + var contentTypes = request.ContentType.Split(ProxyConstants.SemiColonSplit); foreach (var contentType in contentTypes) { var encodingSplit = contentType.Split('='); diff --git a/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs b/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs index 029eb107a..d6191faac 100644 --- a/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs @@ -15,7 +15,7 @@ internal static Encoding GetResponseCharacterEncoding(this Response response) return Encoding.GetEncoding("ISO-8859-1"); //extract the encoding by finding the charset - var contentTypes = response.ContentType.Split(Constants.SemiColonSplit); + var contentTypes = response.ContentType.Split(ProxyConstants.SemiColonSplit); foreach (var contentType in contentTypes) { var encodingSplit = contentType.Split('='); diff --git a/Titanium.Web.Proxy/Extensions/StreamExtensions.cs b/Titanium.Web.Proxy/Extensions/StreamExtensions.cs index c86bcb264..e6ccbc4f8 100644 --- a/Titanium.Web.Proxy/Extensions/StreamExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/StreamExtensions.cs @@ -25,12 +25,12 @@ internal static async Task CopyBytesToStream(this CustomBinaryReader streamReade var totalbytesRead = 0; long bytesToRead; - if (totalBytesToRead < Constants.BUFFER_SIZE) + if (totalBytesToRead < ProxyConstants.BUFFER_SIZE) { bytesToRead = totalBytesToRead; } else - bytesToRead = Constants.BUFFER_SIZE; + bytesToRead = ProxyConstants.BUFFER_SIZE; while (totalbytesRead < totalBytesToRead) diff --git a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs index 26976fa14..bbfca3966 100644 --- a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs +++ b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs @@ -78,12 +78,12 @@ internal async Task> ReadAllLinesAsync() internal async Task ReadBytesAsync(long totalBytesToRead) { - int bytesToRead = Constants.BUFFER_SIZE; + int bytesToRead = ProxyConstants.BUFFER_SIZE; - if (totalBytesToRead < Constants.BUFFER_SIZE) + if (totalBytesToRead < ProxyConstants.BUFFER_SIZE) bytesToRead = (int)totalBytesToRead; - var buffer = new byte[Constants.BUFFER_SIZE]; + var buffer = new byte[ProxyConstants.BUFFER_SIZE]; var bytesRead = 0; var totalBytesRead = 0; @@ -100,7 +100,7 @@ internal async Task ReadBytesAsync(long totalBytesToRead) bytesRead = 0; var remainingBytes = (totalBytesToRead - totalBytesRead); - bytesToRead = remainingBytes > (long)Constants.BUFFER_SIZE ? Constants.BUFFER_SIZE : (int)remainingBytes; + bytesToRead = remainingBytes > (long)ProxyConstants.BUFFER_SIZE ? ProxyConstants.BUFFER_SIZE : (int)remainingBytes; } return outStream.ToArray(); diff --git a/Titanium.Web.Proxy/Helpers/Tcp.cs b/Titanium.Web.Proxy/Helpers/Tcp.cs index cd754aa27..f5e5faf05 100644 --- a/Titanium.Web.Proxy/Helpers/Tcp.cs +++ b/Titanium.Web.Proxy/Helpers/Tcp.cs @@ -50,7 +50,7 @@ internal async static Task SendRaw(Stream clientStream, string httpCmd, List - /// Used to pass in Session object for ServerCertificateValidation Callback - /// - internal class CustomSslStream : SslStream - { - - internal CustomSslStream(Stream innerStream, bool leaveInnerStreamOpen, RemoteCertificateValidationCallback userCertificateValidationCallback, LocalCertificateSelectionCallback clientCertificateSelectionCallback) - :base(innerStream, leaveInnerStreamOpen, userCertificateValidationCallback, clientCertificateSelectionCallback) - { - - } - } -} diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index 06e0c24e1..d885f5c7d 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -102,14 +102,14 @@ private static async Task CreateClient(string hostname, int port, if (isHttps) { - CustomSslStream sslStream = null; + SslStream sslStream = null; if (ProxyServer.UpStreamHttpsProxy != null) { client = new TcpClient(ProxyServer.UpStreamHttpsProxy.HostName, ProxyServer.UpStreamHttpsProxy.Port); stream = (Stream)client.GetStream(); - using (var writer = new StreamWriter(stream, Encoding.ASCII, Constants.BUFFER_SIZE, true)) + using (var writer = new StreamWriter(stream, Encoding.ASCII, ProxyConstants.BUFFER_SIZE, true)) { await writer.WriteLineAsync(string.Format("CONNECT {0}:{1} {2}", hostname, port, version)).ConfigureAwait(false); await writer.WriteLineAsync(string.Format("Host: {0}:{1}", hostname, port)).ConfigureAwait(false); @@ -137,9 +137,9 @@ private static async Task CreateClient(string hostname, int port, try { - sslStream = new CustomSslStream(stream, true, new RemoteCertificateValidationCallback(ProxyServer.ValidateServerCertificate), + sslStream = new SslStream(stream, true, new RemoteCertificateValidationCallback(ProxyServer.ValidateServerCertificate), new LocalCertificateSelectionCallback(ProxyServer.SelectClientCertificate)); - await sslStream.AuthenticateAsClientAsync(hostname, null, Constants.SupportedProtocols, false).ConfigureAwait(false); + await sslStream.AuthenticateAsClientAsync(hostname, null, ProxyConstants.SupportedSslProtocols, false).ConfigureAwait(false); stream = (Stream)sslStream; } catch diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index b08f5ea9d..a2768f63c 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -41,7 +41,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient } //break up the line into three components (method, remote URL & Http Version) - var httpCmdSplit = httpCmd.Split(Constants.SpaceSplit, 3); + var httpCmdSplit = httpCmd.Split(ProxyConstants.SpaceSplit, 3); var httpVerb = httpCmdSplit[0]; @@ -86,7 +86,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient var certificate = await CertManager.CreateCertificate(httpRemoteUri.Host, false); //Successfully managed to authenticate the client using the fake certificate await sslStream.AuthenticateAsServerAsync(certificate, false, - Constants.SupportedProtocols, false).ConfigureAwait(false); + ProxyConstants.SupportedSslProtocols, false).ConfigureAwait(false); //HTTPS server created - we can now decrypt the client's traffic clientStream = sslStream; @@ -198,12 +198,12 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http } var args = new SessionEventArgs(); - args.Client.TcpClient = client; + args.Client = client; try { //break up the line into three components (method, remote URL & Http Version) - var httpCmdSplit = httpCmd.Split(Constants.SpaceSplit, 3); + var httpCmdSplit = httpCmd.Split(ProxyConstants.SpaceSplit, 3); var httpMethod = httpCmdSplit[0]; @@ -223,7 +223,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http string tmpLine; while (!string.IsNullOrEmpty(tmpLine = await clientStreamReader.ReadLineAsync().ConfigureAwait(false))) { - var header = tmpLine.Split(Constants.ColonSplit, 2); + var header = tmpLine.Split(ProxyConstants.ColonSplit, 2); args.WebSession.Request.RequestHeaders.Add(new HttpHeader(header[0], header[1])); } @@ -240,9 +240,9 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http args.WebSession.Request.Method = httpMethod; args.WebSession.Request.HttpVersion = version; - args.Client.ClientStream = clientStream; - args.Client.ClientStreamReader = clientStreamReader; - args.Client.ClientStreamWriter = clientStreamWriter; + args.ProxyClient.ClientStream = clientStream; + args.ProxyClient.ClientStreamReader = clientStreamReader; + args.ProxyClient.ClientStreamWriter = clientStreamWriter; PrepareRequestHeaders(args.WebSession.Request.RequestHeaders, args.WebSession); @@ -292,14 +292,14 @@ await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHe if (args.WebSession.Request.Is100Continue) { await WriteResponseStatus(args.WebSession.Response.HttpVersion, "100", - "Continue", args.Client.ClientStreamWriter); - await args.Client.ClientStreamWriter.WriteLineAsync(); + "Continue", args.ProxyClient.ClientStreamWriter); + await args.ProxyClient.ClientStreamWriter.WriteLineAsync(); } else if (args.WebSession.Request.ExpectationFailed) { await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", - "Expectation Failed", args.Client.ClientStreamWriter); - await args.Client.ClientStreamWriter.WriteLineAsync(); + "Expectation Failed", args.ProxyClient.ClientStreamWriter); + await args.ProxyClient.ClientStreamWriter.WriteLineAsync(); } if (!args.WebSession.Request.ExpectContinue) @@ -415,7 +415,7 @@ private static async Task SendClientRequestBody(SessionEventArgs args) { try { - await args.Client.ClientStreamReader.CopyBytesToStream(postStream, args.WebSession.Request.ContentLength).ConfigureAwait(false); + await args.ProxyClient.ClientStreamReader.CopyBytesToStream(postStream, args.WebSession.Request.ContentLength).ConfigureAwait(false); } catch { @@ -427,7 +427,7 @@ private static async Task SendClientRequestBody(SessionEventArgs args) { try { - await args.Client.ClientStreamReader.CopyBytesToStreamChunked(postStream).ConfigureAwait(false); + await args.ProxyClient.ClientStreamReader.CopyBytesToStreamChunked(postStream).ConfigureAwait(false); } catch { @@ -496,7 +496,7 @@ internal static X509Certificate SelectClientCertificate( string[] acceptableIssuers) { X509Certificate clientCertificate = null; - var customSslStream = sender as CustomSslStream; + var customSslStream = sender as SslStream; if (acceptableIssuers != null && acceptableIssuers.Length > 0 && diff --git a/Titanium.Web.Proxy/ResponseHandler.cs b/Titanium.Web.Proxy/ResponseHandler.cs index 233ddfd21..6f4308e53 100644 --- a/Titanium.Web.Proxy/ResponseHandler.cs +++ b/Titanium.Web.Proxy/ResponseHandler.cs @@ -45,18 +45,18 @@ public static async Task HandleHttpSessionResponse(SessionEventArgs args) if (args.WebSession.Response.Is100Continue) { await WriteResponseStatus(args.WebSession.Response.HttpVersion, "100", - "Continue", args.Client.ClientStreamWriter); - await args.Client.ClientStreamWriter.WriteLineAsync(); + "Continue", args.ProxyClient.ClientStreamWriter); + await args.ProxyClient.ClientStreamWriter.WriteLineAsync(); } else if (args.WebSession.Response.ExpectationFailed) { await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", - "Expectation Failed", args.Client.ClientStreamWriter); - await args.Client.ClientStreamWriter.WriteLineAsync(); + "Expectation Failed", args.ProxyClient.ClientStreamWriter); + await args.ProxyClient.ClientStreamWriter.WriteLineAsync(); } await WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession.Response.ResponseStatusCode, - args.WebSession.Response.ResponseStatusDescription, args.Client.ClientStreamWriter); + args.WebSession.Response.ResponseStatusDescription, args.ProxyClient.ClientStreamWriter); if (args.WebSession.Response.ResponseBodyRead) { @@ -73,24 +73,24 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession. args.WebSession.Response.ContentLength = -1; } - await WriteResponseHeaders(args.Client.ClientStreamWriter, args.WebSession.Response.ResponseHeaders).ConfigureAwait(false); - await WriteResponseBody(args.Client.ClientStream, args.WebSession.Response.ResponseBody, isChunked).ConfigureAwait(false); + await WriteResponseHeaders(args.ProxyClient.ClientStreamWriter, args.WebSession.Response.ResponseHeaders).ConfigureAwait(false); + await WriteResponseBody(args.ProxyClient.ClientStream, args.WebSession.Response.ResponseBody, isChunked).ConfigureAwait(false); } else { - await WriteResponseHeaders(args.Client.ClientStreamWriter, args.WebSession.Response.ResponseHeaders); + await WriteResponseHeaders(args.ProxyClient.ClientStreamWriter, args.WebSession.Response.ResponseHeaders); if (args.WebSession.Response.IsChunked || args.WebSession.Response.ContentLength > 0 || (args.WebSession.Response.HttpVersion.Major == 1 && args.WebSession.Response.HttpVersion.Minor == 0)) - await WriteResponseBody(args.WebSession.ServerConnection.StreamReader, args.Client.ClientStream, args.WebSession.Response.IsChunked, args.WebSession.Response.ContentLength).ConfigureAwait(false); + await WriteResponseBody(args.WebSession.ServerConnection.StreamReader, args.ProxyClient.ClientStream, args.WebSession.Response.IsChunked, args.WebSession.Response.ContentLength).ConfigureAwait(false); } - await args.Client.ClientStream.FlushAsync(); + await args.ProxyClient.ClientStream.FlushAsync(); } catch { - Dispose(args.Client.TcpClient, args.Client.ClientStream, args.Client.ClientStreamReader, args.Client.ClientStreamWriter, args); + Dispose(args.ProxyClient.TcpClient, args.ProxyClient.ClientStream, args.ProxyClient.ClientStreamReader, args.ProxyClient.ClientStreamWriter, args); } finally { @@ -164,12 +164,12 @@ private static async Task WriteResponseBody(CustomBinaryReader inStreamReader, S if (ContentLength == -1) ContentLength = long.MaxValue; - int bytesToRead = Constants.BUFFER_SIZE; + int bytesToRead = ProxyConstants.BUFFER_SIZE; - if (ContentLength < Constants.BUFFER_SIZE) + if (ContentLength < ProxyConstants.BUFFER_SIZE) bytesToRead = (int)ContentLength; - var buffer = new byte[Constants.BUFFER_SIZE]; + var buffer = new byte[ProxyConstants.BUFFER_SIZE]; var bytesRead = 0; var totalBytesRead = 0; @@ -184,7 +184,7 @@ private static async Task WriteResponseBody(CustomBinaryReader inStreamReader, S bytesRead = 0; var remainingBytes = (ContentLength - totalBytesRead); - bytesToRead = remainingBytes > (long)Constants.BUFFER_SIZE ? Constants.BUFFER_SIZE : (int)remainingBytes; + bytesToRead = remainingBytes > (long)ProxyConstants.BUFFER_SIZE ? ProxyConstants.BUFFER_SIZE : (int)remainingBytes; } } else @@ -206,17 +206,17 @@ private static async Task WriteResponseBodyChunked(CustomBinaryReader inStreamRe var chunkHeadBytes = Encoding.ASCII.GetBytes(chunkSize.ToString("x2")); await outStream.WriteAsync(chunkHeadBytes, 0, chunkHeadBytes.Length).ConfigureAwait(false); - await outStream.WriteAsync(Constants.NewLineBytes, 0, Constants.NewLineBytes.Length).ConfigureAwait(false); + await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length).ConfigureAwait(false); await outStream.WriteAsync(buffer, 0, chunkSize).ConfigureAwait(false); - await outStream.WriteAsync(Constants.NewLineBytes, 0, Constants.NewLineBytes.Length).ConfigureAwait(false); + await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length).ConfigureAwait(false); await inStreamReader.ReadLineAsync().ConfigureAwait(false); } else { await inStreamReader.ReadLineAsync().ConfigureAwait(false); - await outStream.WriteAsync(Constants.ChunkEnd, 0, Constants.ChunkEnd.Length).ConfigureAwait(false); + await outStream.WriteAsync(ProxyConstants.ChunkEnd, 0, ProxyConstants.ChunkEnd.Length).ConfigureAwait(false); break; } } @@ -227,11 +227,11 @@ private static async Task WriteResponseBodyChunked(byte[] data, Stream outStream var chunkHead = Encoding.ASCII.GetBytes(data.Length.ToString("x2")); await outStream.WriteAsync(chunkHead, 0, chunkHead.Length).ConfigureAwait(false); - await outStream.WriteAsync(Constants.NewLineBytes, 0, Constants.NewLineBytes.Length).ConfigureAwait(false); + await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length).ConfigureAwait(false); await outStream.WriteAsync(data, 0, data.Length).ConfigureAwait(false); - await outStream.WriteAsync(Constants.NewLineBytes, 0, Constants.NewLineBytes.Length).ConfigureAwait(false); + await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length).ConfigureAwait(false); - await outStream.WriteAsync(Constants.ChunkEnd, 0, Constants.ChunkEnd.Length).ConfigureAwait(false); + await outStream.WriteAsync(ProxyConstants.ChunkEnd, 0, ProxyConstants.ChunkEnd.Length).ConfigureAwait(false); } diff --git a/Titanium.Web.Proxy/Shared/Constants.cs b/Titanium.Web.Proxy/Shared/ProxyConstants.cs similarity index 70% rename from Titanium.Web.Proxy/Shared/Constants.cs rename to Titanium.Web.Proxy/Shared/ProxyConstants.cs index 5b9934c01..e5583718c 100644 --- a/Titanium.Web.Proxy/Shared/Constants.cs +++ b/Titanium.Web.Proxy/Shared/ProxyConstants.cs @@ -1,17 +1,14 @@ using System; using System.Security.Authentication; using System.Text; -using System.Text.RegularExpressions; namespace Titanium.Web.Proxy.Shared { /// /// Literals shared by Proxy Server /// - internal class Constants - { - internal static readonly int BUFFER_SIZE = 8192; - + internal class ProxyConstants + { internal static readonly char[] SpaceSplit = { ' ' }; internal static readonly char[] ColonSplit = { ':' }; internal static readonly char[] SemiColonSplit = { ';' }; @@ -21,6 +18,8 @@ internal class Constants internal static readonly byte[] ChunkEnd = Encoding.ASCII.GetBytes(0.ToString("x2") + Environment.NewLine + Environment.NewLine); - internal static SslProtocols SupportedProtocols = SslProtocols.Tls | SslProtocols.Tls11 | SslProtocols.Tls12 | SslProtocols.Ssl3; + public static SslProtocols SupportedSslProtocols = SslProtocols.Tls | SslProtocols.Tls11 | SslProtocols.Tls12 | SslProtocols.Ssl3; + + public static readonly int BUFFER_SIZE = 8192; } } diff --git a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj index 4327e0a95..e5007a17b 100644 --- a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj +++ b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj @@ -74,7 +74,6 @@ - @@ -88,7 +87,7 @@ - + From af36eceecc82bfe6a7c3233e4ecc683965361f88 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 11 Jun 2016 15:20:20 -0400 Subject: [PATCH 20/29] Expose both IP & port --- Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs | 4 ++-- Titanium.Web.Proxy/RequestHandler.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs index 9158ef8e0..2aa1d0122 100644 --- a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs @@ -41,7 +41,7 @@ internal SessionEventArgs() public bool IsHttps => WebSession.Request.RequestUri.Scheme == Uri.UriSchemeHttps; - public IPAddress ClientIpAddress => ((IPEndPoint)Client.Client.RemoteEndPoint).Address; + public EndPoint ClientEndPoint => (IPEndPoint)TcpClient.Client.RemoteEndPoint; /// /// A web session corresponding to a single request/response sequence @@ -52,7 +52,7 @@ internal SessionEventArgs() /// /// Reference to client connection /// - internal TcpClient Client { get; set; } + internal TcpClient TcpClient { get; set; } /// diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index a2768f63c..6cf8df3aa 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -198,7 +198,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http } var args = new SessionEventArgs(); - args.Client = client; + args.TcpClient = client; try { From 3ef9595d3f67d5cc7d709da557490def3746be45 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 11 Jun 2016 15:22:09 -0400 Subject: [PATCH 21/29] EndPoint to IpEndPoint --- Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs index 2aa1d0122..b4229974b 100644 --- a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs @@ -41,7 +41,7 @@ internal SessionEventArgs() public bool IsHttps => WebSession.Request.RequestUri.Scheme == Uri.UriSchemeHttps; - public EndPoint ClientEndPoint => (IPEndPoint)TcpClient.Client.RemoteEndPoint; + public IPEndPoint ClientEndPoint => (IPEndPoint)TcpClient.Client.RemoteEndPoint; /// /// A web session corresponding to a single request/response sequence From 8bc50e765d40d2272fc9151deb047768aad9cf23 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 11 Jun 2016 15:46:33 -0400 Subject: [PATCH 22/29] Fix bug in dispose param --- Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs | 9 ++------- Titanium.Web.Proxy/Http/HttpWebClient.cs | 3 +++ Titanium.Web.Proxy/Network/ProxyClient.cs | 8 ++++---- Titanium.Web.Proxy/RequestHandler.cs | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs index b4229974b..b48b9477a 100644 --- a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs @@ -31,7 +31,7 @@ internal SessionEventArgs() } /// - /// Holds a reference to server connection + /// Holds a reference to client /// internal ProxyClient ProxyClient { get; set; } @@ -41,7 +41,7 @@ internal SessionEventArgs() public bool IsHttps => WebSession.Request.RequestUri.Scheme == Uri.UriSchemeHttps; - public IPEndPoint ClientEndPoint => (IPEndPoint)TcpClient.Client.RemoteEndPoint; + public IPEndPoint ClientEndPoint => (IPEndPoint)ProxyClient.TcpClient.Client.RemoteEndPoint; /// /// A web session corresponding to a single request/response sequence @@ -49,11 +49,6 @@ internal SessionEventArgs() /// public HttpWebClient WebSession { get; set; } - /// - /// Reference to client connection - /// - internal TcpClient TcpClient { get; set; } - /// /// implement any cleanup here diff --git a/Titanium.Web.Proxy/Http/HttpWebClient.cs b/Titanium.Web.Proxy/Http/HttpWebClient.cs index cd4d7e950..79586c3a0 100644 --- a/Titanium.Web.Proxy/Http/HttpWebClient.cs +++ b/Titanium.Web.Proxy/Http/HttpWebClient.cs @@ -11,6 +11,9 @@ namespace Titanium.Web.Proxy.Http { public class HttpWebClient { + /// + /// Connection to server + /// internal TcpConnection ServerConnection { get; set; } public Request Request { get; set; } diff --git a/Titanium.Web.Proxy/Network/ProxyClient.cs b/Titanium.Web.Proxy/Network/ProxyClient.cs index c39dd72ba..630af4fd1 100644 --- a/Titanium.Web.Proxy/Network/ProxyClient.cs +++ b/Titanium.Web.Proxy/Network/ProxyClient.cs @@ -10,22 +10,22 @@ namespace Titanium.Web.Proxy.Network public class ProxyClient { /// - /// TcpClient used to communicate with server + /// TcpClient used to communicate with client /// internal TcpClient TcpClient { get; set; } /// - /// holds the stream to server + /// holds the stream to client /// internal Stream ClientStream { get; set; } /// - /// Used to read line by line from server + /// Used to read line by line from client /// internal CustomBinaryReader ClientStreamReader { get; set; } /// - /// used to write line by line to server + /// used to write line by line to client /// internal StreamWriter ClientStreamWriter { get; set; } diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 6cf8df3aa..1e6290c85 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -198,7 +198,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http } var args = new SessionEventArgs(); - args.TcpClient = client; + args.ProxyClient.TcpClient = client; try { From a4e2e3edebb01161fc0d907fc1f0a2b528ef1b07 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Sat, 11 Jun 2016 15:48:32 -0400 Subject: [PATCH 23/29] comments --- Titanium.Web.Proxy/Network/ProxyClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Titanium.Web.Proxy/Network/ProxyClient.cs b/Titanium.Web.Proxy/Network/ProxyClient.cs index 630af4fd1..6fa59ec1a 100644 --- a/Titanium.Web.Proxy/Network/ProxyClient.cs +++ b/Titanium.Web.Proxy/Network/ProxyClient.cs @@ -5,7 +5,7 @@ namespace Titanium.Web.Proxy.Network { /// - /// This class wraps Tcp connection to Server + /// This class wraps Tcp connection to client /// public class ProxyClient { From f59fb6a39a66e475dfa5119369406708b871f3a3 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Wed, 15 Jun 2016 23:03:58 -0400 Subject: [PATCH 24/29] clean up;add comments --- .../Compression/DeflateCompression.cs | 2 +- .../Compression/GZipCompression.cs | 2 +- .../Compression/ZlibCompression.cs | 2 +- .../Decompression/DeflateDecompression.cs | 2 +- .../Decompression/GZipDecompression.cs | 2 +- .../Decompression/ZlibDecompression.cs | 2 +- .../EventArguments/SessionEventArgs.cs | 44 +++--- .../Extensions/ByteArrayExtensions.cs | 27 ++++ .../Extensions/HttpWebRequestExtensions.cs | 6 +- .../Extensions/HttpWebResponseExtensions.cs | 5 + .../Extensions/StreamExtensions.cs | 143 +++++++++++++++++- .../Extensions/TcpExtensions.cs | 5 + .../Helpers/CertificateManager.cs | 2 +- .../Helpers/CustomBinaryReader.cs | 54 +++---- Titanium.Web.Proxy/Helpers/Tcp.cs | 2 +- Titanium.Web.Proxy/Http/HttpWebClient.cs | 12 +- .../Network/TcpConnectionManager.cs | 20 +-- Titanium.Web.Proxy/RequestHandler.cs | 63 ++++---- Titanium.Web.Proxy/ResponseHandler.cs | 109 ++----------- Titanium.Web.Proxy/Titanium.Web.Proxy.csproj | 1 + 20 files changed, 293 insertions(+), 212 deletions(-) create mode 100644 Titanium.Web.Proxy/Extensions/ByteArrayExtensions.cs diff --git a/Titanium.Web.Proxy/Compression/DeflateCompression.cs b/Titanium.Web.Proxy/Compression/DeflateCompression.cs index 25677ac04..032c97142 100644 --- a/Titanium.Web.Proxy/Compression/DeflateCompression.cs +++ b/Titanium.Web.Proxy/Compression/DeflateCompression.cs @@ -12,7 +12,7 @@ public async Task Compress(byte[] responseBody) { using (var zip = new DeflateStream(ms, CompressionMode.Compress, true)) { - await zip.WriteAsync(responseBody, 0, responseBody.Length).ConfigureAwait(false); + await zip.WriteAsync(responseBody, 0, responseBody.Length); } return ms.ToArray(); diff --git a/Titanium.Web.Proxy/Compression/GZipCompression.cs b/Titanium.Web.Proxy/Compression/GZipCompression.cs index 62614c610..893008b8a 100644 --- a/Titanium.Web.Proxy/Compression/GZipCompression.cs +++ b/Titanium.Web.Proxy/Compression/GZipCompression.cs @@ -12,7 +12,7 @@ public async Task Compress(byte[] responseBody) { using (var zip = new GZipStream(ms, CompressionMode.Compress, true)) { - await zip.WriteAsync(responseBody, 0, responseBody.Length).ConfigureAwait(false); + await zip.WriteAsync(responseBody, 0, responseBody.Length); } return ms.ToArray(); diff --git a/Titanium.Web.Proxy/Compression/ZlibCompression.cs b/Titanium.Web.Proxy/Compression/ZlibCompression.cs index 757abf8fb..fa8622bba 100644 --- a/Titanium.Web.Proxy/Compression/ZlibCompression.cs +++ b/Titanium.Web.Proxy/Compression/ZlibCompression.cs @@ -12,7 +12,7 @@ public async Task Compress(byte[] responseBody) { using (var zip = new ZlibStream(ms, CompressionMode.Compress, true)) { - await zip.WriteAsync(responseBody, 0, responseBody.Length).ConfigureAwait(false); + await zip.WriteAsync(responseBody, 0, responseBody.Length); } return ms.ToArray(); diff --git a/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs b/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs index e7fddd3b0..175745f27 100644 --- a/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs @@ -20,7 +20,7 @@ public async Task Decompress(byte[] compressedArray) int read; while ((read = await decompressor.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) > 0) { - await output.WriteAsync(buffer, 0, read).ConfigureAwait(false); + await output.WriteAsync(buffer, 0, read); } return output.ToArray(); diff --git a/Titanium.Web.Proxy/Decompression/GZipDecompression.cs b/Titanium.Web.Proxy/Decompression/GZipDecompression.cs index e18414c53..61a591399 100644 --- a/Titanium.Web.Proxy/Decompression/GZipDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/GZipDecompression.cs @@ -17,7 +17,7 @@ public async Task Decompress(byte[] compressedArray) int read; while ((read = await decompressor.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) > 0) { - await output.WriteAsync(buffer, 0, read).ConfigureAwait(false); + await output.WriteAsync(buffer, 0, read); } return output.ToArray(); } diff --git a/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs b/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs index 24d797149..a81a0ca39 100644 --- a/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs @@ -19,7 +19,7 @@ public async Task Decompress(byte[] compressedArray) int read; while ((read = await decompressor.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) > 0) { - await output.WriteAsync(buffer, 0, read).ConfigureAwait(false); + await output.WriteAsync(buffer, 0, read); } return output.ToArray(); } diff --git a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs index b48b9477a..701f79a96 100644 --- a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs @@ -81,7 +81,7 @@ private async Task ReadRequestBody() //For chunked request we need to read data as they arrive, until we reach a chunk end symbol if (WebSession.Request.IsChunked) { - await this.ProxyClient.ClientStreamReader.CopyBytesToStreamChunked(requestBodyStream).ConfigureAwait(false); + await this.ProxyClient.ClientStreamReader.CopyBytesToStreamChunked(requestBodyStream); } else { @@ -89,13 +89,13 @@ private async Task ReadRequestBody() if (WebSession.Request.ContentLength > 0) { //If not chunked then its easy just read the amount of bytes mentioned in content length header of response - await this.ProxyClient.ClientStreamReader.CopyBytesToStream(requestBodyStream, WebSession.Request.ContentLength).ConfigureAwait(false); + await this.ProxyClient.ClientStreamReader.CopyBytesToStream(requestBodyStream, WebSession.Request.ContentLength); } else if(WebSession.Request.HttpVersion.Major == 1 && WebSession.Request.HttpVersion.Minor == 0) - await WebSession.ServerConnection.StreamReader.CopyBytesToStream(requestBodyStream, long.MaxValue).ConfigureAwait(false); + await WebSession.ServerConnection.StreamReader.CopyBytesToStream(requestBodyStream, long.MaxValue); } - WebSession.Request.RequestBody = await GetDecompressedResponseBody(WebSession.Request.ContentEncoding, requestBodyStream.ToArray()).ConfigureAwait(false); + WebSession.Request.RequestBody = await GetDecompressedResponseBody(WebSession.Request.ContentEncoding, requestBodyStream.ToArray()); } //Now set the flag to true @@ -118,21 +118,21 @@ private async Task ReadResponseBody() //If chuncked the read chunk by chunk until we hit chunk end symbol if (WebSession.Response.IsChunked) { - await WebSession.ServerConnection.StreamReader.CopyBytesToStreamChunked(responseBodyStream).ConfigureAwait(false); + await WebSession.ServerConnection.StreamReader.CopyBytesToStreamChunked(responseBodyStream); } else { if (WebSession.Response.ContentLength > 0) { //If not chunked then its easy just read the amount of bytes mentioned in content length header of response - await WebSession.ServerConnection.StreamReader.CopyBytesToStream(responseBodyStream, WebSession.Response.ContentLength).ConfigureAwait(false); + await WebSession.ServerConnection.StreamReader.CopyBytesToStream(responseBodyStream, WebSession.Response.ContentLength); } else if(WebSession.Response.HttpVersion.Major == 1 && WebSession.Response.HttpVersion.Minor == 0) - await WebSession.ServerConnection.StreamReader.CopyBytesToStream(responseBodyStream, long.MaxValue).ConfigureAwait(false); + await WebSession.ServerConnection.StreamReader.CopyBytesToStream(responseBodyStream, long.MaxValue); } - WebSession.Response.ResponseBody = await GetDecompressedResponseBody(WebSession.Response.ContentEncoding, responseBodyStream.ToArray()).ConfigureAwait(false); + WebSession.Response.ResponseBody = await GetDecompressedResponseBody(WebSession.Response.ContentEncoding, responseBodyStream.ToArray()); } //set this to true for caching @@ -149,7 +149,7 @@ public async Task GetRequestBody() if (WebSession.Request.RequestLocked) throw new Exception("You cannot call this function after request is made to server."); - await ReadRequestBody().ConfigureAwait(false); + await ReadRequestBody(); return WebSession.Request.RequestBody; } /// @@ -162,7 +162,7 @@ public async Task GetRequestBodyAsString() throw new Exception("You cannot call this function after request is made to server."); - await ReadRequestBody().ConfigureAwait(false); + await ReadRequestBody(); //Use the encoding specified in request to decode the byte[] data to string return WebSession.Request.RequestBodyString ?? (WebSession.Request.RequestBodyString = WebSession.Request.Encoding.GetString(WebSession.Request.RequestBody)); @@ -180,7 +180,7 @@ public async Task SetRequestBody(byte[] body) //syphon out the request body from client before setting the new body if (!WebSession.Request.RequestBodyRead) { - await ReadRequestBody().ConfigureAwait(false); + await ReadRequestBody(); } WebSession.Request.RequestBody = body; @@ -203,7 +203,7 @@ public async Task SetRequestBodyString(string body) //syphon out the request body from client before setting the new body if (!WebSession.Request.RequestBodyRead) { - await ReadRequestBody().ConfigureAwait(false); + await ReadRequestBody(); } await SetRequestBody(WebSession.Request.Encoding.GetBytes(body)); @@ -219,7 +219,7 @@ public async Task GetResponseBody() if (!WebSession.Request.RequestLocked) throw new Exception("You cannot call this function before request is made to server."); - await ReadResponseBody().ConfigureAwait(false); + await ReadResponseBody(); return WebSession.Response.ResponseBody; } @@ -232,7 +232,7 @@ public async Task GetResponseBodyAsString() if (!WebSession.Request.RequestLocked) throw new Exception("You cannot call this function before request is made to server."); - await GetResponseBody().ConfigureAwait(false); + await GetResponseBody(); return WebSession.Response.ResponseBodyString ?? (WebSession.Response.ResponseBodyString = WebSession.Response.Encoding.GetString(WebSession.Response.ResponseBody)); } @@ -249,7 +249,7 @@ public async Task SetResponseBody(byte[] body) //syphon out the response body from server before setting the new body if (WebSession.Response.ResponseBody == null) { - await GetResponseBody().ConfigureAwait(false); + await GetResponseBody(); } WebSession.Response.ResponseBody = body; @@ -273,12 +273,12 @@ public async Task SetResponseBodyString(string body) //syphon out the response body from server before setting the new body if (WebSession.Response.ResponseBody == null) { - await GetResponseBody().ConfigureAwait(false); + await GetResponseBody(); } var bodyBytes = WebSession.Response.Encoding.GetBytes(body); - await SetResponseBody(bodyBytes).ConfigureAwait(false); + await SetResponseBody(bodyBytes); } private async Task GetDecompressedResponseBody(string encodingType, byte[] responseBodyStream) @@ -286,7 +286,7 @@ private async Task GetDecompressedResponseBody(string encodingType, byte var decompressionFactory = new DecompressionFactory(); var decompressor = decompressionFactory.Create(encodingType); - return await decompressor.Decompress(responseBodyStream).ConfigureAwait(false); + return await decompressor.Decompress(responseBodyStream); } @@ -306,7 +306,7 @@ public async Task Ok(string html) var result = Encoding.Default.GetBytes(html); - await Ok(result).ConfigureAwait(false); + await Ok(result); } /// @@ -322,7 +322,7 @@ public async Task Ok(byte[] result) response.HttpVersion = WebSession.Request.HttpVersion; response.ResponseBody = result; - await Respond(response).ConfigureAwait(false); + await Respond(response); WebSession.Request.CancelRequest = true; } @@ -335,7 +335,7 @@ public async Task Redirect(string url) response.ResponseHeaders.Add(new Models.HttpHeader("Location", url)); response.ResponseBody = Encoding.ASCII.GetBytes(string.Empty); - await Respond(response).ConfigureAwait(false); + await Respond(response); WebSession.Request.CancelRequest = true; } @@ -350,7 +350,7 @@ public async Task Respond(Response response) WebSession.Response = response; - await ProxyServer.HandleHttpSessionResponse(this).ConfigureAwait(false); + await ProxyServer.HandleHttpSessionResponse(this); } } diff --git a/Titanium.Web.Proxy/Extensions/ByteArrayExtensions.cs b/Titanium.Web.Proxy/Extensions/ByteArrayExtensions.cs new file mode 100644 index 000000000..6082b529e --- /dev/null +++ b/Titanium.Web.Proxy/Extensions/ByteArrayExtensions.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Titanium.Web.Proxy.Extensions +{ + public static class ByteArrayExtensions + { + /// + /// Get the sub array from byte of data + /// + /// + /// + /// + /// + /// + public static T[] SubArray(this T[] data, int index, int length) + { + T[] result = new T[length]; + Array.Copy(data, index, result, 0, length); + return result; + } + + } +} diff --git a/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs b/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs index 1e096a416..dd7b61cab 100644 --- a/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/HttpWebRequestExtensions.cs @@ -9,7 +9,11 @@ namespace Titanium.Web.Proxy.Extensions /// internal static class HttpWebRequestExtensions { - //Get encoding of the HTTP request + /// + /// parse the character encoding of request from request headers + /// + /// + /// internal static Encoding GetEncoding(this Request request) { try diff --git a/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs b/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs index d6191faac..10c4357e6 100644 --- a/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/HttpWebResponseExtensions.cs @@ -6,6 +6,11 @@ namespace Titanium.Web.Proxy.Extensions { internal static class HttpWebResponseExtensions { + /// + /// Gets the character encoding of response from response headers + /// + /// + /// internal static Encoding GetResponseCharacterEncoding(this Response response) { try diff --git a/Titanium.Web.Proxy/Extensions/StreamExtensions.cs b/Titanium.Web.Proxy/Extensions/StreamExtensions.cs index e6ccbc4f8..fbc8037b3 100644 --- a/Titanium.Web.Proxy/Extensions/StreamExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/StreamExtensions.cs @@ -8,8 +8,15 @@ namespace Titanium.Web.Proxy.Extensions { - internal static class StreamHelper + internal static class StreamExtensions { + /// + /// Copy streams asynchronously with an initial data inserted to the beginning of stream + /// + /// + /// + /// + /// internal static async Task CopyToAsync(this Stream input, string initialData, Stream output) { if (!string.IsNullOrEmpty(initialData)) @@ -19,7 +26,13 @@ internal static async Task CopyToAsync(this Stream input, string initialData, St } await input.CopyToAsync(output); } - + /// + /// copies the specified bytes to the stream from the input stream + /// + /// + /// + /// + /// internal static async Task CopyBytesToStream(this CustomBinaryReader streamReader, Stream stream, long totalBytesToRead) { var totalbytesRead = 0; @@ -50,11 +63,18 @@ internal static async Task CopyBytesToStream(this CustomBinaryReader streamReade await stream.WriteAsync(buffer, 0, buffer.Length); } } + + /// + /// Copies the stream chunked + /// + /// + /// + /// internal static async Task CopyBytesToStreamChunked(this CustomBinaryReader clientStreamReader, Stream stream) { while (true) { - var chuchkHead = await clientStreamReader.ReadLineAsync().ConfigureAwait(false); + var chuchkHead = await clientStreamReader.ReadLineAsync(); var chunkSize = int.Parse(chuchkHead, NumberStyles.HexNumber); if (chunkSize != 0) @@ -62,14 +82,127 @@ internal static async Task CopyBytesToStreamChunked(this CustomBinaryReader clie var buffer = await clientStreamReader.ReadBytesAsync(chunkSize); await stream.WriteAsync(buffer, 0, buffer.Length); //chunk trail - await clientStreamReader.ReadLineAsync().ConfigureAwait(false); + await clientStreamReader.ReadLineAsync(); + } + else + { + await clientStreamReader.ReadLineAsync(); + break; + } + } + } + /// + /// Writes the byte array body to the given stream; optionally chunked + /// + /// + /// + /// + /// + internal static async Task WriteResponseBody(this Stream clientStream, byte[] data, bool isChunked) + { + if (!isChunked) + { + await clientStream.WriteAsync(data, 0, data.Length); + } + else + await WriteResponseBodyChunked(data, clientStream); + } + /// + /// Copies the specified content length number of bytes to the output stream from the given inputs stream + /// optionally chunked + /// + /// + /// + /// + /// + /// + internal static async Task WriteResponseBody(this CustomBinaryReader inStreamReader, Stream outStream, bool isChunked, long ContentLength) + { + if (!isChunked) + { + //http 1.0 + if (ContentLength == -1) + ContentLength = long.MaxValue; + + int bytesToRead = ProxyConstants.BUFFER_SIZE; + + if (ContentLength < ProxyConstants.BUFFER_SIZE) + bytesToRead = (int)ContentLength; + + var buffer = new byte[ProxyConstants.BUFFER_SIZE]; + + var bytesRead = 0; + var totalBytesRead = 0; + + while ((bytesRead += await inStreamReader.BaseStream.ReadAsync(buffer, 0, bytesToRead).ConfigureAwait(false)) > 0) + { + await outStream.WriteAsync(buffer, 0, bytesRead); + totalBytesRead += bytesRead; + + if (totalBytesRead == ContentLength) + break; + + bytesRead = 0; + var remainingBytes = (ContentLength - totalBytesRead); + bytesToRead = remainingBytes > (long)ProxyConstants.BUFFER_SIZE ? ProxyConstants.BUFFER_SIZE : (int)remainingBytes; + } + } + else + await WriteResponseBodyChunked(inStreamReader, outStream); + } + + /// + /// Copies the streams chunked + /// + /// + /// + /// + internal static async Task WriteResponseBodyChunked(this CustomBinaryReader inStreamReader, Stream outStream) + { + while (true) + { + var chunkHead = await inStreamReader.ReadLineAsync(); + var chunkSize = int.Parse(chunkHead, NumberStyles.HexNumber); + + if (chunkSize != 0) + { + var buffer = await inStreamReader.ReadBytesAsync(chunkSize); + + var chunkHeadBytes = Encoding.ASCII.GetBytes(chunkSize.ToString("x2")); + + await outStream.WriteAsync(chunkHeadBytes, 0, chunkHeadBytes.Length); + await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length); + + await outStream.WriteAsync(buffer, 0, chunkSize); + await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length); + + await inStreamReader.ReadLineAsync(); } else { - await clientStreamReader.ReadLineAsync().ConfigureAwait(false); + await inStreamReader.ReadLineAsync(); + await outStream.WriteAsync(ProxyConstants.ChunkEnd, 0, ProxyConstants.ChunkEnd.Length); break; } } } + /// + /// Copies the given input bytes to output stream chunked + /// + /// + /// + /// + internal static async Task WriteResponseBodyChunked(this byte[] data, Stream outStream) + { + var chunkHead = Encoding.ASCII.GetBytes(data.Length.ToString("x2")); + + await outStream.WriteAsync(chunkHead, 0, chunkHead.Length); + await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length); + await outStream.WriteAsync(data, 0, data.Length); + await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length); + + await outStream.WriteAsync(ProxyConstants.ChunkEnd, 0, ProxyConstants.ChunkEnd.Length); + } + } } \ No newline at end of file diff --git a/Titanium.Web.Proxy/Extensions/TcpExtensions.cs b/Titanium.Web.Proxy/Extensions/TcpExtensions.cs index 4d1ce7adb..102924cf3 100644 --- a/Titanium.Web.Proxy/Extensions/TcpExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/TcpExtensions.cs @@ -4,6 +4,11 @@ namespace Titanium.Web.Proxy.Extensions { internal static class TcpExtensions { + /// + /// verifies if the underlying socket is connected before using a TcpClient connection + /// + /// + /// internal static bool IsConnected(this Socket client) { // This is how you can determine whether a socket is still connected. diff --git a/Titanium.Web.Proxy/Helpers/CertificateManager.cs b/Titanium.Web.Proxy/Helpers/CertificateManager.cs index 8bf52beb9..e19cdcbab 100644 --- a/Titanium.Web.Proxy/Helpers/CertificateManager.cs +++ b/Titanium.Web.Proxy/Helpers/CertificateManager.cs @@ -248,7 +248,7 @@ internal async void ClearIdleCertificates() } finally { semaphoreLock.Release(); } - await Task.Delay(1000 * 60 * 3).ConfigureAwait(false); + await Task.Delay(1000 * 60 * 3); } } diff --git a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs index bbfca3966..129ad2e52 100644 --- a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs +++ b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs @@ -1,15 +1,14 @@ using System; using System.Collections.Generic; using System.IO; -using System.Net.Security; -using System.Net.Sockets; using System.Text; using System.Threading.Tasks; -using Titanium.Web.Proxy.Network; +using Titanium.Web.Proxy.Extensions; using Titanium.Web.Proxy.Shared; namespace Titanium.Web.Proxy.Helpers { + /// /// A custom binary reader that would allo us to read string line by line /// using the specified encoding @@ -18,10 +17,11 @@ namespace Titanium.Web.Proxy.Helpers internal class CustomBinaryReader : IDisposable { private Stream stream; - + private Encoding encoding; internal CustomBinaryReader(Stream stream) { this.stream = stream; + this.encoding = Encoding.UTF8; } internal Stream BaseStream => stream; @@ -32,32 +32,34 @@ internal CustomBinaryReader(Stream stream) /// internal async Task ReadLineAsync() { - var readBuffer = new StringBuilder(); - - try + using (var readBuffer = new MemoryStream()) { - var lastChar = default(char); - var buffer = new byte[1]; - - while ((await this.stream.ReadAsync(buffer, 0, 1)) > 0) + try { - if (lastChar == '\r' && buffer[0] == '\n') - { - return await Task.FromResult(readBuffer.Remove(readBuffer.Length - 1, 1).ToString()); - } - if (buffer[0] == '\0') + var lastChar = default(char); + var buffer = new byte[1]; + + while ((await this.stream.ReadAsync(buffer, 0, 1)) > 0) { - return await Task.FromResult(readBuffer.ToString()); + if (lastChar == '\r' && buffer[0] == '\n') + { + var result = readBuffer.ToArray(); + return encoding.GetString(result.SubArray(0, result.Length - 1)); + } + if (buffer[0] == '\0') + { + return encoding.GetString(readBuffer.ToArray()); + } + await readBuffer.WriteAsync(buffer,0,1); + lastChar = (char)buffer[0]; } - readBuffer.Append((char)buffer[0]); - lastChar = (char)buffer[0]; - } - return await Task.FromResult(readBuffer.ToString()); - } - catch (IOException) - { - return await Task.FromResult(readBuffer.ToString()); + return encoding.GetString(readBuffer.ToArray()); + } + catch (IOException) + { + return encoding.GetString(readBuffer.ToArray()); + } } } @@ -92,7 +94,7 @@ internal async Task ReadBytesAsync(long totalBytesToRead) { while ((bytesRead += await this.stream.ReadAsync(buffer, 0, bytesToRead).ConfigureAwait(false)) > 0) { - await outStream.WriteAsync(buffer, 0, bytesRead).ConfigureAwait(false); + await outStream.WriteAsync(buffer, 0, bytesRead); totalBytesRead += bytesRead; if (totalBytesRead == totalBytesToRead) diff --git a/Titanium.Web.Proxy/Helpers/Tcp.cs b/Titanium.Web.Proxy/Helpers/Tcp.cs index f5e5faf05..cfd002090 100644 --- a/Titanium.Web.Proxy/Helpers/Tcp.cs +++ b/Titanium.Web.Proxy/Helpers/Tcp.cs @@ -72,7 +72,7 @@ internal async static Task SendRaw(Stream clientStream, string httpCmd, List responseLines = await ServerConnection.StreamReader.ReadAllLinesAsync().ConfigureAwait(false); + List responseLines = await ServerConnection.StreamReader.ReadAllLinesAsync(); for (int index = 0; index < responseLines.Count; ++index) { diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index d885f5c7d..0a90d7385 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -79,7 +79,7 @@ internal static async Task GetClient(string hostname, int port, b } if (cached == null) - cached = await CreateClient(hostname, port, isHttps, version).ConfigureAwait(false); + cached = await CreateClient(hostname, port, isHttps, version); if (cachedConnections == null || cachedConnections.Count() <= 2) { @@ -111,22 +111,22 @@ private static async Task CreateClient(string hostname, int port, using (var writer = new StreamWriter(stream, Encoding.ASCII, ProxyConstants.BUFFER_SIZE, true)) { - await writer.WriteLineAsync(string.Format("CONNECT {0}:{1} {2}", hostname, port, version)).ConfigureAwait(false); - await writer.WriteLineAsync(string.Format("Host: {0}:{1}", hostname, port)).ConfigureAwait(false); - await writer.WriteLineAsync("Connection: Keep-Alive").ConfigureAwait(false); - await writer.WriteLineAsync().ConfigureAwait(false); - await writer.FlushAsync().ConfigureAwait(false); + await writer.WriteLineAsync(string.Format("CONNECT {0}:{1} {2}", hostname, port, version)); + await writer.WriteLineAsync(string.Format("Host: {0}:{1}", hostname, port)); + await writer.WriteLineAsync("Connection: Keep-Alive"); + await writer.WriteLineAsync(); + await writer.FlushAsync(); writer.Close(); } using (var reader = new CustomBinaryReader(stream)) { - var result = await reader.ReadLineAsync().ConfigureAwait(false); + var result = await reader.ReadLineAsync(); if (!result.ToLower().Contains("200 connection established")) throw new Exception("Upstream proxy failed to create a secure tunnel"); - await reader.ReadAllLinesAsync().ConfigureAwait(false); + await reader.ReadAllLinesAsync(); } } else @@ -139,7 +139,7 @@ private static async Task CreateClient(string hostname, int port, { sslStream = new SslStream(stream, true, new RemoteCertificateValidationCallback(ProxyServer.ValidateServerCertificate), new LocalCertificateSelectionCallback(ProxyServer.SelectClientCertificate)); - await sslStream.AuthenticateAsClientAsync(hostname, null, ProxyConstants.SupportedSslProtocols, false).ConfigureAwait(false); + await sslStream.AuthenticateAsClientAsync(hostname, null, ProxyConstants.SupportedSslProtocols, false); stream = (Stream)sslStream; } catch @@ -224,7 +224,7 @@ internal async static void ClearIdleConnections() } finally { connectionAccessLock.Release(); } - await Task.Delay(1000 * 60 * 3).ConfigureAwait(false); + await Task.Delay(1000 * 60); } } diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 1e6290c85..48fe3cd1c 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -32,7 +32,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient try { //read the first line HTTP command - var httpCmd = await clientStreamReader.ReadLineAsync().ConfigureAwait(false); + var httpCmd = await clientStreamReader.ReadLineAsync(); if (string.IsNullOrEmpty(httpCmd)) { @@ -68,25 +68,23 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient if (httpVerb.ToUpper() == "CONNECT" && !excluded && httpRemoteUri.Port != 80) { httpRemoteUri = new Uri("https://" + httpCmdSplit[1]); - await clientStreamReader.ReadAllLinesAsync().ConfigureAwait(false); - - await WriteConnectResponse(clientStreamWriter, version).ConfigureAwait(false); - + await clientStreamReader.ReadAllLinesAsync(); + await WriteConnectResponse(clientStreamWriter, version); SslStream sslStream = null; try { - var tunnelClient = await TcpConnectionManager.GetClient(httpRemoteUri.Host, httpRemoteUri.Port, true, version).ConfigureAwait(false); + var tunnelClient = await TcpConnectionManager.GetClient(httpRemoteUri.Host, httpRemoteUri.Port, true, version); await TcpConnectionManager.ReleaseClient(tunnelClient); sslStream = new SslStream(clientStream, true); var certificate = await CertManager.CreateCertificate(httpRemoteUri.Host, false); //Successfully managed to authenticate the client using the fake certificate await sslStream.AuthenticateAsServerAsync(certificate, false, - ProxyConstants.SupportedSslProtocols, false).ConfigureAwait(false); + ProxyConstants.SupportedSslProtocols, false); //HTTPS server created - we can now decrypt the client's traffic clientStream = sslStream; @@ -104,26 +102,24 @@ await sslStream.AuthenticateAsServerAsync(certificate, false, return; } - - httpCmd = await clientStreamReader.ReadLineAsync().ConfigureAwait(false); + httpCmd = await clientStreamReader.ReadLineAsync(); } else if (httpVerb.ToUpper() == "CONNECT") { - await clientStreamReader.ReadAllLinesAsync().ConfigureAwait(false); - await WriteConnectResponse(clientStreamWriter, version).ConfigureAwait(false); + await clientStreamReader.ReadAllLinesAsync(); + await WriteConnectResponse(clientStreamWriter, version); await TcpHelper.SendRaw(clientStream, null, null, httpRemoteUri.Host, httpRemoteUri.Port, - false).ConfigureAwait(false); + false); Dispose(client, clientStream, clientStreamReader, clientStreamWriter, null); return; } //Now create the request - await HandleHttpSessionRequest(client, httpCmd, clientStream, clientStreamReader, clientStreamWriter, - httpRemoteUri.Scheme == Uri.UriSchemeHttps ? true : false).ConfigureAwait(false); + httpRemoteUri.Scheme == Uri.UriSchemeHttps ? true : false); } catch { @@ -155,7 +151,7 @@ private static async void HandleClient(TransparentProxyEndPoint endPoint, TcpCli { //Successfully managed to authenticate the client using the fake certificate await sslStream.AuthenticateAsServerAsync(certificate, false, - SslProtocols.Tls, false).ConfigureAwait(false); + SslProtocols.Tls, false); clientStreamReader = new CustomBinaryReader(sslStream); clientStreamWriter = new StreamWriter(sslStream); @@ -177,11 +173,11 @@ await sslStream.AuthenticateAsServerAsync(certificate, false, clientStreamReader = new CustomBinaryReader(clientStream); } - var httpCmd = await clientStreamReader.ReadLineAsync().ConfigureAwait(false); + var httpCmd = await clientStreamReader.ReadLineAsync(); //Now create the request await HandleHttpSessionRequest(tcpClient, httpCmd, clientStream, clientStreamReader, clientStreamWriter, - true).ConfigureAwait(false); + true); } private static async Task HandleHttpSessionRequest(TcpClient client, string httpCmd, Stream clientStream, @@ -234,7 +230,6 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http Dispose(client, clientStream, clientStreamReader, clientStreamWriter, null); break; } - #endif args.WebSession.Request.RequestUri = httpRemoteUri; @@ -244,7 +239,6 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http args.ProxyClient.ClientStreamReader = clientStreamReader; args.ProxyClient.ClientStreamWriter = clientStreamWriter; - PrepareRequestHeaders(args.WebSession.Request.RequestHeaders, args.WebSession); args.WebSession.Request.Host = args.WebSession.Request.RequestUri.Authority; @@ -259,19 +253,19 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http handlerTasks[i] = ((Func)invocationList[i])(null, args); } - await Task.WhenAll(handlerTasks).ConfigureAwait(false); + await Task.WhenAll(handlerTasks); } if (args.WebSession.Request.UpgradeToWebSocket) { await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHeaders, - httpRemoteUri.Host, httpRemoteUri.Port, args.IsHttps).ConfigureAwait(false); + httpRemoteUri.Host, httpRemoteUri.Port, args.IsHttps); Dispose(client, clientStream, clientStreamReader, clientStreamWriter, args); return; } //construct the web request that we are going to issue on behalf of the client. - connection = await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version).ConfigureAwait(false); + connection = await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version); args.WebSession.Request.RequestLocked = true; @@ -285,7 +279,7 @@ await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHe if (args.WebSession.Request.ExpectContinue) { args.WebSession.SetConnection(connection); - await args.WebSession.SendRequest().ConfigureAwait(false); + await args.WebSession.SendRequest(); } if (Enable100ContinueBehaviour) @@ -305,7 +299,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", if (!args.WebSession.Request.ExpectContinue) { args.WebSession.SetConnection(connection); - await args.WebSession.SendRequest().ConfigureAwait(false); + await args.WebSession.SendRequest(); } //If request was modified by user @@ -319,7 +313,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", args.WebSession.Request.ContentLength = args.WebSession.Request.RequestBody.Length; var newStream = args.WebSession.ServerConnection.Stream; - await newStream.WriteAsync(args.WebSession.Request.RequestBody, 0, args.WebSession.Request.RequestBody.Length).ConfigureAwait(false); + await newStream.WriteAsync(args.WebSession.Request.RequestBody, 0, args.WebSession.Request.RequestBody.Length); } else { @@ -328,14 +322,14 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", //If its a post/put request, then read the client html body and send it to server if (httpMethod.ToUpper() == "POST" || httpMethod.ToUpper() == "PUT") { - await SendClientRequestBody(args).ConfigureAwait(false); + await SendClientRequestBody(args); } } } if (!args.WebSession.Request.ExpectationFailed) { - await HandleHttpSessionResponse(args).ConfigureAwait(false); + await HandleHttpSessionResponse(args); } //if connection is closing exit @@ -348,7 +342,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", await TcpConnectionManager.ReleaseClient(connection); // read the next request - httpCmd = await clientStreamReader.ReadLineAsync().ConfigureAwait(false); + httpCmd = await clientStreamReader.ReadLineAsync(); } catch @@ -363,10 +357,10 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, Version httpVersion) { - await clientStreamWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2}", httpVersion.Major, httpVersion.Minor, "200 Connection established")).ConfigureAwait(false); - await clientStreamWriter.WriteLineAsync(string.Format("Timestamp: {0}", DateTime.Now)).ConfigureAwait(false); - await clientStreamWriter.WriteLineAsync().ConfigureAwait(false); - await clientStreamWriter.FlushAsync().ConfigureAwait(false); + await clientStreamWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2}", httpVersion.Major, httpVersion.Minor, "200 Connection established")); + await clientStreamWriter.WriteLineAsync(string.Format("Timestamp: {0}", DateTime.Now)); + await clientStreamWriter.WriteLineAsync(); + await clientStreamWriter.FlushAsync(); } private static void PrepareRequestHeaders(List requestHeaders, HttpWebClient webRequest) @@ -405,6 +399,7 @@ private static void FixRequestProxyHeaders(List headers) headers.RemoveAll(x => x.Name.ToLower() == "proxy-connection"); } + //This is called when the request is PUT/POST to read the body private static async Task SendClientRequestBody(SessionEventArgs args) { @@ -415,7 +410,7 @@ private static async Task SendClientRequestBody(SessionEventArgs args) { try { - await args.ProxyClient.ClientStreamReader.CopyBytesToStream(postStream, args.WebSession.Request.ContentLength).ConfigureAwait(false); + await args.ProxyClient.ClientStreamReader.CopyBytesToStream(postStream, args.WebSession.Request.ContentLength); } catch { @@ -427,7 +422,7 @@ private static async Task SendClientRequestBody(SessionEventArgs args) { try { - await args.ProxyClient.ClientStreamReader.CopyBytesToStreamChunked(postStream).ConfigureAwait(false); + await args.ProxyClient.ClientStreamReader.CopyBytesToStreamChunked(postStream); } catch { diff --git a/Titanium.Web.Proxy/ResponseHandler.cs b/Titanium.Web.Proxy/ResponseHandler.cs index 6f4308e53..f35f688a2 100644 --- a/Titanium.Web.Proxy/ResponseHandler.cs +++ b/Titanium.Web.Proxy/ResponseHandler.cs @@ -1,16 +1,13 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Net.Sockets; -using System.Text; using Titanium.Web.Proxy.EventArguments; -using Titanium.Web.Proxy.Helpers; using Titanium.Web.Proxy.Models; using Titanium.Web.Proxy.Compression; -using Titanium.Web.Proxy.Shared; using System.Threading.Tasks; +using Titanium.Web.Proxy.Extensions; namespace Titanium.Web.Proxy { @@ -19,7 +16,7 @@ partial class ProxyServer //Called asynchronously when a request was successfully and we received the response public static async Task HandleHttpSessionResponse(SessionEventArgs args) { - await args.WebSession.ReceiveResponse().ConfigureAwait(false); + await args.WebSession.ReceiveResponse(); try { @@ -37,7 +34,7 @@ public static async Task HandleHttpSessionResponse(SessionEventArgs args) handlerTasks[i] = ((Func)invocationList[i])(null, args); } - await Task.WhenAll(handlerTasks).ConfigureAwait(false); + await Task.WhenAll(handlerTasks); } args.WebSession.Response.ResponseLocked = true; @@ -65,7 +62,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession. if (contentEncoding != null) { - args.WebSession.Response.ResponseBody = await GetCompressedResponseBody(contentEncoding, args.WebSession.Response.ResponseBody).ConfigureAwait(false); + args.WebSession.Response.ResponseBody = await GetCompressedResponseBody(contentEncoding, args.WebSession.Response.ResponseBody); if (isChunked == false) args.WebSession.Response.ContentLength = args.WebSession.Response.ResponseBody.Length; @@ -73,8 +70,8 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession. args.WebSession.Response.ContentLength = -1; } - await WriteResponseHeaders(args.ProxyClient.ClientStreamWriter, args.WebSession.Response.ResponseHeaders).ConfigureAwait(false); - await WriteResponseBody(args.ProxyClient.ClientStream, args.WebSession.Response.ResponseBody, isChunked).ConfigureAwait(false); + await WriteResponseHeaders(args.ProxyClient.ClientStreamWriter, args.WebSession.Response.ResponseHeaders); + await args.ProxyClient.ClientStream.WriteResponseBody(args.WebSession.Response.ResponseBody, isChunked); } else { @@ -82,7 +79,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession. if (args.WebSession.Response.IsChunked || args.WebSession.Response.ContentLength > 0 || (args.WebSession.Response.HttpVersion.Major == 1 && args.WebSession.Response.HttpVersion.Minor == 0)) - await WriteResponseBody(args.WebSession.ServerConnection.StreamReader, args.ProxyClient.ClientStream, args.WebSession.Response.IsChunked, args.WebSession.Response.ContentLength).ConfigureAwait(false); + await args.WebSession.ServerConnection.StreamReader.WriteResponseBody(args.ProxyClient.ClientStream, args.WebSession.Response.IsChunked, args.WebSession.Response.ContentLength); } await args.ProxyClient.ClientStream.FlushAsync(); @@ -102,7 +99,7 @@ private static async Task GetCompressedResponseBody(string encodingType, { var compressionFactory = new CompressionFactory(); var compressor = compressionFactory.Create(encodingType); - return await compressor.Compress(responseBodyStream).ConfigureAwait(false); + return await compressor.Compress(responseBodyStream); } @@ -145,95 +142,7 @@ private static void FixResponseProxyHeaders(List headers) headers.RemoveAll(x => x.Name.ToLower() == "proxy-connection"); } - - private static async Task WriteResponseBody(Stream clientStream, byte[] data, bool isChunked) - { - if (!isChunked) - { - await clientStream.WriteAsync(data, 0, data.Length).ConfigureAwait(false); - } - else - await WriteResponseBodyChunked(data, clientStream).ConfigureAwait(false); - } - - private static async Task WriteResponseBody(CustomBinaryReader inStreamReader, Stream outStream, bool isChunked, long ContentLength) - { - if (!isChunked) - { - //http 1.0 - if (ContentLength == -1) - ContentLength = long.MaxValue; - - int bytesToRead = ProxyConstants.BUFFER_SIZE; - - if (ContentLength < ProxyConstants.BUFFER_SIZE) - bytesToRead = (int)ContentLength; - - var buffer = new byte[ProxyConstants.BUFFER_SIZE]; - - var bytesRead = 0; - var totalBytesRead = 0; - - while ((bytesRead += await inStreamReader.BaseStream.ReadAsync(buffer, 0, bytesToRead).ConfigureAwait(false)) > 0) - { - await outStream.WriteAsync(buffer, 0, bytesRead).ConfigureAwait(false); - totalBytesRead += bytesRead; - - if (totalBytesRead == ContentLength) - break; - - bytesRead = 0; - var remainingBytes = (ContentLength - totalBytesRead); - bytesToRead = remainingBytes > (long)ProxyConstants.BUFFER_SIZE ? ProxyConstants.BUFFER_SIZE : (int)remainingBytes; - } - } - else - await WriteResponseBodyChunked(inStreamReader, outStream).ConfigureAwait(false); - } - - //Send chunked response - private static async Task WriteResponseBodyChunked(CustomBinaryReader inStreamReader, Stream outStream) - { - while (true) - { - var chunkHead = await inStreamReader.ReadLineAsync().ConfigureAwait(false); - var chunkSize = int.Parse(chunkHead, NumberStyles.HexNumber); - - if (chunkSize != 0) - { - var buffer = await inStreamReader.ReadBytesAsync(chunkSize).ConfigureAwait(false); - - var chunkHeadBytes = Encoding.ASCII.GetBytes(chunkSize.ToString("x2")); - - await outStream.WriteAsync(chunkHeadBytes, 0, chunkHeadBytes.Length).ConfigureAwait(false); - await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length).ConfigureAwait(false); - - await outStream.WriteAsync(buffer, 0, chunkSize).ConfigureAwait(false); - await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length).ConfigureAwait(false); - - await inStreamReader.ReadLineAsync().ConfigureAwait(false); - } - else - { - await inStreamReader.ReadLineAsync().ConfigureAwait(false); - await outStream.WriteAsync(ProxyConstants.ChunkEnd, 0, ProxyConstants.ChunkEnd.Length).ConfigureAwait(false); - break; - } - } - } - - private static async Task WriteResponseBodyChunked(byte[] data, Stream outStream) - { - var chunkHead = Encoding.ASCII.GetBytes(data.Length.ToString("x2")); - - await outStream.WriteAsync(chunkHead, 0, chunkHead.Length).ConfigureAwait(false); - await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length).ConfigureAwait(false); - await outStream.WriteAsync(data, 0, data.Length).ConfigureAwait(false); - await outStream.WriteAsync(ProxyConstants.NewLineBytes, 0, ProxyConstants.NewLineBytes.Length).ConfigureAwait(false); - - await outStream.WriteAsync(ProxyConstants.ChunkEnd, 0, ProxyConstants.ChunkEnd.Length).ConfigureAwait(false); - } - + private static void Dispose(TcpClient client, IDisposable clientStream, IDisposable clientStreamReader, IDisposable clientStreamWriter, IDisposable args) diff --git a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj index e5007a17b..cadbfb1b0 100644 --- a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj +++ b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj @@ -62,6 +62,7 @@ + From 6fb5f0bf1a9dfc8ef5a3a060e25787525db87fc9 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Wed, 15 Jun 2016 23:55:44 -0400 Subject: [PATCH 25/29] Add comments --- .../ProxyTestController.cs | 8 +- .../Compression/CompressionFactory.cs | 5 +- .../Compression/DeflateCompression.cs | 5 +- .../Compression/GZipCompression.cs | 5 +- .../Compression/ICompression.cs | 3 + .../Compression/ZlibCompression.cs | 5 +- .../Decompression/DecompressionFactory.cs | 3 + .../Decompression/DefaultDecompression.cs | 4 + .../Decompression/DeflateDecompression.cs | 5 +- .../Decompression/GZipDecompression.cs | 5 +- .../Decompression/IDecompression.cs | 6 +- .../Decompression/ZlibDecompression.cs | 5 +- .../CertificateSelectionEventArgs.cs | 3 + .../CertificateValidationEventArgs.cs | 3 + .../EventArguments/SessionEventArgs.cs | 1 - .../Extensions/ByteArrayExtensions.cs | 4 - .../Extensions/StreamExtensions.cs | 8 +- .../Helpers/CustomBinaryReader.cs | 19 +- Titanium.Web.Proxy/Helpers/SystemProxy.cs | 47 +- Titanium.Web.Proxy/Helpers/Tcp.cs | 16 +- Titanium.Web.Proxy/Http/HttpWebClient.cs | 24 +- Titanium.Web.Proxy/Http/Request.cs | 3 + Titanium.Web.Proxy/Http/Response.cs | 4 +- .../Http/Responses/OkResponse.cs | 7 +- .../Http/Responses/RedirectResponse.cs | 3 + Titanium.Web.Proxy/Models/EndPoint.cs | 11 + Titanium.Web.Proxy/Models/ExternalProxy.cs | 12 +- .../Network/CachedCertificate.cs | 25 + .../CertificateManager.cs | 553 +++++++++--------- Titanium.Web.Proxy/Network/TcpConnection.cs | 45 ++ .../Network/TcpConnectionManager.cs | 74 ++- Titanium.Web.Proxy/ProxyServer.cs | 94 ++- Titanium.Web.Proxy/RequestHandler.cs | 6 +- Titanium.Web.Proxy/Titanium.Web.Proxy.csproj | 4 +- 34 files changed, 680 insertions(+), 345 deletions(-) create mode 100644 Titanium.Web.Proxy/Network/CachedCertificate.cs rename Titanium.Web.Proxy/{Helpers => Network}/CertificateManager.cs (84%) create mode 100644 Titanium.Web.Proxy/Network/TcpConnection.cs diff --git a/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs b/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs index dc2ba74af..97779fd87 100644 --- a/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs +++ b/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs @@ -12,10 +12,10 @@ public class ProxyTestController { public void StartProxy() { - ProxyServer.BeforeRequest += OnRequest; - ProxyServer.BeforeResponse += OnResponse; - ProxyServer.ServerCertificateValidationCallback += OnCertificateValidation; - ProxyServer.ClientCertificateSelectionCallback += OnCertificateSelection; + //ProxyServer.BeforeRequest += OnRequest; + //ProxyServer.BeforeResponse += OnResponse; + //ProxyServer.ServerCertificateValidationCallback += OnCertificateValidation; + //ProxyServer.ClientCertificateSelectionCallback += OnCertificateSelection; //Exclude Https addresses you don't want to proxy //Usefull for clients that use certificate pinning diff --git a/Titanium.Web.Proxy/Compression/CompressionFactory.cs b/Titanium.Web.Proxy/Compression/CompressionFactory.cs index 48ba33f01..5ee9355fc 100644 --- a/Titanium.Web.Proxy/Compression/CompressionFactory.cs +++ b/Titanium.Web.Proxy/Compression/CompressionFactory.cs @@ -1,6 +1,9 @@ namespace Titanium.Web.Proxy.Compression { - class CompressionFactory + /// + /// A factory to generate the compression methods based on the type of compression + /// + internal class CompressionFactory { public ICompression Create(string type) { diff --git a/Titanium.Web.Proxy/Compression/DeflateCompression.cs b/Titanium.Web.Proxy/Compression/DeflateCompression.cs index 032c97142..bf4eb855e 100644 --- a/Titanium.Web.Proxy/Compression/DeflateCompression.cs +++ b/Titanium.Web.Proxy/Compression/DeflateCompression.cs @@ -4,7 +4,10 @@ namespace Titanium.Web.Proxy.Compression { - class DeflateCompression : ICompression + /// + /// Concrete implementation of deflate compression + /// + internal class DeflateCompression : ICompression { public async Task Compress(byte[] responseBody) { diff --git a/Titanium.Web.Proxy/Compression/GZipCompression.cs b/Titanium.Web.Proxy/Compression/GZipCompression.cs index 893008b8a..5e14746bf 100644 --- a/Titanium.Web.Proxy/Compression/GZipCompression.cs +++ b/Titanium.Web.Proxy/Compression/GZipCompression.cs @@ -4,7 +4,10 @@ namespace Titanium.Web.Proxy.Compression { - class GZipCompression : ICompression + /// + /// concreate implementation of gzip compression + /// + internal class GZipCompression : ICompression { public async Task Compress(byte[] responseBody) { diff --git a/Titanium.Web.Proxy/Compression/ICompression.cs b/Titanium.Web.Proxy/Compression/ICompression.cs index a0b8ce31f..4dd590208 100644 --- a/Titanium.Web.Proxy/Compression/ICompression.cs +++ b/Titanium.Web.Proxy/Compression/ICompression.cs @@ -2,6 +2,9 @@ namespace Titanium.Web.Proxy.Compression { + /// + /// An inteface for http compression + /// interface ICompression { Task Compress(byte[] responseBody); diff --git a/Titanium.Web.Proxy/Compression/ZlibCompression.cs b/Titanium.Web.Proxy/Compression/ZlibCompression.cs index fa8622bba..80ed1fc53 100644 --- a/Titanium.Web.Proxy/Compression/ZlibCompression.cs +++ b/Titanium.Web.Proxy/Compression/ZlibCompression.cs @@ -4,7 +4,10 @@ namespace Titanium.Web.Proxy.Compression { - class ZlibCompression : ICompression + /// + /// concrete implementation of zlib compression + /// + internal class ZlibCompression : ICompression { public async Task Compress(byte[] responseBody) { diff --git a/Titanium.Web.Proxy/Decompression/DecompressionFactory.cs b/Titanium.Web.Proxy/Decompression/DecompressionFactory.cs index 4548fc147..af6a42209 100644 --- a/Titanium.Web.Proxy/Decompression/DecompressionFactory.cs +++ b/Titanium.Web.Proxy/Decompression/DecompressionFactory.cs @@ -1,5 +1,8 @@ namespace Titanium.Web.Proxy.Decompression { + /// + /// A factory to generate the de-compression methods based on the type of compression + /// internal class DecompressionFactory { internal IDecompression Create(string type) diff --git a/Titanium.Web.Proxy/Decompression/DefaultDecompression.cs b/Titanium.Web.Proxy/Decompression/DefaultDecompression.cs index e8fa483c0..cbbc2d6ef 100644 --- a/Titanium.Web.Proxy/Decompression/DefaultDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/DefaultDecompression.cs @@ -2,6 +2,10 @@ namespace Titanium.Web.Proxy.Decompression { + + /// + /// When no compression is specified just return the byte array + /// internal class DefaultDecompression : IDecompression { public Task Decompress(byte[] compressedArray) diff --git a/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs b/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs index 175745f27..12d18d0f6 100644 --- a/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/DeflateDecompression.cs @@ -5,6 +5,9 @@ namespace Titanium.Web.Proxy.Decompression { + /// + /// concrete implementation of deflate de-compression + /// internal class DeflateDecompression : IDecompression { public async Task Decompress(byte[] compressedArray) @@ -18,7 +21,7 @@ public async Task Decompress(byte[] compressedArray) using (var output = new MemoryStream()) { int read; - while ((read = await decompressor.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) > 0) + while ((read = await decompressor.ReadAsync(buffer, 0, buffer.Length)) > 0) { await output.WriteAsync(buffer, 0, read); } diff --git a/Titanium.Web.Proxy/Decompression/GZipDecompression.cs b/Titanium.Web.Proxy/Decompression/GZipDecompression.cs index 61a591399..27972308f 100644 --- a/Titanium.Web.Proxy/Decompression/GZipDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/GZipDecompression.cs @@ -5,6 +5,9 @@ namespace Titanium.Web.Proxy.Decompression { + /// + /// concrete implementation of gzip de-compression + /// internal class GZipDecompression : IDecompression { public async Task Decompress(byte[] compressedArray) @@ -15,7 +18,7 @@ public async Task Decompress(byte[] compressedArray) using (var output = new MemoryStream()) { int read; - while ((read = await decompressor.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) > 0) + while ((read = await decompressor.ReadAsync(buffer, 0, buffer.Length)) > 0) { await output.WriteAsync(buffer, 0, read); } diff --git a/Titanium.Web.Proxy/Decompression/IDecompression.cs b/Titanium.Web.Proxy/Decompression/IDecompression.cs index f215bd1ad..670a0816c 100644 --- a/Titanium.Web.Proxy/Decompression/IDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/IDecompression.cs @@ -1,8 +1,10 @@ -using System.IO; -using System.Threading.Tasks; +using System.Threading.Tasks; namespace Titanium.Web.Proxy.Decompression { + /// + /// An interface for decompression + /// internal interface IDecompression { Task Decompress(byte[] compressedArray); diff --git a/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs b/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs index a81a0ca39..cac856d1c 100644 --- a/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs +++ b/Titanium.Web.Proxy/Decompression/ZlibDecompression.cs @@ -5,6 +5,9 @@ namespace Titanium.Web.Proxy.Decompression { + /// + /// concrete implemetation of zlib de-compression + /// internal class ZlibDecompression : IDecompression { public async Task Decompress(byte[] compressedArray) @@ -17,7 +20,7 @@ public async Task Decompress(byte[] compressedArray) using (var output = new MemoryStream()) { int read; - while ((read = await decompressor.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) > 0) + while ((read = await decompressor.ReadAsync(buffer, 0, buffer.Length)) > 0) { await output.WriteAsync(buffer, 0, read); } diff --git a/Titanium.Web.Proxy/EventArguments/CertificateSelectionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/CertificateSelectionEventArgs.cs index eed8cc774..167f31bd6 100644 --- a/Titanium.Web.Proxy/EventArguments/CertificateSelectionEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/CertificateSelectionEventArgs.cs @@ -3,6 +3,9 @@ namespace Titanium.Web.Proxy.EventArguments { + /// + /// An argument passed on to user for client certificate selection during mutual SSL authentication + /// public class CertificateSelectionEventArgs : EventArgs, IDisposable { public object sender { get; internal set; } diff --git a/Titanium.Web.Proxy/EventArguments/CertificateValidationEventArgs.cs b/Titanium.Web.Proxy/EventArguments/CertificateValidationEventArgs.cs index 6704cc33c..3f6c013fd 100644 --- a/Titanium.Web.Proxy/EventArguments/CertificateValidationEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/CertificateValidationEventArgs.cs @@ -4,6 +4,9 @@ namespace Titanium.Web.Proxy.EventArguments { + /// + /// An argument passed on to the user for validating the server certificate during SSL authentication + /// public class CertificateValidationEventArgs : EventArgs, IDisposable { public X509Certificate Certificate { get; internal set; } diff --git a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs index 701f79a96..935eafcf3 100644 --- a/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs +++ b/Titanium.Web.Proxy/EventArguments/SessionEventArgs.cs @@ -9,7 +9,6 @@ using System.Threading.Tasks; using Titanium.Web.Proxy.Network; using System.Net; -using System.Net.Sockets; namespace Titanium.Web.Proxy.EventArguments { diff --git a/Titanium.Web.Proxy/Extensions/ByteArrayExtensions.cs b/Titanium.Web.Proxy/Extensions/ByteArrayExtensions.cs index 6082b529e..d355ce329 100644 --- a/Titanium.Web.Proxy/Extensions/ByteArrayExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/ByteArrayExtensions.cs @@ -1,8 +1,4 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Titanium.Web.Proxy.Extensions { diff --git a/Titanium.Web.Proxy/Extensions/StreamExtensions.cs b/Titanium.Web.Proxy/Extensions/StreamExtensions.cs index fbc8037b3..8e9149b1f 100644 --- a/Titanium.Web.Proxy/Extensions/StreamExtensions.cs +++ b/Titanium.Web.Proxy/Extensions/StreamExtensions.cs @@ -1,5 +1,4 @@ -using System; -using System.Globalization; +using System.Globalization; using System.IO; using System.Text; using System.Threading.Tasks; @@ -8,6 +7,9 @@ namespace Titanium.Web.Proxy.Extensions { + /// + /// Extensions used for Stream and CustomBinaryReader objects + /// internal static class StreamExtensions { /// @@ -134,7 +136,7 @@ internal static async Task WriteResponseBody(this CustomBinaryReader inStreamRea var bytesRead = 0; var totalBytesRead = 0; - while ((bytesRead += await inStreamReader.BaseStream.ReadAsync(buffer, 0, bytesToRead).ConfigureAwait(false)) > 0) + while ((bytesRead += await inStreamReader.BaseStream.ReadAsync(buffer, 0, bytesToRead)) > 0) { await outStream.WriteAsync(buffer, 0, bytesRead); totalBytesRead += bytesRead; diff --git a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs index 129ad2e52..1d003516a 100644 --- a/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs +++ b/Titanium.Web.Proxy/Helpers/CustomBinaryReader.cs @@ -18,9 +18,12 @@ internal class CustomBinaryReader : IDisposable { private Stream stream; private Encoding encoding; + internal CustomBinaryReader(Stream stream) { this.stream = stream; + + //default to UTF-8 this.encoding = Encoding.UTF8; } @@ -41,16 +44,21 @@ internal async Task ReadLineAsync() while ((await this.stream.ReadAsync(buffer, 0, 1)) > 0) { + //if new line if (lastChar == '\r' && buffer[0] == '\n') { var result = readBuffer.ToArray(); return encoding.GetString(result.SubArray(0, result.Length - 1)); } + //end of stream if (buffer[0] == '\0') { return encoding.GetString(readBuffer.ToArray()); } + await readBuffer.WriteAsync(buffer,0,1); + + //store last char for new line comparison lastChar = (char)buffer[0]; } @@ -58,7 +66,7 @@ internal async Task ReadLineAsync() } catch (IOException) { - return encoding.GetString(readBuffer.ToArray()); + throw; } } } @@ -71,13 +79,18 @@ internal async Task> ReadAllLinesAsync() { string tmpLine; var requestLines = new List(); - while (!string.IsNullOrEmpty(tmpLine = await ReadLineAsync().ConfigureAwait(false))) + while (!string.IsNullOrEmpty(tmpLine = await ReadLineAsync())) { requestLines.Add(tmpLine); } return requestLines; } + /// + /// Read the specified number of raw bytes from the base stream + /// + /// + /// internal async Task ReadBytesAsync(long totalBytesToRead) { int bytesToRead = ProxyConstants.BUFFER_SIZE; @@ -92,7 +105,7 @@ internal async Task ReadBytesAsync(long totalBytesToRead) using (var outStream = new MemoryStream()) { - while ((bytesRead += await this.stream.ReadAsync(buffer, 0, bytesToRead).ConfigureAwait(false)) > 0) + while ((bytesRead += await this.stream.ReadAsync(buffer, 0, bytesToRead)) > 0) { await outStream.WriteAsync(buffer, 0, bytesRead); totalBytesRead += bytesRead; diff --git a/Titanium.Web.Proxy/Helpers/SystemProxy.cs b/Titanium.Web.Proxy/Helpers/SystemProxy.cs index f46426dcd..36597e01d 100644 --- a/Titanium.Web.Proxy/Helpers/SystemProxy.cs +++ b/Titanium.Web.Proxy/Helpers/SystemProxy.cs @@ -5,8 +5,12 @@ using System.Collections.Generic; using System.Linq; +/// +/// Helper classes for setting system proxy settings +/// namespace Titanium.Web.Proxy.Helpers { + internal static class NativeMethods { [DllImport("wininet.dll")] @@ -14,7 +18,7 @@ internal static extern bool InternetSetOption(IntPtr hInternet, int dwOption, In int dwBufferLength); } - internal class HttpSystemProxyValue + internal class HttpSystemProxyValue { internal string HostName { get; set; } internal int Port { get; set; } @@ -59,14 +63,16 @@ internal static void SetHttpProxy(string hostname, int port) Refresh(); } - + /// + /// Remove the http proxy setting from current machine + /// internal static void RemoveHttpProxy() { var reg = Registry.CurrentUser.OpenSubKey( "Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings", true); if (reg != null) { - if (reg.GetValue("ProxyServer")!=null) + if (reg.GetValue("ProxyServer") != null) { var exisitingContent = reg.GetValue("ProxyServer") as string; @@ -89,11 +95,16 @@ internal static void RemoveHttpProxy() Refresh(); } + /// + /// Set the HTTPS proxy server for current machine + /// + /// + /// internal static void SetHttpsProxy(string hostname, int port) { var reg = Registry.CurrentUser.OpenSubKey( "Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings", true); - + if (reg != null) { prepareRegistry(reg); @@ -116,6 +127,9 @@ internal static void SetHttpsProxy(string hostname, int port) Refresh(); } + /// + /// Removes the https proxy setting to nothing + /// internal static void RemoveHttpsProxy() { var reg = Registry.CurrentUser.OpenSubKey( @@ -139,13 +153,16 @@ internal static void RemoveHttpsProxy() reg.SetValue("ProxyEnable", 0); reg.SetValue("ProxyServer", string.Empty); } - + } } Refresh(); } + /// + /// Removes all types of proxy settings (both http & https) + /// internal static void DisableAllProxy() { var reg = Registry.CurrentUser.OpenSubKey( @@ -159,6 +176,12 @@ internal static void DisableAllProxy() Refresh(); } + + /// + /// Get the current system proxy setting values + /// + /// + /// private static List GetSystemProxyValues(string prevServerValue) { var result = new List(); @@ -187,6 +210,11 @@ private static List GetSystemProxyValues(string prevServer return result; } + /// + /// Parses the system proxy setting string + /// + /// + /// private static HttpSystemProxyValue parseProxyValue(string value) { var tmp = Regex.Replace(value, @"\s+", " ").Trim().ToLower(); @@ -213,7 +241,10 @@ private static HttpSystemProxyValue parseProxyValue(string value) return null; } - + /// + /// Prepares the proxy server registry (create empty values if they don't exist) + /// + /// private static void prepareRegistry(RegistryKey reg) { if (reg.GetValue("ProxyEnable") == null) @@ -227,8 +258,10 @@ private static void prepareRegistry(RegistryKey reg) } } - + /// + /// Refresh the settings so that the system know about a change in proxy setting + /// private static void Refresh() { NativeMethods.InternetSetOption(IntPtr.Zero, InternetOptionSettingsChanged, IntPtr.Zero, 0); diff --git a/Titanium.Web.Proxy/Helpers/Tcp.cs b/Titanium.Web.Proxy/Helpers/Tcp.cs index cfd002090..4353d5943 100644 --- a/Titanium.Web.Proxy/Helpers/Tcp.cs +++ b/Titanium.Web.Proxy/Helpers/Tcp.cs @@ -12,11 +12,24 @@ namespace Titanium.Web.Proxy.Helpers { + internal class TcpHelper { + /// + /// relays the input clientStream to the server at the specified host name & port with the given httpCmd & headers as prefix + /// Usefull for websocket requests + /// + /// + /// + /// + /// + /// + /// + /// internal async static Task SendRaw(Stream clientStream, string httpCmd, List requestHeaders, string hostName, int tunnelPort, bool isHttps) { + //prepare the prefix content StringBuilder sb = null; if (httpCmd != null || requestHeaders != null) { @@ -38,7 +51,7 @@ internal async static Task SendRaw(Stream clientStream, string httpCmd, Listclient & client=>server data if (sb != null) sendRelay = clientStream.CopyToAsync(sb.ToString(), tunnelStream); else diff --git a/Titanium.Web.Proxy/Http/HttpWebClient.cs b/Titanium.Web.Proxy/Http/HttpWebClient.cs index dddb8146b..75aeaf750 100644 --- a/Titanium.Web.Proxy/Http/HttpWebClient.cs +++ b/Titanium.Web.Proxy/Http/HttpWebClient.cs @@ -9,6 +9,9 @@ namespace Titanium.Web.Proxy.Http { + /// + /// Used to communicate with the server over HTTP(S) + /// public class HttpWebClient { /// @@ -19,6 +22,9 @@ public class HttpWebClient public Request Request { get; set; } public Response Response { get; set; } + /// + /// Is Https? + /// public bool IsHttps { get @@ -27,6 +33,10 @@ public bool IsHttps } } + /// + /// Set the tcp connection to server used by this webclient + /// + /// internal void SetConnection(TcpConnection Connection) { Connection.LastAccess = DateTime.Now; @@ -39,19 +49,24 @@ internal HttpWebClient() this.Response = new Response(); } + /// + /// Prepare & send the http(s) request + /// + /// internal async Task SendRequest() { Stream stream = ServerConnection.Stream; StringBuilder requestLines = new StringBuilder(); - + + //prepare the request & headers requestLines.AppendLine(string.Join(" ", new string[3] { this.Request.Method, this.Request.RequestUri.PathAndQuery, string.Format("HTTP/{0}.{1}",this.Request.HttpVersion.Major, this.Request.HttpVersion.Minor) })); - + //write request headers foreach (HttpHeader httpHeader in this.Request.RequestHeaders) { requestLines.AppendLine(httpHeader.Name + ':' + httpHeader.Value); @@ -87,6 +102,10 @@ internal async Task SendRequest() } } + /// + /// Receive & parse the http response from server + /// + /// internal async Task ReceiveResponse() { //return if this is already read @@ -130,6 +149,7 @@ internal async Task ReceiveResponse() return; } + //read response headers List responseLines = await ServerConnection.StreamReader.ReadAllLinesAsync(); for (int index = 0; index < responseLines.Count; ++index) diff --git a/Titanium.Web.Proxy/Http/Request.cs b/Titanium.Web.Proxy/Http/Request.cs index 4bfecffb1..1af198b27 100644 --- a/Titanium.Web.Proxy/Http/Request.cs +++ b/Titanium.Web.Proxy/Http/Request.cs @@ -7,6 +7,9 @@ namespace Titanium.Web.Proxy.Http { + /// + /// A HTTP(S) request object + /// public class Request { public string Method { get; set; } diff --git a/Titanium.Web.Proxy/Http/Response.cs b/Titanium.Web.Proxy/Http/Response.cs index d08c8ec9c..1b8b830b5 100644 --- a/Titanium.Web.Proxy/Http/Response.cs +++ b/Titanium.Web.Proxy/Http/Response.cs @@ -8,7 +8,9 @@ namespace Titanium.Web.Proxy.Http { - + /// + /// Http(s) response object + /// public class Response { public string ResponseStatusCode { get; set; } diff --git a/Titanium.Web.Proxy/Http/Responses/OkResponse.cs b/Titanium.Web.Proxy/Http/Responses/OkResponse.cs index c23333ee5..eddec08b9 100644 --- a/Titanium.Web.Proxy/Http/Responses/OkResponse.cs +++ b/Titanium.Web.Proxy/Http/Responses/OkResponse.cs @@ -1,7 +1,8 @@ -using Titanium.Web.Proxy.Network; - -namespace Titanium.Web.Proxy.Http.Responses +namespace Titanium.Web.Proxy.Http.Responses { + /// + /// 200 Ok response + /// public class OkResponse : Response { public OkResponse() diff --git a/Titanium.Web.Proxy/Http/Responses/RedirectResponse.cs b/Titanium.Web.Proxy/Http/Responses/RedirectResponse.cs index 212f8cac3..db41f5eac 100644 --- a/Titanium.Web.Proxy/Http/Responses/RedirectResponse.cs +++ b/Titanium.Web.Proxy/Http/Responses/RedirectResponse.cs @@ -2,6 +2,9 @@ namespace Titanium.Web.Proxy.Http.Responses { + /// + /// Redirect response + /// public class RedirectResponse : Response { public RedirectResponse() diff --git a/Titanium.Web.Proxy/Models/EndPoint.cs b/Titanium.Web.Proxy/Models/EndPoint.cs index 736bf9984..a3e27ee67 100644 --- a/Titanium.Web.Proxy/Models/EndPoint.cs +++ b/Titanium.Web.Proxy/Models/EndPoint.cs @@ -4,6 +4,9 @@ namespace Titanium.Web.Proxy.Models { + /// + /// An abstract endpoint where the proxy listens + /// public abstract class ProxyEndPoint { public ProxyEndPoint(IPAddress IpAddress, int Port, bool EnableSsl) @@ -20,6 +23,10 @@ public ProxyEndPoint(IPAddress IpAddress, int Port, bool EnableSsl) internal TcpListener listener { get; set; } } + /// + /// A proxy endpoint that the client is aware of + /// So client application know that it is communicating with a proxy server + /// public class ExplicitProxyEndPoint : ProxyEndPoint { internal bool IsSystemHttpProxy { get; set; } @@ -34,6 +41,10 @@ public ExplicitProxyEndPoint(IPAddress IpAddress, int Port, bool EnableSsl) } } + /// + /// A proxy end point client is not aware of + /// Usefull when requests are redirected to this proxy end point through port forwarding + /// public class TransparentProxyEndPoint : ProxyEndPoint { //Name of the Certificate need to be sent (same as the hostname we want to proxy) diff --git a/Titanium.Web.Proxy/Models/ExternalProxy.cs b/Titanium.Web.Proxy/Models/ExternalProxy.cs index 1c8ae8f6e..7ed00a796 100644 --- a/Titanium.Web.Proxy/Models/ExternalProxy.cs +++ b/Titanium.Web.Proxy/Models/ExternalProxy.cs @@ -1,12 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Text; -using System.Threading.Tasks; - -namespace Titanium.Web.Proxy.Models +namespace Titanium.Web.Proxy.Models { + /// + /// An upstream proxy this proxy uses if any + /// public class ExternalProxy { public string HostName { get; set; } diff --git a/Titanium.Web.Proxy/Network/CachedCertificate.cs b/Titanium.Web.Proxy/Network/CachedCertificate.cs new file mode 100644 index 000000000..86645581a --- /dev/null +++ b/Titanium.Web.Proxy/Network/CachedCertificate.cs @@ -0,0 +1,25 @@ +using System; +using System.Security.Cryptography.X509Certificates; + +namespace Titanium.Web.Proxy.Network +{ + /// + /// An object that holds the cached certificate + /// + internal class CachedCertificate + { + internal X509Certificate2 Certificate { get; set; } + + /// + /// last time this certificate was used + /// Usefull in determining its cache lifetime + /// + internal DateTime LastAccess { get; set; } + + internal CachedCertificate() + { + LastAccess = DateTime.Now; + } + + } +} diff --git a/Titanium.Web.Proxy/Helpers/CertificateManager.cs b/Titanium.Web.Proxy/Network/CertificateManager.cs similarity index 84% rename from Titanium.Web.Proxy/Helpers/CertificateManager.cs rename to Titanium.Web.Proxy/Network/CertificateManager.cs index e19cdcbab..63af72392 100644 --- a/Titanium.Web.Proxy/Helpers/CertificateManager.cs +++ b/Titanium.Web.Proxy/Network/CertificateManager.cs @@ -1,264 +1,291 @@ -using System; -using System.IO; -using System.Diagnostics; -using System.Collections.Generic; -using System.Security.Cryptography.X509Certificates; -using System.Reflection; -using System.Threading.Tasks; -using System.Threading; -using System.Linq; - -namespace Titanium.Web.Proxy.Helpers -{ - internal class CachedCertificate - { - internal X509Certificate2 Certificate { get; set; } - - internal DateTime LastAccess { get; set; } - - internal CachedCertificate() - { - LastAccess = DateTime.Now; - } - - } - internal class CertificateManager : IDisposable - { - private const string CertCreateFormat = - "-ss {0} -n \"CN={1}, O={2}\" -sky {3} -cy {4} -m 120 -a sha256 -eku 1.3.6.1.5.5.7.3.1 {5}"; - - private readonly IDictionary certificateCache; - private static SemaphoreSlim semaphoreLock = new SemaphoreSlim(1); - - internal string Issuer { get; private set; } - internal string RootCertificateName { get; private set; } - - internal X509Store MyStore { get; private set; } - internal X509Store RootStore { get; private set; } - - internal CertificateManager(string issuer, string rootCertificateName) - { - Issuer = issuer; - RootCertificateName = rootCertificateName; - - MyStore = new X509Store(StoreName.My, StoreLocation.CurrentUser); - RootStore = new X509Store(StoreName.Root, StoreLocation.CurrentUser); - - certificateCache = new Dictionary(); - } - - /// - /// Attempts to move a self-signed certificate to the root store. - /// - /// true if succeeded, else false - internal async Task CreateTrustedRootCertificate() - { - X509Certificate2 rootCertificate = - await CreateCertificate(RootStore, RootCertificateName, true); - - return rootCertificate != null; - } - /// - /// Attempts to remove the self-signed certificate from the root store. - /// - /// true if succeeded, else false - internal bool DestroyTrustedRootCertificate() - { - return DestroyCertificate(RootStore, RootCertificateName, false); - } - - internal X509Certificate2Collection FindCertificates(string certificateSubject) - { - return FindCertificates(MyStore, certificateSubject); - } - protected virtual X509Certificate2Collection FindCertificates(X509Store store, string certificateSubject) - { - X509Certificate2Collection discoveredCertificates = store.Certificates - .Find(X509FindType.FindBySubjectDistinguishedName, certificateSubject, false); - - return discoveredCertificates.Count > 0 ? - discoveredCertificates : null; - } - - internal async Task CreateCertificate(string certificateName, bool isRootCertificate) - { - return await CreateCertificate(MyStore, certificateName, isRootCertificate); - } - - protected async virtual Task CreateCertificate(X509Store store, string certificateName, bool isRootCertificate) - { - await semaphoreLock.WaitAsync(); - - try - { - if (certificateCache.ContainsKey(certificateName)) - { - var cached = certificateCache[certificateName]; - cached.LastAccess = DateTime.Now; - return cached.Certificate; - } - - X509Certificate2 certificate = null; - store.Open(OpenFlags.ReadWrite); - string certificateSubject = string.Format("CN={0}, O={1}", certificateName, Issuer); - - X509Certificate2Collection certificates; - - if (isRootCertificate) - { - certificates = FindCertificates(store, certificateSubject); - - if (certificates != null) - certificate = certificates[0]; - } - - if (certificate == null) - { - string[] args = new[] { - GetCertificateCreateArgs(store, certificateName) }; - - await CreateCertificate(args); - certificates = FindCertificates(store, certificateSubject); - - //remove it from store - if (!isRootCertificate) - DestroyCertificate(certificateName); - - if (certificates != null) - certificate = certificates[0]; - } - - store.Close(); - - if (certificate != null && !certificateCache.ContainsKey(certificateName)) - certificateCache.Add(certificateName, new CachedCertificate() { Certificate = certificate }); - - return certificate; - } - finally - { - semaphoreLock.Release(); - } - } - - protected virtual Task CreateCertificate(string[] args) - { - - // there is no non-generic TaskCompletionSource - var tcs = new TaskCompletionSource(); - - var process = new Process(); - - string file = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "makecert.exe"); - - if (!File.Exists(file)) - throw new Exception("Unable to locate 'makecert.exe'."); - - process.StartInfo.Verb = "runas"; - process.StartInfo.Arguments = args != null ? args[0] : string.Empty; - process.StartInfo.CreateNoWindow = true; - process.StartInfo.UseShellExecute = false; - process.StartInfo.FileName = file; - process.EnableRaisingEvents = true; - - process.Exited += (sender, processArgs) => - { - tcs.SetResult(process.ExitCode); - process.Dispose(); - }; - - bool started = process.Start(); - if (!started) - { - //you may allow for the process to be re-used (started = false) - //but I'm not sure about the guarantees of the Exited event in such a case - throw new InvalidOperationException("Could not start process: " + process); - } - - return tcs.Task; - - } - - internal bool DestroyCertificate(string certificateName) - { - return DestroyCertificate(MyStore, certificateName, false); - } - - protected virtual bool DestroyCertificate(X509Store store, string certificateName, bool removeFromCache) - { - X509Certificate2Collection certificates = null; - - store.Open(OpenFlags.ReadWrite); - string certificateSubject = string.Format("CN={0}, O={1}", certificateName, Issuer); - - certificates = FindCertificates(store, certificateSubject); - if (certificates != null) - { - store.RemoveRange(certificates); - certificates = FindCertificates(store, certificateSubject); - } - - store.Close(); - if (removeFromCache && - certificateCache.ContainsKey(certificateName)) - { - certificateCache.Remove(certificateName); - } - return certificates == null; - } - - protected virtual string GetCertificateCreateArgs(X509Store store, string certificateName) - { - bool isRootCertificate = - (certificateName == RootCertificateName); - - string certCreatArgs = string.Format(CertCreateFormat, - store.Name, certificateName, Issuer, - isRootCertificate ? "signature" : "exchange", - isRootCertificate ? "authority" : "end", - isRootCertificate ? "-h 1 -r" : string.Format("-pe -in \"{0}\" -is Root", RootCertificateName)); - - return certCreatArgs; - } - - private static bool clearCertificates { get; set; } - - internal void StopClearIdleCertificates() - { - clearCertificates = false; - } - - internal async void ClearIdleCertificates() - { - clearCertificates = true; - while (clearCertificates) - { - await semaphoreLock.WaitAsync(); - - try - { - var cutOff = DateTime.Now.AddMinutes(-1 * ProxyServer.CertificateCacheTimeOutMinutes); - - var outdated = certificateCache - .Where(x => x.Value.LastAccess < cutOff) - .ToList(); - - foreach (var cache in outdated) - certificateCache.Remove(cache.Key); - } - finally { semaphoreLock.Release(); } - - await Task.Delay(1000 * 60 * 3); - } - } - - public void Dispose() - { - if (MyStore != null) - MyStore.Close(); - - if (RootStore != null) - RootStore.Close(); - } - } +using System; +using System.IO; +using System.Diagnostics; +using System.Collections.Generic; +using System.Security.Cryptography.X509Certificates; +using System.Reflection; +using System.Threading.Tasks; +using System.Threading; +using System.Linq; + +namespace Titanium.Web.Proxy.Network +{ + /// + /// A class to manage SSL certificates used by this proxy server + /// + internal class CertificateManager : IDisposable + { + private const string CertCreateFormat = + "-ss {0} -n \"CN={1}, O={2}\" -sky {3} -cy {4} -m 120 -a sha256 -eku 1.3.6.1.5.5.7.3.1 {5}"; + + private readonly IDictionary certificateCache; + private static SemaphoreSlim semaphoreLock = new SemaphoreSlim(1); + + internal string Issuer { get; private set; } + internal string RootCertificateName { get; private set; } + + internal X509Store MyStore { get; private set; } + internal X509Store RootStore { get; private set; } + + internal CertificateManager(string issuer, string rootCertificateName) + { + Issuer = issuer; + RootCertificateName = rootCertificateName; + + MyStore = new X509Store(StoreName.My, StoreLocation.CurrentUser); + RootStore = new X509Store(StoreName.Root, StoreLocation.CurrentUser); + + certificateCache = new Dictionary(); + } + + /// + /// Attempts to move a self-signed certificate to the root store. + /// + /// true if succeeded, else false + internal async Task CreateTrustedRootCertificate() + { + X509Certificate2 rootCertificate = + await CreateCertificate(RootStore, RootCertificateName, true); + + return rootCertificate != null; + } + /// + /// Attempts to remove the self-signed certificate from the root store. + /// + /// true if succeeded, else false + internal bool DestroyTrustedRootCertificate() + { + return DestroyCertificate(RootStore, RootCertificateName, false); + } + + internal X509Certificate2Collection FindCertificates(string certificateSubject) + { + return FindCertificates(MyStore, certificateSubject); + } + protected virtual X509Certificate2Collection FindCertificates(X509Store store, string certificateSubject) + { + X509Certificate2Collection discoveredCertificates = store.Certificates + .Find(X509FindType.FindBySubjectDistinguishedName, certificateSubject, false); + + return discoveredCertificates.Count > 0 ? + discoveredCertificates : null; + } + + internal async Task CreateCertificate(string certificateName, bool isRootCertificate) + { + return await CreateCertificate(MyStore, certificateName, isRootCertificate); + } + + /// + /// Create an SSL certificate + /// + /// + /// + /// + /// + protected async virtual Task CreateCertificate(X509Store store, string certificateName, bool isRootCertificate) + { + await semaphoreLock.WaitAsync(); + + try + { + if (certificateCache.ContainsKey(certificateName)) + { + var cached = certificateCache[certificateName]; + cached.LastAccess = DateTime.Now; + return cached.Certificate; + } + + X509Certificate2 certificate = null; + store.Open(OpenFlags.ReadWrite); + string certificateSubject = string.Format("CN={0}, O={1}", certificateName, Issuer); + + X509Certificate2Collection certificates; + + if (isRootCertificate) + { + certificates = FindCertificates(store, certificateSubject); + + if (certificates != null) + certificate = certificates[0]; + } + + if (certificate == null) + { + string[] args = new[] { + GetCertificateCreateArgs(store, certificateName) }; + + await CreateCertificate(args); + certificates = FindCertificates(store, certificateSubject); + + //remove it from store + if (!isRootCertificate) + DestroyCertificate(certificateName); + + if (certificates != null) + certificate = certificates[0]; + } + + store.Close(); + + if (certificate != null && !certificateCache.ContainsKey(certificateName)) + certificateCache.Add(certificateName, new CachedCertificate() { Certificate = certificate }); + + return certificate; + } + finally + { + semaphoreLock.Release(); + } + } + + /// + /// Create certificate using makecert.exe + /// + /// + /// + protected virtual Task CreateCertificate(string[] args) + { + + // there is no non-generic TaskCompletionSource + var tcs = new TaskCompletionSource(); + + var process = new Process(); + + string file = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "makecert.exe"); + + if (!File.Exists(file)) + throw new Exception("Unable to locate 'makecert.exe'."); + + process.StartInfo.Verb = "runas"; + process.StartInfo.Arguments = args != null ? args[0] : string.Empty; + process.StartInfo.CreateNoWindow = true; + process.StartInfo.UseShellExecute = false; + process.StartInfo.FileName = file; + process.EnableRaisingEvents = true; + + process.Exited += (sender, processArgs) => + { + tcs.SetResult(process.ExitCode); + process.Dispose(); + }; + + bool started = process.Start(); + if (!started) + { + //you may allow for the process to be re-used (started = false) + //but I'm not sure about the guarantees of the Exited event in such a case + throw new InvalidOperationException("Could not start process: " + process); + } + + return tcs.Task; + + } + /// + /// Destroy an SSL certificate from local store + /// + /// + /// + internal bool DestroyCertificate(string certificateName) + { + return DestroyCertificate(MyStore, certificateName, false); + } + + /// + /// Destroy certificate from the specified store + /// optionally also remove from proxy certificate cache + /// + /// + /// + /// + /// + protected virtual bool DestroyCertificate(X509Store store, string certificateName, bool removeFromCache) + { + X509Certificate2Collection certificates = null; + + store.Open(OpenFlags.ReadWrite); + string certificateSubject = string.Format("CN={0}, O={1}", certificateName, Issuer); + + certificates = FindCertificates(store, certificateSubject); + if (certificates != null) + { + store.RemoveRange(certificates); + certificates = FindCertificates(store, certificateSubject); + } + + store.Close(); + if (removeFromCache && + certificateCache.ContainsKey(certificateName)) + { + certificateCache.Remove(certificateName); + } + return certificates == null; + } + /// + /// Create the neccessary arguments for makecert.exe to create the required certificate + /// + /// + /// + /// + protected virtual string GetCertificateCreateArgs(X509Store store, string certificateName) + { + bool isRootCertificate = + (certificateName == RootCertificateName); + + string certCreatArgs = string.Format(CertCreateFormat, + store.Name, certificateName, Issuer, + isRootCertificate ? "signature" : "exchange", + isRootCertificate ? "authority" : "end", + isRootCertificate ? "-h 1 -r" : string.Format("-pe -in \"{0}\" -is Root", RootCertificateName)); + + return certCreatArgs; + } + + private static bool clearCertificates { get; set; } + + /// + /// Stops the certificate cache clear process + /// + internal void StopClearIdleCertificates() + { + clearCertificates = false; + } + + /// + /// A method to clear outdated certificates + /// + internal async void ClearIdleCertificates() + { + clearCertificates = true; + while (clearCertificates) + { + await semaphoreLock.WaitAsync(); + + try + { + var cutOff = DateTime.Now.AddMinutes(-1 * ProxyServer.CertificateCacheTimeOutMinutes); + + var outdated = certificateCache + .Where(x => x.Value.LastAccess < cutOff) + .ToList(); + + foreach (var cache in outdated) + certificateCache.Remove(cache.Key); + } + finally { semaphoreLock.Release(); } + + //after a minute come back to check for outdated certificates in cache + await Task.Delay(1000 * 60); + } + } + + public void Dispose() + { + if (MyStore != null) + MyStore.Close(); + + if (RootStore != null) + RootStore.Close(); + } + } } \ No newline at end of file diff --git a/Titanium.Web.Proxy/Network/TcpConnection.cs b/Titanium.Web.Proxy/Network/TcpConnection.cs new file mode 100644 index 000000000..9330a2f17 --- /dev/null +++ b/Titanium.Web.Proxy/Network/TcpConnection.cs @@ -0,0 +1,45 @@ +using System; +using System.IO; +using System.Net.Sockets; +using Titanium.Web.Proxy.Helpers; + +namespace Titanium.Web.Proxy.Network +{ + /// + /// An object that holds TcpConnection to a particular server & port + /// + public class TcpConnection + { + internal string HostName { get; set; } + internal int port { get; set; } + + internal bool IsHttps { get; set; } + + /// + /// Http version + /// + internal Version Version { get; set; } + + internal TcpClient TcpClient { get; set; } + + /// + /// used to read lines from server + /// + internal CustomBinaryReader StreamReader { get; set; } + + /// + /// Server stream + /// + internal Stream Stream { get; set; } + + /// + /// Last time this connection was used + /// + internal DateTime LastAccess { get; set; } + + internal TcpConnection() + { + LastAccess = DateTime.Now; + } + } +} diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index 0a90d7385..3f54e6e2c 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -10,44 +10,41 @@ using System.Threading; using Titanium.Web.Proxy.Extensions; using Titanium.Web.Proxy.Shared; -using System.Security.Cryptography.X509Certificates; -using Titanium.Web.Proxy.EventArguments; -using Titanium.Web.Proxy.Models; namespace Titanium.Web.Proxy.Network { - public class TcpConnection - { - internal string HostName { get; set; } - internal int port { get; set; } - internal bool IsHttps { get; set; } - internal Version Version { get; set; } - - internal TcpClient TcpClient { get; set; } - internal CustomBinaryReader StreamReader { get; set; } - internal Stream Stream { get; set; } - - internal DateTime LastAccess { get; set; } - - internal TcpConnection() - { - LastAccess = DateTime.Now; - } - } - + /// + /// A class that manages Tcp Connection to server used by this proxy server + /// internal class TcpConnectionManager { + /// + /// Connection cache + /// static Dictionary> connectionCache = new Dictionary>(); - static SemaphoreSlim connectionAccessLock = new SemaphoreSlim(1); + /// + /// A lock to manage concurrency + /// + static SemaphoreSlim connectionAccessLock = new SemaphoreSlim(1); + /// + /// Get a TcpConnection to the specified host, port optionally HTTPS and a particular HTTP version + /// + /// + /// + /// + /// + /// internal static async Task GetClient(string hostname, int port, bool isHttps, Version version) { List cachedConnections = null; TcpConnection cached = null; + //Get a unique string to identify this connection var key = GetConnectionKey(hostname, port, isHttps, version); + while (true) { await connectionAccessLock.WaitAsync(); @@ -76,6 +73,7 @@ internal static async Task GetClient(string hostname, int port, b if (cached == null) break; + } if (cached == null) @@ -90,11 +88,27 @@ internal static async Task GetClient(string hostname, int port, b return cached; } + /// + /// Get a string to identfiy the connection to a particular host, port + /// + /// + /// + /// + /// + /// internal static string GetConnectionKey(string hostname, int port, bool isHttps, Version version) { return string.Format("{0}:{1}:{2}:{3}:{4}", hostname.ToLower(), port, isHttps, version.Major, version.Minor); } + /// + /// Create connection to a particular host/port optionally with SSL and a particular HTTP version + /// + /// + /// + /// + /// + /// private static async Task CreateClient(string hostname, int port, bool isHttps, Version version) { TcpClient client; @@ -104,6 +118,7 @@ private static async Task CreateClient(string hostname, int port, { SslStream sslStream = null; + //If this proxy uses another external proxy then create a tunnel request for HTTPS connections if (ProxyServer.UpStreamHttpsProxy != null) { client = new TcpClient(ProxyServer.UpStreamHttpsProxy.HostName, ProxyServer.UpStreamHttpsProxy.Port); @@ -175,7 +190,11 @@ private static async Task CreateClient(string hostname, int port, }; } - + /// + /// Returns a Tcp Connection back to cache for reuse by other requests + /// + /// + /// internal static async Task ReleaseClient(TcpConnection connection) { @@ -199,11 +218,17 @@ internal static async Task ReleaseClient(TcpConnection connection) private static bool clearConenctions { get; set; } + /// + /// Stop clearing idle connections + /// internal static void StopClearIdleConnections() { clearConenctions = false; } + /// + /// A method to clear idle connections + /// internal async static void ClearIdleConnections() { clearConenctions = true; @@ -224,6 +249,7 @@ internal async static void ClearIdleConnections() } finally { connectionAccessLock.Release(); } + //every minute run this await Task.Delay(1000 * 60); } diff --git a/Titanium.Web.Proxy/ProxyServer.cs b/Titanium.Web.Proxy/ProxyServer.cs index 52adbd12e..9434f74e7 100644 --- a/Titanium.Web.Proxy/ProxyServer.cs +++ b/Titanium.Web.Proxy/ProxyServer.cs @@ -22,17 +22,41 @@ public partial class ProxyServer static ProxyServer() { ProxyEndPoints = new List(); + + //default values ConnectionCacheTimeOutMinutes = 3; CertificateCacheTimeOutMinutes = 60; } + /// + /// Manages certificates used by this proxy + /// private static CertificateManager CertManager { get; set; } + /// + /// Does the root certificate used by this proxy is trusted by the machine? + /// private static bool certTrusted { get; set; } + + /// + /// Is the proxy currently running + /// private static bool proxyRunning { get; set; } + /// + /// Name of the root certificate issuer + /// public static string RootCertificateIssuerName { get; set; } + + /// + /// Name of the root certificate + /// public static string RootCertificateName { get; set; } + + /// + /// Does this proxy uses the HTTP protocol 100 continue behaviour strictly? + /// Broken 100 contunue implementations on server/client may cause problems if enabled + /// public static bool Enable100ContinueBehaviour { get; set; } /// @@ -45,13 +69,11 @@ static ProxyServer() /// public static int CertificateCacheTimeOutMinutes { get; set; } - /// /// Intercept request to server /// public static event Func BeforeRequest; - /// /// Intercept response from server /// @@ -77,20 +99,33 @@ static ProxyServer() /// public static event Func ClientCertificateSelectionCallback; + /// + /// A list of IpAddress & port this proxy is listening to + /// public static List ProxyEndPoints { get; set; } + /// + /// Initialize the proxy + /// public static void Initialize() { TcpConnectionManager.ClearIdleConnections(); CertManager.ClearIdleCertificates(); } + /// + /// Quit the proxy + /// public static void Quit() { TcpConnectionManager.StopClearIdleConnections(); CertManager.StopClearIdleCertificates(); } + /// + /// Add a proxy end point + /// + /// public static void AddEndPoint(ProxyEndPoint endPoint) { ProxyEndPoints.Add(endPoint); @@ -99,6 +134,11 @@ public static void AddEndPoint(ProxyEndPoint endPoint) Listen(endPoint); } + /// + /// Remove a proxy end point + /// Will throw error if the end point does'nt exist + /// + /// public static void RemoveEndPoint(ProxyEndPoint endPoint) { @@ -111,10 +151,13 @@ public static void RemoveEndPoint(ProxyEndPoint endPoint) QuitListen(endPoint); } - + /// + /// Set the given explicit end point as the default proxy server for current machine + /// + /// public static void SetAsSystemHttpProxy(ExplicitProxyEndPoint endPoint) { - VerifyProxy(endPoint); + ValidateEndPointAsSystemProxy(endPoint); //clear any settings previously added ProxyEndPoints.OfType().ToList().ForEach(x => x.IsSystemHttpProxy = false); @@ -130,14 +173,21 @@ public static void SetAsSystemHttpProxy(ExplicitProxyEndPoint endPoint) } + /// + /// Remove any HTTP proxy setting of current machien + /// public static void DisableSystemHttpProxy() { SystemProxyHelper.RemoveHttpProxy(); } + /// + /// Set the given explicit end point as the default proxy server for current machine + /// + /// public static void SetAsSystemHttpsProxy(ExplicitProxyEndPoint endPoint) { - VerifyProxy(endPoint); + ValidateEndPointAsSystemProxy(endPoint); if (!endPoint.EnableSsl) { @@ -164,16 +214,25 @@ public static void SetAsSystemHttpsProxy(ExplicitProxyEndPoint endPoint) Console.WriteLine("Set endpoint at Ip {1} and port: {2} as System HTTPS Proxy", endPoint.GetType().Name, endPoint.IpAddress, endPoint.Port); } + /// + /// Remove any HTTPS proxy setting for current machine + /// public static void DisableSystemHttpsProxy() { SystemProxyHelper.RemoveHttpsProxy(); } + /// + /// Clear all proxy settings for current machine + /// public static void DisableAllSystemProxies() { SystemProxyHelper.DisableAllProxy(); } + /// + /// Start this proxy server + /// public static void Start() { if (proxyRunning) @@ -197,6 +256,9 @@ public static void Start() proxyRunning = true; } + /// + /// Stop this proxy server + /// public static void Stop() { if (!proxyRunning) @@ -224,6 +286,10 @@ public static void Stop() proxyRunning = false; } + /// + /// Listen on the given end point on local machine + /// + /// private static void Listen(ProxyEndPoint endPoint) { endPoint.listener = new TcpListener(endPoint.IpAddress, endPoint.Port); @@ -234,13 +300,20 @@ private static void Listen(ProxyEndPoint endPoint) endPoint.listener.BeginAcceptTcpClient(OnAcceptConnection, endPoint); } + /// + /// Quit listening on the given end point + /// + /// private static void QuitListen(ProxyEndPoint endPoint) { endPoint.listener.Stop(); } - - private static void VerifyProxy(ExplicitProxyEndPoint endPoint) + /// + /// Verifiy if its safe to set this end point as System proxy + /// + /// + private static void ValidateEndPointAsSystemProxy(ExplicitProxyEndPoint endPoint) { if (ProxyEndPoints.Contains(endPoint) == false) throw new Exception("Cannot set endPoints not added to proxy as system proxy"); @@ -249,12 +322,17 @@ private static void VerifyProxy(ExplicitProxyEndPoint endPoint) throw new Exception("Cannot set system proxy settings before proxy has been started."); } + /// + /// When a connection is received from client act + /// + /// private static void OnAcceptConnection(IAsyncResult asyn) { var endPoint = (ProxyEndPoint)asyn.AsyncState; try { + //based on end point type call appropriate request handlers var client = endPoint.listener.EndAcceptTcpClient(asyn); if (endPoint.GetType() == typeof(TransparentProxyEndPoint)) HandleClient(endPoint as TransparentProxyEndPoint, client); @@ -273,7 +351,7 @@ private static void OnAcceptConnection(IAsyncResult asyn) } catch { - + //Other errors are discarded to keep the proxy running } } diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 48fe3cd1c..8b0636de4 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -59,10 +59,10 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient { version = new Version(1, 0); } - } - var excluded = endPoint.ExcludedHttpsHostNameRegex != null ? endPoint.ExcludedHttpsHostNameRegex.Any(x => Regex.IsMatch(httpRemoteUri.Host, x)) : false; + var excluded = endPoint.ExcludedHttpsHostNameRegex != null ? + endPoint.ExcludedHttpsHostNameRegex.Any(x => Regex.IsMatch(httpRemoteUri.Host, x)) : false; //Client wants to create a secure tcp tunnel (its a HTTPS request) if (httpVerb.ToUpper() == "CONNECT" && !excluded && httpRemoteUri.Port != 80) @@ -217,7 +217,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http args.WebSession.Request.RequestHeaders = new List(); string tmpLine; - while (!string.IsNullOrEmpty(tmpLine = await clientStreamReader.ReadLineAsync().ConfigureAwait(false))) + while (!string.IsNullOrEmpty(tmpLine = await clientStreamReader.ReadLineAsync())) { var header = tmpLine.Split(ProxyConstants.ColonSplit, 2); args.WebSession.Request.RequestHeaders.Add(new HttpHeader(header[0], header[1])); diff --git a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj index cadbfb1b0..8b0a7cc30 100644 --- a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj +++ b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj @@ -63,11 +63,12 @@ + - + @@ -75,6 +76,7 @@ + From 4bcd22fc02ae5bdecc705d936260cfa49ed4bb56 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Wed, 15 Jun 2016 23:56:03 -0400 Subject: [PATCH 26/29] undo test change --- .../ProxyTestController.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs b/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs index 97779fd87..dc2ba74af 100644 --- a/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs +++ b/Examples/Titanium.Web.Proxy.Examples.Basic/ProxyTestController.cs @@ -12,10 +12,10 @@ public class ProxyTestController { public void StartProxy() { - //ProxyServer.BeforeRequest += OnRequest; - //ProxyServer.BeforeResponse += OnResponse; - //ProxyServer.ServerCertificateValidationCallback += OnCertificateValidation; - //ProxyServer.ClientCertificateSelectionCallback += OnCertificateSelection; + ProxyServer.BeforeRequest += OnRequest; + ProxyServer.BeforeResponse += OnResponse; + ProxyServer.ServerCertificateValidationCallback += OnCertificateValidation; + ProxyServer.ClientCertificateSelectionCallback += OnCertificateSelection; //Exclude Https addresses you don't want to proxy //Usefull for clients that use certificate pinning From 9bac69a91eaebe2a79d3c47aede9986b040cf0e9 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Thu, 16 Jun 2016 00:32:13 -0400 Subject: [PATCH 27/29] add more comments --- Titanium.Web.Proxy/Http/Request.cs | 59 +++++++++++++++- Titanium.Web.Proxy/Http/Response.cs | 40 ++++++++++- .../Network/TcpConnectionManager.cs | 2 +- Titanium.Web.Proxy/ProxyServer.cs | 2 +- Titanium.Web.Proxy/RequestHandler.cs | 70 ++++++++++++++++--- Titanium.Web.Proxy/ResponseHandler.cs | 43 +++++++++++- 6 files changed, 200 insertions(+), 16 deletions(-) diff --git a/Titanium.Web.Proxy/Http/Request.cs b/Titanium.Web.Proxy/Http/Request.cs index 1af198b27..0cf076257 100644 --- a/Titanium.Web.Proxy/Http/Request.cs +++ b/Titanium.Web.Proxy/Http/Request.cs @@ -12,10 +12,24 @@ namespace Titanium.Web.Proxy.Http /// public class Request { + /// + /// Request Method + /// public string Method { get; set; } + + /// + /// Request HTTP Uri + /// public Uri RequestUri { get; set; } + + /// + /// Request Http Version + /// public Version HttpVersion { get; set; } + /// + /// Request Http hostanem + /// internal string Host { get @@ -35,6 +49,9 @@ internal string Host } } + /// + /// Request content encoding + /// internal string ContentEncoding { get @@ -50,6 +67,9 @@ internal string ContentEncoding } } + /// + /// Request content-length + /// public long ContentLength { get @@ -68,7 +88,6 @@ public long ContentLength } set { - var header = RequestHeaders.FirstOrDefault(x => x.Name.ToLower() == "content-length"); if (value >= 0) @@ -88,6 +107,9 @@ public long ContentLength } } + /// + /// Request content-type + /// public string ContentType { get @@ -109,6 +131,9 @@ public string ContentType } + /// + /// Is request body send as chunked bytes + /// public bool IsChunked { get @@ -140,6 +165,9 @@ public bool IsChunked } } + /// + /// Does this request has a 100-continue header? + /// public bool ExpectContinue { get @@ -150,19 +178,37 @@ public bool ExpectContinue } } + /// + /// Request Url + /// public string Url { get { return RequestUri.OriginalString; } } + /// + /// Encoding for this request + /// internal Encoding Encoding { get { return this.GetEncoding(); } } /// /// Terminates the underlying Tcp Connection to client after current request /// internal bool CancelRequest { get; set; } + /// + /// Request body as byte array + /// internal byte[] RequestBody { get; set; } + + /// + /// request body as string + /// internal string RequestBodyString { get; set; } + + internal bool RequestBodyRead { get; set; } internal bool RequestLocked { get; set; } + /// + /// Does this request has an upgrade to websocket header? + /// internal bool UpgradeToWebSocket { get @@ -179,8 +225,19 @@ internal bool UpgradeToWebSocket } } + /// + /// Request heade collection + /// public List RequestHeaders { get; set; } + + /// + /// Does server responsed positively for 100 continue request + /// public bool Is100Continue { get; internal set; } + + /// + /// Server responsed negatively for the request for 100 continue + /// public bool ExpectationFailed { get; internal set; } public Request() diff --git a/Titanium.Web.Proxy/Http/Response.cs b/Titanium.Web.Proxy/Http/Response.cs index 1b8b830b5..fc2254b69 100644 --- a/Titanium.Web.Proxy/Http/Response.cs +++ b/Titanium.Web.Proxy/Http/Response.cs @@ -18,7 +18,9 @@ public class Response internal Encoding Encoding { get { return this.GetResponseCharacterEncoding(); } } - + /// + /// Content encoding for this response + /// internal string ContentEncoding { get @@ -35,6 +37,10 @@ internal string ContentEncoding } internal Version HttpVersion { get; set; } + + /// + /// Keep the connection alive? + /// internal bool ResponseKeepAlive { get @@ -51,6 +57,9 @@ internal bool ResponseKeepAlive } } + /// + /// Content type of this response + /// public string ContentType { get @@ -67,6 +76,9 @@ public string ContentType } } + /// + /// Length of response body + /// internal long ContentLength { get @@ -105,6 +117,9 @@ internal long ContentLength } } + /// + /// Response transfer-encoding is chunked? + /// internal bool IsChunked { get @@ -143,14 +158,37 @@ internal bool IsChunked } } + /// + /// Collection of all response headers + /// public List ResponseHeaders { get; set; } + /// + /// Response network stream + /// internal Stream ResponseStream { get; set; } + + /// + /// response body contenst as byte array + /// internal byte[] ResponseBody { get; set; } + + /// + /// response body as string + /// internal string ResponseBodyString { get; set; } + internal bool ResponseBodyRead { get; set; } internal bool ResponseLocked { get; set; } + + /// + /// Is response 100-continue + /// public bool Is100Continue { get; internal set; } + + /// + /// expectation failed returned by server? + /// public bool ExpectationFailed { get; internal set; } public Response() diff --git a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs index 3f54e6e2c..e8524f293 100644 --- a/Titanium.Web.Proxy/Network/TcpConnectionManager.cs +++ b/Titanium.Web.Proxy/Network/TcpConnectionManager.cs @@ -44,7 +44,6 @@ internal static async Task GetClient(string hostname, int port, b //Get a unique string to identify this connection var key = GetConnectionKey(hostname, port, isHttps, version); - while (true) { await connectionAccessLock.WaitAsync(); @@ -76,6 +75,7 @@ internal static async Task GetClient(string hostname, int port, b } + if (cached == null) cached = await CreateClient(hostname, port, isHttps, version); diff --git a/Titanium.Web.Proxy/ProxyServer.cs b/Titanium.Web.Proxy/ProxyServer.cs index 9434f74e7..5a3f3ec28 100644 --- a/Titanium.Web.Proxy/ProxyServer.cs +++ b/Titanium.Web.Proxy/ProxyServer.cs @@ -351,7 +351,7 @@ private static void OnAcceptConnection(IAsyncResult asyn) } catch { - //Other errors are discarded to keep the proxy running + //Other errors are discarded to keep proxy running } } diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 8b0636de4..735784d90 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -18,7 +18,9 @@ namespace Titanium.Web.Proxy { - + /// + /// Handle the requesr + /// partial class ProxyServer { //This is called when client is aware of proxy @@ -43,6 +45,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient //break up the line into three components (method, remote URL & Http Version) var httpCmdSplit = httpCmd.Split(ProxyConstants.SpaceSplit, 3); + //Find the request Verb var httpVerb = httpCmdSplit[0]; if (httpVerb.ToUpper() == "CONNECT") @@ -50,6 +53,7 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient else httpRemoteUri = new Uri(httpCmdSplit[1]); + //parse the HTTP version Version version = new Version(1, 1); if (httpCmdSplit.Length == 3) { @@ -76,7 +80,8 @@ private static async void HandleClient(ExplicitProxyEndPoint endPoint, TcpClient try { - + //create the Tcp Connection to server and then release it to connection cache + //Just doing what CONNECT request is asking as to do var tunnelClient = await TcpConnectionManager.GetClient(httpRemoteUri.Host, httpRemoteUri.Port, true, version); await TcpConnectionManager.ReleaseClient(tunnelClient); @@ -88,7 +93,6 @@ await sslStream.AuthenticateAsServerAsync(certificate, false, //HTTPS server created - we can now decrypt the client's traffic clientStream = sslStream; - clientStreamReader = new CustomBinaryReader(sslStream); clientStreamWriter = new StreamWriter(sslStream); @@ -102,14 +106,19 @@ await sslStream.AuthenticateAsServerAsync(certificate, false, return; } + //Now read the actual HTTPS request line httpCmd = await clientStreamReader.ReadLineAsync(); } + //Sorry cannot do a HTTPS request decrypt to port 80 at this time else if (httpVerb.ToUpper() == "CONNECT") { + //Cyphen out CONNECT request headers await clientStreamReader.ReadAllLinesAsync(); + //write back successfull CONNECT response await WriteConnectResponse(clientStreamWriter, version); + //Just relay the request/response without decrypting it await TcpHelper.SendRaw(clientStream, null, null, httpRemoteUri.Host, httpRemoteUri.Port, false); @@ -179,12 +188,23 @@ await sslStream.AuthenticateAsServerAsync(certificate, false, await HandleHttpSessionRequest(tcpClient, httpCmd, clientStream, clientStreamReader, clientStreamWriter, true); } - + /// + /// This is the core request handler method for a particular connection from client + /// + /// + /// + /// + /// + /// + /// + /// private static async Task HandleHttpSessionRequest(TcpClient client, string httpCmd, Stream clientStream, CustomBinaryReader clientStreamReader, StreamWriter clientStreamWriter, bool isHttps) { TcpConnection connection = null; + //Loop through each subsequest request on this particular client connection + //(assuming HTTP connection is kept alive by client) while (true) { if (string.IsNullOrEmpty(httpCmd)) @@ -203,6 +223,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http var httpMethod = httpCmdSplit[0]; + //find the request HTTP version Version version = new Version(1, 1); if (httpCmdSplit.Length == 3) { @@ -216,6 +237,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http args.WebSession.Request.RequestHeaders = new List(); + //Read the request headers string tmpLine; while (!string.IsNullOrEmpty(tmpLine = await clientStreamReader.ReadLineAsync())) { @@ -225,6 +247,8 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http var httpRemoteUri = new Uri(!isHttps ? httpCmdSplit[1] : (string.Concat("https://", args.WebSession.Request.Host, httpCmdSplit[1]))); #if DEBUG + //Just ignore local requests while Debugging + //Its annoying if (httpRemoteUri.Host.Contains("localhost")) { Dispose(client, clientStream, clientStreamReader, clientStreamWriter, null); @@ -242,7 +266,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http PrepareRequestHeaders(args.WebSession.Request.RequestHeaders, args.WebSession); args.WebSession.Request.Host = args.WebSession.Request.RequestUri.Authority; - //If requested interception + //If user requested interception do it if (BeforeRequest != null) { Delegate[] invocationList = BeforeRequest.GetInvocationList(); @@ -256,6 +280,7 @@ private static async Task HandleHttpSessionRequest(TcpClient client, string http await Task.WhenAll(handlerTasks); } + //if upgrading to websocket then relay the requet without reading the contents if (args.WebSession.Request.UpgradeToWebSocket) { await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHeaders, @@ -267,21 +292,24 @@ await TcpHelper.SendRaw(clientStream, httpCmd, args.WebSession.Request.RequestHe //construct the web request that we are going to issue on behalf of the client. connection = await TcpConnectionManager.GetClient(args.WebSession.Request.RequestUri.Host, args.WebSession.Request.RequestUri.Port, args.IsHttps, version); - args.WebSession.Request.RequestLocked = true; + //If request was cancelled by user then dispose the client if (args.WebSession.Request.CancelRequest) { Dispose(client, clientStream, clientStreamReader, clientStreamWriter, args); break; } + //if expect continue is enabled then send the headers first + //and see if server would return 100 conitinue if (args.WebSession.Request.ExpectContinue) { args.WebSession.SetConnection(connection); await args.WebSession.SendRequest(); } + //If 100 continue was the response inform that to the client if (Enable100ContinueBehaviour) if (args.WebSession.Request.Is100Continue) { @@ -296,6 +324,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", await args.ProxyClient.ClientStreamWriter.WriteLineAsync(); } + //If expect continue is not enabled then set the connectio and send request headers if (!args.WebSession.Request.ExpectContinue) { args.WebSession.SetConnection(connection); @@ -327,6 +356,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", } } + //If not expectation failed response was returned by server then parse response if (!args.WebSession.Request.ExpectationFailed) { await HandleHttpSessionResponse(args); @@ -339,9 +369,10 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", return; } + //send the tcp connection to server back to connection cache for reuse await TcpConnectionManager.ReleaseClient(connection); - // read the next request + // read the next request httpCmd = await clientStreamReader.ReadLineAsync(); } @@ -355,6 +386,12 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", } + /// + /// Write successfull CONNECT response to client + /// + /// + /// + /// private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, Version httpVersion) { await clientStreamWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2}", httpVersion.Major, httpVersion.Minor, "200 Connection established")); @@ -363,6 +400,11 @@ private static async Task WriteConnectResponse(StreamWriter clientStreamWriter, await clientStreamWriter.FlushAsync(); } + /// + /// prepare the request headers so that we can avoid encodings not parsable by this proxy + /// + /// + /// private static void PrepareRequestHeaders(List requestHeaders, HttpWebClient webRequest) { for (var i = 0; i < requestHeaders.Count; i++) @@ -381,6 +423,11 @@ private static void PrepareRequestHeaders(List requestHeaders, HttpW FixRequestProxyHeaders(requestHeaders); webRequest.Request.RequestHeaders = requestHeaders; } + + /// + /// Fix proxy specific headers + /// + /// private static void FixRequestProxyHeaders(List headers) { //If proxy-connection close was returned inform to close the connection @@ -400,7 +447,11 @@ private static void FixRequestProxyHeaders(List headers) headers.RemoveAll(x => x.Name.ToLower() == "proxy-connection"); } - //This is called when the request is PUT/POST to read the body + /// + /// This is called when the request is PUT/POST to read the body + /// + /// + /// private static async Task SendClientRequestBody(SessionEventArgs args) { // End the operation @@ -445,6 +496,7 @@ internal static bool ValidateServerCertificate( X509Chain chain, SslPolicyErrors sslPolicyErrors) { + //if user callback is registered then do it if (ServerCertificateValidationCallback != null) { var args = new CertificateValidationEventArgs(); @@ -511,7 +563,7 @@ internal static X509Certificate SelectClientCertificate( localCertificates.Count > 0) clientCertificate = localCertificates[0]; - + //If user call back is registered if (ClientCertificateSelectionCallback != null) { var args = new CertificateSelectionEventArgs(); diff --git a/Titanium.Web.Proxy/ResponseHandler.cs b/Titanium.Web.Proxy/ResponseHandler.cs index f35f688a2..0a39bbf5f 100644 --- a/Titanium.Web.Proxy/ResponseHandler.cs +++ b/Titanium.Web.Proxy/ResponseHandler.cs @@ -11,11 +11,15 @@ namespace Titanium.Web.Proxy { + /// + /// Handle the response from server + /// partial class ProxyServer { //Called asynchronously when a request was successfully and we received the response public static async Task HandleHttpSessionResponse(SessionEventArgs args) { + //read response & headers from server await args.WebSession.ReceiveResponse(); try @@ -23,7 +27,7 @@ public static async Task HandleHttpSessionResponse(SessionEventArgs args) if (!args.WebSession.Response.ResponseBodyRead) args.WebSession.Response.ResponseStream = args.WebSession.ServerConnection.Stream; - + //If client request call back then do it if (BeforeResponse != null && !args.WebSession.Response.ResponseLocked) { Delegate[] invocationList = BeforeResponse.GetInvocationList(); @@ -39,6 +43,7 @@ public static async Task HandleHttpSessionResponse(SessionEventArgs args) args.WebSession.Response.ResponseLocked = true; + //Write back to client 100-conitinue response if that's what server returned if (args.WebSession.Response.Is100Continue) { await WriteResponseStatus(args.WebSession.Response.HttpVersion, "100", @@ -52,6 +57,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", await args.ProxyClient.ClientStreamWriter.WriteLineAsync(); } + //Write back response status await WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession.Response.ResponseStatusCode, args.WebSession.Response.ResponseStatusDescription, args.ProxyClient.ClientStreamWriter); @@ -95,6 +101,12 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession. } } + /// + /// get the compressed response body from give response bytes + /// + /// + /// + /// private static async Task GetCompressedResponseBody(string encodingType, byte[] responseBodyStream) { var compressionFactory = new CompressionFactory(); @@ -102,13 +114,26 @@ private static async Task GetCompressedResponseBody(string encodingType, return await compressor.Compress(responseBodyStream); } - + /// + /// Write response status + /// + /// + /// + /// + /// + /// private static async Task WriteResponseStatus(Version version, string code, string description, StreamWriter responseWriter) { await responseWriter.WriteLineAsync(string.Format("HTTP/{0}.{1} {2} {3}", version.Major, version.Minor, code, description)); } + /// + /// Write response headers to client + /// + /// + /// + /// private static async Task WriteResponseHeaders(StreamWriter responseWriter, List headers) { if (headers != null) @@ -124,6 +149,11 @@ private static async Task WriteResponseHeaders(StreamWriter responseWriter, List await responseWriter.WriteLineAsync(); await responseWriter.FlushAsync(); } + + /// + /// Fix the proxy specific headers before sending response headers to client + /// + /// private static void FixResponseProxyHeaders(List headers) { //If proxy-connection close was returned inform to close the connection @@ -143,7 +173,14 @@ private static void FixResponseProxyHeaders(List headers) headers.RemoveAll(x => x.Name.ToLower() == "proxy-connection"); } - + /// + /// Handle dispose of a client/server session + /// + /// + /// + /// + /// + /// private static void Dispose(TcpClient client, IDisposable clientStream, IDisposable clientStreamReader, IDisposable clientStreamWriter, IDisposable args) { From 7cf9be8b06008689fc48093e836c7710beb4c9e0 Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Thu, 16 Jun 2016 00:34:21 -0400 Subject: [PATCH 28/29] fix comment --- Titanium.Web.Proxy/ResponseHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Titanium.Web.Proxy/ResponseHandler.cs b/Titanium.Web.Proxy/ResponseHandler.cs index 0a39bbf5f..81190492c 100644 --- a/Titanium.Web.Proxy/ResponseHandler.cs +++ b/Titanium.Web.Proxy/ResponseHandler.cs @@ -27,7 +27,7 @@ public static async Task HandleHttpSessionResponse(SessionEventArgs args) if (!args.WebSession.Response.ResponseBodyRead) args.WebSession.Response.ResponseStream = args.WebSession.ServerConnection.Stream; - //If client request call back then do it + //If user requested call back then do it if (BeforeResponse != null && !args.WebSession.Response.ResponseLocked) { Delegate[] invocationList = BeforeResponse.GetInvocationList(); From 3042d211abb4ca2f7fc5b99fd85ca8afede5ed4e Mon Sep 17 00:00:00 2001 From: justcoding121 Date: Thu, 16 Jun 2016 00:42:08 -0400 Subject: [PATCH 29/29] Move certificate handler --- Titanium.Web.Proxy/CertificateHandler.cs | 121 +++++++++++++++++++ Titanium.Web.Proxy/RequestHandler.cs | 109 ----------------- Titanium.Web.Proxy/ResponseHandler.cs | 2 +- Titanium.Web.Proxy/Titanium.Web.Proxy.csproj | 1 + 4 files changed, 123 insertions(+), 110 deletions(-) create mode 100644 Titanium.Web.Proxy/CertificateHandler.cs diff --git a/Titanium.Web.Proxy/CertificateHandler.cs b/Titanium.Web.Proxy/CertificateHandler.cs new file mode 100644 index 000000000..448e9f56f --- /dev/null +++ b/Titanium.Web.Proxy/CertificateHandler.cs @@ -0,0 +1,121 @@ +using System; +using System.Linq; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using System.Threading.Tasks; +using Titanium.Web.Proxy.EventArguments; + +namespace Titanium.Web.Proxy +{ + public partial class ProxyServer + { + /// + /// Call back to override server certificate validation + /// + /// + /// + /// + /// + /// + internal static bool ValidateServerCertificate( + object sender, + X509Certificate certificate, + X509Chain chain, + SslPolicyErrors sslPolicyErrors) + { + //if user callback is registered then do it + if (ServerCertificateValidationCallback != null) + { + var args = new CertificateValidationEventArgs(); + + args.Certificate = certificate; + args.Chain = chain; + args.SslPolicyErrors = sslPolicyErrors; + + + Delegate[] invocationList = ServerCertificateValidationCallback.GetInvocationList(); + Task[] handlerTasks = new Task[invocationList.Length]; + + for (int i = 0; i < invocationList.Length; i++) + { + handlerTasks[i] = ((Func)invocationList[i])(null, args); + } + + Task.WhenAll(handlerTasks).Wait(); + + return args.IsValid; + } + + if (sslPolicyErrors == SslPolicyErrors.None) + return true; + + //By default + //do not allow this client to communicate with unauthenticated servers. + return false; + } + + /// + /// Call back to select client certificate used for mutual authentication + /// + /// + /// + /// + /// + /// + internal static X509Certificate SelectClientCertificate( + object sender, + string targetHost, + X509CertificateCollection localCertificates, + X509Certificate remoteCertificate, + string[] acceptableIssuers) + { + X509Certificate clientCertificate = null; + var customSslStream = sender as SslStream; + + if (acceptableIssuers != null && + acceptableIssuers.Length > 0 && + localCertificates != null && + localCertificates.Count > 0) + { + // Use the first certificate that is from an acceptable issuer. + foreach (X509Certificate certificate in localCertificates) + { + string issuer = certificate.Issuer; + if (Array.IndexOf(acceptableIssuers, issuer) != -1) + clientCertificate = certificate; + } + } + + if (localCertificates != null && + localCertificates.Count > 0) + clientCertificate = localCertificates[0]; + + //If user call back is registered + if (ClientCertificateSelectionCallback != null) + { + var args = new CertificateSelectionEventArgs(); + + args.targetHost = targetHost; + args.localCertificates = localCertificates; + args.remoteCertificate = remoteCertificate; + args.acceptableIssuers = acceptableIssuers; + args.clientCertificate = clientCertificate; + + Delegate[] invocationList = ClientCertificateSelectionCallback.GetInvocationList(); + Task[] handlerTasks = new Task[invocationList.Length]; + + for (int i = 0; i < invocationList.Length; i++) + { + handlerTasks[i] = ((Func)invocationList[i])(null, args); + } + + Task.WhenAll(handlerTasks).Wait(); + + return args.clientCertificate; + } + + return clientCertificate; + + } + } +} diff --git a/Titanium.Web.Proxy/RequestHandler.cs b/Titanium.Web.Proxy/RequestHandler.cs index 735784d90..7309fb0fb 100644 --- a/Titanium.Web.Proxy/RequestHandler.cs +++ b/Titanium.Web.Proxy/RequestHandler.cs @@ -482,114 +482,5 @@ private static async Task SendClientRequestBody(SessionEventArgs args) } } - /// - /// Call back to override server certificate validation - /// - /// - /// - /// - /// - /// - internal static bool ValidateServerCertificate( - object sender, - X509Certificate certificate, - X509Chain chain, - SslPolicyErrors sslPolicyErrors) - { - //if user callback is registered then do it - if (ServerCertificateValidationCallback != null) - { - var args = new CertificateValidationEventArgs(); - - args.Certificate = certificate; - args.Chain = chain; - args.SslPolicyErrors = sslPolicyErrors; - - - Delegate[] invocationList = ServerCertificateValidationCallback.GetInvocationList(); - Task[] handlerTasks = new Task[invocationList.Length]; - - for (int i = 0; i < invocationList.Length; i++) - { - handlerTasks[i] = ((Func)invocationList[i])(null, args); - } - - Task.WhenAll(handlerTasks).Wait(); - - return args.IsValid; - } - - if (sslPolicyErrors == SslPolicyErrors.None) - return true; - - //By default - //do not allow this client to communicate with unauthenticated servers. - return false; - } - - /// - /// Call back to select client certificate used for mutual authentication - /// - /// - /// - /// - /// - /// - internal static X509Certificate SelectClientCertificate( - object sender, - string targetHost, - X509CertificateCollection localCertificates, - X509Certificate remoteCertificate, - string[] acceptableIssuers) - { - X509Certificate clientCertificate = null; - var customSslStream = sender as SslStream; - - if (acceptableIssuers != null && - acceptableIssuers.Length > 0 && - localCertificates != null && - localCertificates.Count > 0) - { - // Use the first certificate that is from an acceptable issuer. - foreach (X509Certificate certificate in localCertificates) - { - string issuer = certificate.Issuer; - if (Array.IndexOf(acceptableIssuers, issuer) != -1) - clientCertificate = certificate; - } - } - - if (localCertificates != null && - localCertificates.Count > 0) - clientCertificate = localCertificates[0]; - - //If user call back is registered - if (ClientCertificateSelectionCallback != null) - { - var args = new CertificateSelectionEventArgs(); - - args.targetHost = targetHost; - args.localCertificates = localCertificates; - args.remoteCertificate = remoteCertificate; - args.acceptableIssuers = acceptableIssuers; - args.clientCertificate = clientCertificate; - - Delegate[] invocationList = ClientCertificateSelectionCallback.GetInvocationList(); - Task[] handlerTasks = new Task[invocationList.Length]; - - for (int i = 0; i < invocationList.Length; i++) - { - handlerTasks[i] = ((Func)invocationList[i])(null, args); - } - - Task.WhenAll(handlerTasks).Wait(); - - return args.clientCertificate; - } - - return clientCertificate; - - } - } } \ No newline at end of file diff --git a/Titanium.Web.Proxy/ResponseHandler.cs b/Titanium.Web.Proxy/ResponseHandler.cs index 81190492c..304d07a1d 100644 --- a/Titanium.Web.Proxy/ResponseHandler.cs +++ b/Titanium.Web.Proxy/ResponseHandler.cs @@ -57,7 +57,7 @@ await WriteResponseStatus(args.WebSession.Response.HttpVersion, "417", await args.ProxyClient.ClientStreamWriter.WriteLineAsync(); } - //Write back response status + //Write back response status to client await WriteResponseStatus(args.WebSession.Response.HttpVersion, args.WebSession.Response.ResponseStatusCode, args.WebSession.Response.ResponseStatusDescription, args.ProxyClient.ClientStreamWriter); diff --git a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj index 8b0a7cc30..10d28b011 100644 --- a/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj +++ b/Titanium.Web.Proxy/Titanium.Web.Proxy.csproj @@ -49,6 +49,7 @@ +