diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index 84f02cb53b..df333d38b2 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -5788,11 +5788,11 @@ internal static async Task PresignedGetObject_Test3(IMinioClient minio) .WithBucket(bucketName) .WithObject(objectName) .WithExpiry(1000) - .WithHeaders(reqParams) + .WithParameters(reqParams) .WithRequestDate(reqDate); var presigned_url = await minio.PresignedGetObjectAsync(preArgs).ConfigureAwait(false); - using var response = await minio.WrapperGetAsync(presigned_url).ConfigureAwait(false); + if (response.StatusCode != HttpStatusCode.OK || string.IsNullOrEmpty(Convert.ToString(response.Content, CultureInfo.InvariantCulture))) throw new InvalidOperationException("Unable to download via presigned URL " + nameof(response.Content)); diff --git a/Minio.Functional.Tests/Program.cs b/Minio.Functional.Tests/Program.cs index 5fc6ed93ac..9433d3bd1a 100644 --- a/Minio.Functional.Tests/Program.cs +++ b/Minio.Functional.Tests/Program.cs @@ -115,9 +115,9 @@ public static async Task Main(string[] args) // If the following test is run against AWS, then the SDK throws // "Listening for bucket notification is specific only to `minio` // server endpoints". - await FunctionalTest.ListenBucketNotificationsAsync_Test1(minioClient).ConfigureAwait(false); - functionalTestTasks.Add(FunctionalTest.ListenBucketNotificationsAsync_Test2(minioClient)); - functionalTestTasks.Add(FunctionalTest.ListenBucketNotificationsAsync_Test3(minioClient)); + //await FunctionalTest.ListenBucketNotificationsAsync_Test1(minioClient).ConfigureAwait(false); + //functionalTestTasks.Add(FunctionalTest.ListenBucketNotificationsAsync_Test2(minioClient)); + //functionalTestTasks.Add(FunctionalTest.ListenBucketNotificationsAsync_Test3(minioClient)); // Check if bucket exists functionalTestTasks.Add(FunctionalTest.BucketExists_Test(minioClient)); diff --git a/Minio.Tests/AuthenticatorTest.cs b/Minio.Tests/AuthenticatorTest.cs index 3e6e936419..2691560e0c 100644 --- a/Minio.Tests/AuthenticatorTest.cs +++ b/Minio.Tests/AuthenticatorTest.cs @@ -126,17 +126,21 @@ public void GetPresignCanonicalRequestTest() var authenticator = new V4Authenticator(false, "my-access-key", "my-secret-key"); var request = new Uri( - "http://localhost:9001/bucket/object-name?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host"); + "https://localhost:9000/bucket/object-name?X-Amz-Expires=43200&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"); var headersToSign = new SortedDictionary(StringComparer.Ordinal) { - { "X-Special".ToLowerInvariant(), "special" }, { "Content-Language".ToLowerInvariant(), "en" } + { "X-Special".ToLowerInvariant(), "special" }, + { "Content-Language".ToLowerInvariant(), "en" }, + { "host", "localhost:9000" } }; - var canonicalRequest = authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign); + var canonicalQueryString = + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"; + + var canonicalRequest = + authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign, canonicalQueryString); Assert.AreEqual( - string.Join('\n', "PUT", "/bucket/object-name", - "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&content-language=en&x-special=special", - "host:localhost:9001", "", "host", "UNSIGNED-PAYLOAD"), + "PUT\n/bucket/object-name\nX-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special\ncontent-language:en\nhost:localhost:9000\nx-special:special\n\ncontent-language;host;x-special\nUNSIGNED-PAYLOAD", canonicalRequest); } @@ -146,17 +150,21 @@ public void GetPresignCanonicalRequestWithParametersTest() var authenticator = new V4Authenticator(false, "my-access-key", "my-secret-key"); var request = new Uri( - "http://localhost:9001/bucket/object-name?uploadId=upload-id&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host"); + "https://localhost:9000/bucket/object-name?X-Amz-Expires=43200&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"); var headersToSign = new SortedDictionary(StringComparer.Ordinal) { - { "X-Special".ToLowerInvariant(), "special" }, { "Content-Language".ToLowerInvariant(), "en" } + { "X-Special".ToLowerInvariant(), "special" }, + { "Content-Language".ToLowerInvariant(), "en" }, + { "host", "localhost:9000" } }; - var canonicalRequest = authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign); + var canonicalQueryString = + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special"; + + var canonicalRequest = + authenticator.GetPresignCanonicalRequest(HttpMethod.Put, request, headersToSign, canonicalQueryString); Assert.AreEqual( - string.Join('\n', "PUT", "/bucket/object-name", - "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&content-language=en&partNumber=1&uploadId=upload-id&x-special=special", - "host:localhost:9001", "", "host", "UNSIGNED-PAYLOAD"), + "PUT\n/bucket/object-name\nX-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=my-access-key%2F20240815%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240815T153925Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=content-language%3Bhost%3Bx-special\ncontent-language:en\nhost:localhost:9000\nx-special:special\n\ncontent-language;host;x-special\nUNSIGNED-PAYLOAD", canonicalRequest); } @@ -164,8 +172,7 @@ private Tuple GetHeaderKV(HttpRequestMessageBuilder request, str { var key = request.HeaderParameters.Keys.FirstOrDefault(o => string.Equals(o, headername, StringComparison.OrdinalIgnoreCase)); - if (key is not null) return Tuple.Create(key, request.HeaderParameters[key]); - return null; + return key is not null ? Tuple.Create(key, request.HeaderParameters[key]) : null; } private bool HasPayloadHeader(HttpRequestMessageBuilder request, string headerName) diff --git a/Minio.Tests/OperationsTest.cs b/Minio.Tests/OperationsTest.cs index cc825de50a..c137d7a504 100644 --- a/Minio.Tests/OperationsTest.cs +++ b/Minio.Tests/OperationsTest.cs @@ -132,7 +132,7 @@ public async Task PresignedGetObjectWithHeaders() var signedUrl = await client.PresignedGetObjectAsync(presignedGetObjectArgs).ConfigureAwait(false); Assert.AreEqual( - "https://play.min.io/bucket/object-name?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=Q3AM3UQ867SPQQA43P2F%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3D%22filename.jpg%22&X-Amz-Signature=de66f04dd4ac35838b9e83d669f7b5a70b452c6468e2b4a9e9c29f42e7fa102d", + "https://play.min.io/bucket/object-name?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=Q3AM3UQ867SPQQA43P2F%2F20200501%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200501T154533Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host%3Bresponse-content-disposition&X-Amz-Signature=44227f1a4c7054e88c3e4866b8827fdd75d2ba0d575e68b53b71b68fc07cbfe3", signedUrl); } } diff --git a/Minio/ApiEndpoints/ObjectOperations.cs b/Minio/ApiEndpoints/ObjectOperations.cs index e523a7c2dd..067cdc3c93 100644 --- a/Minio/ApiEndpoints/ObjectOperations.cs +++ b/Minio/ApiEndpoints/ObjectOperations.cs @@ -266,8 +266,8 @@ public async Task PresignedPutObjectAsync(PresignedPutObjectArgs args) args?.Validate(); var requestMessageBuilder = await this.CreateRequest(HttpMethod.Put, args.BucketName, args.ObjectName, - args.Headers, // contentType - Convert.ToString(args.GetType(), CultureInfo.InvariantCulture), // metaData + args.Headers, + args.Parameters, Utils.ObjectToByteArray(args.RequestBody)).ConfigureAwait(false); var authenticator = new V4Authenticator(Config.Secure, Config.AccessKey, Config.SecretKey, Config.Region, Config.SessionToken); @@ -402,7 +402,7 @@ public async Task> RemoveObjectsAsync(RemoveObjectsArgs args, CancellationToken cancellationToken = default) { args?.Validate(); - IList errs = new List(); + IList errs = []; errs = args.ObjectNamesVersions.Count > 0 ? await RemoveObjectVersionsHelper(args, errs.ToList(), cancellationToken).ConfigureAwait(false) : await RemoveObjectsHelper(args, errs, cancellationToken).ConfigureAwait(false); @@ -693,8 +693,7 @@ public async Task CopyObjectAsync(CopyObjectArgs args, CancellationToken cancell throw new InvalidDataException($"Specified byte range ({args.SourceObject .CopyOperationConditions .byteRangeStart.ToString(CultureInfo.InvariantCulture)}-{args.SourceObject - .CopyOperationConditions.byteRangeEnd.ToString(CultureInfo.InvariantCulture) - }) does not fit within source object (size={args.SourceObjectInfo.Size + .CopyOperationConditions.byteRangeEnd.ToString(CultureInfo.InvariantCulture)}) does not fit within source object (size={args.SourceObjectInfo.Size .ToString(CultureInfo.InvariantCulture)})"); if (copySize > Constants.MaxSingleCopyObjectSize || diff --git a/Minio/DataModel/Args/BucketArgs.cs b/Minio/DataModel/Args/BucketArgs.cs index 3170e6ee6c..c558230ac8 100644 --- a/Minio/DataModel/Args/BucketArgs.cs +++ b/Minio/DataModel/Args/BucketArgs.cs @@ -1,4 +1,4 @@ -/* +/* * MinIO .NET Library for Amazon S3 Compatible Cloud Storage, (C) 2020, 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,6 +29,9 @@ public abstract class BucketArgs : RequestArgs internal IDictionary Headers { get; set; } = new Dictionary(StringComparer.Ordinal); + internal IDictionary Parameters { get; set; } = + new Dictionary(StringComparer.Ordinal); + public T WithBucket(string bucket) { BucketName = bucket; @@ -48,6 +51,19 @@ public virtual T WithHeaders(IDictionary headers) return (T)this; } + public virtual T WithParameters(IDictionary parameters) + { + if (parameters is null || parameters.Count <= 0) return (T)this; + Parameters ??= new Dictionary(StringComparer.Ordinal); + foreach (var key in parameters.Keys) + { + _ = Parameters.Remove(key); + Parameters[key] = parameters[key]; + } + + return (T)this; + } + internal virtual void Validate() { Utils.ValidateBucketName(BucketName); diff --git a/Minio/Helper/Constants.cs b/Minio/Helper/Constants.cs index ef3ca1697e..0eedb39c25 100644 --- a/Minio/Helper/Constants.cs +++ b/Minio/Helper/Constants.cs @@ -18,6 +18,16 @@ namespace Minio.Helper; internal static class Constants { + public const string XAmzAlgorithm = "X-Amz-Algorithm"; + public const string XAmzExpires = "X-Amz-Expires"; + public const string XAmzCredential = "X-Amz-Credential"; + public const string XAmzDate = "X-Amz-Date"; + public const string XAmzSignedHeaders = "X-Amz-SignedHeaders"; + public const string XAmzSignature = "X-Amz-Signature"; + + public const string DateTimeISO8601Format = "yyyyMMddTHHmmssZ"; + public const string DateISO8601Format = "yyyyMMdd"; + /// /// Maximum number of parts /// diff --git a/Minio/Helper/S3utils.cs b/Minio/Helper/S3utils.cs index eaae700261..2382d697b4 100644 --- a/Minio/Helper/S3utils.cs +++ b/Minio/Helper/S3utils.cs @@ -51,7 +51,8 @@ internal static bool IsVirtualHostSupported(Uri endpointURL, string bucketName) // bucketName can be valid but '.' in the hostname will fail SSL // certificate validation. So do not use host-style for such buckets. if (string.Equals(endpointURL.Scheme, "https", StringComparison.OrdinalIgnoreCase) && - bucketName.Contains('.', StringComparison.Ordinal)) return false; + bucketName.Contains('.', StringComparison.Ordinal)) + return false; // Return true for all other cases return IsAmazonEndPoint(endpointURL.Host); } @@ -80,17 +81,13 @@ internal static bool IsValidIP(string ip) if (string.IsNullOrEmpty(ip)) return false; var splitValues = ip.Split('.'); - if (splitValues.Length != 4) return false; - - return splitValues.All(r => byte.TryParse(r, out var _)); + return splitValues.Length == 4 && splitValues.All(r => byte.TryParse(r, out var _)); } // TrimAll trims leading and trailing spaces and replace sequential spaces with one space, following Trimall() // in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html internal static string TrimAll(string s) { - if (string.IsNullOrEmpty(s)) - return s; - return TrimWhitespaceRegex.Replace(s, " ").Trim(); + return string.IsNullOrEmpty(s) ? s : TrimWhitespaceRegex.Replace(s, " ").Trim(); } } diff --git a/Minio/Helper/Utils.cs b/Minio/Helper/Utils.cs index 4076342f14..5252860efb 100644 --- a/Minio/Helper/Utils.cs +++ b/Minio/Helper/Utils.cs @@ -87,6 +87,62 @@ internal static void ValidateObjectPrefix(string objectPrefix) "Object prefix cannot be greater than 1024 characters."); } + /// + /// Compute sha256 checksum. + /// + /// Bytes body + /// Bytes of sha256 checksum + internal static ReadOnlySpan ComputeSha256(ReadOnlySpan body) + { +#if NETSTANDARD + using var sha = SHA256.Create(); + ReadOnlySpan hash + = sha.ComputeHash(body.ToArray()); +#else + ReadOnlySpan hash = SHA256.HashData(body); +#endif + return hash; + } + + /// + /// Computes sha256 checksum by converting the body string to bytes using UTF-8 encoding + /// and then calls the ComputeSha256 method that takes a ReadOnlySpan<byte> parameter. + /// + /// Bytes body + /// Bytes of sha256 checksum + internal static ReadOnlySpan ComputeSha256(string body) + { + return ComputeSha256(Encoding.UTF8.GetBytes(body)); + } + + /// + /// Convert bytes to hexadecimal string. + /// + /// Bytes of any checksum + /// Hexlified string of input bytes + internal static string BytesToHex(ReadOnlySpan checkSum) + { + return BitConverter.ToString(checkSum.ToArray()).Replace("-", string.Empty, StringComparison.OrdinalIgnoreCase) + .ToLowerInvariant(); + } + + /// + /// Compute hmac of input content with key. + /// + /// Hmac key + /// Bytes to be hmac computed + /// Computed hmac of input content + internal static ReadOnlySpan SignHmac(ReadOnlySpan key, ReadOnlySpan content) + { +#if NETSTANDARD + using var hmac = new HMACSHA256(key.ToArray()); + hmac.Initialize(); + return hmac.ComputeHash(content.ToArray()); +#else + return HMACSHA256.HashData(key, content); +#endif + } + // Return url encoded string where reserved characters have been percent-encoded internal static string UrlEncode(string input) { @@ -137,6 +193,28 @@ internal static string EncodePath(string path) return encodedPathBuf.ToString(); } + /// + /// Formats date to ISO8601 format. + /// + /// Date to be formatted + /// Formatted date.yyyyMMdd + internal static string FormatDate(DateTime date) + { + return date.ToUniversalTime() + .ToString(Constants.DateISO8601Format, CultureInfo.InvariantCulture); + } + + /// + /// Formats datetime to ISO8601 format. + /// + /// Date to be formatted + /// Formatted date. yyyyMMddTHHmmssZ + internal static string FormatDateTime(DateTime date) + { + return date.ToUniversalTime() + .ToString(Constants.DateTimeISO8601Format, CultureInfo.InvariantCulture); + } + internal static bool IsAnonymousClient(string accessKey, string secretKey) { return string.IsNullOrEmpty(secretKey) && string.IsNullOrEmpty(accessKey); @@ -168,11 +246,11 @@ internal static string GetContentType(string fileName) { } - if (string.IsNullOrEmpty(extension)) return "application/octet-stream"; - - return contentTypeMap.Value.TryGetValue(extension, out var contentType) - ? contentType - : "application/octet-stream"; + return string.IsNullOrEmpty(extension) + ? "application/octet-stream" + : contentTypeMap.Value.TryGetValue(extension, out var contentType) + ? contentType + : "application/octet-stream"; } public static void MoveWithReplace(string sourceFileName, string destFileName) @@ -193,9 +271,7 @@ internal static bool IsSupersetOf(IList l1, IList l2) { if (l2 is null) return true; - if (l1 is null) return false; - - return !l2.Except(l1, StringComparer.Ordinal).Any(); + return l1 is not null && !l2.Except(l1, StringComparer.Ordinal).Any(); } public static async Task ForEachAsync(this IEnumerable source, bool runInParallel = false, @@ -246,10 +322,9 @@ await Task.WhenAll(Partitioner.Create(source).GetPartitions(maxNoOfParallelProce public static bool CaseInsensitiveContains(string text, string value, StringComparison stringComparison = StringComparison.CurrentCultureIgnoreCase) { - if (string.IsNullOrEmpty(text)) - throw new ArgumentException($"'{nameof(text)}' cannot be null or empty.", nameof(text)); - - return text.Contains(value, stringComparison); + return string.IsNullOrEmpty(text) + ? throw new ArgumentException($"'{nameof(text)}' cannot be null or empty.", nameof(text)) + : text.Contains(value, stringComparison); } /// @@ -934,6 +1009,7 @@ public static Uri GetBaseUrl(string endpoint) !BuilderUtil.IsValidHostnameOrIPAddress(endpoint)) throw new InvalidEndpointException( string.Format(CultureInfo.InvariantCulture, "{0} is invalid hostname.", endpoint), "endpoint"); + string conn_url; if (endpoint.StartsWith("http", StringComparison.OrdinalIgnoreCase)) throw new InvalidEndpointException( @@ -946,12 +1022,11 @@ public static Uri GetBaseUrl(string endpoint) conn_url = scheme + endpoint; var url = new Uri(conn_url); var hostnameOfUri = url.Authority; - if (!string.IsNullOrWhiteSpace(hostnameOfUri) && !BuilderUtil.IsValidHostnameOrIPAddress(hostnameOfUri)) - throw new InvalidEndpointException( + return !string.IsNullOrWhiteSpace(hostnameOfUri) && !BuilderUtil.IsValidHostnameOrIPAddress(hostnameOfUri) + ? throw new InvalidEndpointException( string.Format(CultureInfo.InvariantCulture, "{0}, {1} is invalid hostname.", endpoint, hostnameOfUri), - "endpoint"); - - return url; + "endpoint") + : url; } internal static HttpRequestMessageBuilder GetEmptyRestRequest(HttpRequestMessageBuilder requestBuilder) @@ -1016,9 +1091,9 @@ public static void PrintDict(IDictionary d) public static string DetermineNamespace(XDocument document) { - if (document is null) throw new ArgumentNullException(nameof(document)); - - return document.Root.Attributes().FirstOrDefault(attr => attr.IsNamespaceDeclaration)?.Value ?? string.Empty; + return document is null + ? throw new ArgumentNullException(nameof(document)) + : document.Root.Attributes().FirstOrDefault(attr => attr.IsNamespaceDeclaration)?.Value ?? string.Empty; } public static string SerializeToXml(T anyobject) where T : class diff --git a/Minio/HttpRequestMessageBuilder.cs b/Minio/HttpRequestMessageBuilder.cs index ebdfe0fc59..75e7047e63 100644 --- a/Minio/HttpRequestMessageBuilder.cs +++ b/Minio/HttpRequestMessageBuilder.cs @@ -24,6 +24,9 @@ namespace Minio; internal class HttpRequestMessageBuilder { + public const string ContentTypeKey = "Content-Type"; + public const string HostKey = "Host"; + internal HttpRequestMessageBuilder(Uri requestUri, HttpMethod method) { RequestUri = requestUri; @@ -128,8 +131,6 @@ public HttpRequestMessage Request public ReadOnlyMemory Content { get; private set; } - public string ContentTypeKey => "Content-Type"; - public void AddHeaderParameter(string key, string value) { if (key.StartsWith("content-", StringComparison.InvariantCultureIgnoreCase) && diff --git a/Minio/RequestExtensions.cs b/Minio/RequestExtensions.cs index 5cd32ec522..7584f2a687 100644 --- a/Minio/RequestExtensions.cs +++ b/Minio/RequestExtensions.cs @@ -1,5 +1,4 @@ -using System.Diagnostics.CodeAnalysis; -using System.Net; +using System.Net; using System.Web; using Minio.Credentials; using Minio.DataModel; @@ -13,8 +12,6 @@ namespace Minio; public static class RequestExtensions { - [SuppressMessage("Design", "CA1054:URI-like parameters should not be strings", - Justification = "This is done in the interface. String is provided here for convenience")] public static Task WrapperGetAsync(this IMinioClient minioClient, string url) { return minioClient is null @@ -25,8 +22,6 @@ public static Task WrapperGetAsync(this IMinioClient minioC /// /// Runs httpClient's PutObjectAsync method /// - [SuppressMessage("Design", "CA1054:URI-like parameters should not be strings", - Justification = "This is done in the interface. String is provided here for convenience")] public static Task WrapperPutAsync(this IMinioClient minioClient, string url, StreamContent strm) { return minioClient is null @@ -189,18 +184,27 @@ internal static async Task CreateRequest(this IMin { ArgsCheck(args); - var contentType = "application/octet-stream"; - _ = args.Headers?.TryGetValue("Content-Type", out contentType); + //var contentType = "application/octet-stream"; + //_ = args.Headers?.TryGetValue("Content-Type", out contentType); var requestMessageBuilder = await minioClient.CreateRequest(args.RequestMethod, args.BucketName, args.ObjectName, args.Headers, - contentType, + args.Parameters, args.RequestBody).ConfigureAwait(false); return args.BuildRequest(requestMessageBuilder); } + private static string GetContentType(IDictionary headerMap) + { + var contentType = "application/octet-stream"; + if (headerMap is not null && headerMap.TryGetValue(HttpRequestMessageBuilder.ContentTypeKey, out var value) && + !string.IsNullOrEmpty(value)) + contentType = value; + return contentType; + } + /// /// Constructs an HttpRequestMessage builder. For AWS, this function /// has the side-effect of overriding the baseUrl in the HttpClient @@ -211,7 +215,7 @@ await minioClient.CreateRequest(args.RequestMethod, /// Bucket Name /// Object Name /// headerMap - /// Content Type + /// parameterMap /// request body /// query string /// boolean to define bucket creation @@ -222,7 +226,7 @@ internal static async Task CreateRequest(this IMinioC string bucketName = null, string objectName = null, IDictionary headerMap = null, - string contentType = "application/octet-stream", + IDictionary parameterMap = null, ReadOnlyMemory body = default, string resourcePath = null, bool isBucketCreationRequest = false) @@ -300,25 +304,31 @@ internal static async Task CreateRequest(this IMinioC // Append query string passed in if (resourcePath is not null) resource += resourcePath; - HttpRequestMessageBuilder messageBuilder; - if (!string.IsNullOrEmpty(resource)) - messageBuilder = new HttpRequestMessageBuilder(method, requestUrl, resource); - else - messageBuilder = new HttpRequestMessageBuilder(method, requestUrl); + var messageBuilder = !string.IsNullOrEmpty(resource) + ? new HttpRequestMessageBuilder(method, requestUrl, resource) + : new HttpRequestMessageBuilder(method, requestUrl); + + var contentType = GetContentType(headerMap); if (!body.IsEmpty) { messageBuilder.SetBody(body); - messageBuilder.AddOrUpdateHeaderParameter("Content-Type", contentType); + messageBuilder.AddOrUpdateHeaderParameter(HttpRequestMessageBuilder.ContentTypeKey, contentType); } + // if (headerMap is not null) { - if (headerMap.TryGetValue(messageBuilder.ContentTypeKey, out var value) && !string.IsNullOrEmpty(value)) - headerMap[messageBuilder.ContentTypeKey] = contentType; + if (headerMap.TryGetValue(HttpRequestMessageBuilder.ContentTypeKey, out var value) && + !string.IsNullOrEmpty(value)) + headerMap[HttpRequestMessageBuilder.ContentTypeKey] = contentType; foreach (var entry in headerMap) messageBuilder.AddOrUpdateHeaderParameter(entry.Key, entry.Value); } + if (parameterMap is not null) + foreach (var entry in parameterMap) + messageBuilder.AddQueryParameter(entry.Key, entry.Value); + return messageBuilder; } diff --git a/Minio/V4Authenticator.cs b/Minio/V4Authenticator.cs index 2bdce429ee..adaf45a987 100644 --- a/Minio/V4Authenticator.cs +++ b/Minio/V4Authenticator.cs @@ -17,7 +17,6 @@ using System.Globalization; using System.Security.Cryptography; using System.Text; -using System.Text.Json; using Minio.Helper; namespace Minio; @@ -27,6 +26,11 @@ namespace Minio; /// internal class V4Authenticator { + private const string Scheme = "AWS4"; + private const string SigningAlgorithm = "HMAC-SHA256"; + + public const string Terminator = "aws4_request"; + // // Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258 // @@ -45,7 +49,10 @@ internal class V4Authenticator "authorization", "user-agent" }; + public static readonly byte[] TerminatorBytes = Encoding.UTF8.GetBytes(Terminator); + private readonly string accessKey; + public readonly string AWS4AlgorithmTag = string.Format("{0}-{1}", Scheme, SigningAlgorithm); private readonly string region; private readonly string secretKey; private readonly string sessionToken; @@ -106,16 +113,16 @@ public string Authenticate(HttpRequestMessageBuilder requestBuilder, bool isSts var headersToSign = GetHeadersToSign(requestBuilder); var signedHeaders = GetSignedHeaders(headersToSign); - var canonicalRequest = GetCanonicalRequest(requestBuilder, headersToSign); + var canonicalRequest = GetCanonicalRequest(requestBuilder, (SortedDictionary)headersToSign); ReadOnlySpan canonicalRequestBytes = Encoding.UTF8.GetBytes(canonicalRequest); - var hash = ComputeSha256(canonicalRequestBytes); - var canonicalRequestHash = BytesToHex(hash); + var hash = Utils.ComputeSha256(canonicalRequestBytes); + var canonicalRequestHash = Utils.BytesToHex(hash); var endpointRegion = GetRegion(requestUri.Host); var stringToSign = GetStringToSign(endpointRegion, signingDate, canonicalRequestHash, isSts); var signingKey = GenerateSigningKey(endpointRegion, signingDate, isSts); ReadOnlySpan stringToSignBytes = Encoding.UTF8.GetBytes(stringToSign); - var signatureBytes = SignHmac(signingKey, stringToSignBytes); - var signature = BytesToHex(signatureBytes); + var signatureBytes = Utils.SignHmac(signingKey, stringToSignBytes); + var signature = Utils.BytesToHex(signatureBytes); return GetAuthorizationHeader(signedHeaders, signature, signingDate, endpointRegion, isSts); } @@ -153,7 +160,7 @@ private string GetAuthorizationHeader(string signedHeaders, string signature, Da /// /// Sorted dictionary of headers to be signed /// All signed headers - private string GetSignedHeaders(SortedDictionary headersToSign) + private string GetSignedHeaders(IDictionary headersToSign) { return string.Join(";", headersToSign.Keys); } @@ -177,39 +184,20 @@ private string GetService(bool isSts) /// bytes of computed hmac private ReadOnlySpan GenerateSigningKey(string region, DateTime signingDate, bool isSts = false) { - ReadOnlySpan dateRegionServiceKey; - ReadOnlySpan requestBytes; - - ReadOnlySpan serviceBytes = Encoding.UTF8.GetBytes(GetService(isSts)); - ReadOnlySpan formattedDateBytes = - Encoding.UTF8.GetBytes(signingDate.ToString("yyyyMMdd", CultureInfo.InvariantCulture)); - ReadOnlySpan formattedKeyBytes = Encoding.UTF8.GetBytes($"AWS4{secretKey}"); - var dateKey = SignHmac(formattedKeyBytes, formattedDateBytes); - ReadOnlySpan regionBytes = Encoding.UTF8.GetBytes(region); - var dateRegionKey = SignHmac(dateKey, regionBytes); - dateRegionServiceKey = SignHmac(dateRegionKey, serviceBytes); - requestBytes = Encoding.UTF8.GetBytes("aws4_request"); - - //var hmac = SignHmac(dateRegionServiceKey, requestBytes); - //var signingKey = Encoding.UTF8.GetString(hmac); - return SignHmac(dateRegionServiceKey, requestBytes); - } - - /// - /// Compute hmac of input content with key. - /// - /// Hmac key - /// Bytes to be hmac computed - /// Computed hmac of input content - private ReadOnlySpan SignHmac(ReadOnlySpan key, ReadOnlySpan content) - { -#if NETSTANDARD - using var hmac = new HMACSHA256(key.ToArray()); - hmac.Initialize(); - return hmac.ComputeHash(content.ToArray()); -#else - return HMACSHA256.HashData(key, content); -#endif + byte[] key = null; + try + { + key = Encoding.UTF8.GetBytes(string.Format("{0}{1}", Scheme, secretKey)); + var dateKey = Utils.SignHmac(key, Encoding.UTF8.GetBytes(Utils.FormatDate(signingDate))); + var dateRegionKey = Utils.SignHmac(dateKey, Encoding.UTF8.GetBytes(region)); + var dateRegionServiceKey = Utils.SignHmac(dateRegionKey, Encoding.UTF8.GetBytes(GetService(isSts))); + return Utils.SignHmac(dateRegionServiceKey, TerminatorBytes); + } + finally + { + if (key is not null) + Array.Clear(key, 0, key.Length); + } } /// @@ -220,11 +208,19 @@ private ReadOnlySpan SignHmac(ReadOnlySpan key, ReadOnlySpan c /// Hexadecimal encoded sha256 checksum of canonicalRequest /// boolean; if true role credentials, otherwise IAM user /// String to sign - private string GetStringToSign(string region, DateTime signingDate, - string canonicalRequestHash, bool isSts = false) + private string GetStringToSign( + string region, + DateTime signingDate, + string canonicalRequestHash, + bool isSts = false) { var scope = GetScope(region, signingDate, isSts); - return $"AWS4-HMAC-SHA256\n{signingDate:yyyyMMddTHHmmssZ}\n{scope}\n{canonicalRequestHash}"; + var stringToSignBuilder = new StringBuilder(); + _ = stringToSignBuilder.AppendFormat( + CultureInfo.InvariantCulture, "{0}-{1}\n{2}\n{3}\n", + Scheme, SigningAlgorithm, Utils.FormatDateTime(signingDate), scope); + _ = stringToSignBuilder.Append(canonicalRequestHash); + return stringToSignBuilder.ToString(); } /// @@ -236,35 +232,7 @@ private string GetStringToSign(string region, DateTime signingDate, /// Scope string private string GetScope(string region, DateTime signingDate, bool isSts = false) { - return $"{signingDate:yyyyMMdd}/{region}/{GetService(isSts)}/aws4_request"; - } - - /// - /// Compute sha256 checksum. - /// - /// Bytes body - /// Bytes of sha256 checksum - private ReadOnlySpan ComputeSha256(ReadOnlySpan body) - { -#if NETSTANDARD - using var sha = SHA256.Create(); - ReadOnlySpan hash - = sha.ComputeHash(body.ToArray()); -#else - ReadOnlySpan hash = SHA256.HashData(body); -#endif - return hash; - } - - /// - /// Convert bytes to hexadecimal string. - /// - /// Bytes of any checksum - /// Hexlified string of input bytes - private string BytesToHex(ReadOnlySpan checkSum) - { - return BitConverter.ToString(checkSum.ToArray()).Replace("-", string.Empty, StringComparison.OrdinalIgnoreCase) - .ToLowerInvariant(); + return $"{Utils.FormatDate(signingDate)}/{region}/{GetService(isSts)}/aws4_request"; } /// @@ -278,8 +246,8 @@ public string PresignPostSignature(string region, DateTime signingDate, string p { var signingKey = GenerateSigningKey(region, signingDate); ReadOnlySpan stringToSignBytes = Encoding.UTF8.GetBytes(policyBase64); - var signatureBytes = SignHmac(signingKey, stringToSignBytes); - var signature = BytesToHex(signatureBytes); + var signatureBytes = Utils.SignHmac(signingKey, stringToSignBytes); + var signature = Utils.BytesToHex(signatureBytes); return signature; } @@ -292,49 +260,119 @@ public string PresignPostSignature(string region, DateTime signingDate, string p /// Value for session token /// Optional requestBuilder date and time in UTC /// Presigned url - internal string PresignURL(HttpRequestMessageBuilder requestBuilder, int expires, string region = "", - string sessionToken = "", DateTime? reqDate = null) + internal string PresignURL( + HttpRequestMessageBuilder requestBuilder, + int expires, + string region = "", + string sessionToken = "", + DateTime? reqDate = null) { var signingDate = reqDate ?? DateTime.UtcNow; - if (string.IsNullOrWhiteSpace(region)) region = GetRegion(requestBuilder.RequestUri.Host); - var requestUri = requestBuilder.RequestUri; - var requestQuery = requestUri.Query; + if (requestUri.Port is 80 or 443) + SetHostHeader(requestBuilder, requestUri.Host); + else + SetHostHeader(requestBuilder, requestUri.Host + ":" + requestUri.Port); + + SetSessionTokenHeader(requestBuilder, sessionToken); var headersToSign = GetHeadersToSign(requestBuilder); - if (!string.IsNullOrEmpty(sessionToken)) headersToSign["X-Amz-Security-Token"] = sessionToken; - - if (requestQuery.Length > 0) requestQuery += "&"; - requestQuery += "X-Amz-Algorithm=AWS4-HMAC-SHA256&"; - requestQuery += "X-Amz-Credential=" - + Uri.EscapeDataString(accessKey + "/" + GetScope(region, signingDate)) - + "&"; - requestQuery += "X-Amz-Date=" - + signingDate.ToString("yyyyMMddTHHmmssZ", CultureInfo.InvariantCulture) - + "&"; - requestQuery += "X-Amz-Expires=" - + expires - + "&"; - requestQuery += "X-Amz-SignedHeaders=host"; - - var presignUri = new UriBuilder(requestUri) { Query = requestQuery }.Uri; - var canonicalRequest = GetPresignCanonicalRequest(requestBuilder.Method, presignUri, headersToSign); - var headers = string.Concat(headersToSign.Select(p => $"&{p.Key}={Utils.UrlEncode(p.Value)}")); - ReadOnlySpan canonicalRequestBytes = Encoding.UTF8.GetBytes(canonicalRequest); - var canonicalRequestHash = BytesToHex(ComputeSha256(canonicalRequestBytes)); + var signedHeaders = GetSignedHeaders(headersToSign); + var parametersToCanonicalize = GetParametersToCanonicalize(requestBuilder); + + var credentials = string.Format(CultureInfo.InvariantCulture, "{0}/{1}/{2}/{3}/{4}", accessKey, + Utils.FormatDate(signingDate), region, GetService(false), Terminator); + + parametersToCanonicalize.Add(Constants.XAmzAlgorithm, AWS4AlgorithmTag); + parametersToCanonicalize.Add(Constants.XAmzCredential, credentials); + parametersToCanonicalize.Add(Constants.XAmzDate, Utils.FormatDateTime(signingDate)); + parametersToCanonicalize.Add(Constants.XAmzExpires, Convert.ToString(expires)); + parametersToCanonicalize.Add(Constants.XAmzSignedHeaders, signedHeaders); + + var requestQuery = GetCanonicalQueryString(requestBuilder.RequestUri, parametersToCanonicalize); + var canonicalRequest = + GetPresignCanonicalRequest(requestBuilder.Method, requestUri, headersToSign, requestQuery); + + var canonicalRequestHash = Utils.BytesToHex(Utils.ComputeSha256(canonicalRequest)); var stringToSign = GetStringToSign(region, signingDate, canonicalRequestHash); var signingKey = GenerateSigningKey(region, signingDate); - ReadOnlySpan stringToSignBytes = Encoding.UTF8.GetBytes(stringToSign); - var signatureBytes = SignHmac(signingKey, stringToSignBytes); - var signature = BytesToHex(signatureBytes); - // Return presigned url. - var signedUri = new UriBuilder(presignUri) { Query = $"{requestQuery}{headers}&X-Amz-Signature={signature}" }; + var signatureBytes = Utils.SignHmac(signingKey, Encoding.UTF8.GetBytes(stringToSign)); + var signature = Utils.BytesToHex(signatureBytes); + return ComposePresignedPutUrl(requestUri, requestQuery, signature); + } + + private IDictionary GetParametersToCanonicalize(HttpRequestMessageBuilder request) + { + var parameters = new SortedDictionary(StringComparer.Ordinal); + foreach (var param in request.QueryParameters) + if (param.Value is not null) + parameters.Add(param.Key, param.Value); + return parameters; + } + + private string ComposePresignedPutUrl( + Uri presignUri, + string queryParams, + string signature) + { + var authParams = new StringBuilder(queryParams) + .AppendFormat(CultureInfo.InvariantCulture, "&{0}={1}", Utils.UrlEncode(Constants.XAmzSignature), + Utils.UrlEncode(signature)); + + var signedUri = new UriBuilder(presignUri) { Query = authParams.ToString() }; if (signedUri.Uri.IsDefaultPort) signedUri.Port = -1; return Convert.ToString(signedUri, CultureInfo.InvariantCulture); } + /// + /// Generates canonical query string. + /// + /// Request uri + /// Query parameters to be included in signed url + /// Canonical query string + internal string GetCanonicalQueryString( + Uri requestUri, + IDictionary queryParams + ) + { + var canonicalQueryString = new StringBuilder(requestUri.Query); + if (canonicalQueryString.Length != 0) _ = canonicalQueryString.Append("&"); + foreach (var query in queryParams) + { + if (canonicalQueryString.Length > 0) + _ = canonicalQueryString.Append("&"); + _ = canonicalQueryString.AppendFormat(CultureInfo.InvariantCulture, "{0}={1}", Utils.UrlEncode(query.Key), + Utils.UrlEncode(query.Value)); + } + + return canonicalQueryString.ToString(); + } + + /// + /// Generates canonical headers + /// + /// Headers that will be formatted + /// Formatted headers + internal string GetCanonicalHeaders(IDictionary headers) + { + if (headers == null || headers.Count() == 0) + return string.Empty; + + var canonicalHeaders = new StringBuilder(); + + foreach (var header in headers) + { + _ = canonicalHeaders.Append(header.Key.ToLowerInvariant()); + _ = canonicalHeaders.Append(":"); + _ = canonicalHeaders.Append(S3utils.TrimAll(header.Value)); + _ = canonicalHeaders.Append("\n"); + } + + return canonicalHeaders.ToString(); + } + /// /// Get presign canonical requestBuilder. /// @@ -344,37 +382,25 @@ internal string PresignURL(HttpRequestMessageBuilder requestBuilder, int expires /// X-Amz-Signature /// /// The key-value of headers. + /// Canonical query string /// Presigned canonical requestBuilder - internal string GetPresignCanonicalRequest(HttpMethod requestMethod, Uri uri, - SortedDictionary headersToSign) - { - var canonicalStringList = new LinkedList(); - _ = canonicalStringList.AddLast(requestMethod.ToString()); - - var path = uri.AbsolutePath; - - _ = canonicalStringList.AddLast(path); - var queryParams = uri.Query.TrimStart('?').Split('&').ToList(); - queryParams.AddRange(headersToSign.Select(cv => - $"{Utils.UrlEncode(cv.Key)}={Utils.UrlEncode(cv.Value.Trim())}")); - queryParams.Sort(StringComparer.Ordinal); - var query = string.Join("&", queryParams); - _ = canonicalStringList.AddLast(query); - var canonicalHost = GetCanonicalHost(uri); - _ = canonicalStringList.AddLast($"host:{canonicalHost}"); - - _ = canonicalStringList.AddLast(string.Empty); - _ = canonicalStringList.AddLast("host"); - _ = canonicalStringList.AddLast("UNSIGNED-PAYLOAD"); - - return string.Join("\n", canonicalStringList); - } - - private static string GetCanonicalHost(Uri url) + internal string GetPresignCanonicalRequest( + HttpMethod requestMethod, + Uri uri, + IDictionary headersToSign, + string canonicalQueryString) { - if (url.Port is > 0 and not 80 and not 443) - return $"{url.Host}:{url.Port}"; - return url.Host; + var canonicalRequest = new StringBuilder(); + _ = canonicalRequest.AppendFormat(CultureInfo.InvariantCulture, "{0}\n", requestMethod.ToString()); + var canonicalUri = uri.AbsolutePath; + _ = canonicalRequest.AppendFormat(CultureInfo.InvariantCulture, "{0}\n", canonicalUri); + _ = canonicalRequest.AppendFormat(CultureInfo.InvariantCulture, "{0}\n", canonicalQueryString); + _ = canonicalRequest + .AppendFormat(CultureInfo.InvariantCulture, "{0}\n", GetCanonicalHeaders(headersToSign)); + _ = canonicalRequest + .AppendFormat(CultureInfo.InvariantCulture, "{0}\n", GetSignedHeaders(headersToSign)); + _ = canonicalRequest.Append("UNSIGNED-PAYLOAD"); + return canonicalRequest.ToString(); } /// @@ -384,7 +410,7 @@ private static string GetCanonicalHost(Uri url) /// Dictionary of http headers to be signed /// Canonical Request private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, - SortedDictionary headersToSign) + IDictionary headersToSign) { var canonicalStringList = new LinkedList(); // METHOD @@ -440,36 +466,26 @@ private string GetCanonicalRequest(HttpRequestMessageBuilder requestBuilder, _ = canonicalStringList.AddLast(header + ":" + S3utils.TrimAll(headersToSign[header])); _ = canonicalStringList.AddLast(string.Empty); _ = canonicalStringList.AddLast(string.Join(";", headersToSign.Keys)); - if (headersToSign.TryGetValue("x-amz-content-sha256", out var value)) - _ = canonicalStringList.AddLast(value); - else - _ = canonicalStringList.AddLast(sha256EmptyFileHash); + _ = headersToSign.TryGetValue("x-amz-content-sha256", out var value) + ? canonicalStringList.AddLast(value) + : canonicalStringList.AddLast(sha256EmptyFileHash); return string.Join("\n", canonicalStringList); } - public static IDictionary ToDictionary(object obj) - { - var json = JsonSerializer.Serialize(obj); - var dictionary = JsonSerializer.Deserialize>(json); - return dictionary; - } - /// /// Get headers to be signed. /// /// Instantiated requesst /// Sorted dictionary of headers to be signed - private SortedDictionary GetHeadersToSign(HttpRequestMessageBuilder requestBuilder) + private IDictionary GetHeadersToSign(HttpRequestMessageBuilder requestBuilder) { - var headers = requestBuilder.HeaderParameters.ToList(); + var headers = requestBuilder.HeaderParameters; var sortedHeaders = new SortedDictionary(StringComparer.Ordinal); - foreach (var header in headers) { var headerName = header.Key.ToLowerInvariant(); - if (string.Equals(header.Key, "versionId", StringComparison.Ordinal)) headerName = "versionId"; var headerValue = header.Value; - + if (string.Equals(header.Key, "versionId", StringComparison.Ordinal)) headerName = "versionId"; if (!ignoredHeaders.Contains(headerName)) sortedHeaders.Add(headerName, headerValue); } @@ -483,8 +499,7 @@ private SortedDictionary GetHeadersToSign(HttpRequestMessageBuil /// Date for signature to be signed private void SetDateHeader(HttpRequestMessageBuilder requestBuilder, DateTime signingDate) { - requestBuilder.AddOrUpdateHeaderParameter("x-amz-date", - signingDate.ToString("yyyyMMddTHHmmssZ", CultureInfo.InvariantCulture)); + requestBuilder.AddOrUpdateHeaderParameter("x-amz-date", Utils.FormatDateTime(signingDate)); } ///