Added DataLake SDK

sdk/storage/Azure.Storage.Common/src/Constants.cs

@@ -355,6 +355,37 @@ internal static class Share
             }
         }
 
+        /// <summary>
+        /// Data Lake constant values;
+        /// </summary>
+        internal static class DataLake
+        {
+            /// <summary>
+            /// The blob URI suffex.
+            /// </summary>
+            public const string BlobUriSuffix = "blob";
+
+            /// <summary>
+            /// The DFS URI suffex.
+            /// </summary>
+            public const string DfsUriSuffix = "dfs";
+
+            /// <summary>
+            /// The key of the object json object returned for errors.
+            /// </summary>
+            public const string ErrorKey = "error";
+
+            /// <summary>
+            /// The key of the error code returned for errors.
+            /// </summary>
+            public const string ErrorCodeKey = "code";
+
+            /// <summary>
+            /// The key of the error message returned for errors.
+            /// </summary>
+            public const string ErrorMessageKey = "message";
+        }
+
         /// <summary>
         /// Queue constant values
         /// </summary>

sdk/storage/Azure.Storage.Common/tests/Shared/StorageTestBase.cs

@@ -61,6 +61,14 @@ public StorageTestBase(bool async, RecordedTestMode? mode = null)
                 "Storage_TestConfigOAuth",
                 () => TestConfigurations.DefaultTargetOAuthTenant);
 
+        /// <summary>
+        /// Gets the tenant to use for any tests that require authentication
+        /// with Azure AD.
+        /// </summary>
+        public TenantConfiguration TestConfigHierarchicalNamespace => GetTestConfig(
+                "Storage_TestConfigHierarchicalNamespace",
+                () => TestConfigurations.DefaultTargetHierarchicalNamespaceTenant);
+
         /// <summary>
         /// Gets a cache used for storing serialized tenant configurations.  Do
         /// not get values from this directly; use GetTestConfig.
@@ -186,7 +194,7 @@ public TokenCredential GetOAuthCredential(string tenantId, string appId, string
                 Recording.InstrumentClientOptions(
                     new TokenCredentialOptions() { AuthorityHost = authorityHost }));
 
-        public void AssertMetadataEquality(IDictionary<string, string> expected, IDictionary<string, string> actual)
+        public virtual void AssertMetadataEquality(IDictionary<string, string> expected, IDictionary<string, string> actual)
         {
             Assert.IsNotNull(expected, "Expected metadata is null");
             Assert.IsNotNull(actual, "Actual metadata is null");

sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurations.cs

@@ -66,6 +66,12 @@ public class TestConfigurations
         /// </summary>
         private string TargetOAuthTenantName { get; set; }
 
+        /// <summary>
+        /// Gets the name of the tenant in the Tenants dictionary to use for
+        /// any tests that require hierarchical namespace.
+        /// </summary>
+        private string TargetHierarchicalNamespaceTenantName { get; set; }
+
         /// <summary>
         /// Gets the tenant to use by default for our tests.
         /// </summary>
@@ -98,6 +104,12 @@ public class TestConfigurations
         public static TenantConfiguration DefaultTargetOAuthTenant =>
             GetTenant("TargetOAuthTenant", s_configurations.Value.TargetOAuthTenantName);
 
+        /// <summary>
+        /// Gets a tenant to use for any tests that require hierarchical namespace
+        /// </summary>
+        public static TenantConfiguration DefaultTargetHierarchicalNamespaceTenant =>
+            GetTenant("TargetHierarchicalNamespaceTenant", s_configurations.Value.TargetHierarchicalNamespaceTenantName);
+
         /// <summary>
         /// When loading our test configuration, we'll check the
         /// AZ_STORAGE_CONFIG_PATH first.
@@ -193,6 +205,7 @@ private static TestConfigurations ReadFromXml(XDocument doc)
                 TargetPremiumBlobTenantName = Get("TargetPremiumBlobTenant"),
                 TargetPreviewBlobTenantName = Get("TargetPreviewBlobTenant"),
                 TargetOAuthTenantName = Get("TargetOAuthTenant"),
+                TargetHierarchicalNamespaceTenantName = Get("TargetHierarchicalNamespaceTenant"),
                 Tenants =
                     config.Element("TenantConfigurations").Elements("TenantConfiguration")
                     .Select(TenantConfiguration.Parse)

sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurationsTemplate.xml

@@ -4,6 +4,7 @@
   <TargetSecondaryTestTenant>SecondaryTenant</TargetSecondaryTestTenant>
   <TargetPreviewBlobTenant>NotInPreview</TargetPreviewBlobTenant>  <!-- Internal only -->
   <TargetOAuthTenant>OAuthTenant</TargetOAuthTenant>
+  <TargetHierarchicalNamespaceTenant>NamespaceTenant</TargetHierarchicalNamespaceTenant>
 
   <TenantConfigurations>
     <!-- Replace [ACCOUNT] and [ACCOUNT-KEY] -->
@@ -23,6 +24,7 @@
       <ConnectionString>DefaultEndpointsProtocol=https;AccountName=[ACCOUNT];AccountKey=[ACCOUNT-KEY];EndpointSuffix=core.windows.net</ConnectionString>
     </TenantConfiguration>
 
+    <!-- A storage account with Read-only geo-redundant access (RA-GRS) enabled -->
     <!-- Replace [RAGRS-ACCOUNT] and [RAGRS-ACCOUNT-KEY] -->
     <TenantConfiguration>
       <TenantName>SecondaryTenant</TenantName>
@@ -39,16 +41,19 @@
       <TableServiceSecondaryEndpoint>http://[RAGRS-ACCOUNT]-secondary.table.core.windows.net</TableServiceSecondaryEndpoint>
     </TenantConfiguration>
 
+    <!-- A storage account with Oauth access -->
     <!-- Replace [OAUTH-ACCOUNT] and [OAUTH-ACCOUNT-KEY] -->
     <TenantConfiguration>
       <TenantName>OAuthTenant</TenantName>
       <TenantType>Cloud</TenantType>
       <AccountName>[OAUTH-ACCOUNT]</AccountName>
       <AccountKey>[OAUTH-ACCOUNT-KEY]</AccountKey>
+
       <ActiveDirectoryApplicationId>[ActiveDirectoryApplicationId]</ActiveDirectoryApplicationId>
       <ActiveDirectoryApplicationSecret>[ActiveDirectoryApplicationSecret]</ActiveDirectoryApplicationSecret>
       <ActiveDirectoryTenantId>[ActiveDirectoryTenantId]</ActiveDirectoryTenantId>
       <ActiveDirectoryAuthEndpoint>https://login.microsoftonline.com/</ActiveDirectoryAuthEndpoint>
+
       <BlobServiceEndpoint>http://[OAUTH-ACCOUNT].blob.core.windows.net</BlobServiceEndpoint>
       <QueueServiceEndpoint>http://[OAUTH-ACCOUNT].queue.core.windows.net</QueueServiceEndpoint>
       <TableServiceEndpoint>http://[OAUTH-ACCOUNT].table.core.windows.net</TableServiceEndpoint>
@@ -59,7 +64,8 @@
       <TableServiceSecondaryEndpoint>http://[OAUTH-ACCOUNT]-secondary.table.core.windows.net</TableServiceSecondaryEndpoint>
     </TenantConfiguration>
 
-	    <!-- Replace [RAGRS-ACCOUNT] and [RAGRS-ACCOUNT-KEY] -->
+    <!-- A storage account with premium preformance enabled -->
+	<!-- Replace [RAGRS-ACCOUNT] and [RAGRS-ACCOUNT-KEY] -->
     <TenantConfiguration>
       <TenantName>PremiumBlobTenant</TenantName>
       <TenantType>Cloud</TenantType>
@@ -74,5 +80,28 @@
       <FileServiceSecondaryEndpoint>http://[PREMIUM-ACCOUNT]-secondary.file.core.windows.net</FileServiceSecondaryEndpoint>
       <TableServiceSecondaryEndpoint>http://[PREMIUM-ACCOUNT]-secondary.table.core.windows.net</TableServiceSecondaryEndpoint>
     </TenantConfiguration>
+
+    <!-- A storge account with hierarchical namespace enabled and OAuth access -->
+    <!-- Replace [NAMESPACE-ACCOUNT] and [NAMESPACE-ACCOUNT-KEY] -->
+    <TenantConfiguration>
+      <TenantName>NamespaceTenant</TenantName>
+      <TenantType>Cloud</TenantType>
+      <AccountName>[NAMESPACE-ACCOUNT]</AccountName>
+      <AccountKey>[NAMESPACE-ACCOUNT-KEY]</AccountKey>
+
+      <ActiveDirectoryApplicationId>[ActiveDirectoryApplicationId]</ActiveDirectoryApplicationId>
+      <ActiveDirectoryApplicationSecret>[ActiveDirectoryApplicationSecret]</ActiveDirectoryApplicationSecret>
+      <ActiveDirectoryTenantId>[ActiveDirectoryTenantId]</ActiveDirectoryTenantId>
+      <ActiveDirectoryAuthEndpoint>https://login.microsoftonline.com/</ActiveDirectoryAuthEndpoint>
+
+      <BlobServiceEndpoint>http://[NAMESPACE-ACCOUNT].blob.core.windows.net</BlobServiceEndpoint>
+      <QueueServiceEndpoint>http://[NAMESPACE-ACCOUNT].queue.core.windows.net</QueueServiceEndpoint>
+      <TableServiceEndpoint>http://[NAMESPACE-ACCOUNT].table.core.windows.net</TableServiceEndpoint>
+      <FileServiceEndpoint>http://[NAMESPACE-ACCOUNT].file.core.windows.net</FileServiceEndpoint>
+      <BlobServiceSecondaryEndpoint>http://[NAMESPACE-ACCOUNT]-secondary.blob.core.windows.net</BlobServiceSecondaryEndpoint>
+      <QueueServiceSecondaryEndpoint>http://[NAMESPACE-ACCOUNT]-secondary.queue.core.windows.net</QueueServiceSecondaryEndpoint>
+      <FileServiceSecondaryEndpoint>http://[NAMESPACE-ACCOUNT]-secondary.file.core.windows.net</FileServiceSecondaryEndpoint>
+      <TableServiceSecondaryEndpoint>http://[NAMESPACE-ACCOUNT]-secondary.table.core.windows.net</TableServiceSecondaryEndpoint>
+    </TenantConfiguration>
   </TenantConfigurations>
 </TestConfigurations>

sdk/storage/Azure.Storage.Files.DataLake/src/AssemblyInfo.cs

@@ -9,3 +9,4 @@
     "012ea67d2479751a0b8c087a4185541b851bd8b16f8d91b840e51b1cb0ba6fe647997e57429265" +
     "e85ef62d565db50a69ae1647d54d7bd855e4db3d8a91510e5bcbd0edfbbecaa20a7bd9ae74593d" +
     "aa7b11b4")]
+[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]

sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj

@@ -18,7 +18,11 @@
       REST API Reference for Data Lake - https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/filesystem
     </Description>
   </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="System.Text.Json" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Common\src\Azure.Storage.Common.csproj" />
+    <ProjectReference Include="..\..\Azure.Storage.Blobs\src\Azure.Storage.Blobs.csproj" />
   </ItemGroup>
 </Project>

sdk/storage/Azure.Storage.Files.DataLake/src/BlobContainerItemExtensions.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Storage.Blobs.Models;
+using Azure.Storage.Files.DataLake.Models;
+
+namespace Azure.Storage.Files.DataLake
+{
+    internal static class BlobContainerItemExtensions
+    {
+        internal static FileSystemItem ToFileSystemItem(this BlobContainerItem containerItem) =>
+            new FileSystemItem()
+            {
+                Name = containerItem.Name,
+                Properties = new FileSystemProperties()
+                {
+                    LastModified = containerItem.Properties.LastModified,
+                    LeaseStatus = containerItem.Properties.LeaseStatus.HasValue
+                        ? (Models.LeaseStatus)containerItem.Properties.LeaseStatus : default,
+                    LeaseState = containerItem.Properties.LeaseState.HasValue
+                        ? (Models.LeaseState)containerItem.Properties.LeaseState : default,
+                    LeaseDuration = containerItem.Properties.LeaseDuration.HasValue
+                        ? (Models.LeaseDurationType)containerItem.Properties.LeaseDuration : default,
+                    PublicAccess = containerItem.Properties.PublicAccess.HasValue
+                        ? (Models.PublicAccessType)containerItem.Properties.PublicAccess : default,
+                    HasImmutabilityPolicy = containerItem.Properties.HasImmutabilityPolicy,
+                    HasLegalHold = containerItem.Properties.HasLegalHold,
+                    ETag = containerItem.Properties.ETag
+                },
+                Metadata = containerItem.Metadata
+            };
+    }
+}

sdk/storage/Azure.Storage.Files.DataLake/src/BlobDownloadDetailsExtensions.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Storage.Blobs.Models;
+using Azure.Storage.Files.DataLake.Models;
+
+namespace Azure.Storage.Files.DataLake
+{
+    internal static class BlobDownloadDetailsExtensions
+    {
+        internal static FileDownloadDetails ToFileDownloadDetails(this BlobDownloadDetails blobDownloadProperties) =>
+            new FileDownloadDetails()
+            {
+                LastModified = blobDownloadProperties.LastModified,
+                Metadata = blobDownloadProperties.Metadata,
+                ContentRange = blobDownloadProperties.ContentRange,
+                ETag = blobDownloadProperties.ETag,
+                ContentEncoding = blobDownloadProperties.ContentEncoding,
+                ContentDisposition = blobDownloadProperties.ContentDisposition,
+                ContentLanguage = blobDownloadProperties.ContentLanguage,
+                CopyCompletedOn = blobDownloadProperties.CopyCompletedOn,
+                CopyStatusDescription = blobDownloadProperties.CopyStatusDescription,
+                CopyId = blobDownloadProperties.CopyId,
+                CopyProgress = blobDownloadProperties.CopyProgress,
+                CopyStatus = (Models.CopyStatus)blobDownloadProperties.CopyStatus,
+                LeaseDuration = (Models.LeaseDurationType)blobDownloadProperties.LeaseDuration,
+                LeaseState = (Models.LeaseState)blobDownloadProperties.LeaseState,
+                LeaseStatus = (Models.LeaseStatus)blobDownloadProperties.LeaseStatus,
+                AcceptRanges = blobDownloadProperties.AcceptRanges,
+                IsServerEncrypted = blobDownloadProperties.IsServerEncrypted,
+                EncryptionKeySha256 = blobDownloadProperties.EncryptionKeySha256,
+                ContentHash = blobDownloadProperties.BlobContentHash
+            };
+    }
+}

sdk/storage/Azure.Storage.Files.DataLake/src/BlobDownloadInfoExtensions.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Storage.Blobs.Models;
+using Azure.Storage.Files.DataLake.Models;
+
+namespace Azure.Storage.Files.DataLake
+{
+    internal static class BlobDownloadInfoExtensions
+    {
+        internal static FileDownloadInfo ToFileDownloadInfo(this BlobDownloadInfo blobDownloadInfo) =>
+            new FileDownloadInfo()
+            {
+                ContentLength = blobDownloadInfo.ContentLength,
+                Content = blobDownloadInfo.Content,
+                ContentHash = blobDownloadInfo.ContentHash,
+                Properties = blobDownloadInfo.Details.ToFileDownloadDetails()
+            };
+    }
+}

sdk/storage/Azure.Storage.Files.DataLake/src/BlobPropertiesExtensions.cs

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Storage.Blobs.Models;
+using Azure.Storage.Files.DataLake.Models;
+
+namespace Azure.Storage.Files.DataLake
+{
+    internal static class BlobPropertiesExtensions
+    {
+        internal static PathProperties ToPathProperties(this BlobProperties blobProperties) =>
+            new PathProperties()
+            {
+                LastModified = blobProperties.LastModified,
+                CreatedOn = blobProperties.CreatedOn,
+                Metadata = blobProperties.Metadata,
+                CopyCompletedOn = blobProperties.CopyCompletedOn,
+                CopyStatusDescription = blobProperties.CopyStatusDescription,
+                CopyId = blobProperties.CopyId,
+                CopyProgress = blobProperties.CopyProgress,
+                CopySource = blobProperties.CopySource,
+                IsIncrementalCopy = blobProperties.IsIncrementalCopy,
+                LeaseDuration = (Models.LeaseDurationType)blobProperties.LeaseDuration,
+                LeaseStatus = (Models.LeaseStatus)blobProperties.LeaseStatus,
+                LeaseState = (Models.LeaseState)blobProperties.LeaseState,
+                ContentLength = blobProperties.ContentLength,
+                ContentType = blobProperties.ContentType,
+                ETag = blobProperties.ETag,
+                ContentHash = blobProperties.ContentHash,
+                ContentEncoding = blobProperties.ContentEncoding,
+                ContentDisposition = blobProperties.ContentDisposition,
+                ContentLanguage = blobProperties.ContentLanguage,
+                CacheControl = blobProperties.CacheControl,
+                AcceptRanges = blobProperties.AcceptRanges,
+                IsServerEncrypted = blobProperties.IsServerEncrypted,
+                EncryptionKeySha256 = blobProperties.EncryptionKeySha256,
+                AccessTier = blobProperties.AccessTier,
+                ArchiveStatus = blobProperties.ArchiveStatus,
+                AccessTierChangedOn = blobProperties.AccessTierChangedOn
+            };
+    }
+}

sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeClientOptions.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Core;
+using Azure.Storage.Blobs;
+using Azure.Storage.Files.DataLake.Models;
+
+namespace Azure.Storage.Files.DataLake
+{
+#pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion
+    /// <summary>
+    /// Provides the client configuration options for connecting to Azure Data Lake service.
+    /// </summary>
+    public class DataLakeClientOptions : BlobClientOptions
+#pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion
+    {
+
+    }
+}