diff --git a/eng/common/testproxy/target_version.txt b/eng/common/testproxy/target_version.txt index aef19a2487..dedc530f84 100644 --- a/eng/common/testproxy/target_version.txt +++ b/eng/common/testproxy/target_version.txt @@ -1 +1 @@ -1.0.0-dev.20220921.2 +1.0.0-dev.20220922.1 diff --git a/eng/pipelines/templates/jobs/perf.yml b/eng/pipelines/templates/jobs/perf.yml new file mode 100644 index 0000000000..4e1d328449 --- /dev/null +++ b/eng/pipelines/templates/jobs/perf.yml @@ -0,0 +1,44 @@ +parameters: +- name: ServiceDirectory + type: string + default: '' +- name: Services + type: string + default: '' +- name: PackageVersions + type: string + default: '.*' +- name: Tests + type: string + default: '.*' +- name: Arguments + type: string + default: '.*' +- name: Iterations + type: number + default: '5' +- name: AdditionalArguments + type: string + default: '' +- name: EnvVars + type: object + default: [] + +extends: + template: /eng/common/pipelines/templates/jobs/perf.yml + parameters: + Variables: + - template: /eng/pipelines/templates/variables/globals.yml + Language: Cpp + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Services: ${{ parameters.Services }} + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} + EnvVars: ${{ parameters.EnvVars}} + InstallLanguageSteps: + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: Microsoft diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml new file mode 100644 index 0000000000..b782b88239 --- /dev/null +++ b/eng/pipelines/templates/variables/globals.yml @@ -0,0 +1,22 @@ +variables: + # True if 'Enable system diagnostics' is checked when running a pipeline manually + IsDebug: $[coalesce(variables['System.Debug'], 'false')] + + AdditionalOptions: '' + + # Exists if needed in coalesce situations. + DefaultTestGoals: 'surefire:test' + # This will be overwritten by the test matrix, if configured. + TestGoals: $(DefaultTestGoals) + + # This will be overwritten by the test matrix, if configured. + TestOptions: '' + # TestFromSource is one of the cache keys but isn't set until the test matrix + # has been processed. Without a default value it'll be treated as a string literal + # "$(TestFromSource)" instead of true/false. It'll be overwritten when the test + # matrix has been processed + TestFromSource: false + + skipComponentGovernanceDetection: true + DisableDockerDetector: true + Package.EnableSBOMSigning: true diff --git a/sdk/core/perf.yml b/sdk/core/perf.yml new file mode 100644 index 0000000000..a8494272b1 --- /dev/null +++ b/sdk/core/perf.yml @@ -0,0 +1,32 @@ +parameters: +- name: PackageVersions + displayName: PackageVersions (regex of package versions to run) + type: string + default: 'source' +- name: Tests + displayName: Tests (regex of tests to run) + type: string + default: '^(extendedOptions)$' +- name: Arguments + displayName: Arguments (regex of arguments to run) + type: string + default: '.*' +- name: Iterations + displayName: Iterations (times to run each test) + type: number + default: '5' +- name: AdditionalArguments + displayName: AdditionalArguments (passed to PerfAutomation) + type: string + default: ' ' + +extends: + template: /eng/pipelines/templates/jobs/perf.yml + parameters: + ServiceDirectory: core + Services: "^core$" + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp index 7a139f8093..cab73b98ef 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp @@ -130,7 +130,7 @@ id,name,price R"json({"id":"103","name":"apples","price":"99"}|{"id":"106","name":"lemons","price":"69"}|{"id":"110","name":"bananas","price":"39"}|{"id":"112","name":"sapote,mamey","price":"50"}|)json"); } - TEST_F(BlockBlobClientTest, DISABLED_QueryCsvInputArrowOutput_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryCsvInputArrowOutput_LIVEONLY_) { auto const testName(GetTestName()); auto client = GetBlockBlobClient(testName); diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index ef15e55df3..36a75bcd69 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -193,6 +193,67 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using RehydratePriority = Blobs::Models::RehydratePriority; using ArchiveStatus = Blobs::Models::ArchiveStatus; + /** + * @brief The path item returned when listing the paths. + */ + struct PathItem final + { + /** + * The name of the path. + */ + std::string Name; + + /** + * Indicates whether this path is a directory. + */ + bool IsDirectory = false; + + /** + * The data and time the path was last modified. + */ + DateTime LastModified; + + /** + * The size of the file. + */ + int64_t FileSize = int64_t(); + + /** + * The owner of the path. + */ + std::string Owner; + + /** + * The group of the path. + */ + std::string Group; + + /** + * The permission of the path. + */ + std::string Permissions; + + /** + * The name of the encryption scope under which the blob is encrypted. + */ + Nullable EncryptionScope; + + /** + * The creation time of the path. + */ + Nullable CreatedOn; + + /** + * The expiry time of the path. + */ + Nullable ExpiresOn; + + /** + * An HTTP entity tag associated with the path. + */ + std::string ETag; + }; + /** * @brief The properties of the path. */ diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp index d4c0afcb5c..a2d73be699 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp @@ -83,24 +83,24 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { private: std::string m_value; }; - struct PathItem final - { - std::string Name; - bool IsDirectory = false; - DateTime LastModified; - int64_t FileSize = int64_t(); - std::string Owner; - std::string Group; - std::string Permissions; - /** - * The name of the encryption scope under which the blob is encrypted. - */ - Nullable EncryptionScope; - Nullable CreatedOn; - Nullable ExpiresOn; - std::string ETag; - }; namespace _detail { + struct PathItem final + { + std::string Name; + bool IsDirectory = false; + DateTime LastModified; + int64_t FileSize = int64_t(); + std::string Owner; + std::string Group; + std::string Permissions; + /** + * The name of the encryption scope under which the blob is encrypted. + */ + Nullable EncryptionScope; + Nullable CreatedOn; + Nullable ExpiresOn; + std::string ETag; + }; /** * @brief Response type for #Azure::Storage::Files::DataLake::FileSystemClient::ListPaths. */ diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 7e81aa9f0b..3023758b84 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -247,8 +247,31 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { _internal::WithReplicaStatus(context)); ListPathsPagedResponse pagedResponse; - - pagedResponse.Paths = std::move(response.Value.Paths); + const std::string emptyExpiresOnString = "0"; + for (auto& path : response.Value.Paths) + { + Models::PathItem item; + item.Name = std::move(path.Name); + item.IsDirectory = path.IsDirectory; + item.LastModified = std::move(path.LastModified); + item.FileSize = path.FileSize; + item.Owner = std::move(path.Owner); + item.Group = std::move(path.Group); + item.Permissions = std::move(path.Permissions); + item.EncryptionScope = path.EncryptionScope; + item.ETag = std::move(path.ETag); + if (path.CreatedOn.HasValue()) + { + item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.CreatedOn.Value())); + } + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != emptyExpiresOnString) + { + item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.ExpiresOn.Value())); + } + pagedResponse.Paths.push_back(std::move(item)); + } pagedResponse.m_onNextPageFunc = func; pagedResponse.CurrentPageToken = continuationToken; pagedResponse.NextPageToken = response.Value.ContinuationToken; diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index c4e292127f..a6dddbfc15 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -276,18 +276,32 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptionsCopy, _internal::WithReplicaStatus(context)); - const auto emptyExpiresOn - = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime(0); + ListPathsPagedResponse pagedResponse; + const std::string emptyExpiresOnString = "0"; for (auto& path : response.Value.Paths) { - if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() == emptyExpiresOn) + Models::PathItem item; + item.Name = std::move(path.Name); + item.IsDirectory = path.IsDirectory; + item.LastModified = std::move(path.LastModified); + item.FileSize = path.FileSize; + item.Owner = std::move(path.Owner); + item.Group = std::move(path.Group); + item.Permissions = std::move(path.Permissions); + item.EncryptionScope = path.EncryptionScope; + item.ETag = std::move(path.ETag); + if (path.CreatedOn.HasValue()) + { + item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.CreatedOn.Value())); + } + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != emptyExpiresOnString) { - path.ExpiresOn.Reset(); + item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.ExpiresOn.Value())); } + pagedResponse.Paths.push_back(std::move(item)); } - - ListPathsPagedResponse pagedResponse; - pagedResponse.Paths = std::move(response.Value.Paths); pagedResponse.m_onNextPageFunc = func; pagedResponse.CurrentPageToken = continuationToken; pagedResponse.NextPageToken = response.Value.ContinuationToken; diff --git a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp index a8e4e3adf6..d0d006882d 100644 --- a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp +++ b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp @@ -5,6 +5,7 @@ #include +#include #include #include @@ -28,4 +29,48 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam Blobs::BlobClientOptions GetBlobClientOptions(const DataLakeClientOptions& options); + /** + * @brief Provides conversion methods for Win32 FILETIME to an #Azure::DateTime. + * + */ + class Win32FileTimeConverter final { + public: + /** + * @brief Converts Win32 FILETIME to an #Azure::DateTime. + * + * @param win32Filetime The number of 100-nanoseconds since 1601-01-01. + * @return Calculated #Azure::DateTime. + */ + static DateTime Win32FileTimeToDateTime(int64_t win32Filetime) + { + auto t = DateTime(1601) + Azure::_detail::Clock::duration(win32Filetime); + return DateTime(t); + } + + /** + * @brief Converts a DateTime to Win32 FILETIME. + * + * @param dateTime The `%DateTime` to convert. + * @return The number of 100-nanoseconds since 1601-01-01. + */ + static int64_t DateTimeToWin32FileTime(DateTime const& dateTime) + { + return std::chrono::duration_cast(dateTime - DateTime(1601)) + .count(); + } + + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be created. + * + */ + Win32FileTimeConverter() = delete; + + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be destructed, because no + * instance can be created. + * + */ + ~Win32FileTimeConverter() = delete; + }; + }}}}} // namespace Azure::Storage::Files::DataLake::_detail diff --git a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp index 47c7d56686..c105e1428a 100644 --- a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp @@ -94,7 +94,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ? jsonRoot["paths"] : Core::Json::_internal::json::array()) { - Models::PathItem vectorElement2; + Models::_detail::PathItem vectorElement2; vectorElement2.Name = var0["name"].get(); if (var0.count("isDirectory") != 0) { @@ -116,15 +116,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } if (var0.count("creationTime") != 0) { - vectorElement2.CreatedOn - = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(var0["creationTime"].get())); + vectorElement2.CreatedOn = var0["creationTime"].get(); } if (var0.count("expiryTime") != 0) { - vectorElement2.ExpiresOn - = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(var0["expiryTime"].get())); + vectorElement2.ExpiresOn = var0["expiryTime"].get(); } if (var0.count("etag") != 0) { diff --git a/sdk/storage/azure-storage-files-datalake/swagger/README.md b/sdk/storage/azure-storage-files-datalake/swagger/README.md index 9626429b02..b7c915b212 100644 --- a/sdk/storage/azure-storage-files-datalake/swagger/README.md +++ b/sdk/storage/azure-storage-files-datalake/swagger/README.md @@ -229,12 +229,13 @@ directive: where: $.definitions transform: > $.Path["x-ms-client-name"] = "PathItem"; + $.Path["x-namespace"] = "_detail"; $.Path.properties["lastModified"]["format"] = "date-time-rfc1123"; $.Path.properties["contentLength"]["x-ms-client-name"] = "FileSize"; $.Path.properties["isDirectory"]["x-ms-client-default"] = false; $.Path.properties["EncryptionScope"]["x-nullable"] = true; - $.Path.properties["creationTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "CreatedOn", "x-nullable": true}; - $.Path.properties["expiryTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "ExpiresOn", "x-nullable": true}; + $.Path.properties["creationTime"] = {"type": "string", "x-ms-client-name": "CreatedOn", "x-nullable": true}; + $.Path.properties["expiryTime"] = {"type": "string", "x-ms-client-name": "ExpiresOn", "x-nullable": true}; $.Path.properties["etag"] = {"type": "string", "x-ms-format": "string", "x-ms-client-default": "", "x-ms-client-name": "ETag"}; delete $.Path.properties["eTag"]; $.PathList["x-namespace"] = "_detail"; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 2cbf43567a..24f88449a1 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -318,6 +318,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NE(result.end(), iter); EXPECT_EQ(iter->Name, name); EXPECT_EQ(iter->Name.substr(0U, m_directoryA.size()), m_directoryA); + EXPECT_TRUE(iter->CreatedOn.HasValue()); + EXPECT_FALSE(iter->ExpiresOn.HasValue()); } for (const auto& name : m_pathNameSetB) {