From 24bbb4e3ec43fbb9791199f24ea2e340bf7540dc Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 12 May 2022 17:27:49 -0700 Subject: [PATCH 01/22] Initial interface and runtime setup for API --- .../AppInstallerCommonCore.vcxproj | 2 + .../AppInstallerCommonCore.vcxproj.filters | 6 + .../InstallerMetadataCollectionContext.cpp | 128 ++++++++++++++++ .../Public/AppInstallerDownloader.h | 1 + .../InstallerMetadataCollectionContext.h | 42 ++++++ src/AppInstallerCommonCore/ThreadGlobals.cpp | 140 +++++++++--------- src/WinGetUtil/Exports.cpp | 70 ++++++++- src/WinGetUtil/WinGetUtil.h | 43 ++++++ 8 files changed, 361 insertions(+), 71 deletions(-) create mode 100644 src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp create mode 100644 src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj index 043e8091f4..4c73f77c5f 100644 --- a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj +++ b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj @@ -305,6 +305,7 @@ + @@ -388,6 +389,7 @@ + diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters index 86d71438cc..389451e789 100644 --- a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters +++ b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters @@ -195,6 +195,9 @@ Public\winget + + Public\winget + @@ -338,6 +341,9 @@ Source Files + + Source Files + diff --git a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp new file mode 100644 index 0000000000..fc45c7ca84 --- /dev/null +++ b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Public/winget/InstallerMetadataCollectionContext.h" + +#include "Public/AppInstallerDownloader.h" +#include "Public/AppInstallerLogging.h" +#include "Public/AppInstallerStrings.h" + +namespace AppInstaller::Utility +{ + std::unique_ptr InstallerMetadataCollectionContext::FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile) + { + THROW_HR_IF(E_INVALIDARG, file.empty()); + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !std::filesystem::exists(file)); + + std::unique_ptr result = std::make_unique(); + auto threadGlobalsLifetime = result->Initialize(logFile); + + AICLI_LOG(Core, Info, << "Opening InstallerMetadataCollectionContext input file: " << file); + std::ifstream fileStream{ file }; + + result->CollectPreInstallState(ConvertToUTF16(ReadEntireStream(fileStream))); + + return result; + } + + std::unique_ptr InstallerMetadataCollectionContext::FromURI(std::wstring_view uri, const std::filesystem::path& logFile) + { + THROW_HR_IF(E_INVALIDARG, uri.empty()); + + std::unique_ptr result = std::make_unique(); + auto threadGlobalsLifetime = result->Initialize(logFile); + + std::string utf8Uri = ConvertToUTF8(uri); + THROW_HR_IF(E_INVALIDARG, !IsUrlRemote(utf8Uri)); + + AICLI_LOG(Core, Info, << "Downloading InstallerMetadataCollectionContext input file: " << utf8Uri); + + std::ostringstream jsonStream; + ProgressCallback emptyCallback; + + const int MaxRetryCount = 2; + for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount) + { + bool success = false; + try + { + auto downloadHash = DownloadToStream(utf8Uri, jsonStream, DownloadType::InstallerMetadataCollectionInput, emptyCallback); + + success = true; + } + catch (...) + { + if (retryCount < MaxRetryCount - 1) + { + AICLI_LOG(Core, Info, << " Downloading InstallerMetadataCollectionContext input failed, waiting a bit and retrying..."); + Sleep(500); + } + else + { + throw; + } + } + + if (success) + { + break; + } + } + + result->CollectPreInstallState(ConvertToUTF16(jsonStream.str())); + + return result; + } + + std::unique_ptr InstallerMetadataCollectionContext::FromJSON(std::wstring_view json, const std::filesystem::path& logFile) + { + THROW_HR_IF(E_INVALIDARG, json.empty()); + + std::unique_ptr result = std::make_unique(); + auto threadGlobalsLifetime = result->Initialize(logFile); + result->CollectPreInstallState(std::wstring{ json }); + + return result; + } + + std::unique_ptr InstallerMetadataCollectionContext::Initialize(const std::filesystem::path& logFile) + { + auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); + + Logging::Log().SetLevel(Logging::Level::Info); + Logging::Log().EnableChannel(Logging::Channel::All); + Logging::EnableWilFailureTelemetry(); + Logging::AddTraceLogger(); + + if (!logFile.empty()) + { + Logging::AddFileLogger(logFile); + } + + Logging::Telemetry().SetCaller("installer-metadata-collection"); + Logging::Telemetry().LogStartup(); + } + + void InstallerMetadataCollectionContext::CollectPreInstallState(const std::wstring& json) + { + // Parse and validate JSON + web::json::value::parse(json); + + // Collect pre-install system state + } + + void InstallerMetadataCollectionContext::Complete(const std::filesystem::path& output, const std::filesystem::path& diagnostics) + { + auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); + + THROW_HR_IF(E_INVALIDARG, output.empty()); + + // Collect post-install system state + + // Compute metadata match scores + + // Write output + + // Write diagnostics + } +} diff --git a/src/AppInstallerCommonCore/Public/AppInstallerDownloader.h b/src/AppInstallerCommonCore/Public/AppInstallerDownloader.h index 0152346b8c..8518cb94b5 100644 --- a/src/AppInstallerCommonCore/Public/AppInstallerDownloader.h +++ b/src/AppInstallerCommonCore/Public/AppInstallerDownloader.h @@ -22,6 +22,7 @@ namespace AppInstaller::Utility Manifest, WinGetUtil, Installer, + InstallerMetadataCollectionInput, }; // Extra metadata about a download for use by certain downloaders (Delivery Optimization for instance). diff --git a/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h new file mode 100644 index 0000000000..abc66faa4b --- /dev/null +++ b/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once + +#include +#include +#include +#include + +#include + +namespace AppInstaller::Utility +{ + // Contains the functions and data used for collecting metadata from installers. + struct InstallerMetadataCollectionContext + { + InstallerMetadataCollectionContext() = default; + + InstallerMetadataCollectionContext(const InstallerMetadataCollectionContext&) = delete; + InstallerMetadataCollectionContext& operator=(const InstallerMetadataCollectionContext&) = delete; + + InstallerMetadataCollectionContext(InstallerMetadataCollectionContext&&) = default; + InstallerMetadataCollectionContext& operator=(InstallerMetadataCollectionContext&&) = default; + + // Create from various forms of JSON input to prevent type collisions on constructor. + static std::unique_ptr FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile); + static std::unique_ptr FromURI(std::wstring_view uri, const std::filesystem::path& logFile); + static std::unique_ptr FromJSON(std::wstring_view json, const std::filesystem::path& logFile); + + // Completes the collection, writing to the given locations. + void Complete(const std::filesystem::path& output, const std::filesystem::path& diagnostics); + + private: + // Initializes the context runtime, including the log file if provided. + std::unique_ptr Initialize(const std::filesystem::path& logFile); + + // Sets the collection context input and the preinstall state. + void CollectPreInstallState(const std::wstring& json); + + ThreadLocalStorage::ThreadGlobals m_threadGlobals; + }; +} diff --git a/src/AppInstallerCommonCore/ThreadGlobals.cpp b/src/AppInstallerCommonCore/ThreadGlobals.cpp index 13bec7c1fd..3752953bd5 100644 --- a/src/AppInstallerCommonCore/ThreadGlobals.cpp +++ b/src/AppInstallerCommonCore/ThreadGlobals.cpp @@ -1,11 +1,11 @@ -#include "pch.h" -#include "Public/winget/ThreadGlobals.h" - -namespace AppInstaller::ThreadLocalStorage -{ - using namespace AppInstaller::Logging; - - // Set and return Globals for Current Thread +#include "pch.h" +#include "Public/winget/ThreadGlobals.h" + +namespace AppInstaller::ThreadLocalStorage +{ + using namespace AppInstaller::Logging; + + // Set and return Globals for Current Thread static ThreadGlobals* SetOrGetThreadGlobals(bool setThreadGlobals, ThreadGlobals* pThreadGlobals = nullptr); ThreadGlobals::ThreadGlobals(ThreadGlobals& parent, create_sub_thread_globals_t) @@ -17,67 +17,67 @@ namespace AppInstaller::ThreadLocalStorage std::call_once(m_loggerInitOnceFlag, []() {}); } - DiagnosticLogger& ThreadGlobals::GetDiagnosticLogger() - { - return *(m_pDiagnosticLogger); + DiagnosticLogger& ThreadGlobals::GetDiagnosticLogger() + { + return *(m_pDiagnosticLogger); } - TelemetryTraceLogger& ThreadGlobals::GetTelemetryLogger() - { - return *(m_pTelemetryLogger); - } - - std::unique_ptr ThreadGlobals::SetForCurrentThread() - { - Initialize(); - - std::unique_ptr p_prevThreadGlobals = std::make_unique(SetOrGetThreadGlobals(true, this)); - - return p_prevThreadGlobals; - } - - void ThreadGlobals::Initialize() - { - try - { - std::call_once(m_loggerInitOnceFlag, [this]() - { - m_pDiagnosticLogger = std::make_unique(); - m_pTelemetryLogger = std::make_unique(); - - // The above make_unique for TelemetryTraceLogger will either create an object or will throw which is caught below. - m_pTelemetryLogger->Initialize(); - }); - } - catch (...) - { - // May throw std::system_error if any condition prevents calls to call_once from executing as specified - // May throw std::bad_alloc or any exception thrown by the constructor of TelemetryTraceLogger - // Loggers are best effort and shouldn't block core functionality. So eat up the exceptions here - } - } - - ThreadGlobals* ThreadGlobals::GetForCurrentThread() - { - return SetOrGetThreadGlobals(false); - } - - ThreadGlobals* SetOrGetThreadGlobals(bool setThreadGlobals, ThreadGlobals* pThreadGlobals) - { - thread_local AppInstaller::ThreadLocalStorage::ThreadGlobals* t_pThreadGlobals = nullptr; - - if (setThreadGlobals == true) - { - AppInstaller::ThreadLocalStorage::ThreadGlobals* previous_pThreadGlobals = t_pThreadGlobals; - t_pThreadGlobals = pThreadGlobals; - return previous_pThreadGlobals; - } - - return t_pThreadGlobals; - } - - PreviousThreadGlobals::~PreviousThreadGlobals() - { - std::ignore = SetOrGetThreadGlobals(true, m_previous); - } -} + TelemetryTraceLogger& ThreadGlobals::GetTelemetryLogger() + { + return *(m_pTelemetryLogger); + } + + std::unique_ptr ThreadGlobals::SetForCurrentThread() + { + Initialize(); + + std::unique_ptr p_prevThreadGlobals = std::make_unique(SetOrGetThreadGlobals(true, this)); + + return p_prevThreadGlobals; + } + + void ThreadGlobals::Initialize() + { + try + { + std::call_once(m_loggerInitOnceFlag, [this]() + { + m_pDiagnosticLogger = std::make_unique(); + m_pTelemetryLogger = std::make_unique(); + + // The above make_unique for TelemetryTraceLogger will either create an object or will throw which is caught below. + m_pTelemetryLogger->Initialize(); + }); + } + catch (...) + { + // May throw std::system_error if any condition prevents calls to call_once from executing as specified + // May throw std::bad_alloc or any exception thrown by the constructor of TelemetryTraceLogger + // Loggers are best effort and shouldn't block core functionality. So eat up the exceptions here + } + } + + ThreadGlobals* ThreadGlobals::GetForCurrentThread() + { + return SetOrGetThreadGlobals(false); + } + + ThreadGlobals* SetOrGetThreadGlobals(bool setThreadGlobals, ThreadGlobals* pThreadGlobals) + { + thread_local AppInstaller::ThreadLocalStorage::ThreadGlobals* t_pThreadGlobals = nullptr; + + if (setThreadGlobals == true) + { + AppInstaller::ThreadLocalStorage::ThreadGlobals* previous_pThreadGlobals = t_pThreadGlobals; + t_pThreadGlobals = pThreadGlobals; + return previous_pThreadGlobals; + } + + return t_pThreadGlobals; + } + + PreviousThreadGlobals::~PreviousThreadGlobals() + { + std::ignore = SetOrGetThreadGlobals(true, m_previous); + } +} diff --git a/src/WinGetUtil/Exports.cpp b/src/WinGetUtil/Exports.cpp index a3e30ac472..b81e3a0587 100644 --- a/src/WinGetUtil/Exports.cpp +++ b/src/WinGetUtil/Exports.cpp @@ -12,12 +12,21 @@ #include #include #include +#include using namespace AppInstaller::Utility; using namespace AppInstaller::Manifest; using namespace AppInstaller::Repository; using namespace AppInstaller::Repository::Microsoft; +namespace +{ + std::filesystem::path GetPathOrEmpty(WINGET_STRING potentiallyNullPath) + { + return potentiallyNullPath ? std::filesystem::path{ potentiallyNullPath } : std::filesystem::path{}; + } +} + extern "C" { WINGET_UTIL_API WinGetLoggingInit(WINGET_STRING logPath) try @@ -143,7 +152,8 @@ extern "C" WINGET_UTIL_API WinGetSQLiteIndexRemoveManifest( WINGET_SQLITE_INDEX_HANDLE index, - WINGET_STRING manifestPath, WINGET_STRING relativePath) try + WINGET_STRING manifestPath, + WINGET_STRING relativePath) try { THROW_HR_IF(E_INVALIDARG, !index); THROW_HR_IF(E_INVALIDARG, !manifestPath); @@ -335,4 +345,62 @@ extern "C" return S_OK; } CATCH_RETURN() + + WINGET_UTIL_API WinGetBeginInstallerMetadataCollection( + WINGET_STRING inputJSON, + WINGET_STRING logFilePath, + WinGetBeginInstallerMetadataCollectionOptions options, + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE* collectionHandle) try + { + THROW_HR_IF(E_INVALIDARG, !inputJSON); + THROW_HR_IF(E_INVALIDARG, !collectionHandle); + THROW_HR_IF(E_INVALIDARG, !!*collectionHandle); + // Flags specifying what inputJSON means are mutually exclusive + THROW_HR_IF(E_INVALIDARG, !WI_IsClearOrSingleFlagSetInMask(options, + WinGetBeginInstallerMetadataCollectionOption_InputIsFilePath | WinGetBeginInstallerMetadataCollectionOption_InputIsURI)); + + std::unique_ptr result; + + if (WI_IsFlagSet(options, WinGetBeginInstallerMetadataCollectionOption_InputIsFilePath)) + { + result = InstallerMetadataCollectionContext::FromFile(inputJSON, GetPathOrEmpty(logFilePath)); + } + else if (WI_IsFlagSet(options, WinGetBeginInstallerMetadataCollectionOption_InputIsURI)) + { + result = InstallerMetadataCollectionContext::FromURI(inputJSON, GetPathOrEmpty(logFilePath)); + } + else + { + result = InstallerMetadataCollectionContext::FromJSON(inputJSON, GetPathOrEmpty(logFilePath)); + } + + *collectionHandle = static_cast(result.release()); + + return S_OK; + } + CATCH_RETURN() + + WINGET_UTIL_API WinGetCompleteInstallerMetadataCollection( + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE collectionHandle, + WINGET_STRING outputFilePath, + WINGET_STRING diagnosticsFilePath, + WinGetCompleteInstallerMetadataCollectionOptions options) try + { + THROW_HR_IF(E_INVALIDARG, !collectionHandle); + + // Since we always free the handle from calling this function, we can just store it in a unique_ptr from the start + std::unique_ptr context{ reinterpret_cast(collectionHandle) }; + + if (WI_IsFlagSet(options, WinGetCompleteInstallerMetadataCollectionOption_Abandon)) + { + return S_OK; + } + + THROW_HR_IF(E_INVALIDARG, !outputFilePath); + + context->Complete(outputFilePath, GetPathOrEmpty(diagnosticsFilePath)); + + return S_OK; + } + CATCH_RETURN() } diff --git a/src/WinGetUtil/WinGetUtil.h b/src/WinGetUtil/WinGetUtil.h index f5083d7cba..d5f800f986 100644 --- a/src/WinGetUtil/WinGetUtil.h +++ b/src/WinGetUtil/WinGetUtil.h @@ -129,4 +129,47 @@ extern "C" WINGET_STRING versionA, WINGET_STRING versionB, INT* comparisonResult); + + // A handle to the metadata collection object. + typedef void* WINGET_INSTALLER_METADATA_COLLECTION_HANDLE; + + // Option flags for WinGetBeginInstallerMetadataCollection. + enum WinGetBeginInstallerMetadataCollectionOptions + { + WinGetBeginInstallerMetadataCollectionOption_None = 0, + // The inputJSON is a local file path, not a JSON string. + WinGetBeginInstallerMetadataCollectionOption_InputIsFilePath = 0x1, + // The inputJSON is a remote URI, not a JSON string. + WinGetBeginInstallerMetadataCollectionOption_InputIsURI = 0x2, + }; + + DEFINE_ENUM_FLAG_OPERATORS(WinGetBeginInstallerMetadataCollectionOptions); + + // Begins the installer metadata collection process. + // By default, inputJSON is expected to be a JSON string. See the WinGetBeginInstallerMetadataCollectionOptions for more options. + // logFilePath optionally specifies where to write the log file for the collection operation. + // The collectionHandle is owned by the caller and must be passed to WinGetCompleteInstallerMetadataCollection to free it. + WINGET_UTIL_API WinGetBeginInstallerMetadataCollection( + WINGET_STRING inputJSON, + WINGET_STRING logFilePath, + WinGetBeginInstallerMetadataCollectionOptions options, + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE* collectionHandle); + + // Option flags for WinGetCompleteInstallerMetadataCollection. + enum WinGetCompleteInstallerMetadataCollectionOptions + { + WinGetCompleteInstallerMetadataCollectionOption_None = 0, + // Complete will simply free the collection handle without doing any additional work. + WinGetCompleteInstallerMetadataCollectionOption_Abandon = 0x1, + }; + + DEFINE_ENUM_FLAG_OPERATORS(WinGetCompleteInstallerMetadataCollectionOptions); + + // Completes the installer metadata collection process. + // Always frees the collectionHandle; WinGetCompleteInstallerMetadataCollection must be called exactly once for each call to WinGetBeginInstallerMetadataCollection. + WINGET_UTIL_API WinGetCompleteInstallerMetadataCollection( + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE collectionHandle, + WINGET_STRING outputFilePath, + WINGET_STRING diagnosticsFilePath, + WinGetCompleteInstallerMetadataCollectionOptions options); } From 409e0c001cf471735efe1490d33974987701fa1c Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Fri, 13 May 2022 16:53:30 -0700 Subject: [PATCH 02/22] Most of the work of moving the cpprestsdk JSON helpers into common --- .../AppInstallerCommonCore.vcxproj | 20 +-- .../AppInstallerCommonCore.vcxproj.filters | 6 +- .../InstallerMetadataCollectionContext.cpp | 20 +-- src/AppInstallerCommonCore/JsonUtil.cpp | 131 ++++++++++++++++- src/AppInstallerCommonCore/JsonUtil.h | 27 ---- .../InstallerMetadataCollectionContext.h | 4 +- .../Public/winget/JsonSchemaValidation.h | 2 +- .../Public/winget/JsonUtil.h | 52 +++++++ .../Public/winget/ManifestSchemaValidation.h | 2 +- src/AppInstallerCommonCore/UserSettings.cpp | 3 +- src/AppInstallerCommonCore/pch.h | 4 +- .../AppInstallerRepositoryCore.vcxproj | 18 ++- ...AppInstallerRepositoryCore.vcxproj.filters | 6 - .../ConfigurableTestSourceFactory.cpp | 2 +- .../Rest/RestClient.cpp | 6 +- .../Schema/1_0/Json/ManifestDeserializer.h | 2 +- .../Rest/Schema/1_0/RestInterface_1_0.cpp | 16 +-- .../Rest/Schema/JsonHelper.cpp | 133 ------------------ .../Rest/Schema/JsonHelper.h | 34 ----- src/AppInstallerRepositoryCore/pch.h | 2 + 20 files changed, 236 insertions(+), 254 deletions(-) delete mode 100644 src/AppInstallerCommonCore/JsonUtil.h create mode 100644 src/AppInstallerCommonCore/Public/winget/JsonUtil.h delete mode 100644 src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.cpp delete mode 100644 src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.h diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj index 4c73f77c5f..fafc9f07cc 100644 --- a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj +++ b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj @@ -177,9 +177,9 @@ Disabled _DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) true true true @@ -200,7 +200,7 @@ WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) true true true @@ -215,10 +215,10 @@ true true NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) true true true @@ -248,7 +248,7 @@ true true NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD;WINGET_DISABLE_FOR_FUZZING - $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) true stdcpp17 MultiThreaded @@ -285,7 +285,6 @@ - @@ -307,6 +306,7 @@ + diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters index 389451e789..5eac1e5b5c 100644 --- a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters +++ b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters @@ -111,9 +111,6 @@ Public\winget - - Header Files - Public\winget @@ -198,6 +195,9 @@ Public\winget + + Public\winget + diff --git a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp index fc45c7ca84..5f6303acad 100644 --- a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp @@ -15,12 +15,12 @@ namespace AppInstaller::Utility THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !std::filesystem::exists(file)); std::unique_ptr result = std::make_unique(); - auto threadGlobalsLifetime = result->Initialize(logFile); + auto threadGlobalsLifetime = result->InitializeLogging(logFile); AICLI_LOG(Core, Info, << "Opening InstallerMetadataCollectionContext input file: " << file); std::ifstream fileStream{ file }; - result->CollectPreInstallState(ConvertToUTF16(ReadEntireStream(fileStream))); + result->InitializePreinstallState(ConvertToUTF16(ReadEntireStream(fileStream))); return result; } @@ -30,7 +30,7 @@ namespace AppInstaller::Utility THROW_HR_IF(E_INVALIDARG, uri.empty()); std::unique_ptr result = std::make_unique(); - auto threadGlobalsLifetime = result->Initialize(logFile); + auto threadGlobalsLifetime = result->InitializeLogging(logFile); std::string utf8Uri = ConvertToUTF8(uri); THROW_HR_IF(E_INVALIDARG, !IsUrlRemote(utf8Uri)); @@ -69,7 +69,7 @@ namespace AppInstaller::Utility } } - result->CollectPreInstallState(ConvertToUTF16(jsonStream.str())); + result->InitializePreinstallState(ConvertToUTF16(jsonStream.str())); return result; } @@ -79,13 +79,13 @@ namespace AppInstaller::Utility THROW_HR_IF(E_INVALIDARG, json.empty()); std::unique_ptr result = std::make_unique(); - auto threadGlobalsLifetime = result->Initialize(logFile); - result->CollectPreInstallState(std::wstring{ json }); + auto threadGlobalsLifetime = result->InitializeLogging(logFile); + result->InitializePreinstallState(std::wstring{ json }); return result; } - std::unique_ptr InstallerMetadataCollectionContext::Initialize(const std::filesystem::path& logFile) + std::unique_ptr InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile) { auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); @@ -103,10 +103,12 @@ namespace AppInstaller::Utility Logging::Telemetry().LogStartup(); } - void InstallerMetadataCollectionContext::CollectPreInstallState(const std::wstring& json) + void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json) { // Parse and validate JSON - web::json::value::parse(json); + web::json::value inputValue = web::json::value::parse(json); + + //if (inputValue.has_field) // Collect pre-install system state } diff --git a/src/AppInstallerCommonCore/JsonUtil.cpp b/src/AppInstallerCommonCore/JsonUtil.cpp index c4c3090a25..9a34be66ae 100644 --- a/src/AppInstallerCommonCore/JsonUtil.cpp +++ b/src/AppInstallerCommonCore/JsonUtil.cpp @@ -1,10 +1,10 @@ // Copyright(c) Microsoft Corporation. // Licensed under the MIT License. #include "pch.h" -#include "JsonUtil.h" +#include "winget/JsonUtil.h" #include "AppInstallerStrings.h" -namespace AppInstaller::Utility +namespace AppInstaller::JSON { template<> std::optional GetValue(const Json::Value& node) @@ -67,5 +67,130 @@ namespace AppInstaller::Utility return std::nullopt; } -} + utility::string_t GetUtilityString(std::string_view nodeName) + { + return utility::conversions::to_string_t(nodeName.data()); + } + + std::optional> GetJsonValueFromNode(const web::json::value& node, const utility::string_t& keyName) + { + if (node.is_null() || !node.has_field(keyName)) + { + return {}; + } + + return node.at(keyName); + } + + std::optional GetRawStringValueFromJsonValue(const web::json::value& value) + { + if (value.is_null() || !value.is_string()) + { + return {}; + } + + return utility::conversions::to_utf8string(value.as_string()); + } + + std::optional GetRawStringValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) + { + std::optional> jsonValue = GetJsonValueFromNode(node, keyName); + + if (jsonValue) + { + return GetRawStringValueFromJsonValue(jsonValue.value().get()); + } + + return {}; + } + + std::optional GetRawIntValueFromJsonValue(const web::json::value& value) + { + if (value.is_null() || !value.is_integer()) + { + return {}; + } + + return value.as_integer(); + } + + std::optional GetRawIntValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) + { + std::optional> jsonValue = GetJsonValueFromNode(node, keyName); + + if (jsonValue) + { + return GetRawIntValueFromJsonValue(jsonValue.value().get()); + } + + return {}; + } + + std::optional GetRawBoolValueFromJsonValue(const web::json::value& value) + { + if (value.is_null() || !value.is_boolean()) + { + return {}; + } + + return value.as_bool(); + } + + std::optional GetRawBoolValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) + { + std::optional> jsonValue = GetJsonValueFromNode(node, keyName); + + if (jsonValue) + { + return GetRawBoolValueFromJsonValue(jsonValue.value().get()); + } + + return {}; + } + + std::optional> GetRawJsonArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName) + { + std::optional> jsonValue = GetJsonValueFromNode(node, keyName); + + if (!jsonValue || !jsonValue.value().get().is_array()) + { + return {}; + } + + return jsonValue.value().get().as_array(); + } + + std::vector GetRawStringArrayFromJsonNode( + const web::json::value& node, const utility::string_t& keyName) + { + std::optional> arrayValue = GetRawJsonArrayFromJsonNode(node, keyName); + + std::vector result; + if (!arrayValue) + { + return result; + } + + for (auto& value : arrayValue.value().get()) + { + std::optional item = GetRawStringValueFromJsonValue(value); + if (item) + { + result.emplace_back(std::move(item.value())); + } + } + + return result; + } + + bool IsValidNonEmptyStringValue(std::optional& value) + { + if (Utility::IsEmptyOrWhitespace(value.value_or(""))) + { + return false; + } + + return true; + } +} diff --git a/src/AppInstallerCommonCore/JsonUtil.h b/src/AppInstallerCommonCore/JsonUtil.h deleted file mode 100644 index 55431f8e0f..0000000000 --- a/src/AppInstallerCommonCore/JsonUtil.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#pragma once -#include - -#include -#include -#include - -namespace AppInstaller::Utility -{ - template - std::optional GetValue(const Json::Value& node); - - template<> - std::optional GetValue(const Json::Value& node); - - template<> - std::optional GetValue(const Json::Value& node); - - template<> - std::optional GetValue(const Json::Value& node); - - template<> - std::optional> GetValue>(const Json::Value& node); -} diff --git a/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h index abc66faa4b..4597be73dc 100644 --- a/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h @@ -32,10 +32,10 @@ namespace AppInstaller::Utility private: // Initializes the context runtime, including the log file if provided. - std::unique_ptr Initialize(const std::filesystem::path& logFile); + std::unique_ptr InitializeLogging(const std::filesystem::path& logFile); // Sets the collection context input and the preinstall state. - void CollectPreInstallState(const std::wstring& json); + void InitializePreinstallState(const std::wstring& json); ThreadLocalStorage::ThreadGlobals m_threadGlobals; }; diff --git a/src/AppInstallerCommonCore/Public/winget/JsonSchemaValidation.h b/src/AppInstallerCommonCore/Public/winget/JsonSchemaValidation.h index 1c921f8921..d7fa5ebf62 100644 --- a/src/AppInstallerCommonCore/Public/winget/JsonSchemaValidation.h +++ b/src/AppInstallerCommonCore/Public/winget/JsonSchemaValidation.h @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once -#include +#include namespace AppInstaller::JsonSchema { diff --git a/src/AppInstallerCommonCore/Public/winget/JsonUtil.h b/src/AppInstallerCommonCore/Public/winget/JsonUtil.h new file mode 100644 index 0000000000..ae3a4e9276 --- /dev/null +++ b/src/AppInstallerCommonCore/Public/winget/JsonUtil.h @@ -0,0 +1,52 @@ +#pragma once +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include +#include + +#include +#include +#include + +namespace AppInstaller::JSON +{ + // For JSON CPP LIb + template + std::optional GetValue(const Json::Value& node); + + template<> + std::optional GetValue(const Json::Value& node); + + template<> + std::optional GetValue(const Json::Value& node); + + template<> + std::optional GetValue(const Json::Value& node); + + template<> + std::optional> GetValue>(const Json::Value& node); + + // For cpprestsdk JSON + std::optional> GetJsonValueFromNode(const web::json::value& node, const utility::string_t& keyName); + + std::optional GetRawStringValueFromJsonValue(const web::json::value& value); + + std::optional GetRawStringValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + + std::optional GetRawBoolValueFromJsonValue(const web::json::value& value); + + std::optional GetRawBoolValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + + std::optional> GetRawJsonArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + + std::optional GetRawIntValueFromJsonValue(const web::json::value& value); + + std::optional GetRawIntValueFromJsonNode(const web::json::value& value, const utility::string_t& keyName); + + utility::string_t GetUtilityString(std::string_view nodeName); + + std::vector GetRawStringArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + + bool IsValidNonEmptyStringValue(std::optional& value); +} diff --git a/src/AppInstallerCommonCore/Public/winget/ManifestSchemaValidation.h b/src/AppInstallerCommonCore/Public/winget/ManifestSchemaValidation.h index 35488bae24..7e6d2a68c6 100644 --- a/src/AppInstallerCommonCore/Public/winget/ManifestSchemaValidation.h +++ b/src/AppInstallerCommonCore/Public/winget/ManifestSchemaValidation.h @@ -4,7 +4,7 @@ #include "ManifestCommon.h" #include "ManifestValidation.h" -#include +#include namespace AppInstaller::Manifest::YamlParser { diff --git a/src/AppInstallerCommonCore/UserSettings.cpp b/src/AppInstallerCommonCore/UserSettings.cpp index c8b50d3838..1f97df6e7d 100644 --- a/src/AppInstallerCommonCore/UserSettings.cpp +++ b/src/AppInstallerCommonCore/UserSettings.cpp @@ -4,7 +4,7 @@ #include "AppInstallerRuntime.h" #include "AppInstallerLanguageUtilities.h" #include "AppInstallerLogging.h" -#include "JsonUtil.h" +#include "winget/JsonUtil.h" #include "winget/Settings.h" #include "winget/UserSettings.h" @@ -17,6 +17,7 @@ namespace AppInstaller::Settings using namespace Runtime; using namespace Utility; using namespace Logging; + using namespace JSON; static constexpr std::string_view s_SettingEmpty = R"({ diff --git a/src/AppInstallerCommonCore/pch.h b/src/AppInstallerCommonCore/pch.h index 0ef4c263c1..927c69efdd 100644 --- a/src/AppInstallerCommonCore/pch.h +++ b/src/AppInstallerCommonCore/pch.h @@ -19,7 +19,9 @@ #define YAML_DECLARE_STATIC #include -#include +// TODO: See if we can get down to having just one JSON parser... +#include +#include #pragma warning( push ) #pragma warning ( disable : 4458 4100 4702 6031 ) diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj index 0f0df2c4b5..2a68e61c92 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -159,9 +159,9 @@ Disabled _NO_ASYNCRTIMP;_DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) true true true @@ -182,7 +182,7 @@ _NO_ASYNCRTIMP;WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) true true true @@ -197,10 +197,10 @@ true true _NO_ASYNCRTIMP;NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) true true true @@ -290,7 +290,6 @@ - @@ -354,7 +353,6 @@ - diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters index 78d9b98747..72d4e5feaf 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -201,9 +201,6 @@ Rest\Schema\1_0\Json - - Rest\Schema - Rest\Schema @@ -368,9 +365,6 @@ Rest\Schema\1_0\Json - - Rest\Schema - Microsoft\Schema\1_3 diff --git a/src/AppInstallerRepositoryCore/Microsoft/ConfigurableTestSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/ConfigurableTestSourceFactory.cpp index 9dff331335..a171515cd3 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/ConfigurableTestSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/ConfigurableTestSourceFactory.cpp @@ -3,7 +3,7 @@ #include "pch.h" #include "Microsoft/ConfigurableTestSourceFactory.h" -#include +#include using namespace std::string_literals; using namespace std::string_view_literals; diff --git a/src/AppInstallerRepositoryCore/Rest/RestClient.cpp b/src/AppInstallerRepositoryCore/Rest/RestClient.cpp index a019ef91dd..5edf2afc8c 100644 --- a/src/AppInstallerRepositoryCore/Rest/RestClient.cpp +++ b/src/AppInstallerRepositoryCore/Rest/RestClient.cpp @@ -5,8 +5,8 @@ #include "Rest/Schema/1_0/Interface.h" #include "Rest/Schema/1_1/Interface.h" #include "Rest/Schema/HttpClientHelper.h" +#include #include "Rest/Schema/InformationResponseDeserializer.h" -#include "Rest/Schema/JsonHelper.h" #include "Rest/Schema/CommonRestConstants.h" #include "Rest/Schema/RestHelper.h" @@ -34,7 +34,7 @@ namespace AppInstaller::Repository::Rest THROW_HR_IF(APPINSTALLER_CLI_ERROR_CUSTOMHEADER_EXCEEDS_MAXLENGTH, customHeader.value().size() > WindowsPackageManagerHeaderMaxLength); std::unordered_map headers; - headers.emplace(JsonHelper::GetUtilityString(WindowsPackageManagerHeader), JsonHelper::GetUtilityString(customHeader.value())); + headers.emplace(JSON::GetUtilityString(WindowsPackageManagerHeader), JSON::GetUtilityString(customHeader.value())); return headers; } } @@ -68,7 +68,7 @@ namespace AppInstaller::Repository::Rest const utility::string_t& restApi, const std::unordered_map& additionalHeaders, const HttpClientHelper& clientHelper) { // Call information endpoint - utility::string_t endpoint = RestHelper::AppendPathToUri(restApi, JsonHelper::GetUtilityString(InformationGetEndpoint)); + utility::string_t endpoint = RestHelper::AppendPathToUri(restApi, JSON::GetUtilityString(InformationGetEndpoint)); std::optional response = clientHelper.HandleGet(endpoint, additionalHeaders); THROW_HR_IF(APPINSTALLER_CLI_ERROR_UNSUPPORTED_RESTSOURCE, !response); diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h index e60081ae46..6cb8809b35 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h @@ -3,7 +3,7 @@ #pragma once #include #include -#include "Rest\Schema\JsonHelper.h" +#include namespace AppInstaller::Repository::Rest::Schema::V1_0::Json { diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/RestInterface_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/RestInterface_1_0.cpp index b3bb7b9e42..1a1a0d6637 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/RestInterface_1_0.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/RestInterface_1_0.cpp @@ -4,8 +4,8 @@ #include "Rest/Schema/1_0/Interface.h" #include "Rest/Schema/IRestClient.h" #include "Rest/Schema/HttpClientHelper.h" -#include "Rest/Schema/JsonHelper.h" -#include "winget/ManifestValidation.h" +#include +#include #include "Rest/Schema/RestHelper.h" #include "Rest/Schema/CommonRestConstants.h" #include "Rest/Schema/1_0/Json/ManifestDeserializer.h" @@ -25,16 +25,16 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0 utility::string_t GetSearchEndpoint(const std::string& restApiUri) { - return RestHelper::AppendPathToUri(JsonHelper::GetUtilityString(restApiUri), JsonHelper::GetUtilityString(ManifestSearchPostEndpoint)); + return RestHelper::AppendPathToUri(JSON::GetUtilityString(restApiUri), JSON::GetUtilityString(ManifestSearchPostEndpoint)); } utility::string_t GetManifestByVersionEndpoint( const std::string& restApiUri, const std::string& packageId, const std::map& queryParameters) { utility::string_t getManifestEndpoint = RestHelper::AppendPathToUri( - JsonHelper::GetUtilityString(restApiUri), JsonHelper::GetUtilityString(ManifestByVersionAndChannelGetEndpoint)); + JSON::GetUtilityString(restApiUri), JSON::GetUtilityString(ManifestByVersionAndChannelGetEndpoint)); - utility::string_t getManifestWithPackageIdPath = RestHelper::AppendPathToUri(getManifestEndpoint, JsonHelper::GetUtilityString(packageId)); + utility::string_t getManifestWithPackageIdPath = RestHelper::AppendPathToUri(getManifestEndpoint, JSON::GetUtilityString(packageId)); // Create the endpoint with query parameters return RestHelper::AppendQueryParamsToUri(getManifestWithPackageIdPath, queryParameters); @@ -43,10 +43,10 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0 Interface::Interface(const std::string& restApi, const HttpClientHelper& httpClientHelper) : m_restApiUri(restApi), m_httpClientHelper(httpClientHelper) { - THROW_HR_IF(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_URL, !RestHelper::IsValidUri(JsonHelper::GetUtilityString(restApi))); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_URL, !RestHelper::IsValidUri(JSON::GetUtilityString(restApi))); m_searchEndpoint = GetSearchEndpoint(m_restApiUri); - m_requiredRestApiHeaders.emplace(JsonHelper::GetUtilityString(ContractVersion), JsonHelper::GetUtilityString(Version_1_0_0.ToString())); + m_requiredRestApiHeaders.emplace(JSON::GetUtilityString(ContractVersion), JSON::GetUtilityString(Version_1_0_0.ToString())); } Utility::Version Interface::GetVersion() const @@ -80,7 +80,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0 if (!continuationToken.empty()) { AICLI_LOG(Repo, Verbose, << "Received continuation token. Retrieving more results."); - searchHeaders.insert_or_assign(JsonHelper::GetUtilityString(ContinuationToken), continuationToken); + searchHeaders.insert_or_assign(JSON::GetUtilityString(ContinuationToken), continuationToken); } std::optional jsonObject = m_httpClientHelper.HandlePost(m_searchEndpoint, GetValidatedSearchBody(request), searchHeaders); diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.cpp deleted file mode 100644 index 8fd4b38ab3..0000000000 --- a/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#include "pch.h" -#include "JsonHelper.h" - -namespace AppInstaller::Repository::Rest::Schema -{ - utility::string_t JsonHelper::GetUtilityString(std::string_view nodeName) - { - return utility::conversions::to_string_t(nodeName.data()); - } - - std::optional> JsonHelper::GetJsonValueFromNode(const web::json::value& node, const utility::string_t& keyName) - { - if (node.is_null() || !node.has_field(keyName)) - { - return {}; - } - - return node.at(keyName); - } - - std::optional JsonHelper::GetRawStringValueFromJsonValue(const web::json::value& value) - { - if (value.is_null() || !value.is_string()) - { - return {}; - } - - return utility::conversions::to_utf8string(value.as_string()); - } - - std::optional JsonHelper::GetRawStringValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) - { - std::optional> jsonValue = GetJsonValueFromNode(node, keyName); - - if (jsonValue) - { - return GetRawStringValueFromJsonValue(jsonValue.value().get()); - } - - return {}; - } - - std::optional JsonHelper::GetRawIntValueFromJsonValue(const web::json::value& value) - { - if (value.is_null() || !value.is_integer()) - { - return {}; - } - - return value.as_integer(); - } - - std::optional JsonHelper::GetRawIntValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) - { - std::optional> jsonValue = GetJsonValueFromNode(node, keyName); - - if (jsonValue) - { - return GetRawIntValueFromJsonValue(jsonValue.value().get()); - } - - return {}; - } - - std::optional JsonHelper::GetRawBoolValueFromJsonValue(const web::json::value& value) - { - if (value.is_null() || !value.is_boolean()) - { - return {}; - } - - return value.as_bool(); - } - - std::optional JsonHelper::GetRawBoolValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) - { - std::optional> jsonValue = GetJsonValueFromNode(node, keyName); - - if (jsonValue) - { - return GetRawBoolValueFromJsonValue(jsonValue.value().get()); - } - - return {}; - } - - std::optional> JsonHelper::GetRawJsonArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName) - { - std::optional> jsonValue = GetJsonValueFromNode(node, keyName); - - if (!jsonValue || !jsonValue.value().get().is_array()) - { - return {}; - } - - return jsonValue.value().get().as_array(); - } - - std::vector JsonHelper::GetRawStringArrayFromJsonNode( - const web::json::value& node, const utility::string_t& keyName) - { - std::optional> arrayValue = GetRawJsonArrayFromJsonNode(node, keyName); - - std::vector result; - if (!arrayValue) - { - return result; - } - - for (auto& value : arrayValue.value().get()) - { - std::optional item = JsonHelper::GetRawStringValueFromJsonValue(value); - if (item) - { - result.emplace_back(std::move(item.value())); - } - } - - return result; - } - - bool JsonHelper::IsValidNonEmptyStringValue(std::optional& value) - { - if (Utility::IsEmptyOrWhitespace(value.value_or(""))) - { - return false; - } - - return true; - } -} diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.h b/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.h deleted file mode 100644 index ca68c11ac1..0000000000 --- a/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#pragma once -#include -#include "winget/Manifest.h" - -namespace AppInstaller::Repository::Rest::Schema -{ - // Json helper. - struct JsonHelper - { - static std::optional> GetJsonValueFromNode(const web::json::value& node, const utility::string_t& keyName); - - static std::optional GetRawStringValueFromJsonValue(const web::json::value& value); - - static std::optional GetRawStringValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName); - - static std::optional GetRawBoolValueFromJsonValue(const web::json::value& value); - - static std::optional GetRawBoolValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName); - - static std::optional> GetRawJsonArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); - - static std::optional GetRawIntValueFromJsonValue(const web::json::value& value); - - static std::optional GetRawIntValueFromJsonNode(const web::json::value& value, const utility::string_t& keyName); - - static utility::string_t GetUtilityString(std::string_view nodeName); - - static std::vector GetRawStringArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); - - static bool IsValidNonEmptyStringValue(std::optional& value); - }; -} diff --git a/src/AppInstallerRepositoryCore/pch.h b/src/AppInstallerRepositoryCore/pch.h index cb652df5ea..d278411017 100644 --- a/src/AppInstallerRepositoryCore/pch.h +++ b/src/AppInstallerRepositoryCore/pch.h @@ -59,6 +59,8 @@ #include #include +#include + #pragma warning( push ) #pragma warning ( disable : 26495 26439 ) #include From 9141bcc9d4340f3d56acdbefc06b967817777953 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Wed, 18 May 2022 18:43:07 -0700 Subject: [PATCH 03/22] Moving of JSON helpers compiles --- src/AppInstallerCLICore/PackageCollection.h | 2 +- .../AppInstallerCLITests.vcxproj | 12 +-- src/AppInstallerCLITests/CustomHeader.cpp | 6 +- src/AppInstallerCLITests/JsonHelper.cpp | 40 +++---- .../AppInstallerCommonCore.vcxproj | 8 +- .../InstallerMetadataCollectionContext.cpp | 4 + .../Public/winget/JsonUtil.h | 5 + .../Schema/1_0/Json/ManifestDeserializer.h | 4 +- .../1_0/Json/ManifestDeserializer_1_0.cpp | 100 +++++++++--------- .../1_0/Json/SearchRequestSerializer_1_0.cpp | 20 ++-- .../Json/SearchResponseDeserializer_1_0.cpp | 24 ++--- .../1_1/Json/ManifestDeserializer_1_1.cpp | 54 +++++----- .../Rest/Schema/1_1/RestInterface_1_1.cpp | 12 +-- .../InformationResponseDeserializer.cpp | 38 +++---- .../Rest/Schema/RestHelper.cpp | 6 +- 15 files changed, 172 insertions(+), 163 deletions(-) diff --git a/src/AppInstallerCLICore/PackageCollection.h b/src/AppInstallerCLICore/PackageCollection.h index 7ae14d5f2b..32f4c91022 100644 --- a/src/AppInstallerCLICore/PackageCollection.h +++ b/src/AppInstallerCLICore/PackageCollection.h @@ -5,7 +5,7 @@ #include "winget/RepositorySource.h" #include -#include +#include #include diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj index 80b9f29c4c..86b27ac089 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -111,7 +111,7 @@ Disabled _NO_ASYNCRTIMP;_DEBUG;%(PreprocessorDefinitions) - $(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) true @@ -129,7 +129,7 @@ _NO_ASYNCRTIMP;WIN32;%(PreprocessorDefinitions) - $(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) true @@ -148,8 +148,8 @@ true true _NO_ASYNCRTIMP;NDEBUG;%(PreprocessorDefinitions) - $(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) - $(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) + $(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories) true true @@ -269,10 +269,10 @@ true - + true - + true diff --git a/src/AppInstallerCLITests/CustomHeader.cpp b/src/AppInstallerCLITests/CustomHeader.cpp index 16ce925f7e..bc644e3ff8 100644 --- a/src/AppInstallerCLITests/CustomHeader.cpp +++ b/src/AppInstallerCLITests/CustomHeader.cpp @@ -7,7 +7,7 @@ #include "TestSource.h" #include "TestRestRequestHandler.h" #include -#include +#include #include #include @@ -79,7 +79,7 @@ TEST_CASE("RestClient_CustomHeader", "[RestSource][CustomHeader]") }})delimiter"); std::optional customHeader = "Testing custom header"; - auto header = std::make_pair<>(CustomHeaderName, JsonHelper::GetUtilityString(customHeader.value())); + auto header = std::make_pair<>(CustomHeaderName, JSON::GetUtilityString(customHeader.value())); HttpClientHelper helper{ GetCustomHeaderVerificationHandler(web::http::status_codes::OK, sample, header) }; RestClient client = RestClient::Create(utility::conversions::to_utf8string("https://restsource.com/api"), customHeader, std::move(helper)); REQUIRE(client.GetSourceIdentifier() == "Source123"); @@ -127,7 +127,7 @@ TEST_CASE("RestSourceSearch_NoCustomHeader", "[RestSource][CustomHeader]") TEST_CASE("RestSourceSearch_CustomHeaderExceedingSize", "[RestSource][CustomHeader]") { std::string customHeader = "This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. "; - auto header = std::make_pair<>(CustomHeaderName, JsonHelper::GetUtilityString(customHeader)); + auto header = std::make_pair<>(CustomHeaderName, JSON::GetUtilityString(customHeader)); HttpClientHelper helper{ GetCustomHeaderVerificationHandler(web::http::status_codes::OK, sampleSearchResponse, header) }; REQUIRE_THROWS_HR(RestClient::Create(utility::conversions::to_utf8string("https://restsource.com/api"), customHeader, std::move(helper)), diff --git a/src/AppInstallerCLITests/JsonHelper.cpp b/src/AppInstallerCLITests/JsonHelper.cpp index c8db2a7d4a..ab5009205c 100644 --- a/src/AppInstallerCLITests/JsonHelper.cpp +++ b/src/AppInstallerCLITests/JsonHelper.cpp @@ -2,10 +2,10 @@ // Licensed under the MIT License. #include "pch.h" #include "TestCommon.h" -#include "Rest/Schema/JsonHelper.h" +#include #include "cpprest/json.h" -using namespace AppInstaller::Repository::Rest::Schema; +using namespace AppInstaller; web::json::value GetTestJsonObject() { @@ -25,41 +25,41 @@ web::json::value GetTestJsonObject() TEST_CASE("GetUtilityString", "[RestSource]") { - REQUIRE(JsonHelper::GetUtilityString("cpprest") == L"cpprest"); - REQUIRE(JsonHelper::GetUtilityString(" ") == L" "); + REQUIRE(JSON::GetUtilityString("cpprest") == L"cpprest"); + REQUIRE(JSON::GetUtilityString(" ") == L" "); } TEST_CASE("GetJsonValueFromNode", "[RestSource]") { web::json::value jsonObject = GetTestJsonObject(); - std::optional> actual = JsonHelper::GetJsonValueFromNode(jsonObject, L"Key1"); + std::optional> actual = JSON::GetJsonValueFromNode(jsonObject, L"Key1"); REQUIRE(actual); REQUIRE(actual.value().get().as_string() == L"Value1"); - std::optional> absentKey = JsonHelper::GetJsonValueFromNode(jsonObject, L"Key3"); + std::optional> absentKey = JSON::GetJsonValueFromNode(jsonObject, L"Key3"); REQUIRE(!absentKey); web::json::value emptyObject; - std::optional> empty = JsonHelper::GetJsonValueFromNode(emptyObject, L"Key1"); + std::optional> empty = JSON::GetJsonValueFromNode(emptyObject, L"Key1"); REQUIRE(!empty); } TEST_CASE("GetRawStringValueFromJsonValue", "[RestSource]") { - std::optional stringTest = JsonHelper::GetRawStringValueFromJsonValue(web::json::value::string(L"cpprest ")); + std::optional stringTest = JSON::GetRawStringValueFromJsonValue(web::json::value::string(L"cpprest ")); REQUIRE(stringTest); REQUIRE(stringTest.value() == "cpprest "); - std::optional emptyTest = JsonHelper::GetRawStringValueFromJsonValue(web::json::value::string(L" ")); + std::optional emptyTest = JSON::GetRawStringValueFromJsonValue(web::json::value::string(L" ")); REQUIRE(emptyTest); REQUIRE(emptyTest.value() == " "); web::json::value obj; - std::optional nullTest = JsonHelper::GetRawStringValueFromJsonValue(obj); + std::optional nullTest = JSON::GetRawStringValueFromJsonValue(obj); REQUIRE(!nullTest); web::json::value integer = 100; - std::optional mismatchFieldTest = JsonHelper::GetRawStringValueFromJsonValue(integer); + std::optional mismatchFieldTest = JSON::GetRawStringValueFromJsonValue(integer); REQUIRE(!mismatchFieldTest); } @@ -67,47 +67,47 @@ TEST_CASE("GetRawStringValueFromJsonNode", "[RestSource]") { web::json::value jsonObject = GetTestJsonObject(); - std::optional stringTest = JsonHelper::GetRawStringValueFromJsonNode(jsonObject, L"Key1"); + std::optional stringTest = JSON::GetRawStringValueFromJsonNode(jsonObject, L"Key1"); REQUIRE(stringTest); REQUIRE(stringTest.value() == "Value1"); - std::optional emptyTest = JsonHelper::GetRawStringValueFromJsonNode(jsonObject, L"Key3"); + std::optional emptyTest = JSON::GetRawStringValueFromJsonNode(jsonObject, L"Key3"); REQUIRE(!emptyTest); - std::optional mismatchFieldTest = JsonHelper::GetRawStringValueFromJsonNode(jsonObject, L"IntKey"); + std::optional mismatchFieldTest = JSON::GetRawStringValueFromJsonNode(jsonObject, L"IntKey"); REQUIRE(!mismatchFieldTest); } TEST_CASE("GetRawIntValueFromJsonValue", "[RestSource]") { web::json::value jsonObject = 100; - std::optional expected = JsonHelper::GetRawIntValueFromJsonValue(jsonObject); + std::optional expected = JSON::GetRawIntValueFromJsonValue(jsonObject); REQUIRE(expected); REQUIRE(expected.value() == 100); - std::optional mismatchFieldTest = JsonHelper::GetRawIntValueFromJsonValue(web::json::value::string(L"cpprest")); + std::optional mismatchFieldTest = JSON::GetRawIntValueFromJsonValue(web::json::value::string(L"cpprest")); REQUIRE(!mismatchFieldTest); } TEST_CASE("GetRawJsonArrayFromJsonNode", "[RestSource]") { web::json::value jsonObject = GetTestJsonObject(); - std::optional> expected = JsonHelper::GetRawJsonArrayFromJsonNode(jsonObject, L"Array"); + std::optional> expected = JSON::GetRawJsonArrayFromJsonNode(jsonObject, L"Array"); REQUIRE(expected); REQUIRE(expected.value().get().size() == 3); REQUIRE(expected.value().get().at(0).as_string() == L"ArrayValue1"); - std::optional> mismatchFieldTest = JsonHelper::GetRawJsonArrayFromJsonNode(jsonObject, L"Keyword"); + std::optional> mismatchFieldTest = JSON::GetRawJsonArrayFromJsonNode(jsonObject, L"Keyword"); REQUIRE(!mismatchFieldTest); } TEST_CASE("GetRawStringArrayFromJsonNode", "[RestSource]") { web::json::value jsonObject = GetTestJsonObject(); - std::vector expected = JsonHelper::GetRawStringArrayFromJsonNode(jsonObject, L"Array"); + std::vector expected = JSON::GetRawStringArrayFromJsonNode(jsonObject, L"Array"); REQUIRE(expected.size() == 3); REQUIRE(expected[0] == "ArrayValue1"); - std::vector mismatchFieldTest = JsonHelper::GetRawStringArrayFromJsonNode(jsonObject, L"Keyword"); + std::vector mismatchFieldTest = JSON::GetRawStringArrayFromJsonNode(jsonObject, L"Keyword"); REQUIRE(mismatchFieldTest.size() == 0); } diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj index fafc9f07cc..34fc0f43c5 100644 --- a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj +++ b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj @@ -176,7 +176,7 @@ Disabled - _DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD + _NO_ASYNCRTIMP;_DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) @@ -199,7 +199,7 @@ - WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD + _NO_ASYNCRTIMP;WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) true true @@ -214,7 +214,7 @@ MaxSpeed true true - NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD + _NO_ASYNCRTIMP;NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) @@ -247,7 +247,7 @@ MaxSpeed true true - NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD;WINGET_DISABLE_FOR_FUZZING + _NO_ASYNCRTIMP;NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD;WINGET_DISABLE_FOR_FUZZING $(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) true stdcpp17 diff --git a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp index 5f6303acad..15be04c304 100644 --- a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp @@ -101,6 +101,8 @@ namespace AppInstaller::Utility Logging::Telemetry().SetCaller("installer-metadata-collection"); Logging::Telemetry().LogStartup(); + + return threadGlobalsLifetime; } void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json) @@ -119,6 +121,8 @@ namespace AppInstaller::Utility THROW_HR_IF(E_INVALIDARG, output.empty()); + UNREFERENCED_PARAMETER(diagnostics); + // Collect post-install system state // Compute metadata match scores diff --git a/src/AppInstallerCommonCore/Public/winget/JsonUtil.h b/src/AppInstallerCommonCore/Public/winget/JsonUtil.h index ae3a4e9276..50b43a02c0 100644 --- a/src/AppInstallerCommonCore/Public/winget/JsonUtil.h +++ b/src/AppInstallerCommonCore/Public/winget/JsonUtil.h @@ -3,6 +3,11 @@ // Licensed under the MIT License. #pragma once #include + +// Disable dllimport for cpprest JSON in any downstream consumers +#ifndef _NO_ASYNCRTIMP +#define _NO_ASYNCRTIMP +#endif #include #include diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h index 6cb8809b35..f8462b63a2 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h @@ -18,9 +18,9 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json template inline void TryParseStringLocaleField(Manifest::ManifestLocalization& manifestLocale, const web::json::value& localeJsonObject, std::string_view localeJsonFieldName) const { - auto value = JsonHelper::GetRawStringValueFromJsonNode(localeJsonObject, JsonHelper::GetUtilityString(localeJsonFieldName)); + auto value = JSON::GetRawStringValueFromJsonNode(localeJsonObject, JSON::GetUtilityString(localeJsonFieldName)); - if (JsonHelper::IsValidNonEmptyStringValue(value)) + if (JSON::IsValidNonEmptyStringValue(value)) { manifestLocale.Add(value.value()); } diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp index b7da1cc0ec..4db88bd523 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp @@ -5,7 +5,7 @@ #include "Rest/Schema/IRestClient.h" #include "Rest/Schema/HttpClientHelper.h" #include "ManifestDeserializer.h" -#include "Rest/Schema/JsonHelper.h" +#include #include "Rest/Schema/CommonRestConstants.h" using namespace AppInstaller::Manifest; @@ -89,9 +89,9 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json const web::json::value& switchesJsonObject, std::string_view switchJsonFieldName) { - auto value = JsonHelper::GetRawStringValueFromJsonNode(switchesJsonObject, JsonHelper::GetUtilityString(switchJsonFieldName)); + auto value = JSON::GetRawStringValueFromJsonNode(switchesJsonObject, JSON::GetUtilityString(switchJsonFieldName)); - if (JsonHelper::IsValidNonEmptyStringValue(value)) + if (JSON::IsValidNonEmptyStringValue(value)) { installerSwitches[switchType] = value.value(); } @@ -120,7 +120,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json try { std::optional> manifestObject = - JsonHelper::GetJsonValueFromNode(dataJsonObject, JsonHelper::GetUtilityString(Data)); + JSON::GetJsonValueFromNode(dataJsonObject, JSON::GetUtilityString(Data)); if (!manifestObject || manifestObject.value().get().is_null()) { @@ -129,14 +129,14 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } auto& manifestJsonObject = manifestObject.value().get(); - std::optional id = JsonHelper::GetRawStringValueFromJsonNode(manifestJsonObject, JsonHelper::GetUtilityString(PackageIdentifier)); - if (!JsonHelper::IsValidNonEmptyStringValue(id)) + std::optional id = JSON::GetRawStringValueFromJsonNode(manifestJsonObject, JSON::GetUtilityString(PackageIdentifier)); + if (!JSON::IsValidNonEmptyStringValue(id)) { AICLI_LOG(Repo, Error, << "Missing package identifier."); return {}; } - std::optional> versions = JsonHelper::GetRawJsonArrayFromJsonNode(manifestJsonObject, JsonHelper::GetUtilityString(Versions)); + std::optional> versions = JSON::GetRawJsonArrayFromJsonNode(manifestJsonObject, JSON::GetUtilityString(Versions)); if (!versions || versions.value().get().size() == 0) { AICLI_LOG(Repo, Error, << "Missing versions in package: " << id.value()); @@ -149,19 +149,19 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json Manifest::Manifest manifest; manifest.Id = id.value(); - std::optional packageVersion = JsonHelper::GetRawStringValueFromJsonNode(versionItem, JsonHelper::GetUtilityString(PackageVersion)); - if (!JsonHelper::IsValidNonEmptyStringValue(packageVersion)) + std::optional packageVersion = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(PackageVersion)); + if (!JSON::IsValidNonEmptyStringValue(packageVersion)) { AICLI_LOG(Repo, Error, << "Missing package version in package: " << manifest.Id); return {}; } manifest.Version = std::move(packageVersion.value()); - manifest.Channel = JsonHelper::GetRawStringValueFromJsonNode(versionItem, JsonHelper::GetUtilityString(Channel)).value_or(""); + manifest.Channel = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(Channel)).value_or(""); // Default locale std::optional> defaultLocale = - JsonHelper::GetJsonValueFromNode(versionItem, JsonHelper::GetUtilityString(DefaultLocale)); + JSON::GetJsonValueFromNode(versionItem, JSON::GetUtilityString(DefaultLocale)); if (!defaultLocale) { AICLI_LOG(Repo, Error, << "Missing default locale in package: " << manifest.Id); @@ -187,11 +187,11 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json manifest.DefaultLocalization = std::move(defaultLocaleObject.value()); // Moniker is in Default locale - manifest.Moniker = JsonHelper::GetRawStringValueFromJsonNode(defaultLocale.value().get(), JsonHelper::GetUtilityString(Moniker)).value_or(""); + manifest.Moniker = JSON::GetRawStringValueFromJsonNode(defaultLocale.value().get(), JSON::GetUtilityString(Moniker)).value_or(""); } // Installers - std::optional> installers = JsonHelper::GetRawJsonArrayFromJsonNode(versionItem, JsonHelper::GetUtilityString(Installers)); + std::optional> installers = JSON::GetRawJsonArrayFromJsonNode(versionItem, JSON::GetUtilityString(Installers)); if (!installers || installers.value().get().size() == 0) { AICLI_LOG(Repo, Error, << "Missing installers in package: " << manifest.Id); @@ -214,7 +214,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } // Other locales - std::optional> locales = JsonHelper::GetRawJsonArrayFromJsonNode(versionItem, JsonHelper::GetUtilityString(Locales)); + std::optional> locales = JSON::GetRawJsonArrayFromJsonNode(versionItem, JSON::GetUtilityString(Locales)); if (locales) { for (auto& locale : locales.value().get()) @@ -252,8 +252,8 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } Manifest::ManifestLocalization locale; - std::optional packageLocale = JsonHelper::GetRawStringValueFromJsonNode(localeJsonObject, JsonHelper::GetUtilityString(PackageLocale)); - if (!JsonHelper::IsValidNonEmptyStringValue(packageLocale)) + std::optional packageLocale = JSON::GetRawStringValueFromJsonNode(localeJsonObject, JSON::GetUtilityString(PackageLocale)); + if (!JSON::IsValidNonEmptyStringValue(packageLocale)) { AICLI_LOG(Repo, Error, << "Missing package locale."); return {}; @@ -274,7 +274,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json TryParseStringLocaleField(locale, localeJsonObject, CopyrightUrl); TryParseStringLocaleField(locale, localeJsonObject, Description); - auto tags = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(localeJsonObject, JsonHelper::GetUtilityString(Tags))); + auto tags = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(localeJsonObject, JSON::GetUtilityString(Tags))); if (!tags.empty()) { locale.Add(tags); @@ -292,38 +292,38 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json Manifest::ManifestInstaller installer; - installer.Url = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerUrl)).value_or(""); + installer.Url = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerUrl)).value_or(""); - std::optional sha256 = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerSha256)); - if (JsonHelper::IsValidNonEmptyStringValue(sha256)) + std::optional sha256 = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerSha256)); + if (JSON::IsValidNonEmptyStringValue(sha256)) { installer.Sha256 = Utility::SHA256::ConvertToBytes(sha256.value()); } - std::optional arch = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Architecture)); - if (!JsonHelper::IsValidNonEmptyStringValue(arch)) + std::optional arch = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(Architecture)); + if (!JSON::IsValidNonEmptyStringValue(arch)) { AICLI_LOG(Repo, Error, << "Missing installer architecture."); return {}; } installer.Arch = Utility::ConvertToArchitectureEnum(arch.value()); - std::optional installerType = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerType)); - if (!JsonHelper::IsValidNonEmptyStringValue(installerType)) + std::optional installerType = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerType)); + if (!JSON::IsValidNonEmptyStringValue(installerType)) { AICLI_LOG(Repo, Error, << "Missing installer type."); return {}; } installer.InstallerType = ConvertToInstallerType(installerType.value()); - installer.Locale = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerLocale)).value_or(""); + installer.Locale = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerLocale)).value_or(""); // platform - std::optional> platforms = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Platform)); + std::optional> platforms = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(Platform)); if (platforms) { for (auto& platform : platforms.value().get()) { - std::optional platformValue = JsonHelper::GetRawStringValueFromJsonValue(platform); + std::optional platformValue = JSON::GetRawStringValueFromJsonValue(platform); if (platformValue) { installer.Platform.emplace_back(Manifest::ConvertToPlatformEnum(platformValue.value())); @@ -331,26 +331,26 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - installer.MinOSVersion = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(MinimumOSVersion)).value_or(""); - std::optional scope = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Scope)); + installer.MinOSVersion = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(MinimumOSVersion)).value_or(""); + std::optional scope = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(Scope)); if (scope) { installer.Scope = Manifest::ConvertToScopeEnum(scope.value()); } - std::optional signatureSha256 = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(SignatureSha256)); + std::optional signatureSha256 = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(SignatureSha256)); if (signatureSha256) { installer.SignatureSha256 = Utility::SHA256::ConvertToBytes(signatureSha256.value()); } // Install modes - std::optional> installModes = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallModes)); + std::optional> installModes = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallModes)); if (installModes) { for (auto& mode : installModes.value().get()) { - std::optional modeObject = JsonHelper::GetRawStringValueFromJsonValue(mode); + std::optional modeObject = JSON::GetRawStringValueFromJsonValue(mode); if (modeObject) { installer.InstallModes.emplace_back(Manifest::ConvertToInstallModeEnum(modeObject.value())); @@ -361,7 +361,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json // Installer Switches installer.Switches = Manifest::GetDefaultKnownSwitches(installer.InstallerType); std::optional> switches = - JsonHelper::GetJsonValueFromNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerSwitches)); + JSON::GetJsonValueFromNode(installerJsonObject, JSON::GetUtilityString(InstallerSwitches)); if (switches) { const auto& installerSwitches = switches.value().get(); @@ -375,12 +375,12 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } // Installer SuccessCodes - std::optional> installSuccessCodes = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerSuccessCodes)); + std::optional> installSuccessCodes = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerSuccessCodes)); if (installSuccessCodes) { for (auto& code : installSuccessCodes.value().get()) { - std::optional codeValue = JsonHelper::GetRawIntValueFromJsonValue(code); + std::optional codeValue = JSON::GetRawIntValueFromJsonValue(code); if (codeValue) { installer.InstallerSuccessCodes.emplace_back(std::move(codeValue.value())); @@ -388,19 +388,19 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - std::optional updateBehavior = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(UpgradeBehavior)); + std::optional updateBehavior = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(UpgradeBehavior)); if (updateBehavior) { installer.UpdateBehavior = Manifest::ConvertToUpdateBehaviorEnum(updateBehavior.value()); } - installer.Commands = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Commands))); - installer.Protocols = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Protocols))); - installer.FileExtensions = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(FileExtensions))); + installer.Commands = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(Commands))); + installer.Protocols = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(Protocols))); + installer.FileExtensions = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(FileExtensions))); // Dependencies std::optional> dependenciesObject = - JsonHelper::GetJsonValueFromNode(installerJsonObject, JsonHelper::GetUtilityString(Dependencies)); + JSON::GetJsonValueFromNode(installerJsonObject, JSON::GetUtilityString(Dependencies)); if (dependenciesObject) { std::optional dependencyList = DeserializeDependency(dependenciesObject.value().get()); @@ -410,10 +410,10 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - installer.PackageFamilyName = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(PackageFamilyName)).value_or(""); - installer.ProductCode = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(ProductCode)).value_or(""); - installer.Capabilities = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Capabilities))); - installer.RestrictedCapabilities = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(RestrictedCapabilities))); + installer.PackageFamilyName = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(PackageFamilyName)).value_or(""); + installer.ProductCode = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(ProductCode)).value_or(""); + installer.Capabilities = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(Capabilities))); + installer.RestrictedCapabilities = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(RestrictedCapabilities))); return installer; } @@ -427,34 +427,34 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json Manifest::DependencyList dependencyList; - auto wfIds = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(dependenciesObject, JsonHelper::GetUtilityString(WindowsFeatures))); + auto wfIds = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(dependenciesObject, JSON::GetUtilityString(WindowsFeatures))); for (auto&& id : wfIds) { dependencyList.Add(Dependency(DependencyType::WindowsFeature, std::move(id))); }; - const auto& wlIds = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(dependenciesObject, JsonHelper::GetUtilityString(WindowsLibraries))); + const auto& wlIds = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(dependenciesObject, JSON::GetUtilityString(WindowsLibraries))); for (auto id : wlIds) { dependencyList.Add(Dependency(DependencyType::WindowsLibrary, id)); }; - const auto& extIds = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(dependenciesObject, JsonHelper::GetUtilityString(ExternalDependencies))); + const auto& extIds = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(dependenciesObject, JSON::GetUtilityString(ExternalDependencies))); for (auto id : extIds) { dependencyList.Add(Dependency(DependencyType::External, id)); }; // Package Dependencies - std::optional> packageDependencies = JsonHelper::GetRawJsonArrayFromJsonNode(dependenciesObject, JsonHelper::GetUtilityString(PackageDependencies)); + std::optional> packageDependencies = JSON::GetRawJsonArrayFromJsonNode(dependenciesObject, JSON::GetUtilityString(PackageDependencies)); if (packageDependencies) { for (auto& packageDependency : packageDependencies.value().get()) { - std::optional id = JsonHelper::GetRawStringValueFromJsonNode(packageDependency, JsonHelper::GetUtilityString(PackageIdentifier)); + std::optional id = JSON::GetRawStringValueFromJsonNode(packageDependency, JSON::GetUtilityString(PackageIdentifier)); if (id) { - Dependency pkg{ DependencyType::Package, std::move(id.value()) , JsonHelper::GetRawStringValueFromJsonNode(packageDependency, JsonHelper::GetUtilityString(MinimumVersion)).value_or("") }; + Dependency pkg{ DependencyType::Package, std::move(id.value()) , JSON::GetRawStringValueFromJsonNode(packageDependency, JSON::GetUtilityString(MinimumVersion)).value_or("") }; dependencyList.Add(std::move(pkg)); } } diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchRequestSerializer_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchRequestSerializer_1_0.cpp index 7e0b2a1930..1e4e7aec85 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchRequestSerializer_1_0.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchRequestSerializer_1_0.cpp @@ -3,7 +3,7 @@ #include "pch.h" #include "Rest/Schema/IRestClient.h" #include "SearchRequestSerializer.h" -#include "Rest/Schema/JsonHelper.h" +#include #include "Rest/Schema/CommonRestConstants.h" namespace AppInstaller::Repository::Rest::Schema::V1_0::Json @@ -62,12 +62,12 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json web::json::value json_body; if (searchRequest.MaximumResults > 0) { - json_body[JsonHelper::GetUtilityString(MaximumResults)] = searchRequest.MaximumResults; + json_body[JSON::GetUtilityString(MaximumResults)] = searchRequest.MaximumResults; } if (searchRequest.IsForEverything()) { - json_body[JsonHelper::GetUtilityString(FetchAllManifests)] = web::json::value::boolean(true); + json_body[JSON::GetUtilityString(FetchAllManifests)] = web::json::value::boolean(true); return json_body; } @@ -78,7 +78,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json std::optional requestMatchJson = GetRequestMatchJsonObject(requestMatch); if (requestMatchJson) { - json_body[JsonHelper::GetUtilityString(Query)] = std::move(requestMatchJson.value()); + json_body[JSON::GetUtilityString(Query)] = std::move(requestMatchJson.value()); } } @@ -97,7 +97,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - json_body[JsonHelper::GetUtilityString(Filters)] = filters; + json_body[JSON::GetUtilityString(Filters)] = filters; } if (!searchRequest.Inclusions.empty()) @@ -115,7 +115,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - json_body[JsonHelper::GetUtilityString(Inclusions)] = inclusions; + json_body[JSON::GetUtilityString(Inclusions)] = inclusions; } return json_body; @@ -143,7 +143,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json return {}; } - filter[JsonHelper::GetUtilityString(PackageMatchField)] = web::json::value::string(JsonHelper::GetUtilityString(matchField.value())); + filter[JSON::GetUtilityString(PackageMatchField)] = web::json::value::string(JSON::GetUtilityString(matchField.value())); std::optional requestMatchJson = GetRequestMatchJsonObject(packageMatchFilter); if (!requestMatchJson) @@ -152,14 +152,14 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json return {}; } - filter[JsonHelper::GetUtilityString(RequestMatch)] = std::move(requestMatchJson.value()); + filter[JSON::GetUtilityString(RequestMatch)] = std::move(requestMatchJson.value()); return filter; } std::optional SearchRequestSerializer::GetRequestMatchJsonObject(const AppInstaller::Repository::RequestMatch& requestMatch) const { web::json::value match = web::json::value::object(); - match[JsonHelper::GetUtilityString(KeyWord)] = web::json::value::string(JsonHelper::GetUtilityString(requestMatch.Value)); + match[JSON::GetUtilityString(KeyWord)] = web::json::value::string(JSON::GetUtilityString(requestMatch.Value)); std::optional matchType = ConvertMatchTypeToString(requestMatch.Type); if (!matchType) @@ -168,7 +168,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json return {}; } - match[JsonHelper::GetUtilityString(MatchType)] = web::json::value::string(JsonHelper::GetUtilityString(matchType.value())); + match[JSON::GetUtilityString(MatchType)] = web::json::value::string(JSON::GetUtilityString(matchType.value())); return match; } diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchResponseDeserializer_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchResponseDeserializer_1_0.cpp index d70104b8de..c34a08c024 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchResponseDeserializer_1_0.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchResponseDeserializer_1_0.cpp @@ -3,7 +3,7 @@ #include "pch.h" #include "Rest/Schema/IRestClient.h" #include "SearchResponseDeserializer.h" -#include "Rest/Schema/JsonHelper.h" +#include #include "Rest/Schema/RestHelper.h" #include "Rest/Schema/CommonRestConstants.h" @@ -43,7 +43,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json IRestClient::SearchResult result; try { - std::optional> dataArray = JsonHelper::GetRawJsonArrayFromJsonNode(searchResponseObject, JsonHelper::GetUtilityString(Data)); + std::optional> dataArray = JSON::GetRawJsonArrayFromJsonNode(searchResponseObject, JSON::GetUtilityString(Data)); if (!dataArray || dataArray.value().get().size() == 0) { AICLI_LOG(Repo, Verbose, << "No search results returned."); @@ -52,33 +52,33 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json for (auto& manifestItem : dataArray.value().get()) { - std::optional packageId = JsonHelper::GetRawStringValueFromJsonNode(manifestItem, JsonHelper::GetUtilityString(PackageIdentifier)); - std::optional packageName = JsonHelper::GetRawStringValueFromJsonNode(manifestItem, JsonHelper::GetUtilityString(PackageName)); - std::optional publisher = JsonHelper::GetRawStringValueFromJsonNode(manifestItem, JsonHelper::GetUtilityString(Publisher)); + std::optional packageId = JSON::GetRawStringValueFromJsonNode(manifestItem, JSON::GetUtilityString(PackageIdentifier)); + std::optional packageName = JSON::GetRawStringValueFromJsonNode(manifestItem, JSON::GetUtilityString(PackageName)); + std::optional publisher = JSON::GetRawStringValueFromJsonNode(manifestItem, JSON::GetUtilityString(Publisher)); - if (!JsonHelper::IsValidNonEmptyStringValue(packageId) || !JsonHelper::IsValidNonEmptyStringValue(packageName) || !JsonHelper::IsValidNonEmptyStringValue(publisher)) + if (!JSON::IsValidNonEmptyStringValue(packageId) || !JSON::IsValidNonEmptyStringValue(packageName) || !JSON::IsValidNonEmptyStringValue(publisher)) { AICLI_LOG(Repo, Error, << "Missing required package fields in manifest search results."); return {}; } - std::optional> versionValue = JsonHelper::GetRawJsonArrayFromJsonNode(manifestItem, JsonHelper::GetUtilityString(Versions)); + std::optional> versionValue = JSON::GetRawJsonArrayFromJsonNode(manifestItem, JSON::GetUtilityString(Versions)); std::vector versionList; if (versionValue) { for (auto& versionItem : versionValue.value().get()) { - std::optional version = JsonHelper::GetRawStringValueFromJsonNode(versionItem, JsonHelper::GetUtilityString(PackageVersion)); - if (!JsonHelper::IsValidNonEmptyStringValue(version)) + std::optional version = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(PackageVersion)); + if (!JSON::IsValidNonEmptyStringValue(version)) { AICLI_LOG(Repo, Error, << "Received incomplete package version in package: " << packageId.value()); return {}; } - std::string channel = JsonHelper::GetRawStringValueFromJsonNode(versionItem, JsonHelper::GetUtilityString(Channel)).value_or(""); - std::vector packageFamilyNames = RestHelper::GetUniqueItems(JsonHelper::GetRawStringArrayFromJsonNode(versionItem, JsonHelper::GetUtilityString(PackageFamilyNames))); - std::vector productCodes = RestHelper::GetUniqueItems(JsonHelper::GetRawStringArrayFromJsonNode(versionItem, JsonHelper::GetUtilityString(ProductCodes))); + std::string channel = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(Channel)).value_or(""); + std::vector packageFamilyNames = RestHelper::GetUniqueItems(JSON::GetRawStringArrayFromJsonNode(versionItem, JSON::GetUtilityString(PackageFamilyNames))); + std::vector productCodes = RestHelper::GetUniqueItems(JSON::GetRawStringArrayFromJsonNode(versionItem, JSON::GetUtilityString(ProductCodes))); versionList.emplace_back(IRestClient::VersionInfo{ AppInstaller::Utility::VersionAndChannel{std::move(version.value()), std::move(channel)}, {}, std::move(packageFamilyNames), std::move(productCodes)}); diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp index 9b66883c07..29793ee741 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "ManifestDeserializer.h" -#include "Rest/Schema/JsonHelper.h" +#include using namespace AppInstaller::Manifest; @@ -61,22 +61,22 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json { auto& installer = result.value(); - installer.ProductId = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(MSStoreProductIdentifier)).value_or(""); - installer.ReleaseDate = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(ReleaseDate)).value_or(""); - installer.InstallerAbortsTerminal = JsonHelper::GetRawBoolValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerAbortsTerminal)).value_or(false); - installer.InstallLocationRequired = JsonHelper::GetRawBoolValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallLocationRequired)).value_or(false); - installer.RequireExplicitUpgrade = JsonHelper::GetRawBoolValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(RequireExplicitUpgrade)).value_or(false); + installer.ProductId = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(MSStoreProductIdentifier)).value_or(""); + installer.ReleaseDate = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(ReleaseDate)).value_or(""); + installer.InstallerAbortsTerminal = JSON::GetRawBoolValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerAbortsTerminal)).value_or(false); + installer.InstallLocationRequired = JSON::GetRawBoolValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallLocationRequired)).value_or(false); + installer.RequireExplicitUpgrade = JSON::GetRawBoolValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(RequireExplicitUpgrade)).value_or(false); installer.ElevationRequirement = Manifest::ConvertToElevationRequirementEnum( - JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(ElevationRequirement)).value_or("")); + JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(ElevationRequirement)).value_or("")); // list of unsupported OS architectures - std::optional> unsupportedOSArchitectures = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(UnsupportedOSArchitectures)); + std::optional> unsupportedOSArchitectures = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(UnsupportedOSArchitectures)); if (unsupportedOSArchitectures) { for (auto& archValue : unsupportedOSArchitectures.value().get()) { - std::optional arch = JsonHelper::GetRawStringValueFromJsonValue(archValue); - if (JsonHelper::IsValidNonEmptyStringValue(arch)) + std::optional arch = JSON::GetRawStringValueFromJsonValue(archValue); + if (JSON::IsValidNonEmptyStringValue(arch)) { auto archEnum = Utility::ConvertToArchitectureEnum(arch.value()); @@ -95,18 +95,18 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json } // Apps and Features Entries - std::optional> arpEntriesNode = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(AppsAndFeaturesEntries)); + std::optional> arpEntriesNode = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(AppsAndFeaturesEntries)); if (arpEntriesNode) { for (auto& arpEntryNode : arpEntriesNode.value().get()) { AppsAndFeaturesEntry arpEntry; - arpEntry.DisplayName = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(DisplayName)).value_or(""); - arpEntry.Publisher = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(Publisher)).value_or(""); - arpEntry.DisplayVersion = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(DisplayVersion)).value_or(""); - arpEntry.ProductCode = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(ProductCode)).value_or(""); - arpEntry.UpgradeCode = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(UpgradeCode)).value_or(""); - arpEntry.InstallerType = Manifest::ConvertToInstallerTypeEnum(JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(InstallerType)).value_or("")); + arpEntry.DisplayName = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(DisplayName)).value_or(""); + arpEntry.Publisher = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(Publisher)).value_or(""); + arpEntry.DisplayVersion = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(DisplayVersion)).value_or(""); + arpEntry.ProductCode = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(ProductCode)).value_or(""); + arpEntry.UpgradeCode = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(UpgradeCode)).value_or(""); + arpEntry.InstallerType = Manifest::ConvertToInstallerTypeEnum(JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(InstallerType)).value_or("")); // Only add when at least one field is valid if (!arpEntry.DisplayName.empty() || !arpEntry.Publisher.empty() || !arpEntry.DisplayVersion.empty() || @@ -118,23 +118,23 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json } // Markets - std::optional> marketsNode = JsonHelper::GetJsonValueFromNode(installerJsonObject, JsonHelper::GetUtilityString(Markets)); + std::optional> marketsNode = JSON::GetJsonValueFromNode(installerJsonObject, JSON::GetUtilityString(Markets)); if (marketsNode && !marketsNode.value().get().is_null()) { installer.Markets.ExcludedMarkets = V1_0::Json::ManifestDeserializer::ConvertToManifestStringArray( - JsonHelper::GetRawStringArrayFromJsonNode(marketsNode.value().get(), JsonHelper::GetUtilityString(ExcludedMarkets))); + JSON::GetRawStringArrayFromJsonNode(marketsNode.value().get(), JSON::GetUtilityString(ExcludedMarkets))); installer.Markets.AllowedMarkets = V1_0::Json::ManifestDeserializer::ConvertToManifestStringArray( - JsonHelper::GetRawStringArrayFromJsonNode(marketsNode.value().get(), JsonHelper::GetUtilityString(AllowedMarkets))); + JSON::GetRawStringArrayFromJsonNode(marketsNode.value().get(), JSON::GetUtilityString(AllowedMarkets))); } // Expected return codes - std::optional> expectedReturnCodesNode = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(ExpectedReturnCodes)); + std::optional> expectedReturnCodesNode = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(ExpectedReturnCodes)); if (expectedReturnCodesNode) { for (auto& returnCodeNode : expectedReturnCodesNode.value().get()) { - ExpectedReturnCodeEnum returnResponse = Manifest::ConvertToExpectedReturnCodeEnum(JsonHelper::GetRawStringValueFromJsonNode(returnCodeNode, JsonHelper::GetUtilityString(ReturnResponse)).value_or("")); - DWORD installerReturnCode = static_cast(JsonHelper::GetRawIntValueFromJsonNode(returnCodeNode, JsonHelper::GetUtilityString(InstallerReturnCode)).value_or(0)); + ExpectedReturnCodeEnum returnResponse = Manifest::ConvertToExpectedReturnCodeEnum(JSON::GetRawStringValueFromJsonNode(returnCodeNode, JSON::GetUtilityString(ReturnResponse)).value_or("")); + DWORD installerReturnCode = static_cast(JSON::GetRawIntValueFromJsonNode(returnCodeNode, JSON::GetUtilityString(InstallerReturnCode)).value_or(0)); // Only add when it is valid if (installerReturnCode != 0 && returnResponse != ExpectedReturnCodeEnum::Unknown) @@ -175,7 +175,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json TryParseStringLocaleField(locale, localeJsonObject, ReleaseNotesUrl); // Agreements - auto agreementsNode = JsonHelper::GetRawJsonArrayFromJsonNode(localeJsonObject, JsonHelper::GetUtilityString(Agreements)); + auto agreementsNode = JSON::GetRawJsonArrayFromJsonNode(localeJsonObject, JSON::GetUtilityString(Agreements)); if (agreementsNode) { std::vector agreements; @@ -183,9 +183,9 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json { Manifest::Agreement agreementEntry; - agreementEntry.Label = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(AgreementLabel)).value_or(""); - agreementEntry.AgreementText = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(Agreement)).value_or(""); - agreementEntry.AgreementUrl = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(AgreementUrl)).value_or(""); + agreementEntry.Label = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(AgreementLabel)).value_or(""); + agreementEntry.AgreementText = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(Agreement)).value_or(""); + agreementEntry.AgreementUrl = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(AgreementUrl)).value_or(""); if (!agreementEntry.Label.empty() || !agreementEntry.AgreementText.empty() || !agreementEntry.AgreementUrl.empty()) { diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/RestInterface_1_1.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/RestInterface_1_1.cpp index 494055110b..16e6d18062 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/RestInterface_1_1.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/RestInterface_1_1.cpp @@ -4,7 +4,7 @@ #include "Rest/Schema/1_1/Interface.h" #include "Rest/Schema/IRestClient.h" #include "Rest/Schema/HttpClientHelper.h" -#include "Rest/Schema/JsonHelper.h" +#include #include "Rest/Schema/RestHelper.h" #include "Rest/Schema/CommonRestConstants.h" #include "Rest/Schema/1_1/Json/ManifestDeserializer.h" @@ -33,7 +33,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1 const std::unordered_map& additionalHeaders, const HttpClientHelper& httpClientHelper) : V1_0::Interface(restApi, httpClientHelper), m_information(std::move(information)) { - m_requiredRestApiHeaders[JsonHelper::GetUtilityString(ContractVersion)] = JsonHelper::GetUtilityString(Version_1_1_0.ToString()); + m_requiredRestApiHeaders[JSON::GetUtilityString(ContractVersion)] = JSON::GetUtilityString(Version_1_1_0.ToString()); if (!additionalHeaders.empty()) { @@ -140,8 +140,8 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1 if (result.Matches.size() == 0) { - auto requiredPackageMatchFields = JsonHelper::GetRawStringArrayFromJsonNode(searchResponseObject, JsonHelper::GetUtilityString(RequiredPackageMatchFields)); - auto unsupportedPackageMatchFields = JsonHelper::GetRawStringArrayFromJsonNode(searchResponseObject, JsonHelper::GetUtilityString(UnsupportedPackageMatchFields)); + auto requiredPackageMatchFields = JSON::GetRawStringArrayFromJsonNode(searchResponseObject, JSON::GetUtilityString(RequiredPackageMatchFields)); + auto unsupportedPackageMatchFields = JSON::GetRawStringArrayFromJsonNode(searchResponseObject, JSON::GetUtilityString(UnsupportedPackageMatchFields)); if (requiredPackageMatchFields.size() != 0 || unsupportedPackageMatchFields.size() != 0) { @@ -160,8 +160,8 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1 if (result.size() == 0) { - auto requiredQueryParameters = JsonHelper::GetRawStringArrayFromJsonNode(manifestsResponseObject, JsonHelper::GetUtilityString(RequiredQueryParameters)); - auto unsupportedQueryParameters = JsonHelper::GetRawStringArrayFromJsonNode(manifestsResponseObject, JsonHelper::GetUtilityString(UnsupportedQueryParameters)); + auto requiredQueryParameters = JSON::GetRawStringArrayFromJsonNode(manifestsResponseObject, JSON::GetUtilityString(RequiredQueryParameters)); + auto unsupportedQueryParameters = JSON::GetRawStringArrayFromJsonNode(manifestsResponseObject, JSON::GetUtilityString(UnsupportedQueryParameters)); if (requiredQueryParameters.size() != 0 || unsupportedQueryParameters.size() != 0) { diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/InformationResponseDeserializer.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/InformationResponseDeserializer.cpp index 1f02e3d9c8..a6f35cf9a3 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/InformationResponseDeserializer.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/InformationResponseDeserializer.cpp @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "Rest/Schema/IRestClient.h" -#include "Rest/Schema/JsonHelper.h" +#include #include "Rest/Schema/CommonRestConstants.h" #include "InformationResponseDeserializer.h" @@ -47,7 +47,7 @@ namespace AppInstaller::Repository::Rest::Schema return {}; } - std::optional> data = JsonHelper::GetJsonValueFromNode(dataObject, JsonHelper::GetUtilityString(Data)); + std::optional> data = JSON::GetJsonValueFromNode(dataObject, JSON::GetUtilityString(Data)); if (!data) { AICLI_LOG(Repo, Error, << "Missing data"); @@ -55,14 +55,14 @@ namespace AppInstaller::Repository::Rest::Schema } const auto& dataValue = data.value().get(); - std::optional sourceId = JsonHelper::GetRawStringValueFromJsonNode(dataValue, JsonHelper::GetUtilityString(SourceIdentifier)); - if (!JsonHelper::IsValidNonEmptyStringValue(sourceId)) + std::optional sourceId = JSON::GetRawStringValueFromJsonNode(dataValue, JSON::GetUtilityString(SourceIdentifier)); + if (!JSON::IsValidNonEmptyStringValue(sourceId)) { AICLI_LOG(Repo, Error, << "Missing source identifier"); return {}; } - std::vector allVersions = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(ServerSupportedVersions)); + std::vector allVersions = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(ServerSupportedVersions)); if (allVersions.size() == 0) { AICLI_LOG(Repo, Error, << "Missing supported versions."); @@ -71,13 +71,13 @@ namespace AppInstaller::Repository::Rest::Schema IRestClient::Information info{ std::move(sourceId.value()), std::move(allVersions) }; - auto agreements = JsonHelper::GetJsonValueFromNode(dataValue, JsonHelper::GetUtilityString(SourceAgreements)); + auto agreements = JSON::GetJsonValueFromNode(dataValue, JSON::GetUtilityString(SourceAgreements)); if (agreements) { const auto& agreementsValue = agreements.value().get(); - auto agreementsIdentifier = JsonHelper::GetRawStringValueFromJsonNode(agreementsValue, JsonHelper::GetUtilityString(SourceAgreementsIdentifier)); - if (!JsonHelper::IsValidNonEmptyStringValue(agreementsIdentifier)) + auto agreementsIdentifier = JSON::GetRawStringValueFromJsonNode(agreementsValue, JSON::GetUtilityString(SourceAgreementsIdentifier)); + if (!JSON::IsValidNonEmptyStringValue(agreementsIdentifier)) { AICLI_LOG(Repo, Error, << "SourceAgreements node exists but AgreementsIdentifier is missing."); return {}; @@ -85,27 +85,27 @@ namespace AppInstaller::Repository::Rest::Schema info.SourceAgreementsIdentifier = std::move(agreementsIdentifier.value()); - auto agreementsContent = JsonHelper::GetRawJsonArrayFromJsonNode(agreementsValue, JsonHelper::GetUtilityString(SourceAgreementsContent)); + auto agreementsContent = JSON::GetRawJsonArrayFromJsonNode(agreementsValue, JSON::GetUtilityString(SourceAgreementsContent)); if (agreementsContent) { for (auto const& agreementNode : agreementsContent.value().get()) { IRestClient::SourceAgreementEntry agreementEntry; - std::optional label = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(SourceAgreementLabel)); - if (JsonHelper::IsValidNonEmptyStringValue(label)) + std::optional label = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(SourceAgreementLabel)); + if (JSON::IsValidNonEmptyStringValue(label)) { agreementEntry.Label = std::move(label.value()); } - std::optional text = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(SourceAgreementText)); - if (JsonHelper::IsValidNonEmptyStringValue(text)) + std::optional text = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(SourceAgreementText)); + if (JSON::IsValidNonEmptyStringValue(text)) { agreementEntry.Text = std::move(text.value()); } - std::optional url = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(SourceAgreementUrl)); - if (JsonHelper::IsValidNonEmptyStringValue(url)) + std::optional url = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(SourceAgreementUrl)); + if (JSON::IsValidNonEmptyStringValue(url)) { agreementEntry.Url = std::move(url.value()); } @@ -118,10 +118,10 @@ namespace AppInstaller::Repository::Rest::Schema } } - info.RequiredPackageMatchFields = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(RequiredPackageMatchFields)); - info.UnsupportedPackageMatchFields = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(UnsupportedPackageMatchFields)); - info.RequiredQueryParameters = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(RequiredQueryParameters)); - info.UnsupportedQueryParameters = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(UnsupportedQueryParameters)); + info.RequiredPackageMatchFields = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(RequiredPackageMatchFields)); + info.UnsupportedPackageMatchFields = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(UnsupportedPackageMatchFields)); + info.RequiredQueryParameters = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(RequiredQueryParameters)); + info.UnsupportedQueryParameters = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(UnsupportedQueryParameters)); return info; } diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/RestHelper.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/RestHelper.cpp index 327f35d4b8..f49caadcfe 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/RestHelper.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/RestHelper.cpp @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "RestHelper.h" -#include "Rest/Schema/JsonHelper.h" +#include #include "Rest/Schema/CommonRestConstants.h" using namespace AppInstaller::Repository::Rest::Schema; @@ -25,7 +25,7 @@ namespace AppInstaller::Repository::Rest::Schema } // Encode the Uri - return web::uri::encode_uri(JsonHelper::GetUtilityString(uri)); + return web::uri::encode_uri(JSON::GetUtilityString(uri)); } bool RestHelper::IsValidUri(const utility::string_t& restApiUri) @@ -62,7 +62,7 @@ namespace AppInstaller::Repository::Rest::Schema std::optional RestHelper::GetContinuationToken(const web::json::value& jsonObject) { - std::optional continuationToken = JsonHelper::GetRawStringValueFromJsonNode(jsonObject, JsonHelper::GetUtilityString(ContinuationToken)); + std::optional continuationToken = JSON::GetRawStringValueFromJsonNode(jsonObject, JSON::GetUtilityString(ContinuationToken)); if (continuationToken) { From ec7cb5ecb29ebdcc1fa2b8a326619deae12f9f5e Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Wed, 18 May 2022 21:35:27 -0700 Subject: [PATCH 04/22] Needs refactoring to expose JSON manifest parsing --- .../InstallerMetadataCollectionContext.cpp | 104 ++++++++++++++++-- .../InstallerMetadataCollectionContext.h | 21 +++- 2 files changed, 112 insertions(+), 13 deletions(-) diff --git a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp index 15be04c304..9f4f3127ce 100644 --- a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp @@ -4,6 +4,7 @@ #include "Public/winget/InstallerMetadataCollectionContext.h" #include "Public/AppInstallerDownloader.h" +#include "Public/AppInstallerErrors.h" #include "Public/AppInstallerLogging.h" #include "Public/AppInstallerStrings.h" @@ -85,6 +86,23 @@ namespace AppInstaller::Utility return result; } + void InstallerMetadataCollectionContext::Complete(const std::filesystem::path& output, const std::filesystem::path& diagnostics) + { + auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); + + THROW_HR_IF(E_INVALIDARG, output.empty()); + + UNREFERENCED_PARAMETER(diagnostics); + + // Collect post-install system state + + // Compute metadata match scores + + // Write output + + // Write diagnostics + } + std::unique_ptr InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile) { auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); @@ -107,28 +125,92 @@ namespace AppInstaller::Utility void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json) { + AICLI_LOG(Core, Verbose, << "Parsing input JSON:\n" << ConvertToUTF8(json)); + // Parse and validate JSON - web::json::value inputValue = web::json::value::parse(json); + bool jsonException = false; - //if (inputValue.has_field) + try + { + utility::string_t versionFieldName = L"version"; + + web::json::value inputValue = web::json::value::parse(json); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null()); + + auto versionString = JSON::GetRawStringValueFromJsonNode(inputValue, versionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); + + Version version{ versionString.value() }; + AICLI_LOG(Core, Info, << "Parsing input JSON version " << version.ToString()); + + if (version.GetParts()[0].Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + ParseInputJson_1_0(inputValue); + } + } + catch (const web::json::json_exception& exc) + { + AICLI_LOG(Core, Error, << "Exception parsing input JSON: " << exc.what()); + jsonException = true; + } + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, jsonException); // Collect pre-install system state } - void InstallerMetadataCollectionContext::Complete(const std::filesystem::path& output, const std::filesystem::path& diagnostics) + void InstallerMetadataCollectionContext::ParseInputJson_1_0(web::json::value& input) { - auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); - - THROW_HR_IF(E_INVALIDARG, output.empty()); + AICLI_LOG(Core, Info, << "Parsing input JSON 1.0 fields"); + + // Field names + utility::string_t blobVersionFieldName = L"supportedBlobVersion"; + utility::string_t blobSizeFieldName = L"maximumBlobSize"; + utility::string_t blobFieldName = L"currentBlob"; + utility::string_t productRevisionFieldName = L"productRevision"; + utility::string_t installerHashFieldName = L"installerHash"; + utility::string_t currentManifestFieldName = L"currentManifest"; + utility::string_t incomingManifestFieldName = L"incomingManifest"; + + auto blobVersionString = JSON::GetRawStringValueFromJsonNode(input, blobVersionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !blobVersionString); + m_supportedBlobVersion = Version{ blobVersionString.value() }; + + auto blobSizeNumber = JSON::GetRawIntValueFromJsonNode(input, blobSizeFieldName); + if (blobSizeNumber && blobSizeNumber.value() > 0) + { + m_maxBlobSize = static_cast(blobSizeNumber.value()); + } + else + { + m_maxBlobSize = std::numeric_limits::max(); + } - UNREFERENCED_PARAMETER(diagnostics); + auto currentBlobValue = JSON::GetJsonValueFromNode(input, blobFieldName); + if (currentBlobValue) + { + // TODO: Create and parse blob here + m_currentBlob = currentBlobValue.value(); + } - // Collect post-install system state + auto productRevisionNumber = JSON::GetRawIntValueFromJsonNode(input, productRevisionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !productRevisionNumber); + m_productRevision = productRevisionNumber.value(); - // Compute metadata match scores + auto installerHashString = JSON::GetRawStringValueFromJsonNode(input, installerHashFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !installerHashString); + m_installerHash = SHA256::ConvertToBytes(installerHashString.value()); - // Write output + auto currentManifestValue = JSON::GetJsonValueFromNode(input, currentManifestFieldName); + if (currentManifestValue) + { + //m_currentManifest = + } - // Write diagnostics + auto incomingManifestValue = JSON::GetJsonValueFromNode(input, incomingManifestFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !incomingManifestValue); + // m_incomingManifest = } } diff --git a/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h index 4597be73dc..1e1bb90a75 100644 --- a/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h @@ -2,13 +2,18 @@ // Licensed under the MIT License. #pragma once +#include +#include +#include +#include +#include + #include #include +#include #include #include -#include - namespace AppInstaller::Utility { // Contains the functions and data used for collecting metadata from installers. @@ -37,6 +42,18 @@ namespace AppInstaller::Utility // Sets the collection context input and the preinstall state. void InitializePreinstallState(const std::wstring& json); + // Parse version 1.0 of input JSON + void ParseInputJson_1_0(web::json::value& input); + ThreadLocalStorage::ThreadGlobals m_threadGlobals; + + // Parsed input + Version m_supportedBlobVersion; + size_t m_maxBlobSize = 0; + web::json::value m_currentBlob; // TODO: Parse blob as well + int m_productRevision = 0; + SHA256::HashBuffer m_installerHash; + Manifest::Manifest m_currentManifest; + Manifest::Manifest m_incomingManifest; }; } From 8baa2203ba1e488afe6882adfaa7b9ba870dd973 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 19 May 2022 18:13:14 -0700 Subject: [PATCH 05/22] Created public exposure of json parsing; need to move collection context to repository --- .../InstallerMetadataCollectionContext.cpp | 14 +- .../AppInstallerRepositoryCore.vcxproj | 2 + ...AppInstallerRepositoryCore.vcxproj.filters | 6 + .../ManifestJSONParser.cpp | 55 +++++ .../Public/winget/ManifestJSONParser.h | 38 ++++ .../Schema/1_0/Json/ManifestDeserializer.h | 13 +- .../1_0/Json/ManifestDeserializer_1_0.cpp | 209 +++++++++--------- src/WinGetUtil/WinGetUtil.vcxproj | 16 +- 8 files changed, 231 insertions(+), 122 deletions(-) create mode 100644 src/AppInstallerRepositoryCore/ManifestJSONParser.cpp create mode 100644 src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h diff --git a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp index 9f4f3127ce..6e9e8adf2b 100644 --- a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp @@ -8,6 +8,9 @@ #include "Public/AppInstallerLogging.h" #include "Public/AppInstallerStrings.h" +// TODO: Move to self to repository because we can't have a layering inversion +#include + namespace AppInstaller::Utility { std::unique_ptr InstallerMetadataCollectionContext::FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile) @@ -128,8 +131,6 @@ namespace AppInstaller::Utility AICLI_LOG(Core, Verbose, << "Parsing input JSON:\n" << ConvertToUTF8(json)); // Parse and validate JSON - bool jsonException = false; - try { utility::string_t versionFieldName = L"version"; @@ -149,15 +150,18 @@ namespace AppInstaller::Utility // We only have one version currently, so use that as long as the major version is 1 ParseInputJson_1_0(inputValue); } + else + { + AICLI_LOG(Core, Error, << "Don't know how to handle version " << version.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } } catch (const web::json::json_exception& exc) { AICLI_LOG(Core, Error, << "Exception parsing input JSON: " << exc.what()); - jsonException = true; + throw; } - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, jsonException); - // Collect pre-install system state } diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj index 2a68e61c92..5a51c040cf 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -273,6 +273,7 @@ + @@ -311,6 +312,7 @@ NotUsing NotUsing + diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters index 72d4e5feaf..6da60d13b9 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -264,6 +264,9 @@ Public\winget + + Public\winget + @@ -413,6 +416,9 @@ Source Files + + Source Files + diff --git a/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp b/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp new file mode 100644 index 0000000000..2067cc71dc --- /dev/null +++ b/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Public/winget/ManifestJSONParser.h" +#include "Rest/Schema/1_0/Json/ManifestDeserializer.h" +#include "Rest/Schema/1_1/Json/ManifestDeserializer.h" + +namespace AppInstaller::Repository::JSON +{ + struct ManifestJSONParser::impl + { + // The deserializer. We only have one lineage (1.0+) right now. + std::unique_ptr m_deserializer; + }; + + ManifestJSONParser::ManifestJSONParser(const Utility::Version& responseSchemaVersion) + { + const auto& parts = responseSchemaVersion.GetParts(); + THROW_HR_IF(E_INVALIDARG, parts.empty()); + + m_pImpl = std::make_unique(); + + if (parts[0].Integer == 1) + { + if (parts.size() == 1 || parts[1].Integer == 0) + { + m_pImpl->m_deserializer = std::make_unique(); + } + else + { + m_pImpl->m_deserializer = std::make_unique(); + } + } + else + { + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + } + + ManifestJSONParser::ManifestJSONParser(ManifestJSONParser&&) noexcept = default; + ManifestJSONParser& ManifestJSONParser::operator=(ManifestJSONParser&&) noexcept = default; + + ManifestJSONParser::~ManifestJSONParser() = default; + + std::vector ManifestJSONParser::Deserialize(const web::json::value& response) const + { + return m_pImpl->m_deserializer->Deserialize(response); + } + + std::vector ManifestJSONParser::ParseData(const web::json::value& data) const + { + return m_pImpl->m_deserializer->DeserializeData(data); + } + +} diff --git a/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h b/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h new file mode 100644 index 0000000000..bf6a1265d1 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include +#include +#include + +#include +#include + +namespace AppInstaller::Repository::JSON +{ + // Exposes functions for parsing JSON REST responses to manifest requests. + struct ManifestJSONParser + { + ManifestJSONParser(const Utility::Version& responseSchemaVersion); + + ManifestJSONParser(const ManifestJSONParser&) = delete; + ManifestJSONParser& operator=(const ManifestJSONParser&) = delete; + + ManifestJSONParser(ManifestJSONParser&&) noexcept; + ManifestJSONParser& operator=(ManifestJSONParser&&) noexcept; + + ~ManifestJSONParser(); + + // Parses the manifests from the REST response object root. + // May potentially contian multiple versions of the same package. + std::vector Deserialize(const web::json::value& response) const; + + // Parses the manifests from the Data field of the REST response object. + // May potentially contian multiple versions of the same package. + std::vector ParseData(const web::json::value& data) const; + + private: + struct impl; + std::unique_ptr m_pImpl; + }; +} diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h index f8462b63a2..f1c70f75f8 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h @@ -10,24 +10,25 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json // Manifest Deserializer. struct ManifestDeserializer { - // Gets the manifest from the given json object - std::vector Deserialize(const web::json::value& dataJsonObject) const; + // Gets the manifest from the given json object received from a REST request + std::vector Deserialize(const web::json::value& responseJsonObject) const; + + // Gets the manifest from the given json Data field + std::vector DeserializeData(const web::json::value& dataJsonObject) const; protected: template inline void TryParseStringLocaleField(Manifest::ManifestLocalization& manifestLocale, const web::json::value& localeJsonObject, std::string_view localeJsonFieldName) const { - auto value = JSON::GetRawStringValueFromJsonNode(localeJsonObject, JSON::GetUtilityString(localeJsonFieldName)); + auto value = AppInstaller::JSON::GetRawStringValueFromJsonNode(localeJsonObject, AppInstaller::JSON::GetUtilityString(localeJsonFieldName)); - if (JSON::IsValidNonEmptyStringValue(value)) + if (AppInstaller::JSON::IsValidNonEmptyStringValue(value)) { manifestLocale.Add(value.value()); } } - std::optional> DeserializeVersion(const web::json::value& dataJsonObject) const; - virtual std::optional DeserializeLocale(const web::json::value& localeJsonObject) const; virtual std::optional DeserializeInstaller(const web::json::value& installerJsonObject) const; diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp index 4db88bd523..2172fb5d23 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp @@ -98,150 +98,153 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - std::vector ManifestDeserializer::Deserialize(const web::json::value& dataJsonObject) const + std::vector ManifestDeserializer::Deserialize(const web::json::value& responseJsonObject) const { - // Get manifest from json output. - std::optional> manifests = DeserializeVersion(dataJsonObject); - - THROW_HR_IF(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA, !manifests); - - return manifests.value(); - } - - std::optional> ManifestDeserializer::DeserializeVersion(const web::json::value& dataJsonObject) const - { - if (dataJsonObject.is_null()) + if (responseJsonObject.is_null()) { AICLI_LOG(Repo, Error, << "Missing json object."); - return {}; + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } - std::vector manifests; try { std::optional> manifestObject = - JSON::GetJsonValueFromNode(dataJsonObject, JSON::GetUtilityString(Data)); + JSON::GetJsonValueFromNode(responseJsonObject, JSON::GetUtilityString(Data)); if (!manifestObject || manifestObject.value().get().is_null()) { AICLI_LOG(Repo, Verbose, << "No manifest results returned."); - return manifests; + return {}; } - auto& manifestJsonObject = manifestObject.value().get(); - std::optional id = JSON::GetRawStringValueFromJsonNode(manifestJsonObject, JSON::GetUtilityString(PackageIdentifier)); - if (!JSON::IsValidNonEmptyStringValue(id)) + return DeserializeData(manifestObject.value()); + } + catch (const wil::ResultException&) + { + throw; + } + catch (const std::exception& e) + { + AICLI_LOG(Repo, Error, << "Error encountered while deserializing manifest. Reason: " << e.what()); + } + catch (...) + { + AICLI_LOG(Repo, Error, << "Error encountered while deserializing manifest..."); + } + + // If we make it here, there was an exception above that we didn't throw. + // This will convert it into our standard error. + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } + + std::vector ManifestDeserializer::DeserializeData(const web::json::value& dataJsonObject) const + { + THROW_HR_IF(E_INVALIDARG, dataJsonObject.is_null()); + + std::vector manifests; + + std::optional id = JSON::GetRawStringValueFromJsonNode(dataJsonObject, JSON::GetUtilityString(PackageIdentifier)); + if (!JSON::IsValidNonEmptyStringValue(id)) + { + AICLI_LOG(Repo, Error, << "Missing package identifier."); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } + + std::optional> versions = JSON::GetRawJsonArrayFromJsonNode(dataJsonObject, JSON::GetUtilityString(Versions)); + if (!versions || versions.value().get().size() == 0) + { + AICLI_LOG(Repo, Error, << "Missing versions in package: " << id.value()); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } + + const web::json::array versionNodes = versions.value().get(); + for (auto& versionItem : versionNodes) + { + Manifest::Manifest manifest; + manifest.Id = id.value(); + + std::optional packageVersion = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(PackageVersion)); + if (!JSON::IsValidNonEmptyStringValue(packageVersion)) { - AICLI_LOG(Repo, Error, << "Missing package identifier."); - return {}; + AICLI_LOG(Repo, Error, << "Missing package version in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } + manifest.Version = std::move(packageVersion.value()); + + manifest.Channel = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(Channel)).value_or(""); - std::optional> versions = JSON::GetRawJsonArrayFromJsonNode(manifestJsonObject, JSON::GetUtilityString(Versions)); - if (!versions || versions.value().get().size() == 0) + // Default locale + std::optional> defaultLocale = + JSON::GetJsonValueFromNode(versionItem, JSON::GetUtilityString(DefaultLocale)); + if (!defaultLocale) { - AICLI_LOG(Repo, Error, << "Missing versions in package: " << id.value()); - return {}; + AICLI_LOG(Repo, Error, << "Missing default locale in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } - - const web::json::array versionNodes = versions.value().get(); - for (auto& versionItem : versionNodes) + else { - Manifest::Manifest manifest; - manifest.Id = id.value(); - - std::optional packageVersion = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(PackageVersion)); - if (!JSON::IsValidNonEmptyStringValue(packageVersion)) + std::optional defaultLocaleObject = DeserializeLocale(defaultLocale.value().get()); + if (!defaultLocaleObject) { - AICLI_LOG(Repo, Error, << "Missing package version in package: " << manifest.Id); - return {}; + AICLI_LOG(Repo, Error, << "Missing default locale in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } - manifest.Version = std::move(packageVersion.value()); - - manifest.Channel = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(Channel)).value_or(""); - // Default locale - std::optional> defaultLocale = - JSON::GetJsonValueFromNode(versionItem, JSON::GetUtilityString(DefaultLocale)); - if (!defaultLocale) + if (!defaultLocaleObject.value().Contains(Manifest::Localization::PackageName) || + !defaultLocaleObject.value().Contains(Manifest::Localization::Publisher) || + !defaultLocaleObject.value().Contains(Manifest::Localization::ShortDescription)) { - AICLI_LOG(Repo, Error, << "Missing default locale in package: " << manifest.Id); - return {}; + AICLI_LOG(Repo, Error, << "Missing PackageName, Publisher or ShortDescription in default locale: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } - else - { - std::optional defaultLocaleObject = DeserializeLocale(defaultLocale.value().get()); - if (!defaultLocaleObject) - { - AICLI_LOG(Repo, Error, << "Missing default locale in package: " << manifest.Id); - return {}; - } - if (!defaultLocaleObject.value().Contains(Manifest::Localization::PackageName) || - !defaultLocaleObject.value().Contains(Manifest::Localization::Publisher) || - !defaultLocaleObject.value().Contains(Manifest::Localization::ShortDescription)) - { - AICLI_LOG(Repo, Error, << "Missing PackageName, Publisher or ShortDescription in default locale: " << manifest.Id); - return {}; - } - - manifest.DefaultLocalization = std::move(defaultLocaleObject.value()); + manifest.DefaultLocalization = std::move(defaultLocaleObject.value()); - // Moniker is in Default locale - manifest.Moniker = JSON::GetRawStringValueFromJsonNode(defaultLocale.value().get(), JSON::GetUtilityString(Moniker)).value_or(""); - } + // Moniker is in Default locale + manifest.Moniker = JSON::GetRawStringValueFromJsonNode(defaultLocale.value().get(), JSON::GetUtilityString(Moniker)).value_or(""); + } - // Installers - std::optional> installers = JSON::GetRawJsonArrayFromJsonNode(versionItem, JSON::GetUtilityString(Installers)); - if (!installers || installers.value().get().size() == 0) - { - AICLI_LOG(Repo, Error, << "Missing installers in package: " << manifest.Id); - return {}; - } + // Installers + std::optional> installers = JSON::GetRawJsonArrayFromJsonNode(versionItem, JSON::GetUtilityString(Installers)); + if (!installers || installers.value().get().size() == 0) + { + AICLI_LOG(Repo, Error, << "Missing installers in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } - for (auto& installer : installers.value().get()) + for (auto& installer : installers.value().get()) + { + std::optional installerObject = DeserializeInstaller(installer); + if (installerObject) { - std::optional installerObject = DeserializeInstaller(installer); - if (installerObject) - { - manifest.Installers.emplace_back(std::move(installerObject.value())); - } + manifest.Installers.emplace_back(std::move(installerObject.value())); } + } - if (manifest.Installers.size() == 0) - { - AICLI_LOG(Repo, Error, << "Missing valid installers in package: " << manifest.Id); - return {}; - } + if (manifest.Installers.size() == 0) + { + AICLI_LOG(Repo, Error, << "Missing valid installers in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } - // Other locales - std::optional> locales = JSON::GetRawJsonArrayFromJsonNode(versionItem, JSON::GetUtilityString(Locales)); - if (locales) + // Other locales + std::optional> locales = JSON::GetRawJsonArrayFromJsonNode(versionItem, JSON::GetUtilityString(Locales)); + if (locales) + { + for (auto& locale : locales.value().get()) { - for (auto& locale : locales.value().get()) + std::optional localeObject = DeserializeLocale(locale); + if (localeObject) { - std::optional localeObject = DeserializeLocale(locale); - if (localeObject) - { - manifest.Localizations.emplace_back(std::move(localeObject.value())); - } + manifest.Localizations.emplace_back(std::move(localeObject.value())); } } - - manifests.emplace_back(std::move(manifest)); } - return manifests; - } - catch (const std::exception& e) - { - AICLI_LOG(Repo, Error, << "Error encountered while deserializing manifest. Reason: " << e.what()); - } - catch (...) - { - AICLI_LOG(Repo, Error, << "Error encountered while deserializing manifest..."); + manifests.emplace_back(std::move(manifest)); } - return {}; + return manifests; } std::optional ManifestDeserializer::DeserializeLocale(const web::json::value& localeJsonObject) const diff --git a/src/WinGetUtil/WinGetUtil.vcxproj b/src/WinGetUtil/WinGetUtil.vcxproj index d249fc0ed2..16147d5ada 100644 --- a/src/WinGetUtil/WinGetUtil.vcxproj +++ b/src/WinGetUtil/WinGetUtil.vcxproj @@ -162,9 +162,9 @@ Disabled _DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) true true true @@ -206,7 +206,7 @@ WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) true false true @@ -228,10 +228,10 @@ true true NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) true true true From 5b0dcf83144148d7b5376835682ae59bad70ff7b Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Fri, 20 May 2022 16:48:57 -0700 Subject: [PATCH 06/22] Move metadata collector to repository; rename blob; implement more parsing; needs AppsAndFeatures exposed --- .github/actions/spelling/expect.txt | 2 + .../AppInstallerCommonCore.vcxproj | 2 - .../AppInstallerCommonCore.vcxproj.filters | 6 - .../InstallerMetadataCollectionContext.cpp | 220 ----------- .../Public/AppInstallerVersions.h | 4 + src/AppInstallerCommonCore/Versions.cpp | 14 + .../AppInstallerRepositoryCore.vcxproj | 2 + ...AppInstallerRepositoryCore.vcxproj.filters | 6 + .../InstallerMetadataCollectionContext.cpp | 347 ++++++++++++++++++ .../InstallerMetadataCollectionContext.h | 54 ++- .../Public/winget/ManifestJSONParser.h | 4 +- .../1_0/Json/ManifestDeserializer_1_0.cpp | 3 +- src/WinGetUtil/Exports.cpp | 4 +- src/WinGetUtil/WinGetUtil.h | 1 - src/WinGetUtil/WinGetUtil.vcxproj | 16 +- 15 files changed, 434 insertions(+), 251 deletions(-) delete mode 100644 src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp create mode 100644 src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp rename src/{AppInstallerCommonCore => AppInstallerRepositoryCore}/Public/winget/InstallerMetadataCollectionContext.h (56%) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 37d743d84a..6a1fa254b2 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -31,6 +31,7 @@ ashpatil Ashwini asm ASwitch +ASYNCRTIMP Atest ATL AType @@ -99,6 +100,7 @@ deleteifnotneeded desktopappinstaller dirs diskfull +dllimport dnld Dobbeleer dsc diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj index 34fc0f43c5..ad009b8db1 100644 --- a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj +++ b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj @@ -304,7 +304,6 @@ - @@ -389,7 +388,6 @@ - diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters index 5eac1e5b5c..8635282639 100644 --- a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters +++ b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters @@ -192,9 +192,6 @@ Public\winget - - Public\winget - Public\winget @@ -341,9 +338,6 @@ Source Files - - Source Files - diff --git a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp deleted file mode 100644 index 6e9e8adf2b..0000000000 --- a/src/AppInstallerCommonCore/InstallerMetadataCollectionContext.cpp +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#include "pch.h" -#include "Public/winget/InstallerMetadataCollectionContext.h" - -#include "Public/AppInstallerDownloader.h" -#include "Public/AppInstallerErrors.h" -#include "Public/AppInstallerLogging.h" -#include "Public/AppInstallerStrings.h" - -// TODO: Move to self to repository because we can't have a layering inversion -#include - -namespace AppInstaller::Utility -{ - std::unique_ptr InstallerMetadataCollectionContext::FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile) - { - THROW_HR_IF(E_INVALIDARG, file.empty()); - THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !std::filesystem::exists(file)); - - std::unique_ptr result = std::make_unique(); - auto threadGlobalsLifetime = result->InitializeLogging(logFile); - - AICLI_LOG(Core, Info, << "Opening InstallerMetadataCollectionContext input file: " << file); - std::ifstream fileStream{ file }; - - result->InitializePreinstallState(ConvertToUTF16(ReadEntireStream(fileStream))); - - return result; - } - - std::unique_ptr InstallerMetadataCollectionContext::FromURI(std::wstring_view uri, const std::filesystem::path& logFile) - { - THROW_HR_IF(E_INVALIDARG, uri.empty()); - - std::unique_ptr result = std::make_unique(); - auto threadGlobalsLifetime = result->InitializeLogging(logFile); - - std::string utf8Uri = ConvertToUTF8(uri); - THROW_HR_IF(E_INVALIDARG, !IsUrlRemote(utf8Uri)); - - AICLI_LOG(Core, Info, << "Downloading InstallerMetadataCollectionContext input file: " << utf8Uri); - - std::ostringstream jsonStream; - ProgressCallback emptyCallback; - - const int MaxRetryCount = 2; - for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount) - { - bool success = false; - try - { - auto downloadHash = DownloadToStream(utf8Uri, jsonStream, DownloadType::InstallerMetadataCollectionInput, emptyCallback); - - success = true; - } - catch (...) - { - if (retryCount < MaxRetryCount - 1) - { - AICLI_LOG(Core, Info, << " Downloading InstallerMetadataCollectionContext input failed, waiting a bit and retrying..."); - Sleep(500); - } - else - { - throw; - } - } - - if (success) - { - break; - } - } - - result->InitializePreinstallState(ConvertToUTF16(jsonStream.str())); - - return result; - } - - std::unique_ptr InstallerMetadataCollectionContext::FromJSON(std::wstring_view json, const std::filesystem::path& logFile) - { - THROW_HR_IF(E_INVALIDARG, json.empty()); - - std::unique_ptr result = std::make_unique(); - auto threadGlobalsLifetime = result->InitializeLogging(logFile); - result->InitializePreinstallState(std::wstring{ json }); - - return result; - } - - void InstallerMetadataCollectionContext::Complete(const std::filesystem::path& output, const std::filesystem::path& diagnostics) - { - auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); - - THROW_HR_IF(E_INVALIDARG, output.empty()); - - UNREFERENCED_PARAMETER(diagnostics); - - // Collect post-install system state - - // Compute metadata match scores - - // Write output - - // Write diagnostics - } - - std::unique_ptr InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile) - { - auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); - - Logging::Log().SetLevel(Logging::Level::Info); - Logging::Log().EnableChannel(Logging::Channel::All); - Logging::EnableWilFailureTelemetry(); - Logging::AddTraceLogger(); - - if (!logFile.empty()) - { - Logging::AddFileLogger(logFile); - } - - Logging::Telemetry().SetCaller("installer-metadata-collection"); - Logging::Telemetry().LogStartup(); - - return threadGlobalsLifetime; - } - - void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json) - { - AICLI_LOG(Core, Verbose, << "Parsing input JSON:\n" << ConvertToUTF8(json)); - - // Parse and validate JSON - try - { - utility::string_t versionFieldName = L"version"; - - web::json::value inputValue = web::json::value::parse(json); - - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null()); - - auto versionString = JSON::GetRawStringValueFromJsonNode(inputValue, versionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); - - Version version{ versionString.value() }; - AICLI_LOG(Core, Info, << "Parsing input JSON version " << version.ToString()); - - if (version.GetParts()[0].Integer == 1) - { - // We only have one version currently, so use that as long as the major version is 1 - ParseInputJson_1_0(inputValue); - } - else - { - AICLI_LOG(Core, Error, << "Don't know how to handle version " << version.ToString()); - THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); - } - } - catch (const web::json::json_exception& exc) - { - AICLI_LOG(Core, Error, << "Exception parsing input JSON: " << exc.what()); - throw; - } - - // Collect pre-install system state - } - - void InstallerMetadataCollectionContext::ParseInputJson_1_0(web::json::value& input) - { - AICLI_LOG(Core, Info, << "Parsing input JSON 1.0 fields"); - - // Field names - utility::string_t blobVersionFieldName = L"supportedBlobVersion"; - utility::string_t blobSizeFieldName = L"maximumBlobSize"; - utility::string_t blobFieldName = L"currentBlob"; - utility::string_t productRevisionFieldName = L"productRevision"; - utility::string_t installerHashFieldName = L"installerHash"; - utility::string_t currentManifestFieldName = L"currentManifest"; - utility::string_t incomingManifestFieldName = L"incomingManifest"; - - auto blobVersionString = JSON::GetRawStringValueFromJsonNode(input, blobVersionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !blobVersionString); - m_supportedBlobVersion = Version{ blobVersionString.value() }; - - auto blobSizeNumber = JSON::GetRawIntValueFromJsonNode(input, blobSizeFieldName); - if (blobSizeNumber && blobSizeNumber.value() > 0) - { - m_maxBlobSize = static_cast(blobSizeNumber.value()); - } - else - { - m_maxBlobSize = std::numeric_limits::max(); - } - - auto currentBlobValue = JSON::GetJsonValueFromNode(input, blobFieldName); - if (currentBlobValue) - { - // TODO: Create and parse blob here - m_currentBlob = currentBlobValue.value(); - } - - auto productRevisionNumber = JSON::GetRawIntValueFromJsonNode(input, productRevisionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !productRevisionNumber); - m_productRevision = productRevisionNumber.value(); - - auto installerHashString = JSON::GetRawStringValueFromJsonNode(input, installerHashFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !installerHashString); - m_installerHash = SHA256::ConvertToBytes(installerHashString.value()); - - auto currentManifestValue = JSON::GetJsonValueFromNode(input, currentManifestFieldName); - if (currentManifestValue) - { - //m_currentManifest = - } - - auto incomingManifestValue = JSON::GetJsonValueFromNode(input, incomingManifestFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !incomingManifestValue); - // m_incomingManifest = - } -} diff --git a/src/AppInstallerCommonCore/Public/AppInstallerVersions.h b/src/AppInstallerCommonCore/Public/AppInstallerVersions.h index 2af5ff77ce..a8b34c0a67 100644 --- a/src/AppInstallerCommonCore/Public/AppInstallerVersions.h +++ b/src/AppInstallerCommonCore/Public/AppInstallerVersions.h @@ -61,6 +61,7 @@ namespace AppInstaller::Utility // An individual version part in between split characters. struct Part { + Part() = default; Part(const std::string& part); Part(uint64_t integer, std::string other); @@ -75,6 +76,9 @@ namespace AppInstaller::Utility // Gets the part breakdown for a given version; used for tests. const std::vector& GetParts() const { return m_parts; } + // Gets the part at the given index; or the implied zero part if past the end. + const Part& PartAt(size_t index) const; + protected: std::string m_version; std::vector m_parts; diff --git a/src/AppInstallerCommonCore/Versions.cpp b/src/AppInstallerCommonCore/Versions.cpp index 787daf2311..e588fb208f 100644 --- a/src/AppInstallerCommonCore/Versions.cpp +++ b/src/AppInstallerCommonCore/Versions.cpp @@ -164,6 +164,20 @@ namespace AppInstaller::Utility return result; } + const Version::Part& Version::PartAt(size_t index) const + { + static Part s_zero{}; + + if (index < m_parts.size()) + { + return m_parts[index]; + } + else + { + return s_zero; + } + } + Version::Part::Part(const std::string& part) { const char* begin = part.c_str(); diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj index 5a51c040cf..322a8a6b97 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -273,6 +273,7 @@ + @@ -312,6 +313,7 @@ NotUsing NotUsing + diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters index 6da60d13b9..5912f856c7 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -267,6 +267,9 @@ Public\winget + + Public\winget + @@ -419,6 +422,9 @@ Source Files + + Source Files + diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp new file mode 100644 index 0000000000..57a87bd33f --- /dev/null +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -0,0 +1,347 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "winget/InstallerMetadataCollectionContext.h" + +#include +#include +#include +#include + +#include + +using namespace AppInstaller::Utility; + +namespace AppInstaller::Repository::Metadata +{ + + void ProductMetadata::Clear() + { + m_version = {}; + m_productVersionMin = {}; + m_productVersionMax = {}; + m_installerMetadata.clear(); + m_historicalMetadata.clear(); + } + + void ProductMetadata::FromJson(const web::json::value& json) + { + Clear(); + + utility::string_t versionFieldName = L"version"; + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, json.is_null()); + + auto versionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, versionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); + + m_version = Version{ versionString.value() }; + AICLI_LOG(Core, Info, << "Parsing metadata JSON version " << m_version.ToString()); + + if (m_version.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + FromJson_1_0(json); + } + else + { + AICLI_LOG(Core, Error, << "Don't know how to handle metadata version " << m_version.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + } + + void ProductMetadata::FromJson_1_0(const web::json::value& json) + { + AICLI_LOG(Core, Info, << "Parsing metadata JSON 1.0 fields"); + + // Field names + utility::string_t productVersionMinFieldName = L"productVersionMin"; + utility::string_t productVersionMaxFieldName = L"productVersionMax"; + utility::string_t metadataFieldName = L"metadata"; + utility::string_t installerHashFieldName = L"installerHash"; + utility::string_t productRevisionFieldName = L"productRevision"; + utility::string_t versionFieldName = L"version"; + utility::string_t appsAndFeaturesFieldName = L"AppsAndFeaturesEntries"; + utility::string_t historicalFieldName = L"historical"; + + auto productVersionMinString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, productVersionMinFieldName); + if (productVersionMinString) + { + m_productVersionMin = Version{ productVersionMinString.value() }; + } + + auto productVersionMaxString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, productVersionMaxFieldName); + if (productVersionMaxString) + { + m_productVersionMax = Version{ productVersionMaxString.value() }; + } + + // The 1.0 version of metadata uses the 1.1 version of REST + JSON::ManifestJSONParser parser{ Version{ "1.1" } }; + + auto metadataArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, metadataFieldName); + if (metadataArray) + { + for (const auto& item : metadataArray->get()) + { + auto installerHashString = AppInstaller::JSON::GetRawStringValueFromJsonNode(item, installerHashFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !installerHashString); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, m_installerMetadata.find(installerHashString.value()) != m_installerMetadata.end()); + + InstallerMetadata installerMetadata; + + auto productRevisionNumber = AppInstaller::JSON::GetRawIntValueFromJsonNode(item, productRevisionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !productRevisionNumber); + installerMetadata.ProductRevision = productRevisionNumber.value(); + + auto versionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(item, versionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); + installerMetadata.ProductVersion = Version{ versionString.value() }; + + auto appsAndFeatures = AppInstaller::JSON::GetJsonValueFromNode(item, appsAndFeaturesFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures); + installerMetadata.AppsAndFeaturesEntries = parser.ParseAppsAndFeaturesEntries(appsAndFeatures.value()); + + m_installerMetadata[installerHashString.value()] = std::move(installerMetadata); + } + } + + auto historicalArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, historicalFieldName); + if (historicalArray) + { + for (const auto& item : historicalArray->get()) + { + HistoricalMetadata historicalMetadata; + + auto versionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(item, versionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); + historicalMetadata.ProductVersion = Version{ versionString.value() }; + + auto appsAndFeatures = AppInstaller::JSON::GetJsonValueFromNode(item, appsAndFeaturesFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures); + historicalMetadata.AppsAndFeaturesEntries = parser.ParseAppsAndFeaturesEntries(appsAndFeatures.value()); + + m_historicalMetadata.emplace_back(std::move(historicalMetadata)); + } + } + } + + std::unique_ptr InstallerMetadataCollectionContext::FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile) + { + THROW_HR_IF(E_INVALIDARG, file.empty()); + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !std::filesystem::exists(file)); + + std::unique_ptr result = std::make_unique(); + auto threadGlobalsLifetime = result->InitializeLogging(logFile); + + AICLI_LOG(Core, Info, << "Opening InstallerMetadataCollectionContext input file: " << file); + std::ifstream fileStream{ file }; + + result->InitializePreinstallState(ConvertToUTF16(ReadEntireStream(fileStream))); + + return result; + } + + std::unique_ptr InstallerMetadataCollectionContext::FromURI(std::wstring_view uri, const std::filesystem::path& logFile) + { + THROW_HR_IF(E_INVALIDARG, uri.empty()); + + std::unique_ptr result = std::make_unique(); + auto threadGlobalsLifetime = result->InitializeLogging(logFile); + + std::string utf8Uri = ConvertToUTF8(uri); + THROW_HR_IF(E_INVALIDARG, !IsUrlRemote(utf8Uri)); + + AICLI_LOG(Core, Info, << "Downloading InstallerMetadataCollectionContext input file: " << utf8Uri); + + std::ostringstream jsonStream; + ProgressCallback emptyCallback; + + const int MaxRetryCount = 2; + for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount) + { + bool success = false; + try + { + auto downloadHash = DownloadToStream(utf8Uri, jsonStream, DownloadType::InstallerMetadataCollectionInput, emptyCallback); + + success = true; + } + catch (...) + { + if (retryCount < MaxRetryCount - 1) + { + AICLI_LOG(Core, Info, << " Downloading InstallerMetadataCollectionContext input failed, waiting a bit and retrying..."); + Sleep(500); + } + else + { + throw; + } + } + + if (success) + { + break; + } + } + + result->InitializePreinstallState(ConvertToUTF16(jsonStream.str())); + + return result; + } + + std::unique_ptr InstallerMetadataCollectionContext::FromJSON(std::wstring_view json, const std::filesystem::path& logFile) + { + THROW_HR_IF(E_INVALIDARG, json.empty()); + + std::unique_ptr result = std::make_unique(); + auto threadGlobalsLifetime = result->InitializeLogging(logFile); + result->InitializePreinstallState(std::wstring{ json }); + + return result; + } + + void InstallerMetadataCollectionContext::Complete(const std::filesystem::path& output) + { + auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); + + THROW_HR_IF(E_INVALIDARG, output.empty()); + + // Collect post-install system state + + // Compute metadata match scores + + // Write output + + // Write diagnostics + } + + std::unique_ptr InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile) + { + auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); + + Logging::Log().SetLevel(Logging::Level::Info); + Logging::Log().EnableChannel(Logging::Channel::All); + Logging::EnableWilFailureTelemetry(); + Logging::AddTraceLogger(); + + if (!logFile.empty()) + { + Logging::AddFileLogger(logFile); + } + + Logging::Telemetry().SetCaller("installer-metadata-collection"); + Logging::Telemetry().LogStartup(); + + return threadGlobalsLifetime; + } + + void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json) + { + AICLI_LOG(Core, Verbose, << "Parsing input JSON:\n" << ConvertToUTF8(json)); + + // Parse and validate JSON + try + { + utility::string_t versionFieldName = L"version"; + + web::json::value inputValue = web::json::value::parse(json); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null()); + + auto versionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(inputValue, versionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); + + Version version{ versionString.value() }; + AICLI_LOG(Core, Info, << "Parsing input JSON version " << version.ToString()); + + if (version.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + ParseInputJson_1_0(inputValue); + } + else + { + AICLI_LOG(Core, Error, << "Don't know how to handle version " << version.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + } + catch (const web::json::json_exception& exc) + { + AICLI_LOG(Core, Error, << "Exception parsing input JSON: " << exc.what()); + throw; + } + + // Collect pre-install system state + } + + void InstallerMetadataCollectionContext::ParseInputJson_1_0(web::json::value& input) + { + AICLI_LOG(Core, Info, << "Parsing input JSON 1.0 fields"); + + // Field names + utility::string_t metadataVersionFieldName = L"supportedMetadataVersion"; + utility::string_t metadataSizeFieldName = L"maximumMetadataSize"; + utility::string_t metadataFieldName = L"currentMetadata"; + utility::string_t productRevisionFieldName = L"productRevision"; + utility::string_t installerHashFieldName = L"installerHash"; + utility::string_t currentManifestFieldName = L"currentManifest"; + utility::string_t incomingManifestFieldName = L"incomingManifest"; + + auto metadataVersionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(input, metadataVersionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !metadataVersionString); + m_supportedMetadataVersion = Version{ metadataVersionString.value() }; + + auto metadataSizeNumber = AppInstaller::JSON::GetRawIntValueFromJsonNode(input, metadataSizeFieldName); + if (metadataSizeNumber && metadataSizeNumber.value() > 0) + { + m_maxMetadataSize = static_cast(metadataSizeNumber.value()); + } + else + { + m_maxMetadataSize = std::numeric_limits::max(); + } + + auto currentMetadataValue = AppInstaller::JSON::GetJsonValueFromNode(input, metadataFieldName); + if (currentMetadataValue) + { + m_currentMetadata.FromJson(currentMetadataValue.value()); + } + + auto productRevisionNumber = AppInstaller::JSON::GetRawIntValueFromJsonNode(input, productRevisionFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !productRevisionNumber); + m_productRevision = productRevisionNumber.value(); + + auto installerHashString = AppInstaller::JSON::GetRawStringValueFromJsonNode(input, installerHashFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !installerHashString); + m_installerHash = SHA256::ConvertToBytes(installerHashString.value()); + + // The 1.0 version of input uses the 1.1 version of REST + JSON::ManifestJSONParser parser{ Version{ "1.1" }}; + + auto currentManifestValue = AppInstaller::JSON::GetJsonValueFromNode(input, currentManifestFieldName); + if (currentManifestValue) + { + std::vector manifests = parser.ParseData(currentManifestValue.value()); + + if (!manifests.empty()) + { + std::sort(manifests.begin(), manifests.end(), [](const Manifest::Manifest& a, const Manifest::Manifest& b) { return a.Version < b.Version; }); + // Latest version will be sorted to last position by Version < predicate + m_currentManifest = std::move(manifests.back()); + } + } + + auto incomingManifestValue = AppInstaller::JSON::GetJsonValueFromNode(input, incomingManifestFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !incomingManifestValue); + { + std::vector manifests = parser.ParseData(currentManifestValue.value()); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, manifests.empty()); + + std::sort(manifests.begin(), manifests.end(), [](const Manifest::Manifest& a, const Manifest::Manifest& b) { return a.Version < b.Version; }); + // Latest version will be sorted to last position by Version < predicate + m_incomingManifest = std::move(manifests.back()); + } + } +} diff --git a/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h similarity index 56% rename from src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h rename to src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index 1e1bb90a75..1b6d350bc0 100644 --- a/src/AppInstallerCommonCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -2,20 +2,58 @@ // Licensed under the MIT License. #pragma once -#include -#include +#include +#include #include #include #include #include +#include #include #include #include #include -namespace AppInstaller::Utility +namespace AppInstaller::Repository::Metadata { + // The overall metadata that we collect. + struct ProductMetadata + { + ProductMetadata() = default; + + // Removes all stored data. + void Clear(); + + // Load the metadata from an existing JSON blob. + void FromJson(const web::json::value& json); + + // The installer specific metadata that we collect. + struct InstallerMetadata + { + int ProductRevision; + Utility::Version ProductVersion; + std::vector AppsAndFeaturesEntries; + }; + + // Metadata from previous product revisions. + struct HistoricalMetadata + { + Utility::Version ProductVersion; + std::vector AppsAndFeaturesEntries; + }; + + private: + void FromJson_1_0(const web::json::value& json); + + Utility::Version m_version; + Utility::Version m_productVersionMin; + Utility::Version m_productVersionMax; + // Map from installer hash to metadata + std::map m_installerMetadata; + std::vector m_historicalMetadata; + }; + // Contains the functions and data used for collecting metadata from installers. struct InstallerMetadataCollectionContext { @@ -33,7 +71,7 @@ namespace AppInstaller::Utility static std::unique_ptr FromJSON(std::wstring_view json, const std::filesystem::path& logFile); // Completes the collection, writing to the given locations. - void Complete(const std::filesystem::path& output, const std::filesystem::path& diagnostics); + void Complete(const std::filesystem::path& output); private: // Initializes the context runtime, including the log file if provided. @@ -48,11 +86,11 @@ namespace AppInstaller::Utility ThreadLocalStorage::ThreadGlobals m_threadGlobals; // Parsed input - Version m_supportedBlobVersion; - size_t m_maxBlobSize = 0; - web::json::value m_currentBlob; // TODO: Parse blob as well + Utility::Version m_supportedMetadataVersion; + size_t m_maxMetadataSize = 0; + ProductMetadata m_currentMetadata; int m_productRevision = 0; - SHA256::HashBuffer m_installerHash; + Utility::SHA256::HashBuffer m_installerHash; Manifest::Manifest m_currentManifest; Manifest::Manifest m_incomingManifest; }; diff --git a/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h b/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h index bf6a1265d1..960c8601dd 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h +++ b/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h @@ -24,11 +24,11 @@ namespace AppInstaller::Repository::JSON ~ManifestJSONParser(); // Parses the manifests from the REST response object root. - // May potentially contian multiple versions of the same package. + // May potentially contain multiple versions of the same package. std::vector Deserialize(const web::json::value& response) const; // Parses the manifests from the Data field of the REST response object. - // May potentially contian multiple versions of the same package. + // May potentially contain multiple versions of the same package. std::vector ParseData(const web::json::value& data) const; private: diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp index 2172fb5d23..41634d7954 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp @@ -157,8 +157,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } - const web::json::array versionNodes = versions.value().get(); - for (auto& versionItem : versionNodes) + for (auto& versionItem : versions.value().get()) { Manifest::Manifest manifest; manifest.Id = id.value(); diff --git a/src/WinGetUtil/Exports.cpp b/src/WinGetUtil/Exports.cpp index b81e3a0587..1e61a4e13b 100644 --- a/src/WinGetUtil/Exports.cpp +++ b/src/WinGetUtil/Exports.cpp @@ -17,6 +17,7 @@ using namespace AppInstaller::Utility; using namespace AppInstaller::Manifest; using namespace AppInstaller::Repository; +using namespace AppInstaller::Repository::Metadata; using namespace AppInstaller::Repository::Microsoft; namespace @@ -383,7 +384,6 @@ extern "C" WINGET_UTIL_API WinGetCompleteInstallerMetadataCollection( WINGET_INSTALLER_METADATA_COLLECTION_HANDLE collectionHandle, WINGET_STRING outputFilePath, - WINGET_STRING diagnosticsFilePath, WinGetCompleteInstallerMetadataCollectionOptions options) try { THROW_HR_IF(E_INVALIDARG, !collectionHandle); @@ -398,7 +398,7 @@ extern "C" THROW_HR_IF(E_INVALIDARG, !outputFilePath); - context->Complete(outputFilePath, GetPathOrEmpty(diagnosticsFilePath)); + context->Complete(outputFilePath); return S_OK; } diff --git a/src/WinGetUtil/WinGetUtil.h b/src/WinGetUtil/WinGetUtil.h index d5f800f986..71e8c02f4e 100644 --- a/src/WinGetUtil/WinGetUtil.h +++ b/src/WinGetUtil/WinGetUtil.h @@ -170,6 +170,5 @@ extern "C" WINGET_UTIL_API WinGetCompleteInstallerMetadataCollection( WINGET_INSTALLER_METADATA_COLLECTION_HANDLE collectionHandle, WINGET_STRING outputFilePath, - WINGET_STRING diagnosticsFilePath, WinGetCompleteInstallerMetadataCollectionOptions options); } diff --git a/src/WinGetUtil/WinGetUtil.vcxproj b/src/WinGetUtil/WinGetUtil.vcxproj index 16147d5ada..3f577e8bf4 100644 --- a/src/WinGetUtil/WinGetUtil.vcxproj +++ b/src/WinGetUtil/WinGetUtil.vcxproj @@ -162,9 +162,9 @@ Disabled _DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) true true true @@ -206,7 +206,7 @@ WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories) true false true @@ -228,10 +228,10 @@ true true NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) - $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) + $(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories) true true true From 5d09217e8283a8dee7de59b7ea29cee4970cd0b1 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 23 May 2022 15:53:43 -0700 Subject: [PATCH 07/22] Make AppsAndFeaturesEntries parsable from outside --- .../Workflows/InstallFlow.cpp | 2 +- .../InstallerMetadataCollectionContext.cpp | 4 +- .../ManifestJSONParser.cpp | 5 +++ .../Public/winget/ARPCorrelation.h | 2 +- .../InstallerMetadataCollectionContext.h | 2 +- .../Public/winget/ManifestJSONParser.h | 3 ++ .../Schema/1_0/Json/ManifestDeserializer.h | 3 ++ .../1_0/Json/ManifestDeserializer_1_0.cpp | 5 +++ .../Schema/1_1/Json/ManifestDeserializer.h | 2 + .../1_1/Json/ManifestDeserializer_1_1.cpp | 43 +++++++++++-------- 10 files changed, 49 insertions(+), 22 deletions(-) diff --git a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp index 7005af6088..b531340184 100644 --- a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp +++ b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp @@ -520,7 +520,7 @@ namespace AppInstaller::CLI::Workflow return result; }, true); - std::vector> entries; + std::vector entries; for (const auto& entry : arpSource.Search({}).Matches) { diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index 57a87bd33f..f70f0c8c60 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -98,7 +98,7 @@ namespace AppInstaller::Repository::Metadata THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); installerMetadata.ProductVersion = Version{ versionString.value() }; - auto appsAndFeatures = AppInstaller::JSON::GetJsonValueFromNode(item, appsAndFeaturesFieldName); + auto appsAndFeatures = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(item, appsAndFeaturesFieldName); THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures); installerMetadata.AppsAndFeaturesEntries = parser.ParseAppsAndFeaturesEntries(appsAndFeatures.value()); @@ -117,7 +117,7 @@ namespace AppInstaller::Repository::Metadata THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); historicalMetadata.ProductVersion = Version{ versionString.value() }; - auto appsAndFeatures = AppInstaller::JSON::GetJsonValueFromNode(item, appsAndFeaturesFieldName); + auto appsAndFeatures = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(item, appsAndFeaturesFieldName); THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures); historicalMetadata.AppsAndFeaturesEntries = parser.ParseAppsAndFeaturesEntries(appsAndFeatures.value()); diff --git a/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp b/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp index 2067cc71dc..345a7de559 100644 --- a/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp +++ b/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp @@ -52,4 +52,9 @@ namespace AppInstaller::Repository::JSON return m_pImpl->m_deserializer->DeserializeData(data); } + std::vector ManifestJSONParser::ParseAppsAndFeaturesEntries(const web::json::array& data) const + { + return m_pImpl->m_deserializer->DeserializeAppsAndFeaturesEntries(data); + } + } diff --git a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h index 9696495a04..ef497a47d8 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h +++ b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h @@ -28,7 +28,7 @@ namespace AppInstaller::Repository::Correlation // Struct holding all the data from an ARP entry we use for the correlation struct ARPEntry { - ARPEntry(std::shared_ptr entry, bool isNewOrUpdated) : Entry(entry), IsNewOrUpdated(isNewOrUpdated) {} + ARPEntry(std::shared_ptr entry, bool isNewOrUpdated) : Entry(std::move(entry)), IsNewOrUpdated(isNewOrUpdated) {} // Data found in the ARP entry std::shared_ptr Entry; diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index 1b6d350bc0..f36d6f3cc4 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -31,7 +31,7 @@ namespace AppInstaller::Repository::Metadata // The installer specific metadata that we collect. struct InstallerMetadata { - int ProductRevision; + int ProductRevision = 0; Utility::Version ProductVersion; std::vector AppsAndFeaturesEntries; }; diff --git a/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h b/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h index 960c8601dd..740c1c523d 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h +++ b/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h @@ -31,6 +31,9 @@ namespace AppInstaller::Repository::JSON // May potentially contain multiple versions of the same package. std::vector ParseData(const web::json::value& data) const; + // Parses the AppsAndFeaturesEntries node, returning the set of values below it. + std::vector ParseAppsAndFeaturesEntries(const web::json::array& data) const; + private: struct impl; std::unique_ptr m_pImpl; diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h index f1c70f75f8..6bdebad7e3 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h @@ -16,6 +16,9 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json // Gets the manifest from the given json Data field std::vector DeserializeData(const web::json::value& dataJsonObject) const; + // Parses the AppsAndFeaturesEntries node, returning the set of values below it. + virtual std::vector DeserializeAppsAndFeaturesEntries(const web::json::array& entries) const; + protected: template diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp index 41634d7954..ce7cc45762 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp @@ -246,6 +246,11 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json return manifests; } + std::vector ManifestDeserializer::DeserializeAppsAndFeaturesEntries(const web::json::array&) const + { + return {}; + } + std::optional ManifestDeserializer::DeserializeLocale(const web::json::value& localeJsonObject) const { if (localeJsonObject.is_null()) diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h index 36f37a682b..6968f264dc 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h @@ -8,6 +8,8 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json // Manifest Deserializer. struct ManifestDeserializer : public V1_0::Json::ManifestDeserializer { + std::vector DeserializeAppsAndFeaturesEntries(const web::json::array& entries) const override; + protected: std::optional DeserializeInstaller(const web::json::value& installerJsonObject) const override; diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp index 29793ee741..faf5a64d9a 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp @@ -41,6 +41,31 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json constexpr std::string_view AgreementUrl = "AgreementUrl"sv; } + std::vector ManifestDeserializer::DeserializeAppsAndFeaturesEntries(const web::json::array& entries) const + { + std::vector result; + + for (auto& arpEntryNode : entries) + { + AppsAndFeaturesEntry arpEntry; + arpEntry.DisplayName = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(DisplayName)).value_or(""); + arpEntry.Publisher = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(Publisher)).value_or(""); + arpEntry.DisplayVersion = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(DisplayVersion)).value_or(""); + arpEntry.ProductCode = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(ProductCode)).value_or(""); + arpEntry.UpgradeCode = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(UpgradeCode)).value_or(""); + arpEntry.InstallerType = Manifest::ConvertToInstallerTypeEnum(JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(InstallerType)).value_or("")); + + // Only add when at least one field is valid + if (!arpEntry.DisplayName.empty() || !arpEntry.Publisher.empty() || !arpEntry.DisplayVersion.empty() || + !arpEntry.ProductCode.empty() || !arpEntry.UpgradeCode.empty() || arpEntry.InstallerType != InstallerTypeEnum::Unknown) + { + result.emplace_back(std::move(arpEntry)); + } + } + + return result; + } + Manifest::InstallerTypeEnum ManifestDeserializer::ConvertToInstallerType(std::string_view in) const { std::string inStrLower = Utility::ToLower(in); @@ -98,23 +123,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json std::optional> arpEntriesNode = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(AppsAndFeaturesEntries)); if (arpEntriesNode) { - for (auto& arpEntryNode : arpEntriesNode.value().get()) - { - AppsAndFeaturesEntry arpEntry; - arpEntry.DisplayName = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(DisplayName)).value_or(""); - arpEntry.Publisher = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(Publisher)).value_or(""); - arpEntry.DisplayVersion = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(DisplayVersion)).value_or(""); - arpEntry.ProductCode = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(ProductCode)).value_or(""); - arpEntry.UpgradeCode = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(UpgradeCode)).value_or(""); - arpEntry.InstallerType = Manifest::ConvertToInstallerTypeEnum(JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(InstallerType)).value_or("")); - - // Only add when at least one field is valid - if (!arpEntry.DisplayName.empty() || !arpEntry.Publisher.empty() || !arpEntry.DisplayVersion.empty() || - !arpEntry.ProductCode.empty() || !arpEntry.UpgradeCode.empty() || arpEntry.InstallerType != InstallerTypeEnum::Unknown) - { - installer.AppsAndFeaturesEntries.emplace_back(std::move(arpEntry)); - } - } + installer.AppsAndFeaturesEntries = DeserializeAppsAndFeaturesEntries(arpEntriesNode.value()); } // Markets From 250442af73150ae87af4853a322723ac7dec73a2 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 23 May 2022 16:51:28 -0700 Subject: [PATCH 08/22] Part way through refactoring the correlation data to be more reusable --- .../ExecutionContextData.h | 6 +- .../Workflows/InstallFlow.cpp | 42 ++-------- src/AppInstallerCLITests/ARPChanges.cpp | 30 +++---- .../ARPCorrelation.cpp | 79 ++++++++++++------- .../InstallerMetadataCollectionContext.cpp | 1 + .../Public/winget/ARPCorrelation.h | 29 ++++++- .../InstallerMetadataCollectionContext.h | 3 + 7 files changed, 109 insertions(+), 81 deletions(-) diff --git a/src/AppInstallerCLICore/ExecutionContextData.h b/src/AppInstallerCLICore/ExecutionContextData.h index 3498e7513c..4aec126613 100644 --- a/src/AppInstallerCLICore/ExecutionContextData.h +++ b/src/AppInstallerCLICore/ExecutionContextData.h @@ -47,7 +47,7 @@ namespace AppInstaller::CLI::Execution PackagesToInstall, // On import: Sources for the imported packages Sources, - ARPSnapshot, + ARPCorrelationData, CorrelatedAppsAndFeaturesEntries, Dependencies, DependencySource, @@ -186,9 +186,9 @@ namespace AppInstaller::CLI::Execution }; template <> - struct DataMapping + struct DataMapping { - using value_t = std::vector; + using value_t = Repository::Correlation::ARPCorrelationData; }; template <> diff --git a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp index b531340184..7d21f0fe37 100644 --- a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp +++ b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp @@ -512,55 +512,27 @@ namespace AppInstaller::CLI::Workflow if (installer && MightWriteToARP(installer->InstallerType)) { - Source arpSource = context.Reporter.ExecuteWithProgress( - [](IProgressCallback& progress) - { - Repository::Source result = Repository::Source(PredefinedSource::ARP); - result.Open(progress); - return result; - }, true); - - std::vector entries; + Repository::Correlation::ARPCorrelationData data; + data.CapturePreInstallSnapshot(); - for (const auto& entry : arpSource.Search({}).Matches) - { - auto installed = entry.Package->GetInstalledVersion(); - if (installed) - { - entries.emplace_back(std::make_tuple( - entry.Package->GetProperty(PackageProperty::Id), - installed->GetProperty(PackageVersionProperty::Version), - installed->GetProperty(PackageVersionProperty::Channel))); - } - } - - std::sort(entries.begin(), entries.end()); - - context.Add(std::move(entries)); + context.Add(std::move(data)); } } CATCH_LOG() void ReportARPChanges(Execution::Context& context) try { - if (!context.Contains(Execution::Data::ARPSnapshot)) + if (!context.Contains(Execution::Data::ARPCorrelationData)) { return; } const auto& manifest = context.Get(); - const auto& arpSnapshot = context.Get(); + auto& arpCorrelationData = context.Get(); - // Open the ARP source again to get the (potentially) changed ARP entries - Source arpSource = context.Reporter.ExecuteWithProgress( - [](IProgressCallback& progress) - { - Repository::Source result = Repository::Source(PredefinedSource::ARP); - result.Open(progress); - return result; - }, true); + arpCorrelationData.CapturePostInstallSnapshot(); - auto correlationResult = Correlation::FindARPEntryForNewlyInstalledPackage(manifest, arpSnapshot, arpSource); + auto correlationResult = Correlation::FindARPEntryForNewlyInstalledPackage(manifest, arpCorrelationData.GetPreInstallSnapshot(), arpSource); // Store the ARP entry found to match the package to record it in the tracking catalog later if (correlationResult.Package) diff --git a/src/AppInstallerCLITests/ARPChanges.cpp b/src/AppInstallerCLITests/ARPChanges.cpp index 6111bad98a..615894db18 100644 --- a/src/AppInstallerCLITests/ARPChanges.cpp +++ b/src/AppInstallerCLITests/ARPChanges.cpp @@ -230,7 +230,7 @@ TEST_CASE("ARPChanges_MSIX_Ignored", "[ARPChanges][workflow]") context << SnapshotARPEntries; - REQUIRE(!context.Contains(Data::ARPSnapshot)); + REQUIRE(!context.Contains(Data::ARPCorrelationData)); context << ReportARPChanges; @@ -244,9 +244,9 @@ TEST_CASE("ARPChanges_CheckSnapshot", "[ARPChanges][workflow]") context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); - auto snapshot = context.Get(); + auto snapshot = context.Get().GetPreInstallSnapshot(); REQUIRE(context.EverythingResult.Matches.size() == snapshot.size()); @@ -278,7 +278,7 @@ TEST_CASE("ARPChanges_NoChange_NoMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context << ReportARPChanges; context.ExpectEvent(0, 0, 0); @@ -290,7 +290,7 @@ TEST_CASE("ARPChanges_NoChange_SingleMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddMatchResult("MatchId1", "MatchName1", "MatchPublisher1", "MatchVersion1"); @@ -304,7 +304,7 @@ TEST_CASE("ARPChanges_NoChange_MultiMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddMatchResult("MatchId1", "MatchName1", "MatchPublisher1", "MatchVersion1"); context.AddMatchResult("MatchId2", "MatchName2", "MatchPublisher2", "MatchVersion2"); @@ -319,7 +319,7 @@ TEST_CASE("ARPChanges_SingleChange_NoMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); @@ -333,7 +333,7 @@ TEST_CASE("ARPChanges_SingleChange_SingleMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddMatchResult("MatchId1", "MatchName1", "MatchPublisher1", "MatchVersion1"); @@ -348,7 +348,7 @@ TEST_CASE("ARPChanges_SingleChange_MultiMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddMatchResult("MatchId1", "MatchName1", "MatchPublisher1", "MatchVersion1"); @@ -364,7 +364,7 @@ TEST_CASE("ARPChanges_MultiChange_NoMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -379,7 +379,7 @@ TEST_CASE("ARPChanges_MultiChange_SingleMatch_NoOverlap", "[ARPChanges][workflow TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -395,7 +395,7 @@ TEST_CASE("ARPChanges_MultiChange_SingleMatch_Overlap", "[ARPChanges][workflow]" TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -411,7 +411,7 @@ TEST_CASE("ARPChanges_MultiChange_MultiMatch_NoOverlap", "[ARPChanges][workflow] TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -428,7 +428,7 @@ TEST_CASE("ARPChanges_MultiChange_MultiMatch_SingleOverlap", "[ARPChanges][workf TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -445,7 +445,7 @@ TEST_CASE("ARPChanges_MultiChange_MultiMatch_MultiOverlap", "[ARPChanges][workfl TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.MatchResult.Matches.emplace_back(context.EverythingResult.Matches.back()); diff --git a/src/AppInstallerRepositoryCore/ARPCorrelation.cpp b/src/AppInstallerRepositoryCore/ARPCorrelation.cpp index 7a1e54a7c9..64a2a15383 100644 --- a/src/AppInstallerRepositoryCore/ARPCorrelation.cpp +++ b/src/AppInstallerRepositoryCore/ARPCorrelation.cpp @@ -198,32 +198,6 @@ namespace AppInstaller::Repository::Correlation { AICLI_LOG(Repo, Verbose, << "Finding ARP entry matching newly installed package"); - std::vector changedArpEntries; - std::vector existingArpEntries; - - for (auto& entry : arpSource.Search({}).Matches) - { - auto installed = entry.Package->GetInstalledVersion(); - - if (installed) - { - auto entryKey = std::make_tuple( - entry.Package->GetProperty(PackageProperty::Id), - installed->GetProperty(PackageVersionProperty::Version), - installed->GetProperty(PackageVersionProperty::Channel)); - - auto itr = std::lower_bound(arpSnapshot.begin(), arpSnapshot.end(), entryKey); - if (itr == arpSnapshot.end() || *itr != entryKey) - { - changedArpEntries.emplace_back(entry.Package, true); - } - else - { - existingArpEntries.emplace_back(entry.Package, false); - } - } - } - // Also attempt to find the entry based on the manifest data SearchRequest manifestSearchRequest; @@ -393,4 +367,55 @@ namespace AppInstaller::Repository::Correlation return bestMatch ? bestMatch->Entry->GetInstalledVersion() : nullptr; } -} \ No newline at end of file + + void ARPCorrelationData::CapturePreInstallSnapshot() + { + ProgressCallback empty; + Repository::Source preInstallARP = Repository::Source(PredefinedSource::ARP); + preInstallARP.Open(empty); + + for (const auto& entry : preInstallARP.Search({}).Matches) + { + auto installed = entry.Package->GetInstalledVersion(); + if (installed) + { + m_preInstallSnapshot.emplace_back(std::make_tuple( + entry.Package->GetProperty(PackageProperty::Id), + installed->GetProperty(PackageVersionProperty::Version), + installed->GetProperty(PackageVersionProperty::Channel))); + } + } + + std::sort(m_preInstallSnapshot.begin(), m_preInstallSnapshot.end()); + } + + void ARPCorrelationData::CapturePostInstallSnapshot() + { + ProgressCallback empty; + m_postInstallSnapshotSource = Repository::Source(PredefinedSource::ARP); + m_postInstallSnapshotSource.Open(empty); + + for (auto& entry : m_postInstallSnapshotSource.Search({}).Matches) + { + auto installed = entry.Package->GetInstalledVersion(); + + if (installed) + { + auto entryKey = std::make_tuple( + entry.Package->GetProperty(PackageProperty::Id), + installed->GetProperty(PackageVersionProperty::Version), + installed->GetProperty(PackageVersionProperty::Channel)); + + auto itr = std::lower_bound(m_preInstallSnapshot.begin(), m_preInstallSnapshot.end(), entryKey); + if (itr == m_preInstallSnapshot.end() || *itr != entryKey) + { + m_postInstallSnapshot.emplace_back(entry.Package, true); + } + else + { + m_postInstallSnapshot.emplace_back(entry.Package, false); + } + } + } + } +} diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index f70f0c8c60..097bb3f0f2 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -274,6 +274,7 @@ namespace AppInstaller::Repository::Metadata } // Collect pre-install system state + m_correlationData.CapturePreInstallSnapshot(); } void InstallerMetadataCollectionContext::ParseInputJson_1_0(web::json::value& input) diff --git a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h index ef497a47d8..9775bd5dc9 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h +++ b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h @@ -2,7 +2,14 @@ // Licensed under the MIT License. #pragma once +#include #include +#include + +#include +#include +#include +#include namespace AppInstaller { @@ -103,4 +110,24 @@ namespace AppInstaller::Repository::Correlation const AppInstaller::Manifest::Manifest& manifest, const std::vector& arpEntries, IARPMatchConfidenceAlgorithm& algorithm); -} \ No newline at end of file + + // Holds data needed for ARP correlation, as well as functions to run correlation on the collected data. + struct ARPCorrelationData + { + ARPCorrelationData() = default; + + // Captures the ARP state before the package installation. + void CapturePreInstallSnapshot(); + + // Captures the ARP state differences after the package installation. + void CapturePostInstallSnapshot(); + + const std::vector& GetPreInstallSnapshot() const { return m_preInstallSnapshot; } + + private: + std::vector m_preInstallSnapshot; + + Source m_postInstallSnapshotSource; + std::vector m_postInstallSnapshot; + }; +} diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index f36d6f3cc4..afaa418695 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -93,5 +94,7 @@ namespace AppInstaller::Repository::Metadata Utility::SHA256::HashBuffer m_installerHash; Manifest::Manifest m_currentManifest; Manifest::Manifest m_incomingManifest; + + Correlation::ARPCorrelationData m_correlationData; }; } From dbf58e2cc78bc78af8adade55abb566623095af6 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Wed, 25 May 2022 10:34:00 -0700 Subject: [PATCH 09/22] Basic refactor to make the ARP correlation path directly reusable --- .../Workflows/InstallFlow.cpp | 4 +- .../ARPCorrelation.cpp | 236 +++++++++--------- .../InstallerMetadataCollectionContext.cpp | 57 +++-- .../ManifestJSONParser.cpp | 9 +- .../Public/winget/ARPCorrelation.h | 11 +- .../InstallerMetadataCollectionContext.h | 4 +- .../Public/winget/ManifestJSONParser.h | 13 +- .../Schema/1_0/Json/ManifestDeserializer.h | 7 +- .../Schema/1_1/Json/ManifestDeserializer.h | 4 +- 9 files changed, 178 insertions(+), 167 deletions(-) diff --git a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp index 7d21f0fe37..7c526a6f73 100644 --- a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp +++ b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp @@ -514,7 +514,6 @@ namespace AppInstaller::CLI::Workflow { Repository::Correlation::ARPCorrelationData data; data.CapturePreInstallSnapshot(); - context.Add(std::move(data)); } } @@ -531,8 +530,7 @@ namespace AppInstaller::CLI::Workflow auto& arpCorrelationData = context.Get(); arpCorrelationData.CapturePostInstallSnapshot(); - - auto correlationResult = Correlation::FindARPEntryForNewlyInstalledPackage(manifest, arpCorrelationData.GetPreInstallSnapshot(), arpSource); + auto correlationResult = arpCorrelationData.CorrelateForNewlyInstalled(manifest); // Store the ARP entry found to match the package to record it in the tracking catalog later if (correlationResult.Package) diff --git a/src/AppInstallerRepositoryCore/ARPCorrelation.cpp b/src/AppInstallerRepositoryCore/ARPCorrelation.cpp index 64a2a15383..0de437cab2 100644 --- a/src/AppInstallerRepositoryCore/ARPCorrelation.cpp +++ b/src/AppInstallerRepositoryCore/ARPCorrelation.cpp @@ -191,10 +191,109 @@ namespace AppInstaller::Repository::Correlation return bestMatchingScore; } - ARPCorrelationResult FindARPEntryForNewlyInstalledPackage( + // Find the best match using heuristics + std::shared_ptr FindARPEntryForNewlyInstalledPackageWithHeuristics( const Manifest::Manifest& manifest, - const std::vector& arpSnapshot, - Source& arpSource) + const std::vector& arpEntries) + { + // TODO: In the future we can make different passes with different algorithms until we find a match + return FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, arpEntries, IARPMatchConfidenceAlgorithm::Instance()); + } + + std::shared_ptr FindARPEntryForNewlyInstalledPackageWithHeuristics( + const AppInstaller::Manifest::Manifest& manifest, + const std::vector& arpEntries, + IARPMatchConfidenceAlgorithm& algorithm) + { + AICLI_LOG(Repo, Verbose, << "Looking for best match in ARP for manifest " << manifest.Id); + + algorithm.Init(manifest); + + std::optional bestMatch; + double bestScore = 0; + + for (const auto& arpEntry : arpEntries) + { + auto score = algorithm.ComputeConfidence(arpEntry); + AICLI_LOG(Repo, Verbose, << "Match confidence for " << arpEntry.Entry->GetProperty(PackageProperty::Id) << ": " << score); + + if (score < MatchingThreshold) + { + AICLI_LOG(Repo, Verbose, << "Score is lower than threshold"); + continue; + } + + if (!bestMatch || bestScore < score) + { + bestMatch = arpEntry; + bestScore = score; + } + } + + if (bestMatch) + { + AICLI_LOG(Repo, Verbose, << "Best match is " << bestMatch->Entry->GetProperty(PackageProperty::Id)); + } + else + { + AICLI_LOG(Repo, Verbose, << "No ARP entry had a correlation score surpassing the required threshold"); + } + + return bestMatch ? bestMatch->Entry->GetInstalledVersion() : nullptr; + } + + void ARPCorrelationData::CapturePreInstallSnapshot() + { + ProgressCallback empty; + Repository::Source preInstallARP = Repository::Source(PredefinedSource::ARP); + preInstallARP.Open(empty); + + for (const auto& entry : preInstallARP.Search({}).Matches) + { + auto installed = entry.Package->GetInstalledVersion(); + if (installed) + { + m_preInstallSnapshot.emplace_back(std::make_tuple( + entry.Package->GetProperty(PackageProperty::Id), + installed->GetProperty(PackageVersionProperty::Version), + installed->GetProperty(PackageVersionProperty::Channel))); + } + } + + std::sort(m_preInstallSnapshot.begin(), m_preInstallSnapshot.end()); + } + + void ARPCorrelationData::CapturePostInstallSnapshot() + { + ProgressCallback empty; + m_postInstallSnapshotSource = Repository::Source(PredefinedSource::ARP); + m_postInstallSnapshotSource.Open(empty); + + for (auto& entry : m_postInstallSnapshotSource.Search({}).Matches) + { + auto installed = entry.Package->GetInstalledVersion(); + + if (installed) + { + auto entryKey = std::make_tuple( + entry.Package->GetProperty(PackageProperty::Id), + installed->GetProperty(PackageVersionProperty::Version), + installed->GetProperty(PackageVersionProperty::Channel)); + + auto itr = std::lower_bound(m_preInstallSnapshot.begin(), m_preInstallSnapshot.end(), entryKey); + if (itr == m_preInstallSnapshot.end() || *itr != entryKey) + { + m_postInstallSnapshot.emplace_back(entry.Package, true); + } + else + { + m_postInstallSnapshot.emplace_back(entry.Package, false); + } + } + } + } + + ARPCorrelationResult ARPCorrelationData::CorrelateForNewlyInstalled(const Manifest::Manifest& manifest) { AICLI_LOG(Repo, Verbose, << "Finding ARP entry matching newly installed package"); @@ -252,20 +351,23 @@ namespace AppInstaller::Repository::Correlation // Don't execute this search if it would just find everything if (!manifestSearchRequest.IsForEverything()) { - findByManifest = arpSource.Search(manifestSearchRequest); + findByManifest = m_postInstallSnapshotSource.Search(manifestSearchRequest); } // Cross reference the changes with the search results std::vector> packagesInBoth; - for (const auto& change : changedArpEntries) + for (const auto& change : m_postInstallSnapshot) { - for (const auto& byManifest : findByManifest.Matches) + if (change.IsNewOrUpdated) { - if (change.Entry->IsSame(byManifest.Package.get())) + for (const auto& byManifest : findByManifest.Matches) { - packagesInBoth.emplace_back(change.Entry); - break; + if (change.Entry->IsSame(byManifest.Package.get())) + { + packagesInBoth.emplace_back(change.Entry); + break; + } } } } @@ -281,7 +383,7 @@ namespace AppInstaller::Repository::Correlation // Find the package that we are going to log ARPCorrelationResult result; // TODO: Find a good way to consider the other heuristics in these stats. - result.ChangesToARP = changedArpEntries.size(); + result.ChangesToARP = std::count_if(m_postInstallSnapshot.begin(), m_postInstallSnapshot.end(), [](const ARPEntry& e) { return e.IsNewOrUpdated; }); result.MatchesInARP = findByManifest.Matches.size(); result.CountOfIntersectionOfChangesAndMatches = packagesInBoth.size(); @@ -301,121 +403,9 @@ namespace AppInstaller::Repository::Correlation // to try and match the package with some ARP entry by assigning them scores. AICLI_LOG(Repo, Verbose, << "No exact ARP match found. Trying to find one with heuristics"); - std::vector arpEntries; - for (auto&& entry : changedArpEntries) - { - arpEntries.push_back(std::move(entry)); - } - for (auto&& entry : existingArpEntries) - { - arpEntries.push_back(std::move(entry)); - } - - result.Package = FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, arpEntries); + result.Package = FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, m_postInstallSnapshot); } return result; } - - // Find the best match using heuristics - std::shared_ptr FindARPEntryForNewlyInstalledPackageWithHeuristics( - const Manifest::Manifest& manifest, - const std::vector& arpEntries) - { - // TODO: In the future we can make different passes with different algorithms until we find a match - return FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, arpEntries, IARPMatchConfidenceAlgorithm::Instance()); - } - - std::shared_ptr FindARPEntryForNewlyInstalledPackageWithHeuristics( - const AppInstaller::Manifest::Manifest& manifest, - const std::vector& arpEntries, - IARPMatchConfidenceAlgorithm& algorithm) - { - AICLI_LOG(Repo, Verbose, << "Looking for best match in ARP for manifest " << manifest.Id); - - algorithm.Init(manifest); - - std::optional bestMatch; - double bestScore = 0; - - for (const auto& arpEntry : arpEntries) - { - auto score = algorithm.ComputeConfidence(arpEntry); - AICLI_LOG(Repo, Verbose, << "Match confidence for " << arpEntry.Entry->GetProperty(PackageProperty::Id) << ": " << score); - - if (score < MatchingThreshold) - { - AICLI_LOG(Repo, Verbose, << "Score is lower than threshold"); - continue; - } - - if (!bestMatch || bestScore < score) - { - bestMatch = arpEntry; - bestScore = score; - } - } - - if (bestMatch) - { - AICLI_LOG(Repo, Verbose, << "Best match is " << bestMatch->Entry->GetProperty(PackageProperty::Id)); - } - else - { - AICLI_LOG(Repo, Verbose, << "No ARP entry had a correlation score surpassing the required threshold"); - } - - return bestMatch ? bestMatch->Entry->GetInstalledVersion() : nullptr; - } - - void ARPCorrelationData::CapturePreInstallSnapshot() - { - ProgressCallback empty; - Repository::Source preInstallARP = Repository::Source(PredefinedSource::ARP); - preInstallARP.Open(empty); - - for (const auto& entry : preInstallARP.Search({}).Matches) - { - auto installed = entry.Package->GetInstalledVersion(); - if (installed) - { - m_preInstallSnapshot.emplace_back(std::make_tuple( - entry.Package->GetProperty(PackageProperty::Id), - installed->GetProperty(PackageVersionProperty::Version), - installed->GetProperty(PackageVersionProperty::Channel))); - } - } - - std::sort(m_preInstallSnapshot.begin(), m_preInstallSnapshot.end()); - } - - void ARPCorrelationData::CapturePostInstallSnapshot() - { - ProgressCallback empty; - m_postInstallSnapshotSource = Repository::Source(PredefinedSource::ARP); - m_postInstallSnapshotSource.Open(empty); - - for (auto& entry : m_postInstallSnapshotSource.Search({}).Matches) - { - auto installed = entry.Package->GetInstalledVersion(); - - if (installed) - { - auto entryKey = std::make_tuple( - entry.Package->GetProperty(PackageProperty::Id), - installed->GetProperty(PackageVersionProperty::Version), - installed->GetProperty(PackageVersionProperty::Channel)); - - auto itr = std::lower_bound(m_preInstallSnapshot.begin(), m_preInstallSnapshot.end(), entryKey); - if (itr == m_preInstallSnapshot.end() || *itr != entryKey) - { - m_postInstallSnapshot.emplace_back(entry.Package, true); - } - else - { - m_postInstallSnapshot.emplace_back(entry.Package, false); - } - } - } - } } diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index 097bb3f0f2..c8f2eb7098 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -59,7 +59,7 @@ namespace AppInstaller::Repository::Metadata utility::string_t productVersionMaxFieldName = L"productVersionMax"; utility::string_t metadataFieldName = L"metadata"; utility::string_t installerHashFieldName = L"installerHash"; - utility::string_t productRevisionFieldName = L"productRevision"; + utility::string_t submissionIdentifierFieldName = L"submissionIdentifier"; utility::string_t versionFieldName = L"version"; utility::string_t appsAndFeaturesFieldName = L"AppsAndFeaturesEntries"; utility::string_t historicalFieldName = L"historical"; @@ -90,9 +90,9 @@ namespace AppInstaller::Repository::Metadata InstallerMetadata installerMetadata; - auto productRevisionNumber = AppInstaller::JSON::GetRawIntValueFromJsonNode(item, productRevisionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !productRevisionNumber); - installerMetadata.ProductRevision = productRevisionNumber.value(); + auto submissionIdentifierString = AppInstaller::JSON::GetRawStringValueFromJsonNode(item, submissionIdentifierFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !submissionIdentifierString); + installerMetadata.SubmissionIdentifier = submissionIdentifierString.value(); auto versionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(item, versionFieldName); THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); @@ -100,7 +100,7 @@ namespace AppInstaller::Repository::Metadata auto appsAndFeatures = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(item, appsAndFeaturesFieldName); THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures); - installerMetadata.AppsAndFeaturesEntries = parser.ParseAppsAndFeaturesEntries(appsAndFeatures.value()); + installerMetadata.AppsAndFeaturesEntries = parser.DeserializeAppsAndFeaturesEntries(appsAndFeatures.value()); m_installerMetadata[installerHashString.value()] = std::move(installerMetadata); } @@ -119,7 +119,7 @@ namespace AppInstaller::Repository::Metadata auto appsAndFeatures = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(item, appsAndFeaturesFieldName); THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures); - historicalMetadata.AppsAndFeaturesEntries = parser.ParseAppsAndFeaturesEntries(appsAndFeatures.value()); + historicalMetadata.AppsAndFeaturesEntries = parser.DeserializeAppsAndFeaturesEntries(appsAndFeatures.value()); m_historicalMetadata.emplace_back(std::move(historicalMetadata)); } @@ -285,10 +285,12 @@ namespace AppInstaller::Repository::Metadata utility::string_t metadataVersionFieldName = L"supportedMetadataVersion"; utility::string_t metadataSizeFieldName = L"maximumMetadataSize"; utility::string_t metadataFieldName = L"currentMetadata"; - utility::string_t productRevisionFieldName = L"productRevision"; + utility::string_t submissionIdentifierFieldName = L"submissionIdentifier"; utility::string_t installerHashFieldName = L"installerHash"; utility::string_t currentManifestFieldName = L"currentManifest"; - utility::string_t incomingManifestFieldName = L"incomingManifest"; + utility::string_t submissionDataFieldName = L"submissionData"; + utility::string_t defaultLocaleFieldName = L"DefaultLocale"; + utility::string_t localesFieldName = L"Locales"; auto metadataVersionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(input, metadataVersionFieldName); THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !metadataVersionString); @@ -310,9 +312,9 @@ namespace AppInstaller::Repository::Metadata m_currentMetadata.FromJson(currentMetadataValue.value()); } - auto productRevisionNumber = AppInstaller::JSON::GetRawIntValueFromJsonNode(input, productRevisionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !productRevisionNumber); - m_productRevision = productRevisionNumber.value(); + auto submissionIdentifierString = AppInstaller::JSON::GetRawStringValueFromJsonNode(input, submissionIdentifierFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !submissionIdentifierString); + m_submissionIdentifier = submissionIdentifierString.value(); auto installerHashString = AppInstaller::JSON::GetRawStringValueFromJsonNode(input, installerHashFieldName); THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !installerHashString); @@ -324,7 +326,7 @@ namespace AppInstaller::Repository::Metadata auto currentManifestValue = AppInstaller::JSON::GetJsonValueFromNode(input, currentManifestFieldName); if (currentManifestValue) { - std::vector manifests = parser.ParseData(currentManifestValue.value()); + std::vector manifests = parser.DeserializeData(currentManifestValue.value()); if (!manifests.empty()) { @@ -334,15 +336,32 @@ namespace AppInstaller::Repository::Metadata } } - auto incomingManifestValue = AppInstaller::JSON::GetJsonValueFromNode(input, incomingManifestFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !incomingManifestValue); + auto submissionDataValue = AppInstaller::JSON::GetJsonValueFromNode(input, submissionDataFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !submissionDataValue); { - std::vector manifests = parser.ParseData(currentManifestValue.value()); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, manifests.empty()); + auto defaultLocaleValue = AppInstaller::JSON::GetJsonValueFromNode(submissionDataValue.value(), defaultLocaleFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !defaultLocaleValue); - std::sort(manifests.begin(), manifests.end(), [](const Manifest::Manifest& a, const Manifest::Manifest& b) { return a.Version < b.Version; }); - // Latest version will be sorted to last position by Version < predicate - m_incomingManifest = std::move(manifests.back()); + auto defaultLocale = parser.DeserializeLocale(defaultLocaleValue.value()); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, + !defaultLocale || + !defaultLocale->Contains(Manifest::Localization::PackageName) || + !defaultLocale->Contains(Manifest::Localization::Publisher)); + + m_incomingManifest.DefaultLocalization = std::move(defaultLocale).value(); + + auto localesArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(submissionDataValue.value(), localesFieldName); + if (localesArray) + { + for (const auto& locale : localesArray->get()) + { + auto localization = parser.DeserializeLocale(locale); + if (localization) + { + m_incomingManifest.Localizations.emplace_back(std::move(localization).value()); + } + } + } } } } diff --git a/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp b/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp index 345a7de559..00f7683f5f 100644 --- a/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp +++ b/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp @@ -47,14 +47,19 @@ namespace AppInstaller::Repository::JSON return m_pImpl->m_deserializer->Deserialize(response); } - std::vector ManifestJSONParser::ParseData(const web::json::value& data) const + std::vector ManifestJSONParser::DeserializeData(const web::json::value& data) const { return m_pImpl->m_deserializer->DeserializeData(data); } - std::vector ManifestJSONParser::ParseAppsAndFeaturesEntries(const web::json::array& data) const + std::vector ManifestJSONParser::DeserializeAppsAndFeaturesEntries(const web::json::array& data) const { return m_pImpl->m_deserializer->DeserializeAppsAndFeaturesEntries(data); } + std::optional ManifestJSONParser::DeserializeLocale(const web::json::value& locale) const + { + return m_pImpl->m_deserializer->DeserializeLocale(locale); + } + } diff --git a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h index 9775bd5dc9..a2c135b065 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h +++ b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h @@ -94,14 +94,6 @@ namespace AppInstaller::Repository::Correlation std::vector> m_namesAndPublishers; }; - // Finds the ARP entry in the ARP source that matches a newly installed package. - // Takes the package manifest, a snapshot of the ARP before the installation, and the current ARP source. - // Returns the entry in the ARP source, or nullptr if there was no match, plus some stats about the correlation. - ARPCorrelationResult FindARPEntryForNewlyInstalledPackage( - const AppInstaller::Manifest::Manifest& manifest, - const std::vector& arpSnapshot, - AppInstaller::Repository::Source& arpSource); - std::shared_ptr FindARPEntryForNewlyInstalledPackageWithHeuristics( const AppInstaller::Manifest::Manifest& manifest, const std::vector& arpEntries); @@ -122,6 +114,9 @@ namespace AppInstaller::Repository::Correlation // Captures the ARP state differences after the package installation. void CapturePostInstallSnapshot(); + // Correlates the given manifest against the data previously collected with capture calls. + ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest& manifest); + const std::vector& GetPreInstallSnapshot() const { return m_preInstallSnapshot; } private: diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index afaa418695..2e0af58675 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -32,7 +32,7 @@ namespace AppInstaller::Repository::Metadata // The installer specific metadata that we collect. struct InstallerMetadata { - int ProductRevision = 0; + std::string SubmissionIdentifier; Utility::Version ProductVersion; std::vector AppsAndFeaturesEntries; }; @@ -90,7 +90,7 @@ namespace AppInstaller::Repository::Metadata Utility::Version m_supportedMetadataVersion; size_t m_maxMetadataSize = 0; ProductMetadata m_currentMetadata; - int m_productRevision = 0; + std::string m_submissionIdentifier; Utility::SHA256::HashBuffer m_installerHash; Manifest::Manifest m_currentManifest; Manifest::Manifest m_incomingManifest; diff --git a/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h b/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h index 740c1c523d..25e33d7208 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h +++ b/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h @@ -23,16 +23,19 @@ namespace AppInstaller::Repository::JSON ~ManifestJSONParser(); - // Parses the manifests from the REST response object root. + // Deserializes the manifests from the REST response object root. // May potentially contain multiple versions of the same package. std::vector Deserialize(const web::json::value& response) const; - // Parses the manifests from the Data field of the REST response object. + // Deserializes the manifests from the Data field of the REST response object. // May potentially contain multiple versions of the same package. - std::vector ParseData(const web::json::value& data) const; + std::vector DeserializeData(const web::json::value& data) const; - // Parses the AppsAndFeaturesEntries node, returning the set of values below it. - std::vector ParseAppsAndFeaturesEntries(const web::json::array& data) const; + // Deserializes the AppsAndFeaturesEntries node, returning the set of values below it. + std::vector DeserializeAppsAndFeaturesEntries(const web::json::array& data) const; + + // Deserializes the locale node; returning an object if a proper locale was found. + std::optional DeserializeLocale(const web::json::value& locale) const; private: struct impl; diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h index 6bdebad7e3..93feae1a7e 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h @@ -16,9 +16,12 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json // Gets the manifest from the given json Data field std::vector DeserializeData(const web::json::value& dataJsonObject) const; - // Parses the AppsAndFeaturesEntries node, returning the set of values below it. + // Deserializes the AppsAndFeaturesEntries node, returning the set of values below it. virtual std::vector DeserializeAppsAndFeaturesEntries(const web::json::array& entries) const; + // Deserializes the locale; requires that the PackageLocale be set to return an object. + virtual std::optional DeserializeLocale(const web::json::value& localeJsonObject) const; + protected: template @@ -32,8 +35,6 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - virtual std::optional DeserializeLocale(const web::json::value& localeJsonObject) const; - virtual std::optional DeserializeInstaller(const web::json::value& installerJsonObject) const; std::optional DeserializeDependency(const web::json::value& dependenciesJsonObject) const; diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h index 6968f264dc..8e5d574705 100644 --- a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h +++ b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h @@ -10,11 +10,11 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json { std::vector DeserializeAppsAndFeaturesEntries(const web::json::array& entries) const override; + std::optional DeserializeLocale(const web::json::value& localeJsonObject) const override; + protected: std::optional DeserializeInstaller(const web::json::value& installerJsonObject) const override; - std::optional DeserializeLocale(const web::json::value& localeJsonObject) const override; - Manifest::InstallerTypeEnum ConvertToInstallerType(std::string_view in) const override; }; } From 4345add729a768575fdb9fedddf407b8cf2bdcb4 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Wed, 25 May 2022 21:49:50 -0700 Subject: [PATCH 10/22] Output code complete, capstone is to update with the new metadata --- src/AppInstallerCommonCore/JsonUtil.cpp | 5 + .../Public/winget/JsonUtil.h | 2 + .../InstallerMetadataCollectionContext.cpp | 479 +++++++++++++++--- .../InstallerMetadataCollectionContext.h | 60 ++- 4 files changed, 462 insertions(+), 84 deletions(-) diff --git a/src/AppInstallerCommonCore/JsonUtil.cpp b/src/AppInstallerCommonCore/JsonUtil.cpp index 9a34be66ae..bd8c9cd129 100644 --- a/src/AppInstallerCommonCore/JsonUtil.cpp +++ b/src/AppInstallerCommonCore/JsonUtil.cpp @@ -73,6 +73,11 @@ namespace AppInstaller::JSON return utility::conversions::to_string_t(nodeName.data()); } + web::json::value GetStringValue(std::string_view value) + { + return web::json::value::string(Utility::ConvertToUTF16(value)); + } + std::optional> GetJsonValueFromNode(const web::json::value& node, const utility::string_t& keyName) { if (node.is_null() || !node.has_field(keyName)) diff --git a/src/AppInstallerCommonCore/Public/winget/JsonUtil.h b/src/AppInstallerCommonCore/Public/winget/JsonUtil.h index 50b43a02c0..135b51c0f4 100644 --- a/src/AppInstallerCommonCore/Public/winget/JsonUtil.h +++ b/src/AppInstallerCommonCore/Public/winget/JsonUtil.h @@ -51,6 +51,8 @@ namespace AppInstaller::JSON utility::string_t GetUtilityString(std::string_view nodeName); + web::json::value GetStringValue(std::string_view value); + std::vector GetRawStringArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); bool IsValidNonEmptyStringValue(std::optional& value); diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index c8f2eb7098..9cde31c078 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -14,14 +14,101 @@ using namespace AppInstaller::Utility; namespace AppInstaller::Repository::Metadata { + namespace + { + struct ProductMetadataFields_1_0 + { + utility::string_t ProductVersionMin = L"productVersionMin"; + utility::string_t ProductVersionMax = L"productVersionMax"; + utility::string_t Metadata = L"metadata"; + utility::string_t InstallerHash = L"installerHash"; + utility::string_t SubmissionIdentifier = L"submissionIdentifier"; + utility::string_t Version = L"version"; + utility::string_t AppsAndFeaturesEntries = L"AppsAndFeaturesEntries"; + utility::string_t Historical = L"historical"; + + utility::string_t DisplayName = L"DisplayName"; + utility::string_t Publisher = L"Publisher"; + utility::string_t DisplayVersion = L"DisplayVersion"; + utility::string_t ProductCode = L"ProductCode"; + utility::string_t UpgradeCode = L"UpgradeCode"; + utility::string_t InstallerType = L"InstallerType"; + + utility::string_t VersionMin = L"versionMin"; + utility::string_t VersionMax = L"versionMax"; + utility::string_t Names = L"names"; + utility::string_t Publishers = L"publishers"; + utility::string_t ProductCodes = L"productCodes"; + utility::string_t UpgradeCodes = L"upgradeCodes"; + }; + + std::string GetRequiredString(const web::json::value& value, const utility::string_t& field) + { + auto optString = AppInstaller::JSON::GetRawStringValueFromJsonNode(value, field); + if (!optString) + { + AICLI_LOG(Repo, Error, << "Required field '" << Utility::ConvertToUTF8(field) << "' was not present"); + THROW_HR(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE); + } + return std::move(optString).value(); + } + + void AddFieldIfNotEmpty(web::json::value& value, const utility::string_t& field, std::string_view string) + { + if (!string.empty()) + { + value[field] = AppInstaller::JSON::GetStringValue(string); + } + } + + web::json::value CreateStringArray(const std::vector& values) + { + web::json::value result; + size_t index = 0; + + for (const std::string& value : values) + { + result[index++] = AppInstaller::JSON::GetStringValue(value); + } + + return result; + } + + bool AddIfNotPresentAndNotEmpty(std::vector& strings, const std::vector& filter, const std::string& string) + { + if (string.empty() || std::find(filter.begin(), filter.end(), string) != filter.end()) + { + return false; + } + + strings.emplace_back(string); + return true; + } + + bool AddIfNotPresentAndNotEmpty(std::vector& strings, const std::string& string) + { + return AddIfNotPresentAndNotEmpty(strings, strings, string); + } + + void AddIfNotPresent(std::vector& strings, std::vector& filter, const std::vector& inputs) + { + for (const std::string& input : inputs) + { + if (AddIfNotPresentAndNotEmpty(strings, filter, input)) + { + filter.emplace_back(input); + } + } + } + } void ProductMetadata::Clear() { - m_version = {}; - m_productVersionMin = {}; - m_productVersionMax = {}; - m_installerMetadata.clear(); - m_historicalMetadata.clear(); + SchemaVersion = {}; + ProductVersionMin = {}; + ProductVersionMax = {}; + InstallerMetadataMap.clear(); + HistoricalMetadataList.clear(); } void ProductMetadata::FromJson(const web::json::value& json) @@ -32,98 +119,277 @@ namespace AppInstaller::Repository::Metadata THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, json.is_null()); - auto versionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, versionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); - - m_version = Version{ versionString.value() }; - AICLI_LOG(Core, Info, << "Parsing metadata JSON version " << m_version.ToString()); + SchemaVersion = Version{ GetRequiredString(json, versionFieldName) }; + AICLI_LOG(Repo, Info, << "Parsing metadata JSON version " << SchemaVersion.ToString()); - if (m_version.PartAt(0).Integer == 1) + if (SchemaVersion.PartAt(0).Integer == 1) { // We only have one version currently, so use that as long as the major version is 1 FromJson_1_0(json); } else { - AICLI_LOG(Core, Error, << "Don't know how to handle metadata version " << m_version.ToString()); + AICLI_LOG(Repo, Error, << "Don't know how to handle metadata version " << SchemaVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + + // Sort the historical data with oldest last (thus b < a) + std::sort(HistoricalMetadataList.begin(), HistoricalMetadataList.end(), + [](const HistoricalMetadata& a, const HistoricalMetadata& b) { + return b.ProductVersionMin < a.ProductVersionMin; + }); + } + + web::json::value ProductMetadata::ToJson(const Utility::Version& version, size_t maximumSizeInBytes) + { + AICLI_LOG(Repo, Info, << "Creating metadata JSON version " << version.ToString()); + + using ToJsonFunctionPointer = web::json::value(ProductMetadata::*)(); + ToJsonFunctionPointer toJsonFunction = nullptr; + + if (version.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + toJsonFunction = &ProductMetadata::ToJson_1_0; + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to handle metadata version " << version.ToString()); THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); } + + // Constrain the result based on maximum size given + web::json::value result = (this->*toJsonFunction)(); + + while (maximumSizeInBytes) + { + // Determine current size + std::ostringstream temp; + result.serialize(temp); + + std::string tempStr = temp.str(); + if (tempStr.length() > maximumSizeInBytes) + { + if (!DropOldestHistoricalData()) + { + AICLI_LOG(Repo, Error, << "Could not remove any more historical data to get under " << maximumSizeInBytes << " bytes"); + AICLI_LOG(Repo, Info, << " Smallest size was " << tempStr.length() << " bytes with value:\n" << tempStr); + THROW_HR(HRESULT_FROM_WIN32(ERROR_FILE_TOO_LARGE)); + } + result = (this->*toJsonFunction)(); + } + else + { + break; + } + } + + return result; + } + + void ProductMetadata::CopyFrom(const ProductMetadata& source, std::string_view submissionIdentifier) + { + // If the source has no installer metadata, consider it empty + if (source.InstallerMetadataMap.empty()) + { + return; + } + + // With the same submission, just copy over all of the data + if (source.InstallerMetadataMap.begin()->second.SubmissionIdentifier == submissionIdentifier) + { + *this = source; + return; + } + + // This is a new submission, so we must move all of the data to historical and update the older historical data + // First, create a new historical entry for the current metadata + HistoricalMetadata currentHistory; + + currentHistory.ProductVersionMin = source.ProductVersionMin; + currentHistory.ProductVersionMax = source.ProductVersionMax; + + for (const auto& metadataItem : source.InstallerMetadataMap) + { + for (const auto& entry : metadataItem.second.AppsAndFeaturesEntries) + { + AddIfNotPresentAndNotEmpty(currentHistory.Names, entry.DisplayName); + AddIfNotPresentAndNotEmpty(currentHistory.Publishers, entry.Publisher); + AddIfNotPresentAndNotEmpty(currentHistory.ProductCodes, entry.ProductCode); + AddIfNotPresentAndNotEmpty(currentHistory.UpgradeCodes, entry.UpgradeCode); + } + } + + // Copy the data in so that we can continue using currentHistory to track all strings + HistoricalMetadataList.emplace_back(currentHistory); + + // Now, copy over the other historical data, filtering out anything we have seen + for (const auto& historical : source.HistoricalMetadataList) + { + HistoricalMetadata copied; + copied.ProductVersionMin = historical.ProductVersionMin; + copied.ProductVersionMax = historical.ProductVersionMax; + AddIfNotPresent(copied.Names, currentHistory.Names, historical.Names); + AddIfNotPresent(copied.Publishers, currentHistory.Publishers, historical.Publishers); + AddIfNotPresent(copied.ProductCodes, currentHistory.ProductCodes, historical.ProductCodes); + AddIfNotPresent(copied.UpgradeCodes, currentHistory.UpgradeCodes, historical.UpgradeCodes); + + if (!copied.Names.empty() || !copied.Publishers.empty() || !copied.ProductCodes.empty() || !copied.UpgradeCodes.empty()) + { + HistoricalMetadataList.emplace_back(std::move(copied)); + } + } } void ProductMetadata::FromJson_1_0(const web::json::value& json) { - AICLI_LOG(Core, Info, << "Parsing metadata JSON 1.0 fields"); + AICLI_LOG(Repo, Info, << "Parsing metadata JSON 1.0 fields"); - // Field names - utility::string_t productVersionMinFieldName = L"productVersionMin"; - utility::string_t productVersionMaxFieldName = L"productVersionMax"; - utility::string_t metadataFieldName = L"metadata"; - utility::string_t installerHashFieldName = L"installerHash"; - utility::string_t submissionIdentifierFieldName = L"submissionIdentifier"; - utility::string_t versionFieldName = L"version"; - utility::string_t appsAndFeaturesFieldName = L"AppsAndFeaturesEntries"; - utility::string_t historicalFieldName = L"historical"; + ProductMetadataFields_1_0 fields; - auto productVersionMinString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, productVersionMinFieldName); + auto productVersionMinString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, fields.ProductVersionMin); if (productVersionMinString) { - m_productVersionMin = Version{ productVersionMinString.value() }; + ProductVersionMin = Version{ std::move(productVersionMinString).value() }; } - auto productVersionMaxString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, productVersionMaxFieldName); + auto productVersionMaxString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, fields.ProductVersionMax); if (productVersionMaxString) { - m_productVersionMax = Version{ productVersionMaxString.value() }; + ProductVersionMax = Version{ std::move(productVersionMaxString).value() }; } // The 1.0 version of metadata uses the 1.1 version of REST JSON::ManifestJSONParser parser{ Version{ "1.1" } }; - auto metadataArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, metadataFieldName); + std::string submissionIdentifierVerification; + + auto metadataArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, fields.Metadata); if (metadataArray) { for (const auto& item : metadataArray->get()) { - auto installerHashString = AppInstaller::JSON::GetRawStringValueFromJsonNode(item, installerHashFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !installerHashString); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, m_installerMetadata.find(installerHashString.value()) != m_installerMetadata.end()); + std::string installerHashString = GetRequiredString(item, fields.InstallerHash); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, InstallerMetadataMap.find(installerHashString) != InstallerMetadataMap.end()); InstallerMetadata installerMetadata; - auto submissionIdentifierString = AppInstaller::JSON::GetRawStringValueFromJsonNode(item, submissionIdentifierFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !submissionIdentifierString); - installerMetadata.SubmissionIdentifier = submissionIdentifierString.value(); - - auto versionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(item, versionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); - installerMetadata.ProductVersion = Version{ versionString.value() }; + installerMetadata.SubmissionIdentifier = GetRequiredString(item, fields.SubmissionIdentifier); + if (submissionIdentifierVerification.empty()) + { + submissionIdentifierVerification = installerMetadata.SubmissionIdentifier; + } + else if (submissionIdentifierVerification != installerMetadata.SubmissionIdentifier) + { + AICLI_LOG(Repo, Error, << "Different submission identifiers found in metadata: '" << + submissionIdentifierVerification << "' and '" << installerMetadata.SubmissionIdentifier << "'"); + THROW_HR(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE); + } - auto appsAndFeatures = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(item, appsAndFeaturesFieldName); + auto appsAndFeatures = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(item, fields.AppsAndFeaturesEntries); THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures); installerMetadata.AppsAndFeaturesEntries = parser.DeserializeAppsAndFeaturesEntries(appsAndFeatures.value()); - m_installerMetadata[installerHashString.value()] = std::move(installerMetadata); + InstallerMetadataMap[installerHashString] = std::move(installerMetadata); } } - auto historicalArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, historicalFieldName); + auto historicalArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, fields.Historical); if (historicalArray) { for (const auto& item : historicalArray->get()) { HistoricalMetadata historicalMetadata; - auto versionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(item, versionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); - historicalMetadata.ProductVersion = Version{ versionString.value() }; + historicalMetadata.ProductVersionMin = Version{ GetRequiredString(item, fields.VersionMin) }; + historicalMetadata.ProductVersionMax = Version{ GetRequiredString(item, fields.VersionMax) }; + historicalMetadata.Names = AppInstaller::JSON::GetRawStringArrayFromJsonNode(item, fields.Names); + historicalMetadata.Publishers = AppInstaller::JSON::GetRawStringArrayFromJsonNode(item, fields.Publishers); + historicalMetadata.ProductCodes = AppInstaller::JSON::GetRawStringArrayFromJsonNode(item, fields.ProductCodes); + historicalMetadata.UpgradeCodes = AppInstaller::JSON::GetRawStringArrayFromJsonNode(item, fields.UpgradeCodes); - auto appsAndFeatures = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(item, appsAndFeaturesFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures); - historicalMetadata.AppsAndFeaturesEntries = parser.DeserializeAppsAndFeaturesEntries(appsAndFeatures.value()); + HistoricalMetadataList.emplace_back(std::move(historicalMetadata)); + } + } + } + + web::json::value ProductMetadata::ToJson_1_0() + { + AICLI_LOG(Repo, Info, << "Creating metadata JSON 1.0 fields"); + + ProductMetadataFields_1_0 fields; + + web::json::value result; + + result[fields.Version] = web::json::value::string(L"1.0"); + result[fields.ProductVersionMin] = AppInstaller::JSON::GetStringValue(ProductVersionMin.ToString()); + result[fields.ProductVersionMax] = AppInstaller::JSON::GetStringValue(ProductVersionMax.ToString()); + + web::json::value metadataArray = web::json::value::array(); + size_t metadataItemIndex = 0; + for (const auto& item : InstallerMetadataMap) + { + web::json::value itemValue; + + itemValue[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(item.first); + itemValue[fields.SubmissionIdentifier] = AppInstaller::JSON::GetStringValue(item.second.SubmissionIdentifier); - m_historicalMetadata.emplace_back(std::move(historicalMetadata)); + web::json::value appsAndFeaturesArray = web::json::value::array(); + size_t appsAndFeaturesEntryIndex = 0; + for (const auto& entry : item.second.AppsAndFeaturesEntries) + { + web::json::value entryValue; + + AddFieldIfNotEmpty(itemValue, fields.DisplayName, entry.DisplayName); + AddFieldIfNotEmpty(itemValue, fields.Publisher, entry.Publisher); + AddFieldIfNotEmpty(itemValue, fields.DisplayVersion, entry.DisplayVersion); + AddFieldIfNotEmpty(itemValue, fields.ProductCode, entry.ProductCode); + AddFieldIfNotEmpty(itemValue, fields.UpgradeCode, entry.UpgradeCode); + if (entry.InstallerType != Manifest::InstallerTypeEnum::Unknown) + { + itemValue[fields.InstallerType] = AppInstaller::JSON::GetStringValue(Manifest::InstallerTypeToString(entry.InstallerType)); + } + + appsAndFeaturesArray[appsAndFeaturesEntryIndex++] = std::move(entryValue); } + + itemValue[fields.AppsAndFeaturesEntries] = std::move(appsAndFeaturesArray); + + metadataArray[metadataItemIndex++] = std::move(itemValue); } + + result[fields.AppsAndFeaturesEntries] = std::move(metadataArray); + + web::json::value historicalArray = web::json::value::array(); + size_t historicalItemIndex = 0; + for (const auto& item : HistoricalMetadataList) + { + web::json::value itemValue; + + itemValue[fields.VersionMin] = AppInstaller::JSON::GetStringValue(item.ProductVersionMin.ToString()); + itemValue[fields.VersionMax] = AppInstaller::JSON::GetStringValue(item.ProductVersionMax.ToString()); + itemValue[fields.Names] = CreateStringArray(item.Names); + itemValue[fields.Publishers] = CreateStringArray(item.Publishers); + itemValue[fields.ProductCodes] = CreateStringArray(item.ProductCodes); + itemValue[fields.UpgradeCodes] = CreateStringArray(item.UpgradeCodes); + + historicalArray[historicalItemIndex++] = std::move(itemValue); + } + + result[fields.Historical] = std::move(historicalArray); + + return result; + } + + bool ProductMetadata::DropOldestHistoricalData() + { + if (HistoricalMetadataList.empty()) + { + return false; + } + + HistoricalMetadataList.pop_back(); + return true; } std::unique_ptr InstallerMetadataCollectionContext::FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile) @@ -134,7 +400,7 @@ namespace AppInstaller::Repository::Metadata std::unique_ptr result = std::make_unique(); auto threadGlobalsLifetime = result->InitializeLogging(logFile); - AICLI_LOG(Core, Info, << "Opening InstallerMetadataCollectionContext input file: " << file); + AICLI_LOG(Repo, Info, << "Opening InstallerMetadataCollectionContext input file: " << file); std::ifstream fileStream{ file }; result->InitializePreinstallState(ConvertToUTF16(ReadEntireStream(fileStream))); @@ -152,7 +418,7 @@ namespace AppInstaller::Repository::Metadata std::string utf8Uri = ConvertToUTF8(uri); THROW_HR_IF(E_INVALIDARG, !IsUrlRemote(utf8Uri)); - AICLI_LOG(Core, Info, << "Downloading InstallerMetadataCollectionContext input file: " << utf8Uri); + AICLI_LOG(Repo, Info, << "Downloading InstallerMetadataCollectionContext input file: " << utf8Uri); std::ostringstream jsonStream; ProgressCallback emptyCallback; @@ -171,7 +437,7 @@ namespace AppInstaller::Repository::Metadata { if (retryCount < MaxRetryCount - 1) { - AICLI_LOG(Core, Info, << " Downloading InstallerMetadataCollectionContext input failed, waiting a bit and retrying..."); + AICLI_LOG(Repo, Info, << " Downloading InstallerMetadataCollectionContext input failed, waiting a bit and retrying..."); Sleep(500); } else @@ -206,15 +472,39 @@ namespace AppInstaller::Repository::Metadata { auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); - THROW_HR_IF(E_INVALIDARG, output.empty()); + THROW_HR_IF(E_INVALIDARG, !output.has_filename()); // Collect post-install system state + m_correlationData.CapturePostInstallSnapshot(); - // Compute metadata match scores + ComputeOutputData(); + + // Construct output JSON + web::json::value outputJSON; + + AICLI_LOG(Repo, Info, << "Creating output JSON version for input version " << m_inputVersion.ToString()); + + if (m_inputVersion.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + outputJSON = CreateOutputJson_1_0(); + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to output for version " << m_inputVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } // Write output + if (output.has_parent_path()) + { + std::filesystem::create_directories(output.parent_path()); + } + + std::ofstream outputStream{ output }; + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_OPEN_FAILED), !outputStream); - // Write diagnostics + outputJSON.serialize(outputStream); } std::unique_ptr InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile) @@ -239,7 +529,7 @@ namespace AppInstaller::Repository::Metadata void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json) { - AICLI_LOG(Core, Verbose, << "Parsing input JSON:\n" << ConvertToUTF8(json)); + AICLI_LOG(Repo, Info, << "Parsing input JSON:\n" << ConvertToUTF8(json)); // Parse and validate JSON try @@ -250,26 +540,23 @@ namespace AppInstaller::Repository::Metadata THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null()); - auto versionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(inputValue, versionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !versionString); - - Version version{ versionString.value() }; - AICLI_LOG(Core, Info, << "Parsing input JSON version " << version.ToString()); + m_inputVersion = Version{ GetRequiredString(inputValue, versionFieldName) }; + AICLI_LOG(Repo, Info, << "Parsing input JSON version " << m_inputVersion.ToString()); - if (version.PartAt(0).Integer == 1) + if (m_inputVersion.PartAt(0).Integer == 1) { // We only have one version currently, so use that as long as the major version is 1 ParseInputJson_1_0(inputValue); } else { - AICLI_LOG(Core, Error, << "Don't know how to handle version " << version.ToString()); + AICLI_LOG(Repo, Error, << "Don't know how to handle version " << m_inputVersion.ToString()); THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); } } catch (const web::json::json_exception& exc) { - AICLI_LOG(Core, Error, << "Exception parsing input JSON: " << exc.what()); + AICLI_LOG(Repo, Error, << "Exception parsing input JSON: " << exc.what()); throw; } @@ -277,9 +564,19 @@ namespace AppInstaller::Repository::Metadata m_correlationData.CapturePreInstallSnapshot(); } + void InstallerMetadataCollectionContext::ComputeOutputData() + { + // Copy the metadata from the current; this function takes care of moving data to historical if the submission is new. + m_outputMetadata.CopyFrom(m_currentMetadata, m_submissionIdentifier); + + Correlation::ARPCorrelationResult correlationResult = m_correlationData.CorrelateForNewlyInstalled(m_incomingManifest); + + // TODO: Update min/max version, update metadata + } + void InstallerMetadataCollectionContext::ParseInputJson_1_0(web::json::value& input) { - AICLI_LOG(Core, Info, << "Parsing input JSON 1.0 fields"); + AICLI_LOG(Repo, Info, << "Parsing input JSON 1.0 fields"); // Field names utility::string_t metadataVersionFieldName = L"supportedMetadataVersion"; @@ -292,9 +589,7 @@ namespace AppInstaller::Repository::Metadata utility::string_t defaultLocaleFieldName = L"DefaultLocale"; utility::string_t localesFieldName = L"Locales"; - auto metadataVersionString = AppInstaller::JSON::GetRawStringValueFromJsonNode(input, metadataVersionFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !metadataVersionString); - m_supportedMetadataVersion = Version{ metadataVersionString.value() }; + m_supportedMetadataVersion = Version{ GetRequiredString(input, metadataVersionFieldName) }; auto metadataSizeNumber = AppInstaller::JSON::GetRawIntValueFromJsonNode(input, metadataSizeFieldName); if (metadataSizeNumber && metadataSizeNumber.value() > 0) @@ -312,13 +607,8 @@ namespace AppInstaller::Repository::Metadata m_currentMetadata.FromJson(currentMetadataValue.value()); } - auto submissionIdentifierString = AppInstaller::JSON::GetRawStringValueFromJsonNode(input, submissionIdentifierFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !submissionIdentifierString); - m_submissionIdentifier = submissionIdentifierString.value(); - - auto installerHashString = AppInstaller::JSON::GetRawStringValueFromJsonNode(input, installerHashFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !installerHashString); - m_installerHash = SHA256::ConvertToBytes(installerHashString.value()); + m_submissionIdentifier = GetRequiredString(input, submissionIdentifierFieldName); + m_installerHash = GetRequiredString(input, installerHashFieldName); // The 1.0 version of input uses the 1.1 version of REST JSON::ManifestJSONParser parser{ Version{ "1.1" }}; @@ -364,4 +654,47 @@ namespace AppInstaller::Repository::Metadata } } } + + web::json::value InstallerMetadataCollectionContext::CreateOutputJson_1_0() + { + AICLI_LOG(Repo, Info, << "Setting output JSON 1.0 fields"); + + // Field names + utility::string_t versionFieldName = L"version"; + utility::string_t installerHashFieldName = L"installerHash"; + utility::string_t statusFieldName = L"status"; + utility::string_t metadataFieldName = L"metadata"; + utility::string_t diagnosticsFieldName = L"diagnostics"; + + web::json::value result; + + result[versionFieldName] = web::json::value::string(L"1.0"); + result[installerHashFieldName] = AppInstaller::JSON::GetStringValue(m_installerHash); + + // Limit output status to 1.0 known values + OutputStatus statusToUse = m_outputStatus; + if (statusToUse != OutputStatus::Success && statusToUse != OutputStatus::Error && statusToUse != OutputStatus::LowConfidence) + { + statusToUse = OutputStatus::Unknown; + } + result[statusFieldName] = web::json::value::string(ToString(statusToUse)); + + result[metadataFieldName] = m_outputMetadata.ToJson(m_supportedMetadataVersion, m_maxMetadataSize); + result[diagnosticsFieldName] = m_outputDiagnostics; + + return result; + } + + utility::string_t InstallerMetadataCollectionContext::ToString(OutputStatus status) + { + switch (status) + { + case OutputStatus::Success: return L"Success"; + case OutputStatus::Error: return L"Error"; + case OutputStatus::LowConfidence: return L"LowConfidence"; + } + + // For both the status value of Unknown and anything else + return L"Unknown"; + } } diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index 2e0af58675..d1ac83f81c 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -3,7 +3,6 @@ #pragma once #include -#include #include #include #include @@ -29,30 +28,45 @@ namespace AppInstaller::Repository::Metadata // Load the metadata from an existing JSON blob. void FromJson(const web::json::value& json); + // Create a JSON value for the metadata using the given schema version. + web::json::value ToJson(const Utility::Version& version, size_t maximumSizeInBytes); + + // Copies the metadata from the source. If the given submission identifier does not match + // the source, it's data is moved to historical. + void CopyFrom(const ProductMetadata& source, std::string_view submissionIdentifier); + // The installer specific metadata that we collect. struct InstallerMetadata { std::string SubmissionIdentifier; - Utility::Version ProductVersion; std::vector AppsAndFeaturesEntries; }; // Metadata from previous product revisions. struct HistoricalMetadata { - Utility::Version ProductVersion; - std::vector AppsAndFeaturesEntries; + Utility::Version ProductVersionMin; + Utility::Version ProductVersionMax; + std::vector Names; + std::vector Publishers; + std::vector ProductCodes; + std::vector UpgradeCodes; }; + Utility::Version SchemaVersion; + Utility::Version ProductVersionMin; + Utility::Version ProductVersionMax; + // Map from installer hash to metadata + std::map InstallerMetadataMap; + std::vector HistoricalMetadataList; + private: void FromJson_1_0(const web::json::value& json); + web::json::value ToJson_1_0(); - Utility::Version m_version; - Utility::Version m_productVersionMin; - Utility::Version m_productVersionMax; - // Map from installer hash to metadata - std::map m_installerMetadata; - std::vector m_historicalMetadata; + // Removes the historical data with the oldest version. + // Returns true if something was removed; false it not. + bool DropOldestHistoricalData(); }; // Contains the functions and data used for collecting metadata from installers. @@ -81,20 +95,44 @@ namespace AppInstaller::Repository::Metadata // Sets the collection context input and the preinstall state. void InitializePreinstallState(const std::wstring& json); + // Creates the output ProductMetadata and diagnostics objects for output + void ComputeOutputData(); + // Parse version 1.0 of input JSON void ParseInputJson_1_0(web::json::value& input); + // Create version 1.0 of output JSON + web::json::value CreateOutputJson_1_0(); + ThreadLocalStorage::ThreadGlobals m_threadGlobals; // Parsed input + Utility::Version m_inputVersion; Utility::Version m_supportedMetadataVersion; size_t m_maxMetadataSize = 0; ProductMetadata m_currentMetadata; std::string m_submissionIdentifier; - Utility::SHA256::HashBuffer m_installerHash; + std::string m_installerHash; Manifest::Manifest m_currentManifest; Manifest::Manifest m_incomingManifest; Correlation::ARPCorrelationData m_correlationData; + + // Output data + enum class OutputStatus + { + // Version 1.0 status values + Unknown, + Success, + Error, + LowConfidence, + }; + + // Convert status to a JSON string value + static utility::string_t ToString(OutputStatus status); + + OutputStatus m_outputStatus = OutputStatus::Unknown; + ProductMetadata m_outputMetadata; + web::json::value m_outputDiagnostics; }; } From 6cf5105d4fbe49563418f895c4639fce430d0eea Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 26 May 2022 13:25:27 -0700 Subject: [PATCH 11/22] Code complete of initial version --- .github/actions/spelling/allow.txt | 1 + .../Public/AppInstallerVersions.h | 5 + .../InstallerMetadataCollectionContext.cpp | 293 ++++++++++++++---- .../InstallerMetadataCollectionContext.h | 22 +- 4 files changed, 266 insertions(+), 55 deletions(-) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 0ad1f5eb73..57e3261fde 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -109,6 +109,7 @@ denelon depersist deque deserialize +deserializes Deserialize deserializer deserializing diff --git a/src/AppInstallerCommonCore/Public/AppInstallerVersions.h b/src/AppInstallerCommonCore/Public/AppInstallerVersions.h index a8b34c0a67..a72cf678d6 100644 --- a/src/AppInstallerCommonCore/Public/AppInstallerVersions.h +++ b/src/AppInstallerCommonCore/Public/AppInstallerVersions.h @@ -58,6 +58,11 @@ namespace AppInstaller::Utility // Returns a Version that will return true for IsUnknown static Version CreateUnknown(); + // Gets a bool indicating whether the full version string is empty. + // Does not indicate that Parts is empty; for instance when "0.0" is given, + // this will be false while GetParts().empty() would be true. + bool IsEmpty() const { return m_version.empty(); } + // An individual version part in between split characters. struct Part { diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index 9cde31c078..296b7ef0f9 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -42,6 +42,18 @@ namespace AppInstaller::Repository::Metadata utility::string_t UpgradeCodes = L"upgradeCodes"; }; + struct OutputFields_1_0 + { + utility::string_t Version = L"version"; + utility::string_t InstallerHash = L"installerHash"; + utility::string_t Status = L"status"; + utility::string_t Metadata = L"metadata"; + utility::string_t Diagnostics = L"diagnostics"; + + utility::string_t ErrorHR = L"errorHR"; + utility::string_t ErrorText = L"errorText"; + }; + std::string GetRequiredString(const web::json::value& value, const utility::string_t& field) { auto optString = AppInstaller::JSON::GetRawStringValueFromJsonNode(value, field); @@ -474,37 +486,64 @@ namespace AppInstaller::Repository::Metadata THROW_HR_IF(E_INVALIDARG, !output.has_filename()); - // Collect post-install system state - m_correlationData.CapturePostInstallSnapshot(); + if (output.has_parent_path()) + { + std::filesystem::create_directories(output.parent_path()); + } - ComputeOutputData(); + std::ofstream outputStream{ output }; + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_OPEN_FAILED), !outputStream); - // Construct output JSON - web::json::value outputJSON; + CompleteWithThreadGlobalsSet(outputStream); + } - AICLI_LOG(Repo, Info, << "Creating output JSON version for input version " << m_inputVersion.ToString()); + void InstallerMetadataCollectionContext::Complete(std::ostream& output) + { + auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); + CompleteWithThreadGlobalsSet(output); + } - if (m_inputVersion.PartAt(0).Integer == 1) - { - // We only have one version currently, so use that as long as the major version is 1 - outputJSON = CreateOutputJson_1_0(); - } - else + void InstallerMetadataCollectionContext::CompleteWithThreadGlobalsSet(std::ostream& output) + { + web::json::value outputJSON; + + if (!ContainsError()) { - AICLI_LOG(Repo, Error, << "Don't know how to output for version " << m_inputVersion.ToString()); - THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + try + { + // Collect post-install system state + m_correlationData.CapturePostInstallSnapshot(); + + ComputeOutputData(); + + // Construct output JSON + AICLI_LOG(Repo, Info, << "Creating output JSON version for input version " << m_inputVersion.ToString()); + + if (m_inputVersion.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + outputJSON = CreateOutputJson_1_0(); + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to output for version " << m_inputVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + } + catch (...) + { + CollectErrorDataFromException(std::current_exception()); + } } - // Write output - if (output.has_parent_path()) + if (ContainsError()) { - std::filesystem::create_directories(output.parent_path()); + // We only have one version currently + outputJSON = CreateErrorJson_1_0(); } - std::ofstream outputStream{ output }; - THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_OPEN_FAILED), !outputStream); - - outputJSON.serialize(outputStream); + // Write output + outputJSON.serialize(output); } std::unique_ptr InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile) @@ -529,39 +568,46 @@ namespace AppInstaller::Repository::Metadata void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json) { - AICLI_LOG(Repo, Info, << "Parsing input JSON:\n" << ConvertToUTF8(json)); - - // Parse and validate JSON try { - utility::string_t versionFieldName = L"version"; + AICLI_LOG(Repo, Info, << "Parsing input JSON:\n" << ConvertToUTF8(json)); - web::json::value inputValue = web::json::value::parse(json); + // Parse and validate JSON + try + { + utility::string_t versionFieldName = L"version"; - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null()); + web::json::value inputValue = web::json::value::parse(json); - m_inputVersion = Version{ GetRequiredString(inputValue, versionFieldName) }; - AICLI_LOG(Repo, Info, << "Parsing input JSON version " << m_inputVersion.ToString()); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null()); - if (m_inputVersion.PartAt(0).Integer == 1) - { - // We only have one version currently, so use that as long as the major version is 1 - ParseInputJson_1_0(inputValue); + m_inputVersion = Version{ GetRequiredString(inputValue, versionFieldName) }; + AICLI_LOG(Repo, Info, << "Parsing input JSON version " << m_inputVersion.ToString()); + + if (m_inputVersion.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + ParseInputJson_1_0(inputValue); + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to handle version " << m_inputVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } } - else + catch (const web::json::json_exception& exc) { - AICLI_LOG(Repo, Error, << "Don't know how to handle version " << m_inputVersion.ToString()); - THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + AICLI_LOG(Repo, Error, << "Exception parsing input JSON: " << exc.what()); + throw; } + + // Collect pre-install system state + m_correlationData.CapturePreInstallSnapshot(); } - catch (const web::json::json_exception& exc) + catch (...) { - AICLI_LOG(Repo, Error, << "Exception parsing input JSON: " << exc.what()); - throw; + CollectErrorDataFromException(std::current_exception()); } - - // Collect pre-install system state - m_correlationData.CapturePreInstallSnapshot(); } void InstallerMetadataCollectionContext::ComputeOutputData() @@ -571,7 +617,93 @@ namespace AppInstaller::Repository::Metadata Correlation::ARPCorrelationResult correlationResult = m_correlationData.CorrelateForNewlyInstalled(m_incomingManifest); - // TODO: Update min/max version, update metadata + if (correlationResult.Package) + { + m_outputStatus = OutputStatus::Success; + auto& package = correlationResult.Package; + + // Update min and max versions based on the version of the correlated package + Version packageVersion{ package->GetProperty(PackageVersionProperty::Version) }; + + if (m_outputMetadata.ProductVersionMin.IsEmpty() || packageVersion < m_outputMetadata.ProductVersionMin) + { + m_outputMetadata.ProductVersionMin = packageVersion; + } + + if (m_outputMetadata.ProductVersionMax.IsEmpty() || m_outputMetadata.ProductVersionMax < packageVersion) + { + m_outputMetadata.ProductVersionMax = packageVersion; + } + + // Create the AppsAndFeaturesEntry that we need to add + Manifest::AppsAndFeaturesEntry newEntry; + auto packageMetadata = package->GetMetadata(); + + // TODO: Use some amount of normalization here to prevent things like versions being in the name from bloating the data + newEntry.DisplayName = package->GetProperty(PackageVersionProperty::Name).get(); + newEntry.DisplayVersion = packageVersion.ToString(); + if (packageMetadata.count(PackageVersionMetadata::InstalledType)) + { + newEntry.InstallerType = Manifest::ConvertToInstallerTypeEnum(packageMetadata[PackageVersionMetadata::InstalledType]); + } + auto productCodes = package->GetMultiProperty(PackageVersionMultiProperty::ProductCode); + if (!productCodes.empty()) + { + newEntry.ProductCode = std::move(productCodes[0]).get(); + } + newEntry.Publisher = package->GetProperty(PackageVersionProperty::Publisher).get(); + // TODO: Support upgrade code throughout the code base... + //newEntry.UpgradeCode; + + // Add or update the metadata for the installerhash + auto itr = m_outputMetadata.InstallerMetadataMap.find(m_installerHash); + + if (itr == m_outputMetadata.InstallerMetadataMap.end()) + { + // New entry needed + ProductMetadata::InstallerMetadata newMetadata; + + newMetadata.SubmissionIdentifier = m_submissionIdentifier; + newMetadata.AppsAndFeaturesEntries.emplace_back(std::move(newEntry)); + + m_outputMetadata.InstallerMetadataMap[m_installerHash] = std::move(newMetadata); + } + else + { + // Existing entry for installer hash, add/update the entry + auto& appsAndFeatures = itr->second.AppsAndFeaturesEntries; + + // Erase all duplicated data from the new entry + for (const auto& entry : appsAndFeatures) + { +#define WINGET_ERASE_IF_SAME(_value_) if (entry._value_ == newEntry._value_) { newEntry._value_.clear(); } + WINGET_ERASE_IF_SAME(DisplayName); + WINGET_ERASE_IF_SAME(DisplayVersion); + WINGET_ERASE_IF_SAME(ProductCode); + WINGET_ERASE_IF_SAME(Publisher); + WINGET_ERASE_IF_SAME(UpgradeCode); +#undef WINGET_ERASE_IF_SAME + + if (entry.InstallerType == newEntry.InstallerType) + { + newEntry.InstallerType = Manifest::InstallerTypeEnum::Unknown; + } + } + + // If anything remains, add it + if (!newEntry.DisplayName.empty() || !newEntry.DisplayVersion.empty() || !newEntry.ProductCode.empty() || + !newEntry.Publisher.empty() || !newEntry.UpgradeCode.empty() || newEntry.InstallerType != Manifest::InstallerTypeEnum::Unknown) + { + appsAndFeatures.emplace_back(std::move(newEntry)); + } + } + } + else + { + m_outputStatus = OutputStatus::LowConfidence; + + // TODO: Output diagnostics such as the top 10 entries by confidence. + } } void InstallerMetadataCollectionContext::ParseInputJson_1_0(web::json::value& input) @@ -659,17 +791,12 @@ namespace AppInstaller::Repository::Metadata { AICLI_LOG(Repo, Info, << "Setting output JSON 1.0 fields"); - // Field names - utility::string_t versionFieldName = L"version"; - utility::string_t installerHashFieldName = L"installerHash"; - utility::string_t statusFieldName = L"status"; - utility::string_t metadataFieldName = L"metadata"; - utility::string_t diagnosticsFieldName = L"diagnostics"; + OutputFields_1_0 fields; web::json::value result; - result[versionFieldName] = web::json::value::string(L"1.0"); - result[installerHashFieldName] = AppInstaller::JSON::GetStringValue(m_installerHash); + result[fields.Version] = web::json::value::string(L"1.0"); + result[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(m_installerHash); // Limit output status to 1.0 known values OutputStatus statusToUse = m_outputStatus; @@ -677,10 +804,10 @@ namespace AppInstaller::Repository::Metadata { statusToUse = OutputStatus::Unknown; } - result[statusFieldName] = web::json::value::string(ToString(statusToUse)); + result[fields.Status] = web::json::value::string(ToString(statusToUse)); - result[metadataFieldName] = m_outputMetadata.ToJson(m_supportedMetadataVersion, m_maxMetadataSize); - result[diagnosticsFieldName] = m_outputDiagnostics; + result[fields.Metadata] = m_outputMetadata.ToJson(m_supportedMetadataVersion, m_maxMetadataSize); + result[fields.Diagnostics] = m_outputDiagnostics; return result; } @@ -697,4 +824,62 @@ namespace AppInstaller::Repository::Metadata // For both the status value of Unknown and anything else return L"Unknown"; } + + bool InstallerMetadataCollectionContext::ContainsError() const + { + return m_outputStatus == OutputStatus::Error; + } + + void InstallerMetadataCollectionContext::CollectErrorDataFromException(std::exception_ptr exception) + { + m_outputStatus = OutputStatus::Error; + + try + { + std::rethrow_exception(exception); + } + catch (const wil::ResultException& re) + { + m_errorHR = re.GetErrorCode(); + m_errorText = GetUserPresentableMessage(re); + } + catch (const winrt::hresult_error& hre) + { + m_errorHR = hre.code(); + m_errorText = GetUserPresentableMessage(hre); + } + catch (const std::exception& e) + { + m_errorHR = E_FAIL; + m_errorText = GetUserPresentableMessage(e); + } + catch (...) + { + m_errorHR = E_UNEXPECTED; + m_errorText = "An unexpected exception type was thrown."; + } + } + + web::json::value InstallerMetadataCollectionContext::CreateErrorJson_1_0() + { + AICLI_LOG(Repo, Info, << "Setting error JSON 1.0 fields"); + + OutputFields_1_0 fields; + + web::json::value result; + + result[fields.Version] = web::json::value::string(L"1.0"); + result[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(m_installerHash); + result[fields.Status] = web::json::value::string(ToString(OutputStatus::Error)); + result[fields.Metadata] = web::json::value::null(); + + web::json::value error; + + error[fields.ErrorHR] = web::json::value::number(static_cast(m_errorHR)); + error[fields.ErrorText] = AppInstaller::JSON::GetStringValue(m_errorText); + + result[fields.Diagnostics] = std::move(error); + + return result; + } } diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index d1ac83f81c..f06838f5e7 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -85,9 +86,12 @@ namespace AppInstaller::Repository::Metadata static std::unique_ptr FromURI(std::wstring_view uri, const std::filesystem::path& logFile); static std::unique_ptr FromJSON(std::wstring_view json, const std::filesystem::path& logFile); - // Completes the collection, writing to the given locations. + // Completes the collection, writing to the given location. void Complete(const std::filesystem::path& output); + // Completes the collection, writing to the given location. + void Complete(std::ostream& output); + private: // Initializes the context runtime, including the log file if provided. std::unique_ptr InitializeLogging(const std::filesystem::path& logFile); @@ -98,12 +102,24 @@ namespace AppInstaller::Repository::Metadata // Creates the output ProductMetadata and diagnostics objects for output void ComputeOutputData(); + // Callers should set the thread globals before calling this. + void CompleteWithThreadGlobalsSet(std::ostream& output); + // Parse version 1.0 of input JSON void ParseInputJson_1_0(web::json::value& input); // Create version 1.0 of output JSON web::json::value CreateOutputJson_1_0(); + // Determines whether an error has occurred in the context. + bool ContainsError() const; + + // Collects information from the exception for error reporting. + void CollectErrorDataFromException(std::exception_ptr exception); + + // Create version 1.0 of error JSON + web::json::value CreateErrorJson_1_0(); + ThreadLocalStorage::ThreadGlobals m_threadGlobals; // Parsed input @@ -134,5 +150,9 @@ namespace AppInstaller::Repository::Metadata OutputStatus m_outputStatus = OutputStatus::Unknown; ProductMetadata m_outputMetadata; web::json::value m_outputDiagnostics; + + // Error data storage + HRESULT m_errorHR = S_OK; + std::string m_errorText; }; } From 851770cfbf504a8e3d185e81365f2398dcbf675e Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 26 May 2022 15:03:43 -0700 Subject: [PATCH 12/22] A few minor tests and the method to enable detailed testing of the context --- src/AppInstallerCLITests/Versions.cpp | 21 +++++++++++++++++++ .../InstallerMetadataCollectionContext.cpp | 16 ++++++++++---- .../Public/winget/ARPCorrelation.h | 3 ++- .../InstallerMetadataCollectionContext.h | 5 +++-- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/AppInstallerCLITests/Versions.cpp b/src/AppInstallerCLITests/Versions.cpp index 39d91d61a9..554e204281 100644 --- a/src/AppInstallerCLITests/Versions.cpp +++ b/src/AppInstallerCLITests/Versions.cpp @@ -191,3 +191,24 @@ TEST_CASE("VersionUnknownLessThanLatest", "[versions]") { REQUIRE(Version::CreateUnknown() < Version::CreateLatest()); } + +TEST_CASE("VersionIsEmpty", "[versions]") +{ + REQUIRE(Version{}.IsEmpty()); + REQUIRE(Version{""}.IsEmpty()); + REQUIRE(!Version{"1"}.IsEmpty()); + REQUIRE(!Version{"0"}.IsEmpty()); + + Version v{ "1" }; + REQUIRE(!v.IsEmpty()); + v.Assign(""); + REQUIRE(v.IsEmpty()); +} + +TEST_CASE("VersionPartAt", "[versions]") +{ + REQUIRE(Version{}.PartAt(0).Integer == 0); + REQUIRE(Version{"1"}.PartAt(0).Integer == 1); + REQUIRE(Version{"1"}.PartAt(1).Integer == 0); + REQUIRE(Version{"1"}.PartAt(9999).Integer == 0); +} diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index 296b7ef0f9..9b92b72363 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -404,6 +404,14 @@ namespace AppInstaller::Repository::Metadata return true; } + InstallerMetadataCollectionContext::InstallerMetadataCollectionContext() : + m_correlationData(std::make_unique()) + {} + + InstallerMetadataCollectionContext::InstallerMetadataCollectionContext(std::unique_ptr correlationData) : + m_correlationData(std::move(correlationData)) + {} + std::unique_ptr InstallerMetadataCollectionContext::FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile) { THROW_HR_IF(E_INVALIDARG, file.empty()); @@ -512,7 +520,7 @@ namespace AppInstaller::Repository::Metadata try { // Collect post-install system state - m_correlationData.CapturePostInstallSnapshot(); + m_correlationData->CapturePostInstallSnapshot(); ComputeOutputData(); @@ -602,7 +610,7 @@ namespace AppInstaller::Repository::Metadata } // Collect pre-install system state - m_correlationData.CapturePreInstallSnapshot(); + m_correlationData->CapturePreInstallSnapshot(); } catch (...) { @@ -615,7 +623,7 @@ namespace AppInstaller::Repository::Metadata // Copy the metadata from the current; this function takes care of moving data to historical if the submission is new. m_outputMetadata.CopyFrom(m_currentMetadata, m_submissionIdentifier); - Correlation::ARPCorrelationResult correlationResult = m_correlationData.CorrelateForNewlyInstalled(m_incomingManifest); + Correlation::ARPCorrelationResult correlationResult = m_correlationData->CorrelateForNewlyInstalled(m_incomingManifest); if (correlationResult.Package) { @@ -655,7 +663,7 @@ namespace AppInstaller::Repository::Metadata // TODO: Support upgrade code throughout the code base... //newEntry.UpgradeCode; - // Add or update the metadata for the installerhash + // Add or update the metadata for the installer hash auto itr = m_outputMetadata.InstallerMetadataMap.find(m_installerHash); if (itr == m_outputMetadata.InstallerMetadataMap.end()) diff --git a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h index 2e683c516b..e2373fc932 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h +++ b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h @@ -83,6 +83,7 @@ namespace AppInstaller::Repository::Correlation struct ARPCorrelationData { ARPCorrelationData() = default; + virtual ~ARPCorrelationData() = default; // Captures the ARP state before the package installation. void CapturePreInstallSnapshot(); @@ -91,7 +92,7 @@ namespace AppInstaller::Repository::Correlation void CapturePostInstallSnapshot(); // Correlates the given manifest against the data previously collected with capture calls. - ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest& manifest); + virtual ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest& manifest); const std::vector& GetPreInstallSnapshot() const { return m_preInstallSnapshot; } diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index f06838f5e7..94bb6358b5 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -73,7 +73,8 @@ namespace AppInstaller::Repository::Metadata // Contains the functions and data used for collecting metadata from installers. struct InstallerMetadataCollectionContext { - InstallerMetadataCollectionContext() = default; + InstallerMetadataCollectionContext(); + InstallerMetadataCollectionContext(std::unique_ptr correlationData); InstallerMetadataCollectionContext(const InstallerMetadataCollectionContext&) = delete; InstallerMetadataCollectionContext& operator=(const InstallerMetadataCollectionContext&) = delete; @@ -132,7 +133,7 @@ namespace AppInstaller::Repository::Metadata Manifest::Manifest m_currentManifest; Manifest::Manifest m_incomingManifest; - Correlation::ARPCorrelationData m_correlationData; + std::unique_ptr m_correlationData; // Output data enum class OutputStatus From 397368bda7ed2d171ff56ae4b9f5dcd2902de355 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Tue, 31 May 2022 17:45:12 -0700 Subject: [PATCH 13/22] Beginnings of full tests --- .../AppInstallerCLITests.vcxproj | 3 +- .../AppInstallerCLITests.vcxproj.filters | 3 + .../InstallerMetadataCollectionContext.cpp | 96 +++++++++++++++++++ .../InstallerMetadataCollectionContext.cpp | 11 ++- .../InstallerMetadataCollectionContext.h | 4 +- 5 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj index 1504bb03ed..3c3b5d03ea 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -196,6 +196,7 @@ + @@ -560,7 +561,7 @@ true - + true diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters index 092f901c9d..55e48f5b75 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -191,6 +191,9 @@ Source Files + + Source Files + diff --git a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp new file mode 100644 index 0000000000..04df943b71 --- /dev/null +++ b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestCommon.h" + +#include + +using namespace AppInstaller; +using namespace AppInstaller::Repository; +using namespace AppInstaller::Repository::Correlation; +using namespace AppInstaller::Repository::Metadata; + +namespace +{ + struct MinimalDefaults {}; + + struct TestInput + { + TestInput() = default; + + TestInput(MinimalDefaults) + { + + } + + std::optional Version; + std::optional SupportedMetadataVersion; + std::optional MaximumMetadataSize; + std::optional CurrentMetadata; + std::optional SubmissionIdentifier; + std::optional InstallerHash; + // Not currently used + // std::optional CurrentManifest; + // Schema 1.0 only cares about DefaultLocale and Locales + std::optional SubmissionData; + + std::wstring ToJSON() const + { + // TODO: Implement + } + }; + + struct TestOutput + { + TestOutput(std::string_view json) + { + // TODO: Implement + } + + std::optional Version; + std::optional InstallerHash; + std::optional Status; + std::optional Metadata; + std::optional ErrorHR; + std::optional ErrorText; + }; + + struct TestARPCorrelationData : public ARPCorrelationData + { + ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest&) override + { + return CorrelateForNewlyInstalledResult; + } + + ARPCorrelationResult CorrelateForNewlyInstalledResult; + }; + + InstallerMetadataCollectionContext CreateTestContext(std::unique_ptr&& data, const TestInput& input) + { + return { std::move(data), input.ToJSON() }; + } +} + +TEST_CASE("BadInput", "[metadata_collection]") +{ +} + +TEST_CASE("MinimumInput", "[metadata_collection]") +{ +} + +TEST_CASE("LowConfidence", "[metadata_collection]") +{ +} + +TEST_CASE("SameSubmission_SameInstaller", "[metadata_collection]") +{ +} + +TEST_CASE("SameSubmission_NewInstaller", "[metadata_collection]") +{ +} + +TEST_CASE("NewSubmission", "[metadata_collection]") +{ +} diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index 9b92b72363..a8d7fec473 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -408,9 +408,12 @@ namespace AppInstaller::Repository::Metadata m_correlationData(std::make_unique()) {} - InstallerMetadataCollectionContext::InstallerMetadataCollectionContext(std::unique_ptr correlationData) : + InstallerMetadataCollectionContext::InstallerMetadataCollectionContext(std::unique_ptr correlationData, const std::wstring& json) : m_correlationData(std::move(correlationData)) - {} + { + auto threadGlobalsLifetime = InitializeLogging({}); + InitializePreinstallState(json); + } std::unique_ptr InstallerMetadataCollectionContext::FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile) { @@ -477,13 +480,13 @@ namespace AppInstaller::Repository::Metadata return result; } - std::unique_ptr InstallerMetadataCollectionContext::FromJSON(std::wstring_view json, const std::filesystem::path& logFile) + std::unique_ptr InstallerMetadataCollectionContext::FromJSON(const std::wstring& json, const std::filesystem::path& logFile) { THROW_HR_IF(E_INVALIDARG, json.empty()); std::unique_ptr result = std::make_unique(); auto threadGlobalsLifetime = result->InitializeLogging(logFile); - result->InitializePreinstallState(std::wstring{ json }); + result->InitializePreinstallState(json); return result; } diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index 94bb6358b5..63ead606bc 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -74,7 +74,7 @@ namespace AppInstaller::Repository::Metadata struct InstallerMetadataCollectionContext { InstallerMetadataCollectionContext(); - InstallerMetadataCollectionContext(std::unique_ptr correlationData); + InstallerMetadataCollectionContext(std::unique_ptr correlationData, const std::wstring& json); InstallerMetadataCollectionContext(const InstallerMetadataCollectionContext&) = delete; InstallerMetadataCollectionContext& operator=(const InstallerMetadataCollectionContext&) = delete; @@ -85,7 +85,7 @@ namespace AppInstaller::Repository::Metadata // Create from various forms of JSON input to prevent type collisions on constructor. static std::unique_ptr FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile); static std::unique_ptr FromURI(std::wstring_view uri, const std::filesystem::path& logFile); - static std::unique_ptr FromJSON(std::wstring_view json, const std::filesystem::path& logFile); + static std::unique_ptr FromJSON(const std::wstring& json, const std::filesystem::path& logFile); // Completes the collection, writing to the given location. void Complete(const std::filesystem::path& output); From 113d32ef3db89b29f3a9177678fdd18492633b8c Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Fri, 3 Jun 2022 14:57:33 -0700 Subject: [PATCH 14/22] Test progress --- .../InstallerMetadataCollectionContext.cpp | 256 +++++++++++++++++- 1 file changed, 246 insertions(+), 10 deletions(-) diff --git a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp index 04df943b71..5123862f8a 100644 --- a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "TestCommon.h" +#include "TestSource.h" #include @@ -9,18 +10,27 @@ using namespace AppInstaller; using namespace AppInstaller::Repository; using namespace AppInstaller::Repository::Correlation; using namespace AppInstaller::Repository::Metadata; +using namespace TestCommon; namespace { - struct MinimalDefaults {}; + // Indicates to set minimal defaults on TestInput + struct MinimalDefaults_t {} MinimalDefaults; struct TestInput { TestInput() = default; - TestInput(MinimalDefaults) + TestInput(MinimalDefaults_t) { - + Version = "1.0"; + SupportedMetadataVersion = "1.0"; + SubmissionIdentifier = "1"; + InstallerHash = "ABCD"; + SubmissionData = std::make_optional(); + SubmissionData->DefaultLocalization.Locale = "en-us"; + SubmissionData->DefaultLocalization.Add("Name"); + SubmissionData->DefaultLocalization.Add("Publisher"); } std::optional Version; @@ -34,17 +44,123 @@ namespace // Schema 1.0 only cares about DefaultLocale and Locales std::optional SubmissionData; - std::wstring ToJSON() const + std::wstring ToJSON() + { + web::json::value json; + + if (Version) + { + json[L"version"] = JSON::GetStringValue(Version.value()); + } + + if (SupportedMetadataVersion) + { + json[L"supportedMetadataVersion"] = JSON::GetStringValue(SupportedMetadataVersion.value()); + } + + if (MaximumMetadataSize) + { + json[L"maximumMetadataSize"] = MaximumMetadataSize.value(); + } + + if (CurrentMetadata) + { + json[L"currentMetadata"] = CurrentMetadata->ToJson(Utility::Version{ "1.0" }, 0); + } + + if (SubmissionIdentifier) + { + json[L"submissionIdentifier"] = JSON::GetStringValue(SubmissionIdentifier.value()); + } + + if (InstallerHash) + { + json[L"installerHash"] = JSON::GetStringValue(InstallerHash.value()); + } + + if (SubmissionData) + { + web::json::value submissionData; + + submissionData[L"DefaultLocale"] = LocaleToJSON(SubmissionData->DefaultLocalization); + + // TODO: Implement other locales + + json[L"submissionData"] = std::move(submissionData); + } + + return json.serialize(); + } + + private: + web::json::value LocaleToJSON(const Manifest::ManifestLocalization& localization) const { - // TODO: Implement + web::json::value locale; + + locale[L"PackageLocale"] = JSON::GetStringValue(localization.Locale); + + if (localization.Contains(Manifest::Localization::PackageName)) + { + locale[L"PackageName"] = JSON::GetStringValue(localization.Get()); + } + + if (localization.Contains(Manifest::Localization::Publisher)) + { + locale[L"Publisher"] = JSON::GetStringValue(localization.Get()); + } + + // TODO: Implement any other needed fields + + return locale; } }; struct TestOutput { - TestOutput(std::string_view json) + TestOutput(const std::string& json) { - // TODO: Implement + web::json::value input = web::json::value::parse(Utility::ConvertToUTF16(json)); + + auto versionString = JSON::GetRawStringValueFromJsonNode(input, L"version"); + if (versionString) + { + Version = std::move(versionString); + } + + auto installerHashString = JSON::GetRawStringValueFromJsonNode(input, L"installerHash"); + if (installerHashString) + { + InstallerHash = std::move(installerHashString); + } + + auto statusString = JSON::GetRawStringValueFromJsonNode(input, L"status"); + if (statusString) + { + Status = std::move(statusString); + } + + auto metadataValue = JSON::GetJsonValueFromNode(input, L"metadata"); + if (metadataValue) + { + Metadata = std::make_optional(); + Metadata->FromJson(metadataValue->get()); + } + + auto diagnosticsValue = JSON::GetJsonValueFromNode(input, L"diagnostics"); + if (diagnosticsValue) + { + auto errorHRNumber = JSON::GetRawIntValueFromJsonNode(input, L"errorHR"); + if (errorHRNumber) + { + ErrorHR = std::move(errorHRNumber); + } + + auto errorTextString = JSON::GetRawStringValueFromJsonNode(input, L"errorText"); + if (errorTextString) + { + ErrorText = std::move(errorTextString); + } + } } std::optional Version; @@ -53,6 +169,33 @@ namespace std::optional Metadata; std::optional ErrorHR; std::optional ErrorText; + + bool IsError() const + { + return Status && Status.value() == "Error"; + } + + bool IsSuccess() const + { + return Status && Status.value() == "Success"; + } + + bool IsLowConfidence() const + { + return Status && Status.value() == "LowConfidence"; + } + + void ValidateFieldPresence() const + { + REQUIRE(Version); + REQUIRE(InstallerHash); + REQUIRE(Status); + + REQUIRE(IsSuccess() == Metadata.has_value()); + + REQUIRE(IsError() == ErrorHR.has_value()); + REQUIRE(IsError() == ErrorText.has_value()); + } }; struct TestARPCorrelationData : public ARPCorrelationData @@ -65,22 +208,115 @@ namespace ARPCorrelationResult CorrelateForNewlyInstalledResult; }; - InstallerMetadataCollectionContext CreateTestContext(std::unique_ptr&& data, const TestInput& input) + InstallerMetadataCollectionContext CreateTestContext(std::unique_ptr&& data, TestInput& input) { return { std::move(data), input.ToJSON() }; } + + InstallerMetadataCollectionContext CreateTestContext(TestInput& input) + { + return CreateTestContext(std::make_unique(), input); + } + + TestOutput GetOutput(InstallerMetadataCollectionContext& context) + { + std::ostringstream strstr; + context.Complete(strstr); + + return { strstr.str() }; + } + + TestOutput GetOutput(TestInput& input) + { + InstallerMetadataCollectionContext context = CreateTestContext(input); + return GetOutput(context); + } + + void BadInputTest(TestInput& input) + { + TestOutput output = GetOutput(input); + REQUIRE(output.IsError()); + output.ValidateFieldPresence(); + } } -TEST_CASE("BadInput", "[metadata_collection]") +TEST_CASE("MinimumInput", "[metadata_collection]") { + TestInput input(MinimalDefaults); + TestOutput output = GetOutput(input); + REQUIRE(!output.IsError()); + output.ValidateFieldPresence(); + + REQUIRE(output.Version.value() == input.Version.value()); + REQUIRE(output.InstallerHash.value() == input.InstallerHash.value()); } -TEST_CASE("MinimumInput", "[metadata_collection]") +TEST_CASE("BadInput", "[metadata_collection]") { + TestInput input(MinimalDefaults); + +#define RESET_FIELD_SECTION(_field_) \ + SECTION("No " #_field_) \ + { \ + input._field_.reset(); \ + BadInputTest(input); \ + } + + RESET_FIELD_SECTION(Version); + RESET_FIELD_SECTION(SupportedMetadataVersion); + RESET_FIELD_SECTION(SubmissionIdentifier); + RESET_FIELD_SECTION(InstallerHash); + RESET_FIELD_SECTION(SubmissionData); + +#undef RESET_FIELD_SECTION } TEST_CASE("LowConfidence", "[metadata_collection]") { + TestInput input(MinimalDefaults); + // The default test correlation object won't have a package correlation set + TestOutput output = GetOutput(input); + REQUIRE(output.IsLowConfidence()); + REQUIRE(!output.Metadata); + output.ValidateFieldPresence(); +} + +TEST_CASE("NewPackage", "[metadata_collection]") +{ + TestInput input(MinimalDefaults); + auto correlationData = std::make_unique(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add("Test Package Name"); + manifest.DefaultLocalization.Add("Test Publisher"); + manifest.Version = "1.2.3"; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = "{GUID}"; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(Manifest::InstallerTypeEnum::Msi); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared(manifest, metadata); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->ProductVersionMin.ToString() == output.Metadata->ProductVersionMax.ToString()); + REQUIRE(output.Metadata->ProductVersionMin.ToString() == manifest.Version); + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 1); + REQUIRE(output.Metadata->InstallerMetadataMap.count(input.InstallerHash.value()) == 1); + const auto& entry = output.Metadata->InstallerMetadataMap[input.InstallerHash.value()]; + REQUIRE(entry.SubmissionIdentifier == input.SubmissionIdentifier.value()); + REQUIRE(entry.AppsAndFeaturesEntries.size() == 1); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayName == manifest.DefaultLocalization.Get()); + REQUIRE(entry.AppsAndFeaturesEntries[0].Publisher == manifest.DefaultLocalization.Get()); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayVersion == manifest.Version); + REQUIRE(entry.AppsAndFeaturesEntries[0].ProductCode == manifest.Installers[0].ProductCode); + REQUIRE(entry.AppsAndFeaturesEntries[0].InstallerType == Manifest::InstallerTypeEnum::Msi); + REQUIRE(output.Metadata->HistoricalMetadataList.empty()); } TEST_CASE("SameSubmission_SameInstaller", "[metadata_collection]") From 0fe79e024928cdfd934145e0e219e0fdaf26566d Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Fri, 3 Jun 2022 17:29:26 -0700 Subject: [PATCH 15/22] Initial test debugging and more implementation --- .../InstallerMetadataCollectionContext.cpp | 80 ++++++++++++++++++- .../Public/AppInstallerVersions.h | 2 +- src/AppInstallerCommonCore/Versions.cpp | 2 +- .../InstallerMetadataCollectionContext.cpp | 20 +++-- 4 files changed, 90 insertions(+), 14 deletions(-) diff --git a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp index 5123862f8a..e9dbbe369e 100644 --- a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp @@ -33,6 +33,23 @@ namespace SubmissionData->DefaultLocalization.Add("Publisher"); } + TestInput(MinimalDefaults_t, const std::string& productVersion, const std::string& productCode, Manifest::InstallerTypeEnum installerType) : TestInput(MinimalDefaults) + { + CurrentMetadata = std::make_optional(); + CurrentMetadata->SchemaVersion.Assign("1.0"); + CurrentMetadata->ProductVersionMin.Assign(productVersion); + CurrentMetadata->ProductVersionMax.Assign(productVersion); + auto& installerMetadata = CurrentMetadata->InstallerMetadataMap[InstallerHash.value()]; + installerMetadata.SubmissionIdentifier = SubmissionIdentifier.value(); + installerMetadata.AppsAndFeaturesEntries.push_back({}); + auto& entry = installerMetadata.AppsAndFeaturesEntries.back(); + entry.DisplayName = SubmissionData->DefaultLocalization.Get(); + entry.Publisher = SubmissionData->DefaultLocalization.Get(); + entry.DisplayVersion = productVersion; + entry.ProductCode = productCode; + entry.InstallerType = installerType; + } + std::optional Version; std::optional SupportedMetadataVersion; std::optional MaximumMetadataSize; @@ -140,7 +157,7 @@ namespace } auto metadataValue = JSON::GetJsonValueFromNode(input, L"metadata"); - if (metadataValue) + if (metadataValue && !metadataValue->get().is_null()) { Metadata = std::make_optional(); Metadata->FromJson(metadataValue->get()); @@ -149,13 +166,13 @@ namespace auto diagnosticsValue = JSON::GetJsonValueFromNode(input, L"diagnostics"); if (diagnosticsValue) { - auto errorHRNumber = JSON::GetRawIntValueFromJsonNode(input, L"errorHR"); + auto errorHRNumber = JSON::GetRawIntValueFromJsonNode(diagnosticsValue.value(), L"errorHR"); if (errorHRNumber) { ErrorHR = std::move(errorHRNumber); } - auto errorTextString = JSON::GetRawStringValueFromJsonNode(input, L"errorText"); + auto errorTextString = JSON::GetRawStringValueFromJsonNode(diagnosticsValue.value(), L"errorText"); if (errorTextString) { ErrorText = std::move(errorTextString); @@ -291,7 +308,7 @@ TEST_CASE("NewPackage", "[metadata_collection]") manifest.DefaultLocalization.Add("Test Publisher"); manifest.Version = "1.2.3"; manifest.Installers.push_back({}); - manifest.Installers[0].ProductCode = "{GUID}"; + manifest.Installers[0].ProductCode = "{guid}"; IPackageVersion::Metadata metadata; metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(Manifest::InstallerTypeEnum::Msi); @@ -321,6 +338,61 @@ TEST_CASE("NewPackage", "[metadata_collection]") TEST_CASE("SameSubmission_SameInstaller", "[metadata_collection]") { + std::string version = "1.3.5"; + std::string productCode = "{guid}"; + Manifest::InstallerTypeEnum installerType = Manifest::InstallerTypeEnum::Msi; + + TestInput input(MinimalDefaults, version, productCode, installerType); + auto correlationData = std::make_unique(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add("Different Language Name"); + // Same publisher + manifest.DefaultLocalization.Add(input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].Publisher); + manifest.Version = version; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = productCode; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(installerType); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared(manifest, metadata); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->ProductVersionMin.ToString() == output.Metadata->ProductVersionMax.ToString()); + REQUIRE(output.Metadata->ProductVersionMin.ToString() == manifest.Version); + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 1); + REQUIRE(output.Metadata->InstallerMetadataMap.count(input.InstallerHash.value()) == 1); + const auto& entry = output.Metadata->InstallerMetadataMap[input.InstallerHash.value()]; + REQUIRE(entry.SubmissionIdentifier == input.SubmissionIdentifier.value()); + REQUIRE(entry.AppsAndFeaturesEntries.size() == 2); + + // One should have all values, and the other should have just a different name + // Base which one is which off of whether Publisher is set + for (const auto& featureEntry : entry.AppsAndFeaturesEntries) + { + if (featureEntry.Publisher.empty()) + { + REQUIRE(featureEntry.DisplayName == manifest.DefaultLocalization.Get()); + REQUIRE(featureEntry.DisplayVersion.empty()); + REQUIRE(featureEntry.ProductCode.empty()); + REQUIRE(featureEntry.InstallerType == Manifest::InstallerTypeEnum::Unknown); + } + else + { + REQUIRE(featureEntry.DisplayName == input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].DisplayName); + REQUIRE(featureEntry.Publisher == manifest.DefaultLocalization.Get()); + REQUIRE(featureEntry.DisplayVersion == manifest.Version); + REQUIRE(featureEntry.ProductCode == manifest.Installers[0].ProductCode); + REQUIRE(featureEntry.InstallerType == Manifest::InstallerTypeEnum::Msi); + } + } + REQUIRE(output.Metadata->HistoricalMetadataList.empty()); } TEST_CASE("SameSubmission_NewInstaller", "[metadata_collection]") diff --git a/src/AppInstallerCommonCore/Public/AppInstallerVersions.h b/src/AppInstallerCommonCore/Public/AppInstallerVersions.h index a72cf678d6..42adf74919 100644 --- a/src/AppInstallerCommonCore/Public/AppInstallerVersions.h +++ b/src/AppInstallerCommonCore/Public/AppInstallerVersions.h @@ -34,7 +34,7 @@ namespace AppInstaller::Utility Version(std::string&& version, std::string_view splitChars = DefaultSplitChars); // Resets the version's value to the input. - void Assign(std::string&& version, std::string_view splitChars = DefaultSplitChars); + void Assign(std::string version, std::string_view splitChars = DefaultSplitChars); // Gets the full version string used to construct the Version. const std::string& ToString() const { return m_version; } diff --git a/src/AppInstallerCommonCore/Versions.cpp b/src/AppInstallerCommonCore/Versions.cpp index e588fb208f..b370fffa65 100644 --- a/src/AppInstallerCommonCore/Versions.cpp +++ b/src/AppInstallerCommonCore/Versions.cpp @@ -16,7 +16,7 @@ namespace AppInstaller::Utility Assign(std::move(version), splitChars); } - void Version::Assign(std::string&& version, std::string_view splitChars) + void Version::Assign(std::string version, std::string_view splitChars) { m_version = std::move(version); size_t pos = 0; diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index a8d7fec473..9cb91faab0 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -352,14 +352,14 @@ namespace AppInstaller::Repository::Metadata { web::json::value entryValue; - AddFieldIfNotEmpty(itemValue, fields.DisplayName, entry.DisplayName); - AddFieldIfNotEmpty(itemValue, fields.Publisher, entry.Publisher); - AddFieldIfNotEmpty(itemValue, fields.DisplayVersion, entry.DisplayVersion); - AddFieldIfNotEmpty(itemValue, fields.ProductCode, entry.ProductCode); - AddFieldIfNotEmpty(itemValue, fields.UpgradeCode, entry.UpgradeCode); + AddFieldIfNotEmpty(entryValue, fields.DisplayName, entry.DisplayName); + AddFieldIfNotEmpty(entryValue, fields.Publisher, entry.Publisher); + AddFieldIfNotEmpty(entryValue, fields.DisplayVersion, entry.DisplayVersion); + AddFieldIfNotEmpty(entryValue, fields.ProductCode, entry.ProductCode); + AddFieldIfNotEmpty(entryValue, fields.UpgradeCode, entry.UpgradeCode); if (entry.InstallerType != Manifest::InstallerTypeEnum::Unknown) { - itemValue[fields.InstallerType] = AppInstaller::JSON::GetStringValue(Manifest::InstallerTypeToString(entry.InstallerType)); + entryValue[fields.InstallerType] = AppInstaller::JSON::GetStringValue(Manifest::InstallerTypeToString(entry.InstallerType)); } appsAndFeaturesArray[appsAndFeaturesEntryIndex++] = std::move(entryValue); @@ -370,7 +370,7 @@ namespace AppInstaller::Repository::Metadata metadataArray[metadataItemIndex++] = std::move(itemValue); } - result[fields.AppsAndFeaturesEntries] = std::move(metadataArray); + result[fields.Metadata] = std::move(metadataArray); web::json::value historicalArray = web::json::value::array(); size_t historicalItemIndex = 0; @@ -817,7 +817,11 @@ namespace AppInstaller::Repository::Metadata } result[fields.Status] = web::json::value::string(ToString(statusToUse)); - result[fields.Metadata] = m_outputMetadata.ToJson(m_supportedMetadataVersion, m_maxMetadataSize); + if (m_outputStatus == OutputStatus::Success) + { + result[fields.Metadata] = m_outputMetadata.ToJson(m_supportedMetadataVersion, m_maxMetadataSize); + } + result[fields.Diagnostics] = m_outputDiagnostics; return result; From 4f8de0b5baa9bddf5d2e9fc5f7d11f023212d002 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 6 Jun 2022 11:26:57 -0700 Subject: [PATCH 16/22] Finish unit tests --- .../InstallerMetadataCollectionContext.cpp | 127 +++++++++++++++++- 1 file changed, 120 insertions(+), 7 deletions(-) diff --git a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp index e9dbbe369e..c50c54d082 100644 --- a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp @@ -257,7 +257,7 @@ namespace } } -TEST_CASE("MinimumInput", "[metadata_collection]") +TEST_CASE("MetadataCollection_MinimumInput", "[metadata_collection]") { TestInput input(MinimalDefaults); TestOutput output = GetOutput(input); @@ -268,7 +268,7 @@ TEST_CASE("MinimumInput", "[metadata_collection]") REQUIRE(output.InstallerHash.value() == input.InstallerHash.value()); } -TEST_CASE("BadInput", "[metadata_collection]") +TEST_CASE("MetadataCollection_BadInput", "[metadata_collection]") { TestInput input(MinimalDefaults); @@ -288,7 +288,7 @@ TEST_CASE("BadInput", "[metadata_collection]") #undef RESET_FIELD_SECTION } -TEST_CASE("LowConfidence", "[metadata_collection]") +TEST_CASE("MetadataCollection_LowConfidence", "[metadata_collection]") { TestInput input(MinimalDefaults); // The default test correlation object won't have a package correlation set @@ -298,7 +298,7 @@ TEST_CASE("LowConfidence", "[metadata_collection]") output.ValidateFieldPresence(); } -TEST_CASE("NewPackage", "[metadata_collection]") +TEST_CASE("MetadataCollection_NewPackage", "[metadata_collection]") { TestInput input(MinimalDefaults); auto correlationData = std::make_unique(); @@ -336,7 +336,7 @@ TEST_CASE("NewPackage", "[metadata_collection]") REQUIRE(output.Metadata->HistoricalMetadataList.empty()); } -TEST_CASE("SameSubmission_SameInstaller", "[metadata_collection]") +TEST_CASE("MetadataCollection_SameSubmission_SameInstaller", "[metadata_collection]") { std::string version = "1.3.5"; std::string productCode = "{guid}"; @@ -395,10 +395,123 @@ TEST_CASE("SameSubmission_SameInstaller", "[metadata_collection]") REQUIRE(output.Metadata->HistoricalMetadataList.empty()); } -TEST_CASE("SameSubmission_NewInstaller", "[metadata_collection]") +TEST_CASE("MetadataCollection_SameSubmission_NewInstaller", "[metadata_collection]") { + std::string versionPresent = "1.3.5"; + std::string versionIncoming = "1.3.5.1"; + std::string productCodePresent = "{guid}"; + std::string productCodeIncoming = "{guid_different}"; + Manifest::InstallerTypeEnum installerType = Manifest::InstallerTypeEnum::Msi; + + TestInput input(MinimalDefaults, versionPresent, productCodePresent, installerType); + // Change the incoming hash to be new + input.InstallerHash = input.InstallerHash.value() + "_DIFFERENT"; + auto correlationData = std::make_unique(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add("Name (but different architecture)"); + // Same publisher + manifest.DefaultLocalization.Add(input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].Publisher); + manifest.Version = versionIncoming; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = productCodeIncoming; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(installerType); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared(manifest, metadata); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->ProductVersionMin.ToString() == versionPresent); + REQUIRE(output.Metadata->ProductVersionMax.ToString() == versionIncoming); + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 2); + + for (const auto& installerMetadata : output.Metadata->InstallerMetadataMap) + { + const auto& entry = installerMetadata.second; + + REQUIRE(entry.SubmissionIdentifier == input.SubmissionIdentifier.value()); + REQUIRE(entry.AppsAndFeaturesEntries.size() == 1); + + const auto& featureEntry = entry.AppsAndFeaturesEntries.front(); + REQUIRE(featureEntry.Publisher == manifest.DefaultLocalization.Get()); + + if (featureEntry.ProductCode == productCodePresent) + { + REQUIRE(featureEntry.DisplayName == input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].DisplayName); + REQUIRE(featureEntry.DisplayVersion == versionPresent); + REQUIRE(featureEntry.InstallerType == Manifest::InstallerTypeEnum::Msi); + } + else + { + REQUIRE(featureEntry.DisplayName == manifest.DefaultLocalization.Get()); + REQUIRE(featureEntry.DisplayVersion == versionIncoming); + REQUIRE(featureEntry.InstallerType == Manifest::InstallerTypeEnum::Msi); + REQUIRE(featureEntry.ProductCode == productCodeIncoming); + } + } + + REQUIRE(output.Metadata->HistoricalMetadataList.empty()); } -TEST_CASE("NewSubmission", "[metadata_collection]") +TEST_CASE("MetadataCollection_NewSubmission", "[metadata_collection]") { + std::string versionPresent = "1.3.5"; + std::string versionIncoming = "1.4.0"; + std::string productCodePresent = "{guid}"; + std::string productCodeIncoming = "{guid_different}"; + Manifest::InstallerTypeEnum installerType = Manifest::InstallerTypeEnum::Msi; + + TestInput input(MinimalDefaults, versionPresent, productCodePresent, installerType); + input.SubmissionIdentifier = input.SubmissionIdentifier.value() + "_NEW"; + input.InstallerHash = input.InstallerHash.value() + "_DIFFERENT"; + auto correlationData = std::make_unique(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add(input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].DisplayName); + manifest.DefaultLocalization.Add(input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].Publisher); + manifest.Version = versionIncoming; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = productCodeIncoming; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(installerType); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared(manifest, metadata); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->ProductVersionMin.ToString() == output.Metadata->ProductVersionMax.ToString()); + REQUIRE(output.Metadata->ProductVersionMin.ToString() == manifest.Version); + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 1); + REQUIRE(output.Metadata->InstallerMetadataMap.count(input.InstallerHash.value()) == 1); + const auto& entry = output.Metadata->InstallerMetadataMap[input.InstallerHash.value()]; + REQUIRE(entry.SubmissionIdentifier == input.SubmissionIdentifier.value()); + REQUIRE(entry.AppsAndFeaturesEntries.size() == 1); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayName == manifest.DefaultLocalization.Get()); + REQUIRE(entry.AppsAndFeaturesEntries[0].Publisher == manifest.DefaultLocalization.Get()); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayVersion == manifest.Version); + REQUIRE(entry.AppsAndFeaturesEntries[0].ProductCode == manifest.Installers[0].ProductCode); + REQUIRE(entry.AppsAndFeaturesEntries[0].InstallerType == Manifest::InstallerTypeEnum::Msi); + + REQUIRE(output.Metadata->HistoricalMetadataList.size() == 1); + const auto& historicalEntry = output.Metadata->HistoricalMetadataList[0]; + REQUIRE(historicalEntry.ProductVersionMin.ToString() == input.CurrentMetadata->ProductVersionMin.ToString()); + REQUIRE(historicalEntry.ProductVersionMax.ToString() == input.CurrentMetadata->ProductVersionMax.ToString()); + const auto& appsAndFeaturesEntry = input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries.front(); + REQUIRE(historicalEntry.Names.size() == 1); + REQUIRE(historicalEntry.Names[0] == appsAndFeaturesEntry.DisplayName); + REQUIRE(historicalEntry.ProductCodes.size() == 1); + REQUIRE(historicalEntry.ProductCodes[0] == appsAndFeaturesEntry.ProductCode); + REQUIRE(historicalEntry.Publishers.size() == 1); + REQUIRE(historicalEntry.Publishers[0] == appsAndFeaturesEntry.Publisher); } From 7362deb13b1d310bb4df8256460fa7e509541b95 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 9 Jun 2022 17:04:15 -0700 Subject: [PATCH 17/22] Add new exports to def --- src/WinGetUtil/Source.def | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/WinGetUtil/Source.def b/src/WinGetUtil/Source.def index c9789e58b5..5ed4b7eeca 100644 --- a/src/WinGetUtil/Source.def +++ b/src/WinGetUtil/Source.def @@ -15,3 +15,5 @@ EXPORTS WinGetCompareVersions WinGetValidateManifestV2 WinGetValidateManifestDependencies + WinGetBeginInstallerMetadataCollection + WinGetCompleteInstallerMetadataCollection From 2ccf806ee5c71ab28e46a56c13724ea38cee2e87 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 16 Jun 2022 19:38:18 -0700 Subject: [PATCH 18/22] Add merge function prototype --- src/WinGetUtil/WinGetUtil.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/WinGetUtil/WinGetUtil.h b/src/WinGetUtil/WinGetUtil.h index 71e8c02f4e..26a5ff9c73 100644 --- a/src/WinGetUtil/WinGetUtil.h +++ b/src/WinGetUtil/WinGetUtil.h @@ -171,4 +171,18 @@ extern "C" WINGET_INSTALLER_METADATA_COLLECTION_HANDLE collectionHandle, WINGET_STRING outputFilePath, WinGetCompleteInstallerMetadataCollectionOptions options); + + // Option flags for WinGetMergeInstallerMetadata. + enum WinGetMergeInstallerMetadataOptions + { + WinGetMergeInstallerMetadataOptions_None = 0, + }; + + // Merges the given JSON metadata documents into a single one. + WINGET_UTIL_API WinGetMergeInstallerMetadata( + WINGET_STRING inputJSON, + WINGET_STRING_OUT* outputJSON, + UINT32 maximumOutputSizeInBytes, + WINGET_STRING logFilePath, + WinGetMergeInstallerMetadataOptions options); } From b95fdfa06bac6e1a9159c7cb7b616853bb08f728 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 16 Jun 2022 20:32:23 -0700 Subject: [PATCH 19/22] PR feedback --- .github/actions/spelling/allow.txt | 3 +- .../InstallerMetadataCollectionContext.cpp | 6 +-- src/AppInstallerCommonCore/Downloader.cpp | 2 +- src/AppInstallerCommonCore/JsonUtil.cpp | 23 ++++++++++ .../Public/winget/JsonUtil.h | 8 ++-- .../InstallerMetadataCollectionContext.cpp | 46 ++++++++----------- .../Microsoft/SQLiteIndexSource.cpp | 8 +--- .../InstallerMetadataCollectionContext.h | 10 ++-- 8 files changed, 57 insertions(+), 49 deletions(-) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 860e9e2766..815ee4d90d 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -110,9 +110,8 @@ denelon depersist deque deserialize -deserializes -Deserialize deserializer +deserializes deserializing dest devblogs diff --git a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp index c50c54d082..85c3fa1f95 100644 --- a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp @@ -509,9 +509,9 @@ TEST_CASE("MetadataCollection_NewSubmission", "[metadata_collection]") REQUIRE(historicalEntry.ProductVersionMax.ToString() == input.CurrentMetadata->ProductVersionMax.ToString()); const auto& appsAndFeaturesEntry = input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries.front(); REQUIRE(historicalEntry.Names.size() == 1); - REQUIRE(historicalEntry.Names[0] == appsAndFeaturesEntry.DisplayName); + REQUIRE(*historicalEntry.Names.begin() == appsAndFeaturesEntry.DisplayName); REQUIRE(historicalEntry.ProductCodes.size() == 1); - REQUIRE(historicalEntry.ProductCodes[0] == appsAndFeaturesEntry.ProductCode); + REQUIRE(*historicalEntry.ProductCodes.begin() == appsAndFeaturesEntry.ProductCode); REQUIRE(historicalEntry.Publishers.size() == 1); - REQUIRE(historicalEntry.Publishers[0] == appsAndFeaturesEntry.Publisher); + REQUIRE(*historicalEntry.Publishers.begin() == appsAndFeaturesEntry.Publisher); } diff --git a/src/AppInstallerCommonCore/Downloader.cpp b/src/AppInstallerCommonCore/Downloader.cpp index bc3c6478ac..d6ec2c7505 100644 --- a/src/AppInstallerCommonCore/Downloader.cpp +++ b/src/AppInstallerCommonCore/Downloader.cpp @@ -167,7 +167,7 @@ namespace AppInstaller::Utility // Only Installers should be downloaded with DO currently, as: // - Index :: Constantly changing blob at same location is not what DO is for - // - Manifest :: DO overhead is not needed for small files + // - Manifest / InstallerMetadataCollectionInput :: DO overhead is not needed for small files // - WinGetUtil :: Intentionally not using DO at this time if (type == DownloadType::Installer) { diff --git a/src/AppInstallerCommonCore/JsonUtil.cpp b/src/AppInstallerCommonCore/JsonUtil.cpp index bd8c9cd129..a99a91e618 100644 --- a/src/AppInstallerCommonCore/JsonUtil.cpp +++ b/src/AppInstallerCommonCore/JsonUtil.cpp @@ -189,6 +189,29 @@ namespace AppInstaller::JSON return result; } + std::set GetRawStringSetFromJsonNode( + const web::json::value& node, const utility::string_t& keyName) + { + std::optional> arrayValue = GetRawJsonArrayFromJsonNode(node, keyName); + + std::set result; + if (!arrayValue) + { + return result; + } + + for (auto& value : arrayValue.value().get()) + { + std::optional item = GetRawStringValueFromJsonValue(value); + if (item) + { + result.emplace(std::move(item.value())); + } + } + + return result; + } + bool IsValidNonEmptyStringValue(std::optional& value) { if (Utility::IsEmptyOrWhitespace(value.value_or(""))) diff --git a/src/AppInstallerCommonCore/Public/winget/JsonUtil.h b/src/AppInstallerCommonCore/Public/winget/JsonUtil.h index 135b51c0f4..8a63588a04 100644 --- a/src/AppInstallerCommonCore/Public/winget/JsonUtil.h +++ b/src/AppInstallerCommonCore/Public/winget/JsonUtil.h @@ -1,4 +1,3 @@ -#pragma once // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once @@ -16,7 +15,7 @@ namespace AppInstaller::JSON { - // For JSON CPP LIb + // For JSON CPP Lib template std::optional GetValue(const Json::Value& node); @@ -45,6 +44,9 @@ namespace AppInstaller::JSON std::optional> GetRawJsonArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + std::vector GetRawStringArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + std::set GetRawStringSetFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + std::optional GetRawIntValueFromJsonValue(const web::json::value& value); std::optional GetRawIntValueFromJsonNode(const web::json::value& value, const utility::string_t& keyName); @@ -53,7 +55,5 @@ namespace AppInstaller::JSON web::json::value GetStringValue(std::string_view value); - std::vector GetRawStringArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); - bool IsValidNonEmptyStringValue(std::optional& value); } diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index 9cb91faab0..c7fc378235 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -73,7 +73,7 @@ namespace AppInstaller::Repository::Metadata } } - web::json::value CreateStringArray(const std::vector& values) + web::json::value CreateStringArray(const std::set& values) { web::json::value result; size_t index = 0; @@ -86,29 +86,29 @@ namespace AppInstaller::Repository::Metadata return result; } - bool AddIfNotPresentAndNotEmpty(std::vector& strings, const std::vector& filter, const std::string& string) + bool AddIfNotPresentAndNotEmpty(std::set& strings, const std::set& filter, const std::string& string) { - if (string.empty() || std::find(filter.begin(), filter.end(), string) != filter.end()) + if (string.empty() || filter.find(string) != filter.end()) { return false; } - strings.emplace_back(string); + strings.emplace(string); return true; } - bool AddIfNotPresentAndNotEmpty(std::vector& strings, const std::string& string) + bool AddIfNotPresentAndNotEmpty(std::set& strings, const std::string& string) { return AddIfNotPresentAndNotEmpty(strings, strings, string); } - void AddIfNotPresent(std::vector& strings, std::vector& filter, const std::vector& inputs) + void AddIfNotPresent(std::set& strings, std::set& filter, const std::set& inputs) { for (const std::string& input : inputs) { if (AddIfNotPresentAndNotEmpty(strings, filter, input)) { - filter.emplace_back(input); + filter.emplace(input); } } } @@ -152,21 +152,21 @@ namespace AppInstaller::Repository::Metadata }); } - web::json::value ProductMetadata::ToJson(const Utility::Version& version, size_t maximumSizeInBytes) + web::json::value ProductMetadata::ToJson(const Utility::Version& schemaVersion, size_t maximumSizeInBytes) { - AICLI_LOG(Repo, Info, << "Creating metadata JSON version " << version.ToString()); + AICLI_LOG(Repo, Info, << "Creating metadata JSON version " << schemaVersion.ToString()); using ToJsonFunctionPointer = web::json::value(ProductMetadata::*)(); ToJsonFunctionPointer toJsonFunction = nullptr; - if (version.PartAt(0).Integer == 1) + if (schemaVersion.PartAt(0).Integer == 1) { // We only have one version currently, so use that as long as the major version is 1 toJsonFunction = &ProductMetadata::ToJson_1_0; } else { - AICLI_LOG(Repo, Error, << "Don't know how to handle metadata version " << version.ToString()); + AICLI_LOG(Repo, Error, << "Don't know how to handle metadata version " << schemaVersion.ToString()); THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); } @@ -315,10 +315,10 @@ namespace AppInstaller::Repository::Metadata historicalMetadata.ProductVersionMin = Version{ GetRequiredString(item, fields.VersionMin) }; historicalMetadata.ProductVersionMax = Version{ GetRequiredString(item, fields.VersionMax) }; - historicalMetadata.Names = AppInstaller::JSON::GetRawStringArrayFromJsonNode(item, fields.Names); - historicalMetadata.Publishers = AppInstaller::JSON::GetRawStringArrayFromJsonNode(item, fields.Publishers); - historicalMetadata.ProductCodes = AppInstaller::JSON::GetRawStringArrayFromJsonNode(item, fields.ProductCodes); - historicalMetadata.UpgradeCodes = AppInstaller::JSON::GetRawStringArrayFromJsonNode(item, fields.UpgradeCodes); + historicalMetadata.Names = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.Names); + historicalMetadata.Publishers = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.Publishers); + historicalMetadata.ProductCodes = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.ProductCodes); + historicalMetadata.UpgradeCodes = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.UpgradeCodes); HistoricalMetadataList.emplace_back(std::move(historicalMetadata)); } @@ -449,12 +449,10 @@ namespace AppInstaller::Repository::Metadata const int MaxRetryCount = 2; for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount) { - bool success = false; try { auto downloadHash = DownloadToStream(utf8Uri, jsonStream, DownloadType::InstallerMetadataCollectionInput, emptyCallback); - - success = true; + break; } catch (...) { @@ -468,11 +466,6 @@ namespace AppInstaller::Repository::Metadata throw; } } - - if (success) - { - break; - } } result->InitializePreinstallState(ConvertToUTF16(jsonStream.str())); @@ -664,7 +657,6 @@ namespace AppInstaller::Repository::Metadata } newEntry.Publisher = package->GetProperty(PackageVersionProperty::Publisher).get(); // TODO: Support upgrade code throughout the code base... - //newEntry.UpgradeCode; // Add or update the metadata for the installer hash auto itr = m_outputMetadata.InstallerMetadataMap.find(m_installerHash); @@ -810,10 +802,10 @@ namespace AppInstaller::Repository::Metadata result[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(m_installerHash); // Limit output status to 1.0 known values - OutputStatus statusToUse = m_outputStatus; - if (statusToUse != OutputStatus::Success && statusToUse != OutputStatus::Error && statusToUse != OutputStatus::LowConfidence) + OutputStatus statusToUse = OutputStatus::Unknown; + if (m_outputStatus == OutputStatus::Success || m_outputStatus == OutputStatus::Error || m_outputStatus == OutputStatus::LowConfidence) { - statusToUse = OutputStatus::Unknown; + statusToUse = m_outputStatus; } result[fields.Status] = web::json::value::string(ToString(statusToUse)); diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp index 85b76ad1f8..bb74dda174 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp @@ -121,7 +121,6 @@ namespace AppInstaller::Repository::Microsoft const int MaxRetryCount = 2; for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount) { - bool success = false; try { auto downloadHash = Utility::DownloadToStream(fullPath, manifestStream, Utility::DownloadType::Manifest, emptyCallback, !expectedHash.empty()); @@ -132,7 +131,7 @@ namespace AppInstaller::Repository::Microsoft THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); } - success = true; + break; } catch (...) { @@ -146,11 +145,6 @@ namespace AppInstaller::Repository::Microsoft throw; } } - - if (success) - { - break; - } } std::string manifestContents = manifestStream.str(); diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index 63ead606bc..2fb137e201 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -30,7 +30,7 @@ namespace AppInstaller::Repository::Metadata void FromJson(const web::json::value& json); // Create a JSON value for the metadata using the given schema version. - web::json::value ToJson(const Utility::Version& version, size_t maximumSizeInBytes); + web::json::value ToJson(const Utility::Version& schemaVersion, size_t maximumSizeInBytes); // Copies the metadata from the source. If the given submission identifier does not match // the source, it's data is moved to historical. @@ -48,10 +48,10 @@ namespace AppInstaller::Repository::Metadata { Utility::Version ProductVersionMin; Utility::Version ProductVersionMax; - std::vector Names; - std::vector Publishers; - std::vector ProductCodes; - std::vector UpgradeCodes; + std::set Names; + std::set Publishers; + std::set ProductCodes; + std::set UpgradeCodes; }; Utility::Version SchemaVersion; From 1a07763aa3312854bef09192bac2b8ab8e5f38d1 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 16 Jun 2022 22:12:56 -0700 Subject: [PATCH 20/22] Begin implementation of the merge function --- .../InstallerMetadataCollectionContext.cpp | 148 +++++++++++++----- .../InstallerMetadataCollectionContext.h | 9 +- src/WinGetUtil/Exports.cpp | 17 ++ 3 files changed, 137 insertions(+), 37 deletions(-) diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index c7fc378235..9016f02a8e 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -45,6 +45,7 @@ namespace AppInstaller::Repository::Metadata struct OutputFields_1_0 { utility::string_t Version = L"version"; + utility::string_t SubmissionData = L"submissionData"; utility::string_t InstallerHash = L"installerHash"; utility::string_t Status = L"status"; utility::string_t Metadata = L"metadata"; @@ -507,6 +508,51 @@ namespace AppInstaller::Repository::Metadata CompleteWithThreadGlobalsSet(output); } + std::wstring InstallerMetadataCollectionContext::Merge(const std::wstring& json, size_t maximumSizeInBytes, const std::filesystem::path& logFile) + { + ThreadLocalStorage::ThreadGlobals threadGlobals; + auto globalsLifetime = InitializeLogging(threadGlobals, logFile); + + AICLI_LOG(Repo, Info, << "Parsing input JSON:\n" << ConvertToUTF8(json)); + + // Parse and validate JSON + try + { + utility::string_t versionFieldName = L"version"; + + web::json::value inputValue = web::json::value::parse(json); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null()); + + Version inputVersion = Version{ GetRequiredString(inputValue, versionFieldName) }; + AICLI_LOG(Repo, Info, << "Parsing input JSON version " << inputVersion.ToString()); + + web::json::value mergedResult; + + if (inputVersion.PartAt(0).Integer == 1) + { + mergedResult = Merge_1_0(inputValue, maximumSizeInBytes); + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to handle version " << inputVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + + std::wostringstream outputStream; + mergedResult.serialize(outputStream); + + return std::move(outputStream).str(); + } + catch (const web::json::json_exception& exc) + { + AICLI_LOG(Repo, Error, << "Exception parsing input JSON: " << exc.what()); + } + + // We will return within the try or throw a non-json exception, so if we get here it was a json exception. + THROW_HR(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE); + } + void InstallerMetadataCollectionContext::CompleteWithThreadGlobalsSet(std::ostream& output) { web::json::value outputJSON; @@ -550,9 +596,9 @@ namespace AppInstaller::Repository::Metadata outputJSON.serialize(output); } - std::unique_ptr InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile) + std::unique_ptr InstallerMetadataCollectionContext::InitializeLogging(ThreadLocalStorage::ThreadGlobals& threadGlobals, const std::filesystem::path& logFile) { - auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); + auto threadGlobalsLifetime = threadGlobals.SetForCurrentThread(); Logging::Log().SetLevel(Logging::Level::Info); Logging::Log().EnableChannel(Logging::Channel::All); @@ -570,6 +616,11 @@ namespace AppInstaller::Repository::Metadata return threadGlobalsLifetime; } + std::unique_ptr InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile) + { + return InitializeLogging(m_threadGlobals, logFile); + } + void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json) { try @@ -715,56 +766,41 @@ namespace AppInstaller::Repository::Metadata // Field names utility::string_t metadataVersionFieldName = L"supportedMetadataVersion"; - utility::string_t metadataSizeFieldName = L"maximumMetadataSize"; utility::string_t metadataFieldName = L"currentMetadata"; + utility::string_t submissionDataFieldName = L"submissionData"; utility::string_t submissionIdentifierFieldName = L"submissionIdentifier"; + utility::string_t packageDataFieldName = L"packageData"; utility::string_t installerHashFieldName = L"installerHash"; - utility::string_t currentManifestFieldName = L"currentManifest"; - utility::string_t submissionDataFieldName = L"submissionData"; utility::string_t defaultLocaleFieldName = L"DefaultLocale"; utility::string_t localesFieldName = L"Locales"; + // root fields m_supportedMetadataVersion = Version{ GetRequiredString(input, metadataVersionFieldName) }; - auto metadataSizeNumber = AppInstaller::JSON::GetRawIntValueFromJsonNode(input, metadataSizeFieldName); - if (metadataSizeNumber && metadataSizeNumber.value() > 0) - { - m_maxMetadataSize = static_cast(metadataSizeNumber.value()); - } - else - { - m_maxMetadataSize = std::numeric_limits::max(); - } - auto currentMetadataValue = AppInstaller::JSON::GetJsonValueFromNode(input, metadataFieldName); if (currentMetadataValue) { m_currentMetadata.FromJson(currentMetadataValue.value()); } - m_submissionIdentifier = GetRequiredString(input, submissionIdentifierFieldName); - m_installerHash = GetRequiredString(input, installerHashFieldName); + // submissionData fields + auto submissionDataValue = AppInstaller::JSON::GetJsonValueFromNode(input, submissionDataFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !submissionDataValue); + m_submissionData = submissionDataValue.value(); - // The 1.0 version of input uses the 1.1 version of REST - JSON::ManifestJSONParser parser{ Version{ "1.1" }}; + m_submissionIdentifier = GetRequiredString(m_submissionData, submissionIdentifierFieldName); - auto currentManifestValue = AppInstaller::JSON::GetJsonValueFromNode(input, currentManifestFieldName); - if (currentManifestValue) - { - std::vector manifests = parser.DeserializeData(currentManifestValue.value()); + // packageData fields + auto packageDataValue = AppInstaller::JSON::GetJsonValueFromNode(input, packageDataFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !packageDataValue); - if (!manifests.empty()) - { - std::sort(manifests.begin(), manifests.end(), [](const Manifest::Manifest& a, const Manifest::Manifest& b) { return a.Version < b.Version; }); - // Latest version will be sorted to last position by Version < predicate - m_currentManifest = std::move(manifests.back()); - } - } + m_installerHash = GetRequiredString(packageDataValue.value(), installerHashFieldName); + + // The 1.0 version of input uses the 1.1 version of REST + JSON::ManifestJSONParser parser{ Version{ "1.1" }}; - auto submissionDataValue = AppInstaller::JSON::GetJsonValueFromNode(input, submissionDataFieldName); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !submissionDataValue); { - auto defaultLocaleValue = AppInstaller::JSON::GetJsonValueFromNode(submissionDataValue.value(), defaultLocaleFieldName); + auto defaultLocaleValue = AppInstaller::JSON::GetJsonValueFromNode(packageDataValue.value(), defaultLocaleFieldName); THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !defaultLocaleValue); auto defaultLocale = parser.DeserializeLocale(defaultLocaleValue.value()); @@ -775,7 +811,7 @@ namespace AppInstaller::Repository::Metadata m_incomingManifest.DefaultLocalization = std::move(defaultLocale).value(); - auto localesArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(submissionDataValue.value(), localesFieldName); + auto localesArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(packageDataValue.value(), localesFieldName); if (localesArray) { for (const auto& locale : localesArray->get()) @@ -799,6 +835,7 @@ namespace AppInstaller::Repository::Metadata web::json::value result; result[fields.Version] = web::json::value::string(L"1.0"); + result[fields.SubmissionData] = m_submissionData; result[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(m_installerHash); // Limit output status to 1.0 known values @@ -811,7 +848,7 @@ namespace AppInstaller::Repository::Metadata if (m_outputStatus == OutputStatus::Success) { - result[fields.Metadata] = m_outputMetadata.ToJson(m_supportedMetadataVersion, m_maxMetadataSize); + result[fields.Metadata] = m_outputMetadata.ToJson(m_supportedMetadataVersion, 0); } result[fields.Diagnostics] = m_outputDiagnostics; @@ -889,4 +926,45 @@ namespace AppInstaller::Repository::Metadata return result; } + + web::json::value InstallerMetadataCollectionContext::Merge_1_0(web::json::value& input, size_t maximumSizeInBytes) + { + AICLI_LOG(Repo, Info, << "Merging 1.0 input metadatas"); + + utility::string_t metadatasFieldName = L"metadatas"; + + auto metadatasValue = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(input, metadatasFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !metadatasValue); + + std::vector metadatas; + for (const auto& value : metadatasValue->get()) + { + ProductMetadata current; + current.FromJson(value); + metadatas.emplace_back(std::move(current)); + } + + THROW_HR_IF(E_NOT_SET, metadatas.empty()); + + Version maximumSchemaVersion; + + // Require that all merging values use the same submission (and extract the maximum schema version being used) + for (const ProductMetadata& metadata : metadatas) + { + const std::string& firstSubmission = metadatas[0].InstallerMetadataMap.begin()->second.SubmissionIdentifier; + const std::string& metadataSubmission = metadata.InstallerMetadataMap.begin()->second.SubmissionIdentifier; + if (firstSubmission != metadataSubmission) + { + AICLI_LOG(Repo, Info, << "Found submission identifier mismatch: " << firstSubmission << " != " << metadataSubmission); + THROW_HR(E_NOT_VALID_STATE); + } + + if (maximumSchemaVersion < metadata.SchemaVersion) + { + maximumSchemaVersion = metadata.SchemaVersion; + } + } + + // Do the actual merging + } } diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index 2fb137e201..530c5261f1 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -93,8 +93,11 @@ namespace AppInstaller::Repository::Metadata // Completes the collection, writing to the given location. void Complete(std::ostream& output); + static std::wstring Merge(const std::wstring& json, size_t maximumSizeInBytes, const std::filesystem::path& logFile); + private: // Initializes the context runtime, including the log file if provided. + static std::unique_ptr InitializeLogging(ThreadLocalStorage::ThreadGlobals& threadGlobals, const std::filesystem::path& logFile); std::unique_ptr InitializeLogging(const std::filesystem::path& logFile); // Sets the collection context input and the preinstall state. @@ -121,16 +124,18 @@ namespace AppInstaller::Repository::Metadata // Create version 1.0 of error JSON web::json::value CreateErrorJson_1_0(); + // Merge using merge input verision 1.0 + static web::json::value Merge_1_0(web::json::value& input, size_t maximumSizeInBytes); + ThreadLocalStorage::ThreadGlobals m_threadGlobals; // Parsed input Utility::Version m_inputVersion; Utility::Version m_supportedMetadataVersion; - size_t m_maxMetadataSize = 0; ProductMetadata m_currentMetadata; + web::json::value m_submissionData; std::string m_submissionIdentifier; std::string m_installerHash; - Manifest::Manifest m_currentManifest; Manifest::Manifest m_incomingManifest; std::unique_ptr m_correlationData; diff --git a/src/WinGetUtil/Exports.cpp b/src/WinGetUtil/Exports.cpp index a76d41b037..b3ee17c3ce 100644 --- a/src/WinGetUtil/Exports.cpp +++ b/src/WinGetUtil/Exports.cpp @@ -535,4 +535,21 @@ extern "C" return S_OK; } CATCH_RETURN() + + WINGET_UTIL_API WinGetMergeInstallerMetadata( + WINGET_STRING inputJSON, + WINGET_STRING_OUT* outputJSON, + UINT32 maximumOutputSizeInBytes, + WINGET_STRING logFilePath, + WinGetMergeInstallerMetadataOptions) try + { + THROW_HR_IF(E_INVALIDARG, !inputJSON); + THROW_HR_IF(E_INVALIDARG, !outputJSON); + + std::wstring merged = InstallerMetadataCollectionContext::Merge(inputJSON, maximumOutputSizeInBytes, GetPathOrEmpty(logFilePath)); + *outputJSON = ::SysAllocString(merged.c_str()); + + return S_OK; + } + CATCH_RETURN() } From 3411c1c4269dd81545dd20cc0d37600b05b5d27e Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Tue, 21 Jun 2022 20:03:47 -0700 Subject: [PATCH 21/22] Update to new design, adding Merge function --- .github/actions/spelling/allow.txt | 1 + src/AppInstallerCLICore/ExecutionReporter.cpp | 1 + .../InstallerMetadataCollectionContext.cpp | 214 +++++++++++++++--- src/AppInstallerCLITests/main.cpp | 2 +- .../InstallerMetadataCollectionContext.cpp | 103 ++++++--- .../InstallerMetadataCollectionContext.h | 2 +- 6 files changed, 263 insertions(+), 60 deletions(-) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 815ee4d90d..1b60126d46 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -301,6 +301,7 @@ MContext mday memset metadata +metadatas microsoft mimetype Minimatch diff --git a/src/AppInstallerCLICore/ExecutionReporter.cpp b/src/AppInstallerCLICore/ExecutionReporter.cpp index cf6d59d99b..67324b312e 100644 --- a/src/AppInstallerCLICore/ExecutionReporter.cpp +++ b/src/AppInstallerCLICore/ExecutionReporter.cpp @@ -139,6 +139,7 @@ namespace AppInstaller::CLI::Execution std::string response; if (!std::getline(m_in, response)) { + m_in.get(); THROW_HR(APPINSTALLER_CLI_ERROR_PROMPT_INPUT_ERROR); } diff --git a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp index 85c3fa1f95..f95ff4e458 100644 --- a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp @@ -27,10 +27,10 @@ namespace SupportedMetadataVersion = "1.0"; SubmissionIdentifier = "1"; InstallerHash = "ABCD"; - SubmissionData = std::make_optional(); - SubmissionData->DefaultLocalization.Locale = "en-us"; - SubmissionData->DefaultLocalization.Add("Name"); - SubmissionData->DefaultLocalization.Add("Publisher"); + PackageData = std::make_optional(); + PackageData->DefaultLocalization.Locale = "en-us"; + PackageData->DefaultLocalization.Add("Name"); + PackageData->DefaultLocalization.Add("Publisher"); } TestInput(MinimalDefaults_t, const std::string& productVersion, const std::string& productCode, Manifest::InstallerTypeEnum installerType) : TestInput(MinimalDefaults) @@ -43,8 +43,8 @@ namespace installerMetadata.SubmissionIdentifier = SubmissionIdentifier.value(); installerMetadata.AppsAndFeaturesEntries.push_back({}); auto& entry = installerMetadata.AppsAndFeaturesEntries.back(); - entry.DisplayName = SubmissionData->DefaultLocalization.Get(); - entry.Publisher = SubmissionData->DefaultLocalization.Get(); + entry.DisplayName = PackageData->DefaultLocalization.Get(); + entry.Publisher = PackageData->DefaultLocalization.Get(); entry.DisplayVersion = productVersion; entry.ProductCode = productCode; entry.InstallerType = installerType; @@ -52,14 +52,12 @@ namespace std::optional Version; std::optional SupportedMetadataVersion; - std::optional MaximumMetadataSize; std::optional CurrentMetadata; + std::optional SubmissionData; std::optional SubmissionIdentifier; std::optional InstallerHash; - // Not currently used - // std::optional CurrentManifest; // Schema 1.0 only cares about DefaultLocale and Locales - std::optional SubmissionData; + std::optional PackageData; std::wstring ToJSON() { @@ -75,35 +73,39 @@ namespace json[L"supportedMetadataVersion"] = JSON::GetStringValue(SupportedMetadataVersion.value()); } - if (MaximumMetadataSize) - { - json[L"maximumMetadataSize"] = MaximumMetadataSize.value(); - } - if (CurrentMetadata) { json[L"currentMetadata"] = CurrentMetadata->ToJson(Utility::Version{ "1.0" }, 0); } - if (SubmissionIdentifier) + if (SubmissionData) { - json[L"submissionIdentifier"] = JSON::GetStringValue(SubmissionIdentifier.value()); + json[L"submissionData"] = SubmissionData.value(); } - - if (InstallerHash) + else if (SubmissionIdentifier) { - json[L"installerHash"] = JSON::GetStringValue(InstallerHash.value()); + web::json::value submissionData; + submissionData[L"submissionIdentifier"] = JSON::GetStringValue(SubmissionIdentifier.value()); + json[L"submissionData"] = std::move(submissionData); } - if (SubmissionData) + if (InstallerHash || PackageData) { - web::json::value submissionData; + web::json::value packageData; - submissionData[L"DefaultLocale"] = LocaleToJSON(SubmissionData->DefaultLocalization); + if (InstallerHash) + { + packageData[L"installerHash"] = JSON::GetStringValue(InstallerHash.value()); + } - // TODO: Implement other locales + if (PackageData) + { + packageData[L"DefaultLocale"] = LocaleToJSON(PackageData->DefaultLocalization); - json[L"submissionData"] = std::move(submissionData); + // TODO: Implement other locales + } + + json[L"packageData"] = std::move(packageData); } return json.serialize(); @@ -134,7 +136,7 @@ namespace struct TestOutput { - TestOutput(const std::string& json) + TestOutput(const std::string& json) : OriginalJSON(json) { web::json::value input = web::json::value::parse(Utility::ConvertToUTF16(json)); @@ -144,6 +146,12 @@ namespace Version = std::move(versionString); } + auto submissionDataValue = JSON::GetJsonValueFromNode(input, L"submissionData"); + if (submissionDataValue) + { + SubmissionData = submissionDataValue.value(); + } + auto installerHashString = JSON::GetRawStringValueFromJsonNode(input, L"installerHash"); if (installerHashString) { @@ -180,7 +188,10 @@ namespace } } + std::string OriginalJSON; + std::optional Version; + std::optional SubmissionData; std::optional InstallerHash; std::optional Status; std::optional Metadata; @@ -205,6 +216,7 @@ namespace void ValidateFieldPresence() const { REQUIRE(Version); + REQUIRE(SubmissionData); REQUIRE(InstallerHash); REQUIRE(Status); @@ -255,6 +267,71 @@ namespace REQUIRE(output.IsError()); output.ValidateFieldPresence(); } + + ProductMetadata MakeProductMetadata(std::string_view submissionIdentifier = "Submission 1", const std::string& installerHash = "ABCD") + { + ProductMetadata result; + result.SchemaVersion.Assign("1.0"); + result.ProductVersionMin.Assign("1.0"); + result.ProductVersionMax.Assign("1.0"); + auto& installerMetadata = result.InstallerMetadataMap[installerHash]; + installerMetadata.SubmissionIdentifier = submissionIdentifier; + installerMetadata.AppsAndFeaturesEntries.push_back({}); + auto& entry = installerMetadata.AppsAndFeaturesEntries.back(); + entry.DisplayName = "Name"; + entry.Publisher = "Publisher"; + entry.DisplayVersion = "1.0"; + entry.ProductCode = "{guid}"; + entry.InstallerType = Manifest::InstallerTypeEnum::Msi; + return result; + } + + struct TestMerge + { + TestMerge() = default; + + TestMerge(MinimalDefaults_t) + { + Version = "1.0"; + Metadatas = std::make_optional>(); + Metadatas->emplace_back(MakeProductMetadata()); + } + + std::optional Version; + std::optional> Metadatas; + + std::wstring ToJSON() + { + web::json::value json; + + if (Version) + { + json[L"version"] = JSON::GetStringValue(Version.value()); + } + + if (Metadatas) + { + web::json::value metadatasArray; + + if (Metadatas->empty()) + { + metadatasArray = web::json::value::array(); + } + else + { + size_t index = 0; + for (auto& value : Metadatas.value()) + { + metadatasArray[index++] = value.ToJson(value.SchemaVersion, 0); + } + } + + json[L"metadatas"] = std::move(metadatasArray); + } + + return json.serialize(); + } + }; } TEST_CASE("MetadataCollection_MinimumInput", "[metadata_collection]") @@ -268,6 +345,29 @@ TEST_CASE("MetadataCollection_MinimumInput", "[metadata_collection]") REQUIRE(output.InstallerHash.value() == input.InstallerHash.value()); } +TEST_CASE("MetadataCollection_SubmissionDataCopied", "[metadata_collection]") +{ + TestInput input(MinimalDefaults); + + web::json::value submissionData; + std::wstring testValueName = L"testValueName"; + std::string testValueValue = "Test value value"; + submissionData[L"submissionIdentifier"] = JSON::GetStringValue("Required identifier"); + submissionData[testValueName] = JSON::GetStringValue(testValueValue); + + input.SubmissionData = submissionData; + + TestOutput output = GetOutput(input); + REQUIRE(!output.IsError()); + output.ValidateFieldPresence(); + + REQUIRE(output.Version.value() == input.Version.value()); + REQUIRE(output.InstallerHash.value() == input.InstallerHash.value()); + auto outputValue = JSON::GetRawStringValueFromJsonNode(output.SubmissionData.value(), testValueName); + REQUIRE(outputValue); + REQUIRE(outputValue.value() == testValueValue); +} + TEST_CASE("MetadataCollection_BadInput", "[metadata_collection]") { TestInput input(MinimalDefaults); @@ -283,7 +383,7 @@ TEST_CASE("MetadataCollection_BadInput", "[metadata_collection]") RESET_FIELD_SECTION(SupportedMetadataVersion); RESET_FIELD_SECTION(SubmissionIdentifier); RESET_FIELD_SECTION(InstallerHash); - RESET_FIELD_SECTION(SubmissionData); + RESET_FIELD_SECTION(PackageData); #undef RESET_FIELD_SECTION } @@ -515,3 +615,63 @@ TEST_CASE("MetadataCollection_NewSubmission", "[metadata_collection]") REQUIRE(historicalEntry.Publishers.size() == 1); REQUIRE(*historicalEntry.Publishers.begin() == appsAndFeaturesEntry.Publisher); } + +TEST_CASE("MetadataCollection_Merge_Empty", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->clear(); + + REQUIRE_THROWS_HR(InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}), E_NOT_SET); +} + +TEST_CASE("MetadataCollection_Merge_SubmissionMismatch", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->emplace_back(MakeProductMetadata("Submission 2")); + + REQUIRE_THROWS_HR(InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}), E_NOT_VALID_STATE); +} + +TEST_CASE("MetadataCollection_Merge_DifferentInstallers", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->emplace_back(MakeProductMetadata(mergeData.Metadatas->at(0).InstallerMetadataMap.begin()->second.SubmissionIdentifier, "EFGH")); + + std::wstring mergeResult = InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}); + REQUIRE(!mergeResult.empty()); + + ProductMetadata mergeMetadata; + mergeMetadata.FromJson(web::json::value::parse(mergeResult)); + + REQUIRE(mergeMetadata.InstallerMetadataMap.size() == 2); + for (const auto& item : mergeMetadata.InstallerMetadataMap) + { + REQUIRE(item.second.AppsAndFeaturesEntries.size() == 1); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].DisplayName.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].Publisher.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].DisplayVersion.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].ProductCode.empty()); + } +} + +TEST_CASE("MetadataCollection_Merge_SameInstaller", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->emplace_back(MakeProductMetadata()); + + std::wstring mergeResult = InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}); + REQUIRE(!mergeResult.empty()); + + ProductMetadata mergeMetadata; + mergeMetadata.FromJson(web::json::value::parse(mergeResult)); + + REQUIRE(mergeMetadata.InstallerMetadataMap.size() == 1); + for (const auto& item : mergeMetadata.InstallerMetadataMap) + { + REQUIRE(item.second.AppsAndFeaturesEntries.size() == 1); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].DisplayName.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].Publisher.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].DisplayVersion.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].ProductCode.empty()); + } +} diff --git a/src/AppInstallerCLITests/main.cpp b/src/AppInstallerCLITests/main.cpp index 85ae9083f9..d557893111 100644 --- a/src/AppInstallerCLITests/main.cpp +++ b/src/AppInstallerCLITests/main.cpp @@ -154,7 +154,7 @@ int main(int argc, char** argv) Runtime::TestHook_SetPathOverride(Runtime::PathName::LocalState, Runtime::GetPathTo(Runtime::PathName::LocalState) / "Tests"); Runtime::TestHook_SetPathOverride(Runtime::PathName::UserFileSettings, Runtime::GetPathTo(Runtime::PathName::UserFileSettings) / "Tests"); Runtime::TestHook_SetPathOverride(Runtime::PathName::StandardSettings, Runtime::GetPathTo(Runtime::PathName::StandardSettings) / "Tests"); - Runtime::TestHook_SetPathOverride(Runtime::PathName::SecureSettings, Runtime::GetPathTo(Runtime::PathName::Temp) / "WinGet_SecureSettings_Tests"); + Runtime::TestHook_SetPathOverride(Runtime::PathName::SecureSettings, Runtime::GetPathTo(Runtime::PathName::StandardSettings) / "WinGet_SecureSettings_Tests"); int result = Catch::Session().run(static_cast(args.size()), args.data()); diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp index 9016f02a8e..5350888179 100644 --- a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp +++ b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -113,6 +113,33 @@ namespace AppInstaller::Repository::Metadata } } } + + void FilterAndAddToEntries(Manifest::AppsAndFeaturesEntry&& newEntry, std::vector& entries) + { + // Erase all duplicated data from the new entry + for (const auto& entry : entries) + { +#define WINGET_ERASE_IF_SAME(_value_) if (entry._value_ == newEntry._value_) { newEntry._value_.clear(); } + WINGET_ERASE_IF_SAME(DisplayName); + WINGET_ERASE_IF_SAME(DisplayVersion); + WINGET_ERASE_IF_SAME(ProductCode); + WINGET_ERASE_IF_SAME(Publisher); + WINGET_ERASE_IF_SAME(UpgradeCode); +#undef WINGET_ERASE_IF_SAME + + if (entry.InstallerType == newEntry.InstallerType) + { + newEntry.InstallerType = Manifest::InstallerTypeEnum::Unknown; + } + } + + // If anything remains, add it + if (!newEntry.DisplayName.empty() || !newEntry.DisplayVersion.empty() || !newEntry.ProductCode.empty() || + !newEntry.Publisher.empty() || !newEntry.UpgradeCode.empty() || newEntry.InstallerType != Manifest::InstallerTypeEnum::Unknown) + { + entries.emplace_back(std::move(newEntry)); + } + } } void ProductMetadata::Clear() @@ -725,31 +752,7 @@ namespace AppInstaller::Repository::Metadata else { // Existing entry for installer hash, add/update the entry - auto& appsAndFeatures = itr->second.AppsAndFeaturesEntries; - - // Erase all duplicated data from the new entry - for (const auto& entry : appsAndFeatures) - { -#define WINGET_ERASE_IF_SAME(_value_) if (entry._value_ == newEntry._value_) { newEntry._value_.clear(); } - WINGET_ERASE_IF_SAME(DisplayName); - WINGET_ERASE_IF_SAME(DisplayVersion); - WINGET_ERASE_IF_SAME(ProductCode); - WINGET_ERASE_IF_SAME(Publisher); - WINGET_ERASE_IF_SAME(UpgradeCode); -#undef WINGET_ERASE_IF_SAME - - if (entry.InstallerType == newEntry.InstallerType) - { - newEntry.InstallerType = Manifest::InstallerTypeEnum::Unknown; - } - } - - // If anything remains, add it - if (!newEntry.DisplayName.empty() || !newEntry.DisplayVersion.empty() || !newEntry.ProductCode.empty() || - !newEntry.Publisher.empty() || !newEntry.UpgradeCode.empty() || newEntry.InstallerType != Manifest::InstallerTypeEnum::Unknown) - { - appsAndFeatures.emplace_back(std::move(newEntry)); - } + FilterAndAddToEntries(std::move(newEntry), itr->second.AppsAndFeaturesEntries); } } else @@ -913,6 +916,7 @@ namespace AppInstaller::Repository::Metadata web::json::value result; result[fields.Version] = web::json::value::string(L"1.0"); + result[fields.SubmissionData] = m_submissionData; result[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(m_installerHash); result[fields.Status] = web::json::value::string(ToString(OutputStatus::Error)); result[fields.Metadata] = web::json::value::null(); @@ -946,9 +950,7 @@ namespace AppInstaller::Repository::Metadata THROW_HR_IF(E_NOT_SET, metadatas.empty()); - Version maximumSchemaVersion; - - // Require that all merging values use the same submission (and extract the maximum schema version being used) + // Require that all merging values use the same submission for (const ProductMetadata& metadata : metadatas) { const std::string& firstSubmission = metadatas[0].InstallerMetadataMap.begin()->second.SubmissionIdentifier; @@ -958,13 +960,52 @@ namespace AppInstaller::Repository::Metadata AICLI_LOG(Repo, Info, << "Found submission identifier mismatch: " << firstSubmission << " != " << metadataSubmission); THROW_HR(E_NOT_VALID_STATE); } + } + + // Do the actual merging + ProductMetadata resultMetadata; + + // The historical data should be the same across the board, so we can just copy the first one. + resultMetadata.HistoricalMetadataList = metadatas[0].HistoricalMetadataList; + + for (const ProductMetadata& metadata : metadatas) + { + // Get the minimum and maximum versions from the individual values + if (resultMetadata.ProductVersionMin.IsEmpty() || metadata.ProductVersionMin < resultMetadata.ProductVersionMin) + { + resultMetadata.ProductVersionMin = metadata.ProductVersionMin; + } + + if (resultMetadata.ProductVersionMax < metadata.ProductVersionMax) + { + resultMetadata.ProductVersionMax = metadata.ProductVersionMax; + } + + if (resultMetadata.SchemaVersion < metadata.SchemaVersion) + { + resultMetadata.SchemaVersion = metadata.SchemaVersion; + } - if (maximumSchemaVersion < metadata.SchemaVersion) + for (const auto& installerMetadata : metadata.InstallerMetadataMap) { - maximumSchemaVersion = metadata.SchemaVersion; + auto itr = resultMetadata.InstallerMetadataMap.find(installerMetadata.first); + if (itr == resultMetadata.InstallerMetadataMap.end()) + { + // Installer hash not in the result, so just copy it + resultMetadata.InstallerMetadataMap.emplace(installerMetadata); + } + else + { + // Merge into existing installer data + for (const auto& targetEntry : installerMetadata.second.AppsAndFeaturesEntries) + { + FilterAndAddToEntries(Manifest::AppsAndFeaturesEntry{ targetEntry }, itr->second.AppsAndFeaturesEntries); + } + } } } - // Do the actual merging + // Convert to JSON + return resultMetadata.ToJson(resultMetadata.SchemaVersion, maximumSizeInBytes); } } diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h index 530c5261f1..c696be74fa 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h +++ b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -124,7 +124,7 @@ namespace AppInstaller::Repository::Metadata // Create version 1.0 of error JSON web::json::value CreateErrorJson_1_0(); - // Merge using merge input verision 1.0 + // Merge using merge input version 1.0 static web::json::value Merge_1_0(web::json::value& input, size_t maximumSizeInBytes); ThreadLocalStorage::ThreadGlobals m_threadGlobals; From 4335883d55630d0927ed0df31bbe7afbc564bd99 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Wed, 22 Jun 2022 13:13:27 -0700 Subject: [PATCH 22/22] Spellin --- .github/actions/spelling/expect.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index baab97e527..85ac608162 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -109,6 +109,7 @@ dvinns dw ecfr ecfrbrowse +efgh endian enr enums