From 2df545b142ede0ea8bc408d3781103b8c351ba78 Mon Sep 17 00:00:00 2001 From: Thomas Kappler Date: Tue, 28 Jan 2025 14:25:08 +0100 Subject: [PATCH 1/4] A new report listing API versions that are old enough to be removed --- provider/pkg/versioning/build_schema.go | 3 + provider/pkg/versioning/gen.go | 81 +++++++++ reports/oldApiVersions.json | 232 ++++++++++++++++++++++++ 3 files changed, 316 insertions(+) create mode 100644 reports/oldApiVersions.json diff --git a/provider/pkg/versioning/build_schema.go b/provider/pkg/versioning/build_schema.go index d5cea24c9169..31af9c2bf8df 100644 --- a/provider/pkg/versioning/build_schema.go +++ b/provider/pkg/versioning/build_schema.go @@ -47,6 +47,7 @@ type BuildSchemaReports struct { FlattenedPropertyConflicts map[string]map[string]any AllEndpoints map[openapi.ModuleName]map[openapi.ResourceName]map[string]*openapi.Endpoint InactiveDefaultVersions map[openapi.ModuleName][]openapi.ApiVersion + OldApiVersions map[openapi.ModuleName][]openapi.ApiVersion } func (r BuildSchemaReports) WriteTo(outputDir string) ([]string, error) { @@ -65,6 +66,7 @@ func (r BuildSchemaReports) WriteTo(outputDir string) ([]string, error) { "moduleNameErrors.json": r.ModuleNameErrors, "skippedPOSTEndpoints.json": r.SkippedPOSTEndpoints, "typeCaseConflicts.json": r.TypeCaseConflicts, + "oldApiVersions.json": r.OldApiVersions, }) } @@ -122,6 +124,7 @@ func BuildSchema(args BuildSchemaArgs) (*BuildSchemaResult, error) { CurationViolations: versionMetadata.CurationViolations, AllEndpoints: diagnostics.Endpoints, InactiveDefaultVersions: versionMetadata.InactiveDefaultVersions, + OldApiVersions: versionMetadata.GetOldApiVersionsPerModule(), } generationResult, err := gen.PulumiSchema(args.RootDir, modules, versionMetadata, providerVersion) diff --git a/provider/pkg/versioning/gen.go b/provider/pkg/versioning/gen.go index 7bb4f361a34b..be99394de1e4 100644 --- a/provider/pkg/versioning/gen.go +++ b/provider/pkg/versioning/gen.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "slices" "strings" "github.com/pulumi/pulumi-azure-native/v2/provider/pkg/gen" @@ -85,6 +86,86 @@ func (v VersionMetadata) GetAllVersions(moduleName openapi.ModuleName, resource return v.AllResourceVersionsByResource[moduleName][resource] } +// Collect, for each resource, the API versions that are older than the _previous_ default version and that weren't +// explicitly removed. +func (v VersionMetadata) GetOldApiVersionsPerResource() map[openapi.ModuleName][]openapi.ApiVersion { + result := map[openapi.ModuleName][]openapi.ApiVersion{} + for moduleName, resourceVersions := range v.AllResourceVersionsByResource { + // Collect the explicitly removed versions in a set for easy lookup. + removedFromModule := map[openapi.ApiVersion]struct{}{} + for _, version := range v.RemovedVersions[moduleName] { + removedFromModule[version] = struct{}{} + } + + // for resourceName := range resourceVersions { + // previousDefault := v.PreviousDefaultVersions[moduleName][resourceName].ApiVersion + // mustKeep[previousDefault] = struct{}{} + // currentDefault := v.DefaultVersions[moduleName][resourceName].ApiVersion + // mustKeep[currentDefault] = struct{}{} + // } + + for resourceName, versions := range resourceVersions { + oldVersionsSet := map[openapi.ApiVersion]struct{}{} + previousDefault := v.PreviousDefaultVersions[moduleName][resourceName].ApiVersion + for _, version := range versions { + if _, ok := removedFromModule[version]; !ok && openapi.CompareApiVersions(version, previousDefault) < 0 { + oldVersionsSet[version] = struct{}{} + } + } + oldVersions := []openapi.ApiVersion{} + for version := range oldVersionsSet { + oldVersions = append(oldVersions, version) + } + slices.SortStableFunc(oldVersions, openapi.CompareApiVersions) + if len(oldVersions) > 0 { + // result[moduleName] = append(result[moduleName], oldVersions) + } + } + } + return result +} + +// Collect, for each module, the API versions that are older than any _previous_ default version and that weren't +// explicitly removed. +func (v VersionMetadata) GetOldApiVersionsPerModule() map[openapi.ModuleName][]openapi.ApiVersion { + result := map[openapi.ModuleName][]openapi.ApiVersion{} + for moduleName, resourceVersions := range v.AllResourceVersionsByResource { + // Collect the explicitly removed versions in a set for easy lookup. We don't want to list them as removable + // when they're already removed. + removedFromModule := map[openapi.ApiVersion]struct{}{} + for _, version := range v.RemovedVersions[moduleName] { + removedFromModule[version] = struct{}{} + } + + // API versions can be removed by module, not by resource. Determine the oldest previous default version for + // this module. It's the cut-off below which we can safely remove versions. + oldestPreviousDefaultVersion := openapi.ApiVersion("2999-01-01") + for _, version := range v.PreviousDefaultVersions[moduleName] { + if openapi.CompareApiVersions(version.ApiVersion, oldestPreviousDefaultVersion) < 0 { + oldestPreviousDefaultVersion = version.ApiVersion + } + } + + for _, versions := range resourceVersions { + oldVersionsSet := map[openapi.ApiVersion]struct{}{} + for _, version := range versions { + if _, ok := removedFromModule[version]; !ok && openapi.CompareApiVersions(version, oldestPreviousDefaultVersion) < 0 { + oldVersionsSet[version] = struct{}{} + } + } + if len(oldVersionsSet) > 0 { + oldVersions := []openapi.ApiVersion{} + for version := range oldVersionsSet { + oldVersions = append(oldVersions, version) + } + slices.SortStableFunc(oldVersions, openapi.CompareApiVersions) + result[moduleName] = oldVersions + } + } + } + return result +} + func LoadVersionMetadata(rootDir string, modules openapi.AzureModules, majorVersion int) (VersionMetadata, error) { versionSources, err := ReadVersionSources(rootDir, modules, majorVersion) if err != nil { diff --git a/reports/oldApiVersions.json b/reports/oldApiVersions.json new file mode 100644 index 000000000000..24f4ddca992d --- /dev/null +++ b/reports/oldApiVersions.json @@ -0,0 +1,232 @@ +{ + "AadIam": [ + "2017-04-01-preview" + ], + "AnalysisServices": [ + "2017-08-01-beta" + ], + "ApiCenter": [ + "2024-03-01", + "2024-03-15-preview", + "2024-06-01-preview" + ], + "App": [ + "2022-01-01-preview" + ], + "AwsConnector": [ + "2024-12-01" + ], + "AzureDataTransfer": [ + "2024-09-27" + ], + "AzureFleet": [ + "2023-11-01-preview", + "2024-05-01-preview", + "2024-11-01" + ], + "AzureLargeInstance": [ + "2024-08-01-preview" + ], + "AzurePlaywrightService": [ + "2023-10-01-preview", + "2024-02-01-preview", + "2024-08-01-preview", + "2024-12-01" + ], + "BareMetalInfrastructure": [ + "2024-08-01-preview" + ], + "Cloudngfw": [ + "2022-08-29-preview", + "2022-08-29", + "2023-09-01-preview", + "2023-09-01", + "2023-10-10-preview", + "2024-01-19-preview", + "2024-02-07-preview" + ], + "CodeSigning": [ + "2024-02-05-preview", + "2024-09-30-preview" + ], + "Communication": [ + "2020-08-20-preview" + ], + "Community": [ + "2023-11-01" + ], + "Confluent": [ + "2020-03-01-preview" + ], + "ConnectedCache": [ + "2023-05-01-preview" + ], + "Contoso": [ + "2021-10-01-preview" + ], + "DBforMariaDB": [ + "2018-06-01-preview" + ], + "DBforMySQL": [ + "2017-12-01-preview" + ], + "DBforPostgreSQL": [ + "2017-12-01-preview" + ], + "Dashboard": [ + "2021-09-01-preview" + ], + "DataReplication": [ + "2021-02-16-preview" + ], + "DatabaseWatcher": [ + "2023-09-01-preview", + "2024-07-19-preview", + "2024-10-01-preview", + "2025-01-02" + ], + "DesktopVirtualization": [ + "2021-01-14-preview" + ], + "DevCenter": [ + "2022-08-01-preview" + ], + "DevOpsInfrastructure": [ + "2023-10-30-preview", + "2023-12-13-preview", + "2024-03-26-preview", + "2024-04-04-preview", + "2024-10-19" + ], + "DeviceRegistry": [ + "2023-11-01-preview", + "2024-09-01-preview", + "2024-11-01" + ], + "DocumentDB": [ + "2021-01-15" + ], + "DurableTask": [ + "2024-10-01-preview" + ], + "Edge": [ + "2024-02-01-preview" + ], + "EdgeMarketplace": [ + "2023-08-01-preview", + "2023-08-01" + ], + "Elastic": [ + "2020-07-01-preview" + ], + "Fabric": [ + "2023-11-01" + ], + "HealthBot": [ + "2020-12-08-preview" + ], + "HealthDataAIServices": [ + "2024-02-28-preview", + "2024-09-20" + ], + "HealthcareApis": [ + "2021-06-01-preview", + "2021-11-01", + "2022-01-31-preview" + ], + "HybridConnectivity": [ + "2021-10-06-preview" + ], + "Impact": [ + "2024-05-01-preview" + ], + "IntegrationSpaces": [ + "2023-11-14-preview" + ], + "Intune": [ + "2015-01-14-privatepreview" + ], + "IoTFirmwareDefense": [ + "2023-02-08-preview", + "2024-01-10" + ], + "IoTOperations": [ + "2024-07-01-preview" + ], + "IoTOperationsDataProcessor": [ + "2023-10-04-preview" + ], + "IoTOperationsMQ": [ + "2023-10-04-preview" + ], + "IoTOperationsOrchestrator": [ + "2023-10-04-preview" + ], + "KubernetesRuntime": [ + "2023-10-01-preview", + "2024-03-01" + ], + "Logz": [ + "2020-10-01-preview" + ], + "MobileNetwork": [ + "2022-03-01-preview" + ], + "NetworkAnalytics": [ + "2023-11-15" + ], + "NetworkFunction": [ + "2021-09-01-preview" + ], + "OffAzureSpringBoot": [ + "2024-04-01-preview" + ], + "OpenEnergyPlatform": [ + "2021-06-01-preview" + ], + "Orbital": [ + "2022-03-01", + "2022-11-01" + ], + "Portal": [ + "2018-10-01-preview" + ], + "PortalServices": [ + "2024-04-01-preview" + ], + "ProgrammableConnectivity": [ + "2024-01-15-preview" + ], + "Quota": [ + "2023-06-01-preview", + "2024-10-15-preview", + "2024-12-18-preview", + "2025-03-01" + ], + "Scom": [ + "2023-07-07-preview" + ], + "Search": [ + "2020-08-01-preview" + ], + "SecretSyncController": [ + "2024-08-21-preview" + ], + "SecurityAndCompliance": [ + "2021-01-11" + ], + "StandbyPool": [ + "2023-12-01-preview", + "2024-03-01-preview", + "2024-03-01" + ], + "Storage": [ + "2021-01-01" + ], + "StorageActions": [ + "2023-01-01" + ], + "VerifiedId": [ + "2024-01-26-preview" + ] +} \ No newline at end of file From afcdc5e3c92a1e32e5b65e028d109ac5249442cb Mon Sep 17 00:00:00 2001 From: Thomas Kappler Date: Tue, 28 Jan 2025 16:27:46 +0100 Subject: [PATCH 2/4] Fix algorithm, remove unused first draft --- provider/pkg/versioning/gen.go | 69 ++++----------- reports/oldApiVersions.json | 155 +-------------------------------- 2 files changed, 20 insertions(+), 204 deletions(-) diff --git a/provider/pkg/versioning/gen.go b/provider/pkg/versioning/gen.go index be99394de1e4..73eac9a236ca 100644 --- a/provider/pkg/versioning/gen.go +++ b/provider/pkg/versioning/gen.go @@ -86,45 +86,6 @@ func (v VersionMetadata) GetAllVersions(moduleName openapi.ModuleName, resource return v.AllResourceVersionsByResource[moduleName][resource] } -// Collect, for each resource, the API versions that are older than the _previous_ default version and that weren't -// explicitly removed. -func (v VersionMetadata) GetOldApiVersionsPerResource() map[openapi.ModuleName][]openapi.ApiVersion { - result := map[openapi.ModuleName][]openapi.ApiVersion{} - for moduleName, resourceVersions := range v.AllResourceVersionsByResource { - // Collect the explicitly removed versions in a set for easy lookup. - removedFromModule := map[openapi.ApiVersion]struct{}{} - for _, version := range v.RemovedVersions[moduleName] { - removedFromModule[version] = struct{}{} - } - - // for resourceName := range resourceVersions { - // previousDefault := v.PreviousDefaultVersions[moduleName][resourceName].ApiVersion - // mustKeep[previousDefault] = struct{}{} - // currentDefault := v.DefaultVersions[moduleName][resourceName].ApiVersion - // mustKeep[currentDefault] = struct{}{} - // } - - for resourceName, versions := range resourceVersions { - oldVersionsSet := map[openapi.ApiVersion]struct{}{} - previousDefault := v.PreviousDefaultVersions[moduleName][resourceName].ApiVersion - for _, version := range versions { - if _, ok := removedFromModule[version]; !ok && openapi.CompareApiVersions(version, previousDefault) < 0 { - oldVersionsSet[version] = struct{}{} - } - } - oldVersions := []openapi.ApiVersion{} - for version := range oldVersionsSet { - oldVersions = append(oldVersions, version) - } - slices.SortStableFunc(oldVersions, openapi.CompareApiVersions) - if len(oldVersions) > 0 { - // result[moduleName] = append(result[moduleName], oldVersions) - } - } - } - return result -} - // Collect, for each module, the API versions that are older than any _previous_ default version and that weren't // explicitly removed. func (v VersionMetadata) GetOldApiVersionsPerModule() map[openapi.ModuleName][]openapi.ApiVersion { @@ -139,28 +100,34 @@ func (v VersionMetadata) GetOldApiVersionsPerModule() map[openapi.ModuleName][]o // API versions can be removed by module, not by resource. Determine the oldest previous default version for // this module. It's the cut-off below which we can safely remove versions. - oldestPreviousDefaultVersion := openapi.ApiVersion("2999-01-01") + previousDefaultVersions := []openapi.ApiVersion{} for _, version := range v.PreviousDefaultVersions[moduleName] { - if openapi.CompareApiVersions(version.ApiVersion, oldestPreviousDefaultVersion) < 0 { - oldestPreviousDefaultVersion = version.ApiVersion - } + previousDefaultVersions = append(previousDefaultVersions, version.ApiVersion) } + if len(previousDefaultVersions) == 0 { + continue + } + oldestPreviousDefaultVersion := slices.MinFunc(previousDefaultVersions, openapi.CompareApiVersions) + // With oldestPreviousDefaultVersion as cut-off, iterate over all resources and their versions and collect the + // versions that are older than the cut-off. + oldVersionsSet := map[openapi.ApiVersion]struct{}{} for _, versions := range resourceVersions { - oldVersionsSet := map[openapi.ApiVersion]struct{}{} for _, version := range versions { if _, ok := removedFromModule[version]; !ok && openapi.CompareApiVersions(version, oldestPreviousDefaultVersion) < 0 { oldVersionsSet[version] = struct{}{} } } - if len(oldVersionsSet) > 0 { - oldVersions := []openapi.ApiVersion{} - for version := range oldVersionsSet { - oldVersions = append(oldVersions, version) - } - slices.SortStableFunc(oldVersions, openapi.CompareApiVersions) - result[moduleName] = oldVersions + } + + // If there are any old versions, sort them and add them to the result. + if len(oldVersionsSet) > 0 { + oldVersions := []openapi.ApiVersion{} + for version := range oldVersionsSet { + oldVersions = append(oldVersions, version) } + slices.SortStableFunc(oldVersions, openapi.CompareApiVersions) + result[moduleName] = oldVersions } } return result diff --git a/reports/oldApiVersions.json b/reports/oldApiVersions.json index 24f4ddca992d..60cb84014dac 100644 --- a/reports/oldApiVersions.json +++ b/reports/oldApiVersions.json @@ -5,66 +5,17 @@ "AnalysisServices": [ "2017-08-01-beta" ], - "ApiCenter": [ - "2024-03-01", - "2024-03-15-preview", - "2024-06-01-preview" - ], "App": [ "2022-01-01-preview" ], - "AwsConnector": [ - "2024-12-01" - ], - "AzureDataTransfer": [ - "2024-09-27" - ], - "AzureFleet": [ - "2023-11-01-preview", - "2024-05-01-preview", - "2024-11-01" - ], - "AzureLargeInstance": [ - "2024-08-01-preview" - ], - "AzurePlaywrightService": [ - "2023-10-01-preview", - "2024-02-01-preview", - "2024-08-01-preview", - "2024-12-01" - ], - "BareMetalInfrastructure": [ - "2024-08-01-preview" - ], - "Cloudngfw": [ - "2022-08-29-preview", - "2022-08-29", - "2023-09-01-preview", - "2023-09-01", - "2023-10-10-preview", - "2024-01-19-preview", - "2024-02-07-preview" - ], - "CodeSigning": [ - "2024-02-05-preview", - "2024-09-30-preview" - ], "Communication": [ "2020-08-20-preview" ], - "Community": [ - "2023-11-01" - ], "Confluent": [ "2020-03-01-preview" ], - "ConnectedCache": [ - "2023-05-01-preview" - ], - "Contoso": [ - "2021-10-01-preview" - ], "DBforMariaDB": [ + "2018-06-01-privatepreview", "2018-06-01-preview" ], "DBforMySQL": [ @@ -76,60 +27,23 @@ "Dashboard": [ "2021-09-01-preview" ], - "DataReplication": [ - "2021-02-16-preview" - ], - "DatabaseWatcher": [ - "2023-09-01-preview", - "2024-07-19-preview", - "2024-10-01-preview", - "2025-01-02" - ], "DesktopVirtualization": [ "2021-01-14-preview" ], "DevCenter": [ "2022-08-01-preview" ], - "DevOpsInfrastructure": [ - "2023-10-30-preview", - "2023-12-13-preview", - "2024-03-26-preview", - "2024-04-04-preview", - "2024-10-19" - ], - "DeviceRegistry": [ - "2023-11-01-preview", - "2024-09-01-preview", - "2024-11-01" - ], "DocumentDB": [ "2021-01-15" ], - "DurableTask": [ - "2024-10-01-preview" - ], - "Edge": [ - "2024-02-01-preview" - ], - "EdgeMarketplace": [ - "2023-08-01-preview", - "2023-08-01" - ], "Elastic": [ "2020-07-01-preview" ], - "Fabric": [ - "2023-11-01" - ], "HealthBot": [ "2020-12-08-preview" ], - "HealthDataAIServices": [ - "2024-02-28-preview", - "2024-09-20" - ], "HealthcareApis": [ + "2021-01-11", "2021-06-01-preview", "2021-11-01", "2022-01-31-preview" @@ -137,96 +51,31 @@ "HybridConnectivity": [ "2021-10-06-preview" ], - "Impact": [ - "2024-05-01-preview" - ], - "IntegrationSpaces": [ - "2023-11-14-preview" - ], "Intune": [ "2015-01-14-privatepreview" ], - "IoTFirmwareDefense": [ - "2023-02-08-preview", - "2024-01-10" - ], - "IoTOperations": [ - "2024-07-01-preview" - ], - "IoTOperationsDataProcessor": [ - "2023-10-04-preview" - ], - "IoTOperationsMQ": [ - "2023-10-04-preview" - ], - "IoTOperationsOrchestrator": [ - "2023-10-04-preview" - ], - "KubernetesRuntime": [ - "2023-10-01-preview", - "2024-03-01" - ], "Logz": [ "2020-10-01-preview" ], "MobileNetwork": [ "2022-03-01-preview" ], - "NetworkAnalytics": [ - "2023-11-15" - ], "NetworkFunction": [ "2021-09-01-preview" ], - "OffAzureSpringBoot": [ - "2024-04-01-preview" - ], "OpenEnergyPlatform": [ "2021-06-01-preview" ], - "Orbital": [ - "2022-03-01", - "2022-11-01" - ], "Portal": [ "2018-10-01-preview" ], - "PortalServices": [ - "2024-04-01-preview" - ], - "ProgrammableConnectivity": [ - "2024-01-15-preview" - ], - "Quota": [ - "2023-06-01-preview", - "2024-10-15-preview", - "2024-12-18-preview", - "2025-03-01" - ], - "Scom": [ - "2023-07-07-preview" - ], "Search": [ "2020-08-01-preview" ], - "SecretSyncController": [ - "2024-08-21-preview" - ], "SecurityAndCompliance": [ "2021-01-11" ], - "StandbyPool": [ - "2023-12-01-preview", - "2024-03-01-preview", - "2024-03-01" - ], "Storage": [ "2021-01-01" - ], - "StorageActions": [ - "2023-01-01" - ], - "VerifiedId": [ - "2024-01-26-preview" ] } \ No newline at end of file From a1a7ea4b8dbd0528ce461b288dfb6ec6b3b13053 Mon Sep 17 00:00:00 2001 From: Thomas Kappler Date: Tue, 28 Jan 2025 16:40:06 +0100 Subject: [PATCH 3/4] use util.UnsortedKeys Co-authored-by: Daniel Bradley --- provider/pkg/versioning/gen.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/provider/pkg/versioning/gen.go b/provider/pkg/versioning/gen.go index 73eac9a236ca..adba55a3a1a6 100644 --- a/provider/pkg/versioning/gen.go +++ b/provider/pkg/versioning/gen.go @@ -15,6 +15,7 @@ import ( "github.com/pulumi/pulumi-azure-native/v2/provider/pkg/openapi" "github.com/pulumi/pulumi-azure-native/v2/provider/pkg/openapi/paths" "github.com/pulumi/pulumi-azure-native/v2/provider/pkg/providerlist" + "github.com/pulumi/pulumi-azure-native/v2/provider/pkg/util" "gopkg.in/yaml.v3" ) @@ -122,10 +123,7 @@ func (v VersionMetadata) GetOldApiVersionsPerModule() map[openapi.ModuleName][]o // If there are any old versions, sort them and add them to the result. if len(oldVersionsSet) > 0 { - oldVersions := []openapi.ApiVersion{} - for version := range oldVersionsSet { - oldVersions = append(oldVersions, version) - } + oldVersions := util.UnsortedKeys(oldVersionsSet) slices.SortStableFunc(oldVersions, openapi.CompareApiVersions) result[moduleName] = oldVersions } From 748290fe0c5c1b8638c3cdfdff7c2234054c5a37 Mon Sep 17 00:00:00 2001 From: Thomas Kappler Date: Tue, 28 Jan 2025 16:44:32 +0100 Subject: [PATCH 4/4] Document oldApiVersions.json in reports/readme.md --- reports/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/reports/README.md b/reports/README.md index a4589fef07de..bdfbdb114b3d 100644 --- a/reports/README.md +++ b/reports/README.md @@ -47,4 +47,8 @@ This, most likely very unelegant, `jq` query filters the endpoints to only those ```jq jq '[ keys[] as $rp | .[$rp] | keys[] as $path | .[$path][] ] | map(select(.HttpVerbs != null) | select(.HttpVerbs | contains(["POST"])))' allEndpoints.json -``` \ No newline at end of file +``` + +## `oldApiVersions.json` + +This file contains a list of all API versions that are older than the oldest previous default version for each resource provider. These API versions are good candidates for removal. Versions that are already explicitly removed via `vN-removed.json` are not included.