From 7e4805fc434f56d36f2c23d44a2124840da29c69 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 19 Oct 2023 12:05:23 +0200 Subject: [PATCH 1/5] tools/importer-rest-api-specs: adding a test covering the SQL issue (https://github.com/hashicorp/pandora/issues/2842) This happens when the same URI/Path contains multiple operations with different HTTP Methods which use different values for the Constant segments. This means that dependent on the order these are processed, that we can have eventual consistency/flapping between one constant and another - causing https://github.com/hashicorp/pandora/issues/2842 --- .../components/parser/resource_ids_test.go | 89 +++++++++++++++++ .../parser/resourceids/parse_segments.go | 1 + ...fferent_constant_values_per_operation.json | 97 +++++++++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 tools/importer-rest-api-specs/components/parser/testdata/resource_ids_same_uri_different_constant_values_per_operation.json diff --git a/tools/importer-rest-api-specs/components/parser/resource_ids_test.go b/tools/importer-rest-api-specs/components/parser/resource_ids_test.go index df184cdb7a4..42e03119882 100644 --- a/tools/importer-rest-api-specs/components/parser/resource_ids_test.go +++ b/tools/importer-rest-api-specs/components/parser/resource_ids_test.go @@ -1347,6 +1347,95 @@ func TestParseResourceIdContainingTheSegmentsNamedTheSame(t *testing.T) { } } +func TestParseResourceIdsWhereTheSameUriContainsDifferentConstantValuesPerOperation(t *testing.T) { + // Whilst a URI may contain Constants, the values for those constants can differ per HTTP Operation + // as such we need to ensure that these are output as different Resource ID types + // + // In this case there are 2 constants defined, `PlanetNames` and `PlanetEarth` - however `PlanetEarth` is a constant + // with a single value - therefore `PlanetEarth` will be removed. + // The Operation `HEAD /galaxies/{galaxyName}/hello/{planetName}` should remain as-is + // The Operation `DELETE /galaxies/{galaxyName}/hello/{planetName}` should be transformed to `/galaxies/{galaxyName}/hello/Earth` + // This means we should end up with 2 IDs, GalaxyId and PlanetId (with Earth and Mars the Constant PlanetNames in PlanetId) + // + // Experience has shown this is eventually consistent, maybe 100x is overkill, but it'll do for now. + for i := 0; i < 100; i++ { + t.Logf("iteration %d", i) + + result, err := ParseSwaggerFileForTesting(t, "resource_ids_same_uri_different_constant_values_per_operation.json") + if err != nil { + t.Fatalf("parsing: %+v", err) + } + if result == nil { + t.Fatal("result was nil") + } + if len(result.Resources) != 1 { + t.Fatalf("expected 1 resource but got %d", len(result.Resources)) + } + + hello, ok := result.Resources["Hello"] + if !ok { + t.Fatalf("no resources were output with the tag `Hello`") + } + + if len(hello.Constants) != 1 { + t.Fatalf("expected 1 Constant but got %d", len(hello.Constants)) + } + if len(hello.Models) != 0 { + t.Fatalf("expected No Models but got %d", len(hello.Models)) + } + if len(hello.Operations) != 2 { + t.Fatalf("expected 2 Operations but got %d", len(hello.Operations)) + } + if len(hello.ResourceIds) != 2 { + t.Fatalf("expected 2 ResourceIds but got %d", len(hello.ResourceIds)) + } + + // sanity check we're pulling out both values + planetNamesConst, ok := hello.Constants["PlanetNames"] + if !ok { + t.Fatalf("expected a Constant named `PlanetNames` but didn't get one") + } + expected := map[string]string{ + "Earth": "Earth", + "Mars": "Mars", + } + if !reflect.DeepEqual(expected, planetNamesConst.Values) { + t.Fatalf("expected the Constant `PlanetNames` to have 2 values but got %+v", planetNamesConst.Values) + } + + headOperation, ok := hello.Operations["Head"] + if !ok { + t.Fatalf("expected an operation `Head` but didn't get one") + } + if headOperation.UriSuffix != nil { + t.Fatalf("expected UriSuffix to be nil for the Head operation but got %q", *headOperation.UriSuffix) + } + if headOperation.ResourceIdName == nil { + t.Fatalf("expected the ResourceIdName for the Head operation to be `PlanetId` but got nil") + } + if *headOperation.ResourceIdName != "PlanetId" { + t.Fatalf("expected the ResourceIdName for the Head operation to be `PlanetId` but got %q", *headOperation.ResourceIdName) + } + + deleteOperation, ok := hello.Operations["Delete"] + if !ok { + t.Fatalf("expected an operation `Delete` but didn't get one") + } + if deleteOperation.UriSuffix == nil { + t.Fatalf("expected UriSuffix to be `/hello/Earth` for the Delete operation but got nil") + } + if *deleteOperation.UriSuffix != "/hello/Earth" { + t.Fatalf("expected UriSuffix to be `/hello/Earth` for the Delete operation but got %q", *deleteOperation.UriSuffix) + } + if deleteOperation.ResourceIdName == nil { + t.Fatalf("expected the ResourceIdName for the Head operation to be `GalaxyId` but got nil") + } + if *deleteOperation.ResourceIdName != "GalaxyId" { + t.Fatalf("expected the ResourceIdName for the Head operation to be `GalaxyId` but got %q", *deleteOperation.ResourceIdName) + } + } +} + func TestParseResourceIdsCommon(t *testing.T) { result, err := ParseSwaggerFileForTesting(t, "resource_ids_common.json") if err != nil { diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/parse_segments.go b/tools/importer-rest-api-specs/components/parser/resourceids/parse_segments.go index 06d774b630b..a2a1d59514a 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/parse_segments.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/parse_segments.go @@ -131,6 +131,7 @@ func (p *Parser) parseResourceIdFromOperation(uri string, operation *spec.Operat return nil, fmt.Errorf("parsing constant from %q: %+v", uriSegment, err) } + p.logger.Trace(fmt.Sprintf("Found Constant %q with values `%+v`", constant.Name, constant.Details.Values)) if len(constant.Details.Values) == 1 { constantValue := "" for _, v := range constant.Details.Values { diff --git a/tools/importer-rest-api-specs/components/parser/testdata/resource_ids_same_uri_different_constant_values_per_operation.json b/tools/importer-rest-api-specs/components/parser/testdata/resource_ids_same_uri_different_constant_values_per_operation.json new file mode 100644 index 00000000000..d840aec768f --- /dev/null +++ b/tools/importer-rest-api-specs/components/parser/testdata/resource_ids_same_uri_different_constant_values_per_operation.json @@ -0,0 +1,97 @@ +{ + "swagger": "2.0", + "info": { + "title": "Example", + "description": "Example", + "version": "2020-01-01" + }, + "host": "management.mysite.com", + "schemes": [ + "https" + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "security": [], + "securityDefinitions": {}, + "paths": { + "/galaxies/{galaxyName}/hello/{planetName}": { + "head": { + "tags": [ + "Hello" + ], + "operationId": "Hello_Head", + "description": "Example", + "parameters": [ + { + "name": "galaxyName", + "in": "path", + "description": "The name of the galaxy.", + "required": true, + "type": "string" + }, + { + "name": "planetName", + "in": "path", + "description": "The name of the planet.", + "required": true, + "type": "string", + "enum": [ + "Mars", + "Earth" + ], + "x-ms-enum": { + "name": "PlanetNames", + "modelAsString": false + } + } + ], + "responses": { + "200": { + "description": "Success." + } + } + }, + "delete": { + "tags": [ + "Hello" + ], + "operationId": "Hello_Delete", + "description": "Example", + "parameters": [ + { + "name": "galaxyName", + "in": "path", + "description": "The name of the galaxy.", + "required": true, + "type": "string" + }, + { + "name": "planetName", + "in": "path", + "description": "The name of the planet.", + "required": true, + "type": "string", + "enum": [ + "Earth" + ], + "x-ms-enum": { + "name": "PlanetEarth", + "modelAsString": false + } + } + ], + "responses": { + "200": { + "description": "Success." + } + } + } + } + }, + "definitions": {}, + "parameters": {} +} \ No newline at end of file From 52c03c1f11dcf8ab80aee481e5415063540c0ff1 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 19 Oct 2023 14:05:17 +0200 Subject: [PATCH 2/5] tools/importer-rest-api-specs: updating the Resource ID parsing to use Operation ID rather than URI as the identifier This works around an issue where one URI can contain multiple values for the constant segments, meaning that because we were using a map previously, the parsing order of the map (which would be eventually consistent) would cause flapping on the imported data. --- .../components/parser/operations.go | 36 ++---- .../components/parser/parser.go | 6 +- .../resourceids/distinct_resource_ids.go | 9 +- .../parser/resourceids/generate_names.go | 5 +- .../parser/resourceids/generate_names_test.go | 43 +++--- .../components/parser/resourceids/mappings.go | 51 -------- .../components/parser/resourceids/models.go | 28 ++-- .../parser/resourceids/parse_segments.go | 20 +-- .../components/parser/resourceids/parser.go | 122 +++++++++++++++--- .../components/parser/swagger_resources.go | 2 +- .../importer-rest-api-specs/models/models.go | 2 +- 11 files changed, 177 insertions(+), 147 deletions(-) delete mode 100644 tools/importer-rest-api-specs/components/parser/resourceids/mappings.go diff --git a/tools/importer-rest-api-specs/components/parser/operations.go b/tools/importer-rest-api-specs/components/parser/operations.go index c28149477ee..42cfe45c048 100644 --- a/tools/importer-rest-api-specs/components/parser/operations.go +++ b/tools/importer-rest-api-specs/components/parser/operations.go @@ -17,12 +17,12 @@ import ( ) type operationsParser struct { - operations []parsedOperation - urisToResourceIds map[string]resourceids.ParsedOperation - swaggerDefinition *SwaggerDefinition + operations []parsedOperation + operationIdsToParsedOperations map[string]resourceids.ParsedOperation + swaggerDefinition *SwaggerDefinition } -func (d *SwaggerDefinition) parseOperationsWithinTag(tag *string, urisToResourceIds map[string]resourceids.ParsedOperation, resourceProvider *string, found internal.ParseResult) (*map[string]models.OperationDetails, *internal.ParseResult, error) { +func (d *SwaggerDefinition) parseOperationsWithinTag(tag *string, operationIdsToParsedOperations map[string]resourceids.ParsedOperation, resourceProvider *string, found internal.ParseResult) (*map[string]models.OperationDetails, *internal.ParseResult, error) { logger := d.logger.Named("Operations Parser") operations := make(map[string]models.OperationDetails, 0) result := internal.ParseResult{ @@ -32,8 +32,8 @@ func (d *SwaggerDefinition) parseOperationsWithinTag(tag *string, urisToResource result.Append(found) parser := operationsParser{ - urisToResourceIds: urisToResourceIds, - swaggerDefinition: d, + operationIdsToParsedOperations: operationIdsToParsedOperations, + swaggerDefinition: d, } // first find the operations then pull out everything we can @@ -57,7 +57,7 @@ func (d *SwaggerDefinition) parseOperationsWithinTag(tag *string, urisToResource } if existing, hasExisting := operations[operation.name]; hasExisting { - return nil, nil, fmt.Errorf("conflicting operations with the Name %q - first %q %q - second %q %q", operation.name, existing.Method, existing.Uri, parsedOperation.Method, parsedOperation.Uri) + return nil, nil, fmt.Errorf("conflicting operations with the Name %q - first %q %q - second %q %q", operation.name, existing.Method, existing.OperationId, parsedOperation.Method, parsedOperation.OperationId) } if parsedOperation == nil { @@ -76,16 +76,12 @@ func (p operationsParser) parseOperation(operation parsedOperation, resourceProv Models: map[string]models.ModelDetails{}, } - normalizedUri, err := p.normalizedUriForOperation(operation) - if err != nil { - return nil, nil, fmt.Errorf("determining the normalized uri: %+v", err) - } contentType := p.determineContentType(operation) expectedStatusCodes := p.expectedStatusCodesForOperation(operation) paginationField := p.fieldContainingPaginationDetailsForOperation(operation) requestObject, nestedResult, err := p.requestObjectForOperation(operation, result) if err != nil { - return nil, nil, fmt.Errorf("determining request operation for %q (method %q / uri %q): %+v", operation.name, operation.httpMethod, *normalizedUri, err) + return nil, nil, fmt.Errorf("determining request operation for %q (method %q / ID %q): %+v", operation.name, operation.httpMethod, operation.operation.ID, err) } if nestedResult != nil { if err := result.Append(*nestedResult); err != nil { @@ -95,7 +91,7 @@ func (p operationsParser) parseOperation(operation parsedOperation, resourceProv isAListOperation := p.isListOperation(operation) responseResult, nestedResult, err := p.responseObjectForOperation(operation, result) if err != nil { - return nil, nil, fmt.Errorf("determining response operation for %q (method %q / uri %q): %+v", operation.name, operation.httpMethod, *normalizedUri, err) + return nil, nil, fmt.Errorf("determining response operation for %q (method %q / ID %q): %+v", operation.name, operation.httpMethod, operation.operation.ID, err) } if nestedResult != nil { if err := result.Append(*nestedResult); err != nil { @@ -117,7 +113,7 @@ func (p operationsParser) parseOperation(operation parsedOperation, resourceProv } } - resourceId := p.urisToResourceIds[*normalizedUri] + resourceId := p.operationIdsToParsedOperations[operation.operation.ID] usesADifferentResourceProvider, err := resourceIdUsesAResourceProviderOtherThan(resourceId.ResourceId, resourceProvider) if err != nil { return nil, nil, err @@ -133,11 +129,11 @@ func (p operationsParser) parseOperation(operation parsedOperation, resourceProv IsListOperation: isAListOperation, LongRunning: longRunning, Method: strings.ToUpper(operation.httpMethod), + OperationId: operation.operation.ID, Options: *options, RequestObject: requestObject, ResourceIdName: resourceId.ResourceIdName, ResponseObject: responseResult.objectDefinition, - Uri: *normalizedUri, UriSuffix: resourceId.UriSuffix, } @@ -396,16 +392,6 @@ func (p operationsParser) operationShouldBeIgnored(input models.OperationDetails return false } -func (p operationsParser) normalizedUriForOperation(input parsedOperation) (*string, error) { - for key := range p.urisToResourceIds { - if strings.EqualFold(key, input.uri) { - return &key, nil - } - } - - return nil, fmt.Errorf("%q was not found in the normalized uri list", input.uri) -} - func (p operationsParser) requestObjectForOperation(input parsedOperation, known internal.ParseResult) (*models.ObjectDefinition, *internal.ParseResult, error) { // all we should parse out is the top level object - nothing more. diff --git a/tools/importer-rest-api-specs/components/parser/parser.go b/tools/importer-rest-api-specs/components/parser/parser.go index 9ef05ba759a..1269420fcae 100644 --- a/tools/importer-rest-api-specs/components/parser/parser.go +++ b/tools/importer-rest-api-specs/components/parser/parser.go @@ -126,9 +126,9 @@ func (d *SwaggerDefinition) ParseResourceIds(resourceProvider *string) (*resourc func (d *SwaggerDefinition) filterResourceIdsToResourceProvider(input resourceids.ParseResult, resourceProvider string) (*resourceids.ParseResult, error) { output := resourceids.ParseResult{ - OriginalUrisToResourceIDs: input.OriginalUrisToResourceIDs, - NamesToResourceIDs: map[string]models.ParsedResourceId{}, - Constants: input.Constants, + OperationIdsToParsedResourceIds: input.OperationIdsToParsedResourceIds, + NamesToResourceIDs: map[string]models.ParsedResourceId{}, + Constants: input.Constants, } for name := range input.NamesToResourceIDs { diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/distinct_resource_ids.go b/tools/importer-rest-api-specs/components/parser/resourceids/distinct_resource_ids.go index c24b54a0bad..fe73343ba48 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/distinct_resource_ids.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/distinct_resource_ids.go @@ -4,15 +4,16 @@ import "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" func (p *Parser) distinctResourceIds(input map[string]processedResourceId) []models.ParsedResourceId { out := make([]models.ParsedResourceId, 0) - for _, v := range input { - if v.segments == nil { + + for _, operation := range input { + if operation.segments == nil { continue } item := models.ParsedResourceId{ CommonAlias: nil, - Constants: v.constants, - Segments: *v.segments, + Constants: operation.constants, + Segments: *operation.segments, } matchFound := false diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/generate_names.go b/tools/importer-rest-api-specs/components/parser/resourceids/generate_names.go index 5a507c8b86e..0f73e287902 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/generate_names.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/generate_names.go @@ -8,15 +8,14 @@ import ( "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/cleanup" "github.com/hashicorp/pandora/tools/sdk/resourcemanager" - "github.com/hashicorp/go-hclog" "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" ) func (p *Parser) generateNamesForResourceIds(input []models.ParsedResourceId, uriToResourceId map[string]ParsedOperation) (*map[string]models.ParsedResourceId, error) { - return generateNamesForResourceIds(input, p.logger, uriToResourceId) + return generateNamesForResourceIds(input, uriToResourceId) } -func generateNamesForResourceIds(input []models.ParsedResourceId, log hclog.Logger, uriToResourceId map[string]ParsedOperation) (*map[string]models.ParsedResourceId, error) { +func generateNamesForResourceIds(input []models.ParsedResourceId, uriToResourceId map[string]ParsedOperation) (*map[string]models.ParsedResourceId, error) { // now that we have all of the Resource ID's, we then need to go through and determine Unique ID's for those // we need all of them here to avoid conflicts, e.g. AuthorizationRule which can be a NamespaceAuthorizationRule // or an EventHubAuthorizationRule, but is named AuthorizationRule in both diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/generate_names_test.go b/tools/importer-rest-api-specs/components/parser/resourceids/generate_names_test.go index eb953182e4e..82ba49fe2f5 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/generate_names_test.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/generate_names_test.go @@ -5,7 +5,6 @@ import ( "reflect" "testing" - "github.com/hashicorp/go-hclog" "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" "github.com/hashicorp/pandora/tools/sdk/resourcemanager" ) @@ -435,7 +434,7 @@ var redisPatchSchedulesResourceId = models.ParsedResourceId{ func TestResourceIDNamingEmpty(t *testing.T) { uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds([]models.ParsedResourceId{}, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds([]models.ParsedResourceId{}, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -455,7 +454,7 @@ func TestResourceIDNamingSubscriptionId(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -477,7 +476,7 @@ func TestResourceIDNamingSubscriptionIdAndSuffix(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -497,7 +496,7 @@ func TestResourceIDNamingResourceGroupId(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -519,7 +518,7 @@ func TestResourceIDNamingResourceGroupIdAndSuffix(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -539,7 +538,7 @@ func TestResourceIDNamingManagementGroupId(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -561,7 +560,7 @@ func TestResourceIDNamingManagementGroupIdAndSuffix(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -581,7 +580,7 @@ func TestResourceIDNamingEventHubSkuId(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -611,7 +610,7 @@ func TestResourceIDNamingTopLevelScope(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -682,7 +681,7 @@ func TestResourceIDNamingContainingAConstant(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -755,7 +754,7 @@ func TestResourceIDNamingContainingAConstantAndSuffix(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -775,7 +774,7 @@ func TestResourceIdNamingTopLevelResourceId(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -797,7 +796,7 @@ func TestResourceIdNamingTopLevelAndNestedResourceId(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -817,7 +816,7 @@ func TestResourceIdNamingNestedResourceId(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -837,7 +836,7 @@ func TestResourceIdNamingResourceUnderScope(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -859,7 +858,7 @@ func TestResourceIdNamingConflictingTwoLevels(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -890,7 +889,7 @@ func TestResourceIdNamingConflictingWithUpdatingOperation(t *testing.T) { }, } - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -1192,7 +1191,7 @@ func TestResourceIdNamingConflictingMultipleLevels(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -1212,7 +1211,7 @@ func TestResourceIdNamingSignalRId(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -1232,7 +1231,7 @@ func TestResourceIdNamingTrafficManagerEndpoint(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return @@ -1252,7 +1251,7 @@ func TestResourceIDNamingRedisDefaultId(t *testing.T) { } uriToParsedOperation := map[string]ParsedOperation{} - actualNamesToIds, err := generateNamesForResourceIds(input, hclog.NewNullLogger(), uriToParsedOperation) + actualNamesToIds, err := generateNamesForResourceIds(input, uriToParsedOperation) if err != nil { t.Fatalf("error: %+v", err) return diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/mappings.go b/tools/importer-rest-api-specs/components/parser/resourceids/mappings.go deleted file mode 100644 index e26d6e69970..00000000000 --- a/tools/importer-rest-api-specs/components/parser/resourceids/mappings.go +++ /dev/null @@ -1,51 +0,0 @@ -package resourceids - -import ( - "fmt" - - "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" -) - -func (p *Parser) mapProcessedResourceIdsToInputResourceIDs(originalUrisToParsed map[string]processedResourceId, namesToIds map[string]models.ParsedResourceId) (*map[string]ParsedOperation, error) { - out := make(map[string]ParsedOperation, 0) - - for uri, parsed := range originalUrisToParsed { - if parsed.segments == nil { - out[uri] = ParsedOperation{ - ResourceId: nil, - ResourceIdName: nil, - UriSuffix: parsed.uriSuffix, - } - continue - } - - placeholder := models.ParsedResourceId{ - Constants: parsed.constants, - Segments: *parsed.segments, - } - - found := false - for name, id := range namesToIds { - // NOTE: we intentionally use an empty `id` here to avoid comparing on the Alias - other := models.ParsedResourceId{ - Constants: id.Constants, - Segments: id.Segments, - } - if placeholder.Matches(other) { - out[uri] = ParsedOperation{ - ResourceId: &id, - ResourceIdName: &name, - UriSuffix: parsed.uriSuffix, - } - found = true - break - } - } - - if !found { - return nil, fmt.Errorf("couldn't find the processed ID Name for Resource URI %q", uri) - } - } - - return &out, nil -} diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/models.go b/tools/importer-rest-api-specs/components/parser/resourceids/models.go index fbef5d4394a..2e162c2165a 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/models.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/models.go @@ -24,9 +24,13 @@ type ParsedOperation struct { } type ParseResult struct { - // OriginalUrisToResourceIDs is a mapping of the original URI to a ParsedOperation object - // which allows mapping the original URI to the Normalized Resource ID once processed. - OriginalUrisToResourceIDs map[string]ParsedOperation + // OperationIdsToParsedResourceIds is a map of the original Operation IDs to the ParsedOperation + // object containing the parsed Resource ID. + OperationIdsToParsedResourceIds map[string]ParsedOperation + + //// OriginalUrisToResourceIDs is a mapping of the original URI to a ParsedOperation object + //// which allows mapping the original URI to the Normalized Resource ID once processed. + //OriginalUrisToResourceIDs map[string]ParsedOperation // NamesToResourceIDs is a mapping of the ResourceID Names to the Parsed Resource ID objects NamesToResourceIDs map[string]models.ParsedResourceId @@ -43,16 +47,16 @@ func (r *ParseResult) Append(other ParseResult, logger hclog.Logger) error { intermediate.AppendConstants(other.Constants) r.Constants = intermediate.Constants - urisToResourceIDs := make(map[string]ParsedOperation) + operationIdsToParsedOperations := make(map[string]ParsedOperation) // intentional since this can be nil - for k, v := range r.OriginalUrisToResourceIDs { - urisToResourceIDs[k] = v + for k, v := range r.OperationIdsToParsedResourceIds { + operationIdsToParsedOperations[k] = v } - if len(other.OriginalUrisToResourceIDs) > 0 { + if len(other.OperationIdsToParsedResourceIds) > 0 { // first concat the other uris - for k, v := range other.OriginalUrisToResourceIDs { - if existingVal, existing := urisToResourceIDs[k]; existing { + for k, v := range other.OperationIdsToParsedResourceIds { + if existingVal, existing := operationIdsToParsedOperations[k]; existing { matches := false if v.ResourceId != nil && existingVal.ResourceId != nil && v.ResourceId.Matches(*existingVal.ResourceId) { @@ -68,9 +72,9 @@ func (r *ParseResult) Append(other ParseResult, logger hclog.Logger) error { return fmt.Errorf("conflicting Uris with the key %q (First %+v / Second %+v)", k, v, existingVal) } - urisToResourceIDs[k] = v + operationIdsToParsedOperations[k] = v } - r.OriginalUrisToResourceIDs = urisToResourceIDs + r.OperationIdsToParsedResourceIds = operationIdsToParsedOperations // since we have a new list of Resource IDs we also need to go through and regenerate the names // as we may have conflicts etc @@ -92,7 +96,7 @@ func (r *ParseResult) Append(other ParseResult, logger hclog.Logger) error { } // this may cause rename for name conflict, so have to modify the name in OriginalUrisToResourceIDs too - namesToResourceIds, err := generateNamesForResourceIds(combinedResourceIds, logger, r.OriginalUrisToResourceIDs) + namesToResourceIds, err := generateNamesForResourceIds(combinedResourceIds, r.OperationIdsToParsedResourceIds) if err != nil { return fmt.Errorf("regenerating Names : Resource IDs for combined list: %+v", err) } diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/parse_segments.go b/tools/importer-rest-api-specs/components/parser/resourceids/parse_segments.go index a2a1d59514a..761a152b993 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/parse_segments.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/parse_segments.go @@ -4,13 +4,12 @@ import ( "fmt" "strings" + "github.com/go-openapi/spec" "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/cleanup" "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/constants" - internal2 "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/internal" - "github.com/hashicorp/pandora/tools/sdk/resourcemanager" - - "github.com/go-openapi/spec" + "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/internal" "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" + "github.com/hashicorp/pandora/tools/sdk/resourcemanager" ) var knownSegmentsUsedForScope = []string{ @@ -30,13 +29,13 @@ type processedResourceId struct { constants map[string]resourcemanager.ConstantDetails } -func (p *Parser) parseResourceIdsFromOperations() (*map[string]processedResourceId, error) { +func (p *Parser) parseSegmentsForEachOperation() (*map[string]processedResourceId, error) { // TODO: document this - urisToProcessedIds := make(map[string]processedResourceId) + operationIdsToProcessedResourceIds := make(map[string]processedResourceId, 0) for _, operation := range p.swaggerSpecExpanded.Operations() { for uri, operationDetails := range operation { - if internal2.OperationShouldBeIgnored(uri) { + if internal.OperationShouldBeIgnored(uri) { p.logger.Debug(fmt.Sprintf("Ignoring %q", uri)) continue } @@ -46,18 +45,19 @@ func (p *Parser) parseResourceIdsFromOperations() (*map[string]processedResource if err != nil { return nil, fmt.Errorf("parsing Resource ID from Operation for %q: %+v", uri, err) } - urisToProcessedIds[uri] = *resourceId + + operationIdsToProcessedResourceIds[operationDetails.ID] = *resourceId } } - return &urisToProcessedIds, nil + return &operationIdsToProcessedResourceIds, nil } func (p *Parser) parseResourceIdFromOperation(uri string, operation *spec.Operation) (*processedResourceId, error) { // TODO: document this segments := make([]resourcemanager.ResourceIdSegment, 0) - result := internal2.ParseResult{ + result := internal.ParseResult{ Constants: map[string]resourcemanager.ConstantDetails{}, } diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/parser.go b/tools/importer-rest-api-specs/components/parser/resourceids/parser.go index 1c77ad1c15e..e5eb5583cc2 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/parser.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/parser.go @@ -2,38 +2,43 @@ package resourceids import ( "fmt" + "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" ) // Parse takes a list of Swagger Resources and returns a ParseResult, containing // a list of ResourceIDs found within the Swagger Resources. func (p *Parser) Parse() (*ParseResult, error) { - // TODO: replacing static segments, detecting hidden scopes - // TODO: tests for segments - - p.logger.Trace("Parsing Resource IDs from Operations..") - resourceIdsToSegments, err := p.parseResourceIdsFromOperations() + // 1. Go through and map the Operation IDs to the parsed Resource ID + // (which includes the Resource ID and any UriSuffix as needed) + p.logger.Trace("Parsing the segments for each operation..") + operationIdsToSegments, err := p.parseSegmentsForEachOperation() if err != nil { - return nil, fmt.Errorf("parsing Segments from Resource IDs: %+v", err) + return nil, fmt.Errorf("parsing the segments for each operation: %+v", err) } - p.logger.Trace("Identifying Distinct Resource IDs..") - uniqueResourceIds := p.distinctResourceIds(*resourceIdsToSegments) + // 2. Process the list of parsed segments to obtain a unique list of Resource IDs + p.logger.Trace("Determining the list of unique Resource IDs from the parsed input") + uniqueResourceIds := p.distinctResourceIds(*operationIdsToSegments) - p.logger.Trace("Detecting any Common Resource Ids") + // 3. Then we need to find any Common Resource IDs and switch those references out + p.logger.Trace("Generating Names for Resource IDs..") resourceIds := switchOutCommonResourceIDsAsNeeded(uniqueResourceIds) + // 4. We then need to generate a unique Resource ID name for each of the Resource IDs p.logger.Trace("Generating Names for Resource IDs..") namesToResourceIds, err := p.generateNamesForResourceIds(resourceIds, nil) if err != nil { return nil, fmt.Errorf("generating Names for Resource IDs: %+v", err) } - p.logger.Trace("Mapping the Parsed Resource IDs into the originally-processed URIs..") - originalUrisToResourceIds, err := p.mapProcessedResourceIdsToInputResourceIDs(*resourceIdsToSegments, *namesToResourceIds) + // 5. Then we need to work through the list of Resource IDs and Operation IDs to map the data across + p.logger.Trace("Updating the Parsed Operations with the Processed ResourceIds..") + operationIdsToResourceIds, err := p.updateParsedOperationsWithProcessedResourceIds(*operationIdsToSegments, *namesToResourceIds) if err != nil { - return nil, fmt.Errorf("mapping Processed Resource IDs to the Input Resource IDs: %+v", err) + return nil, fmt.Errorf("updating the parsed Operations with the Processed Resource ID information: %+v", err) } + // 6. Finally pull out a unique list of Constants from the parsed Resource IDs p.logger.Trace("Finding Distinct Constants for Resource IDs..") distinctConstants, err := p.findDistinctConstants(*namesToResourceIds) if err != nil { @@ -41,8 +46,95 @@ func (p *Parser) Parse() (*ParseResult, error) { } return &ParseResult{ - OriginalUrisToResourceIDs: *originalUrisToResourceIds, - NamesToResourceIDs: *namesToResourceIds, - Constants: *distinctConstants, + OperationIdsToParsedResourceIds: *operationIdsToResourceIds, + NamesToResourceIDs: *namesToResourceIds, + Constants: *distinctConstants, }, nil + + // ---------------------------------------------------------------------------------------------------- + // ---------------------------------------------------------------------------------------------------- + // ---------------------------------------------------------------------------------------------------- + + //p.logger.Trace("Parsing Resource IDs from Operations..") + //resourceIdsToSegments, err := p.parseSegmentsForEachOperation() + //if err != nil { + // return nil, fmt.Errorf("parsing Segments from Resource IDs: %+v", err) + //} + // + //p.logger.Trace("Identifying Distinct Resource IDs..") + //uniqueResourceIds := p.distinctResourceIds(*resourceIdsToSegments) + // + //p.logger.Trace("Detecting any Common Resource Ids") + //resourceIds := switchOutCommonResourceIDsAsNeeded(uniqueResourceIds) + // + //p.logger.Trace("Generating Names for Resource IDs..") + //namesToResourceIds, err := p.generateNamesForResourceIds(resourceIds, nil) + //if err != nil { + // return nil, fmt.Errorf("generating Names for Resource IDs: %+v", err) + //} + // + //p.logger.Trace("Mapping the Parsed Resource IDs into the originally-processed URIs..") + //originalUrisToResourceIds, err := p.mapProcessedResourceIdsToInputResourceIDs(*resourceIdsToSegments, *namesToResourceIds) + //if err != nil { + // return nil, fmt.Errorf("mapping Processed Resource IDs to the Input Resource IDs: %+v", err) + //} + // + //p.logger.Trace("Finding Distinct Constants for Resource IDs..") + //distinctConstants, err := p.findDistinctConstants(*namesToResourceIds) + //if err != nil { + // return nil, fmt.Errorf("finding Distinct Constants for Resource IDs: %+v", err) + //} + // + //return &ParseResult{ + // OriginalUrisToResourceIDs: *originalUrisToResourceIds, + // NamesToResourceIDs: *namesToResourceIds, + // Constants: *distinctConstants, + //}, nil +} + +func (p *Parser) updateParsedOperationsWithProcessedResourceIds(operationIdsToSegments map[string]processedResourceId, namesToResourceIds map[string]models.ParsedResourceId) (*map[string]ParsedOperation, error) { + output := make(map[string]ParsedOperation) + + for operationId, operation := range operationIdsToSegments { + p.logger.Trace(fmt.Sprintf("Processing Operation ID %q", operationId)) + if operation.segments == nil { + if operation.uriSuffix == nil { + return nil, fmt.Errorf("the Operation ID %q had no Segments and no UriSuffix", operationId) + } + + output[operationId] = ParsedOperation{ + UriSuffix: operation.uriSuffix, + } + continue + } + + placeholder := models.ParsedResourceId{ + Constants: operation.constants, + Segments: *operation.segments, + } + + found := false + for name, resourceId := range namesToResourceIds { + // NOTE: we intentionally use an empty `id` here to avoid comparing on the Alias + other := models.ParsedResourceId{ + Constants: resourceId.Constants, + Segments: resourceId.Segments, + } + if placeholder.Matches(other) { + output[operationId] = ParsedOperation{ + ResourceId: &resourceId, + ResourceIdName: &name, + UriSuffix: operation.uriSuffix, + } + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("couldn't find the Processed Resource Id for the Operation Id %q", operationId) + } + } + + return &output, nil } diff --git a/tools/importer-rest-api-specs/components/parser/swagger_resources.go b/tools/importer-rest-api-specs/components/parser/swagger_resources.go index 8d38032cf94..12e58e36147 100644 --- a/tools/importer-rest-api-specs/components/parser/swagger_resources.go +++ b/tools/importer-rest-api-specs/components/parser/swagger_resources.go @@ -24,7 +24,7 @@ func (d *SwaggerDefinition) parseResourcesWithinSwaggerTag(tag *string, resource } // pull out the operations and any inlined/top-level constants/models - operations, nestedResult, err := d.parseOperationsWithinTag(tag, resourceIds.OriginalUrisToResourceIDs, resourceProvider, result) + operations, nestedResult, err := d.parseOperationsWithinTag(tag, resourceIds.OperationIdsToParsedResourceIds, resourceProvider, result) if err != nil { return nil, fmt.Errorf("finding operations: %+v", err) } diff --git a/tools/importer-rest-api-specs/models/models.go b/tools/importer-rest-api-specs/models/models.go index c5d887d0289..a4f58d3ab07 100644 --- a/tools/importer-rest-api-specs/models/models.go +++ b/tools/importer-rest-api-specs/models/models.go @@ -31,11 +31,11 @@ type OperationDetails struct { IsListOperation bool LongRunning bool Method string + OperationId string Options map[string]OperationOption RequestObject *ObjectDefinition ResourceIdName *string ResponseObject *ObjectDefinition - Uri string UriSuffix *string } From b3fcecee92d2d9f118cddeb823767ce4c135a410 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 19 Oct 2023 14:08:14 +0200 Subject: [PATCH 3/5] tools/importer-rest-api-specs: removing the older `apply overrides` logic This was used to pull data from the Data API to allow overriding existing data, which is no longer a technique we're planning to use. --- .../components/differ/apply.go | 73 ---------- .../components/differ/client.go | 18 --- .../components/differ/load_existing.go | 134 ------------------ .../components/differ/stages.go | 16 --- .../components/transformer/api_to_models.go | 1 - .../pipeline/task_apply_overrides.go | 33 ----- 6 files changed, 275 deletions(-) delete mode 100644 tools/importer-rest-api-specs/components/differ/apply.go delete mode 100644 tools/importer-rest-api-specs/components/differ/client.go delete mode 100644 tools/importer-rest-api-specs/components/differ/load_existing.go delete mode 100644 tools/importer-rest-api-specs/components/differ/stages.go delete mode 100644 tools/importer-rest-api-specs/pipeline/task_apply_overrides.go diff --git a/tools/importer-rest-api-specs/components/differ/apply.go b/tools/importer-rest-api-specs/components/differ/apply.go deleted file mode 100644 index 1f3f0f36539..00000000000 --- a/tools/importer-rest-api-specs/components/differ/apply.go +++ /dev/null @@ -1,73 +0,0 @@ -package differ - -import ( - "fmt" - - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" -) - -func (d Differ) ApplyFromExistingAPIDefinitions(existing models.AzureApiDefinition, parsed models.AzureApiDefinition, logger hclog.Logger) (models.AzureApiDefinition, error) { - // todo we should work through and ensure that all Existing items are present within Parsed - // Each of the Stages to apply can be found in ApplyStages() - - // starting with copying Test Template - logger.Trace("looping through Existing Resources to apply Test Templates to the Parsed Terraform Resources..") - for resourceName, resource := range existing.Resources { - logger.Trace(fmt.Sprintf("found Resource %q in Service %q..", resourceName, existing.ServiceName)) - parsedResource, ok := parsed.Resources[resourceName] - if !ok { - return parsed, fmt.Errorf("unable to find the API Resource %q in the newly parsed api definitions", resourceName) - } - - if existingTerraform := resource.Terraform; existingTerraform != nil { - if parsedTerraform := parsedResource.Terraform; parsedTerraform != nil { - for existingTerraformResourceName, existingTerraformResource := range existingTerraform.Resources { - logger.Trace(fmt.Sprintf("found Terraform Resource %q in Resource %q..", existingTerraformResourceName, resourceName)) - parsedTerraformResource, ok := parsedTerraform.Resources[existingTerraformResourceName] - if !ok { - return parsed, fmt.Errorf("unable to find the Terraform Resource %q in newly parsed api definitions", resourceName) - } - - logger.Trace(fmt.Sprintf("applying existing Documentation from Terraform Resource %q in Resource %q..", existingTerraformResourceName, resourceName)) - parsedTerraformResource.Documentation = existingTerraformResource.Documentation - - logger.Trace(fmt.Sprintf("applying existing Generate Fields from Terraform Resource %q in Resource %q..", existingTerraformResourceName, resourceName)) - parsedTerraformResource.Generate = existingTerraformResource.Generate - parsedTerraformResource.GenerateModel = existingTerraformResource.GenerateModel - parsedTerraformResource.GenerateSchema = existingTerraformResource.GenerateSchema - parsedTerraformResource.GenerateIdValidation = existingTerraformResource.GenerateIdValidation - parsedTerraformResource.ReadMethod.Generate = existingTerraformResource.ReadMethod.Generate - parsedTerraformResource.ReadMethod.TimeoutInMinutes = existingTerraformResource.ReadMethod.TimeoutInMinutes - parsedTerraformResource.CreateMethod.Generate = existingTerraformResource.CreateMethod.Generate - parsedTerraformResource.CreateMethod.TimeoutInMinutes = existingTerraformResource.CreateMethod.TimeoutInMinutes - parsedTerraformResource.UpdateMethod.Generate = existingTerraformResource.UpdateMethod.Generate - parsedTerraformResource.UpdateMethod.TimeoutInMinutes = existingTerraformResource.UpdateMethod.TimeoutInMinutes - parsedTerraformResource.DeleteMethod.Generate = existingTerraformResource.DeleteMethod.Generate - parsedTerraformResource.DeleteMethod.TimeoutInMinutes = existingTerraformResource.DeleteMethod.TimeoutInMinutes - - if existingBasicConfig := existingTerraformResource.Tests.BasicConfiguration; existingBasicConfig != "" { - logger.Trace("applying Existing Basic Test Config from the Existing Terraform Resource to the Parsed Terraform Resource..") - parsedTerraformResource.Tests.BasicConfiguration = existingBasicConfig - } - if existingImportConfig := existingTerraformResource.Tests.RequiresImportConfiguration; existingImportConfig != "" { - logger.Trace("applying Existing Requires Import Test Config from the Existing Terraform Resource to the Parsed Terraform Resource..") - parsedTerraformResource.Tests.RequiresImportConfiguration = existingImportConfig - } - if existingCompleteConfig := existingTerraformResource.Tests.CompleteConfiguration; existingCompleteConfig != nil { - logger.Trace("applying Existing Complete Test Config from the Existing Terraform Resource to the Parsed Terraform Resource..") - parsedTerraformResource.Tests.CompleteConfiguration = existingCompleteConfig - } - if existingTemplate := existingTerraformResource.Tests.TemplateConfiguration; existingTemplate != nil { - logger.Trace("applying Existing Test Template from the Existing Terraform Resource to the Parsed Terraform Resource..") - parsedTerraformResource.Tests.TemplateConfiguration = existingTemplate - } - logger.Trace("applying Other Tests from the Existing Terraform Resource to the Parsed Terraform Resource..") - parsedTerraformResource.Tests.OtherTests = existingTerraformResource.Tests.OtherTests - } - } - } - } - - return parsed, nil -} diff --git a/tools/importer-rest-api-specs/components/differ/client.go b/tools/importer-rest-api-specs/components/differ/client.go deleted file mode 100644 index 1e5acad07b2..00000000000 --- a/tools/importer-rest-api-specs/components/differ/client.go +++ /dev/null @@ -1,18 +0,0 @@ -package differ - -import ( - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/pandora/tools/sdk/resourcemanager" -) - -type Differ struct { - client resourcemanager.Client - logger hclog.Logger -} - -func NewDiffer(dataApiEndpoint string, logger hclog.Logger) Differ { - return Differ{ - client: resourcemanager.NewResourceManagerClient(dataApiEndpoint), - logger: logger, - } -} diff --git a/tools/importer-rest-api-specs/components/differ/load_existing.go b/tools/importer-rest-api-specs/components/differ/load_existing.go deleted file mode 100644 index b42908d2227..00000000000 --- a/tools/importer-rest-api-specs/components/differ/load_existing.go +++ /dev/null @@ -1,134 +0,0 @@ -package differ - -import ( - "fmt" - "strings" - - "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/transformer" - "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" - "github.com/hashicorp/pandora/tools/sdk/resourcemanager" -) - -func (d *Differ) RetrieveExistingService(serviceName, apiVersion string) (*models.AzureApiDefinition, error) { - services, err := d.client.Services().Get() - if err != nil { - return nil, fmt.Errorf("retrieving Services from Data API: %+v", err) - } - if services == nil { - return nil, nil - } - - var existingService *models.AzureApiDefinition - for name, service := range *services { - if !strings.EqualFold(name, serviceName) { - continue - } - - serviceDetails, err := d.client.ServiceDetails().Get(service) - if err != nil { - return nil, fmt.Errorf("retrieving Service Details for %q: %+v", serviceName, err) - } - if serviceDetails == nil { - return nil, nil - } - - terraformDetails, err := d.client.Terraform().Get(*serviceDetails) - if err != nil { - return nil, fmt.Errorf("retrieving Terraform Details for Service %q: %+v", serviceName, err) - } - - for version, versionSummary := range serviceDetails.Versions { - if !strings.EqualFold(version, apiVersion) { - continue - } - - versionDetails, err := d.client.ServiceVersion().Get(versionSummary) - if err != nil { - return nil, fmt.Errorf("retrieving Details for Service %q Version %q: %+v", serviceName, apiVersion, err) - } - if versionDetails == nil { - continue - } - - resources := make(map[string]models.AzureApiResource) - - for resourceName, resourceSummary := range versionDetails.Resources { - resourceOperations, err := d.client.ApiOperations().Get(resourceSummary) - if err != nil { - return nil, fmt.Errorf("retrieving API Operations for Service %q Version %q Resource %q: %+v", serviceName, apiVersion, resourceName, err) - } - if resourceOperations == nil { - continue - } - - resourceSchema, err := d.client.ApiSchema().Get(resourceSummary) - if err != nil { - return nil, fmt.Errorf("retrieving API Schema for Service %q Version %q Resource %q: %+v", serviceName, apiVersion, resourceName, err) - } - if resourceSchema == nil { - continue - } - - mappedModels, err := transformer.MapApiModelsToModelDetails(resourceSchema.Models) - if err != nil { - return nil, fmt.Errorf("mapping Models for Resource %q / Service %q / Version %q: %+v", resourceName, serviceName, apiVersion, err) - } - - mappedOperations, err := transformer.MapApiOperationsToOperationDetails(resourceOperations.Operations) - if err != nil { - return nil, fmt.Errorf("mapping Operations for Resource %q / Service %q / Version %q: %+v", resourceName, serviceName, apiVersion, err) - } - - mappedResourceIds, err := transformer.MapApiResourceIdDefinitionsToParsedResourceIds(resourceSchema.ResourceIds, resourceSchema.Constants) - if err != nil { - return nil, fmt.Errorf("mapping Resource ID's for Resource %q / Service %q / Version %q: %+v", resourceName, serviceName, apiVersion, err) - } - - filteredTerraform := filterTerraformDataToApiVersionAndResource(terraformDetails, apiVersion, resourceName) - - resources[resourceName] = models.AzureApiResource{ - Constants: resourceSchema.Constants, - Models: *mappedModels, - Operations: *mappedOperations, - ResourceIds: *mappedResourceIds, - Terraform: filteredTerraform, - } - } - - existingService = &models.AzureApiDefinition{ - ServiceName: serviceName, - ApiVersion: version, - Resources: resources, - } - } - } - - // TODO: map across the ResourceProvider and the TerraformPackageName - - // TODO: if the parser returns a single `parser.ParsedData` then this can too - but for now emulate the same result - return existingService, nil -} - -func filterTerraformDataToApiVersionAndResource(input *resourcemanager.TerraformDetails, apiVersion string, resourceName string) *resourcemanager.TerraformDetails { - if input == nil { - return nil - } - - out := resourcemanager.TerraformDetails{ - DataSources: map[string]resourcemanager.TerraformDataSourceDetails{}, - Resources: map[string]resourcemanager.TerraformResourceDetails{}, - } - //for k, v := range input.DataSources { - // if v.ApiVersion == apiVersion && v.Resource == resourceName { - // out.DataSources[k] = v - // } - //} - - for k, v := range input.Resources { - if v.ApiVersion == apiVersion && v.Resource == resourceName { - out.Resources[k] = v - } - } - - return &out -} diff --git a/tools/importer-rest-api-specs/components/differ/stages.go b/tools/importer-rest-api-specs/components/differ/stages.go deleted file mode 100644 index 1b2742010e9..00000000000 --- a/tools/importer-rest-api-specs/components/differ/stages.go +++ /dev/null @@ -1,16 +0,0 @@ -package differ - -import ( - "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" -) - -type ApplyStage interface { - Apply(existing models.AzureApiDefinition, parsed models.AzureApiDefinition) (*models.AzureApiDefinition, error) -} - -var ApplyStages = []ApplyStage{ - // TODO: DefaultImpliedValuesStage{}, - where there's a default implied value in the API - // TODO: DetectBreakingChangesStage{}, - // TODO: FieldRenamerStage{}, - handling where a field has a different Name and JsonName (casing-aside) - // TODO: ResourceIdRenamerStage{}, - handling where a ResourceID has been renamed -} diff --git a/tools/importer-rest-api-specs/components/transformer/api_to_models.go b/tools/importer-rest-api-specs/components/transformer/api_to_models.go index 54be400c09b..c2d10ae9371 100644 --- a/tools/importer-rest-api-specs/components/transformer/api_to_models.go +++ b/tools/importer-rest-api-specs/components/transformer/api_to_models.go @@ -184,7 +184,6 @@ func MapApiOperationsToOperationDetails(input map[string]resourcemanager.ApiOper RequestObject: requestObject, ResourceIdName: v.ResourceIdName, ResponseObject: responseObject, - Uri: "", // intentionally not mapped, since this should probably be removed in time UriSuffix: v.UriSuffix, } } diff --git a/tools/importer-rest-api-specs/pipeline/task_apply_overrides.go b/tools/importer-rest-api-specs/pipeline/task_apply_overrides.go deleted file mode 100644 index a7eb40cf8cc..00000000000 --- a/tools/importer-rest-api-specs/pipeline/task_apply_overrides.go +++ /dev/null @@ -1,33 +0,0 @@ -package pipeline - -import ( - "fmt" - - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/differ" - "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" -) - -func (pipelineTask) applyOverridesFromExistingData(data models.AzureApiDefinition, dataApiEndpoint *string, logger hclog.Logger) (*models.AzureApiDefinition, error) { - if dataApiEndpoint != nil { - logger.Trace("Retrieving current Data and Schema from the Data API..") - - differ := differ.NewDiffer(*dataApiEndpoint, logger.Named("Data API Differ")) - existingApiDefinitions, err := differ.RetrieveExistingService(data.ServiceName, data.ApiVersion) - if err != nil { - return nil, fmt.Errorf("retrieving data from Data API: %+v", err) - } - - logger.Trace("Applying Overrides from the Existing API Definitions to the Parsed Swagger Data..") - data, err = differ.ApplyFromExistingAPIDefinitions(*existingApiDefinitions, data, logger) - if err != nil { - return nil, fmt.Errorf("applying Overrides from the existing API Definitions: %+v", err) - } - - logger.Trace("Applied Overrides from the Existing API Definitions to the Parsed Swagger Data.") - } else { - logger.Trace("Skipping retrieving current schema from Data API..") - } - - return &data, nil -} From 9d717e6d3f74b7c8631cb838558b3f9d771b855c Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 19 Oct 2023 15:47:51 +0200 Subject: [PATCH 4/5] tools/importer-rest-api-specs: removing an unnecessary log output --- .../components/parser/cleanup/pluralise_helper.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/importer-rest-api-specs/components/parser/cleanup/pluralise_helper.go b/tools/importer-rest-api-specs/components/parser/cleanup/pluralise_helper.go index 47ef5c80151..eb08b35135c 100644 --- a/tools/importer-rest-api-specs/components/parser/cleanup/pluralise_helper.go +++ b/tools/importer-rest-api-specs/components/parser/cleanup/pluralise_helper.go @@ -2,7 +2,6 @@ package cleanup import ( "fmt" - "log" "strings" "github.com/gertd/go-pluralize" @@ -31,7 +30,6 @@ func GetSingular(input string) string { } for _, v := range invariablePlurals() { if strings.EqualFold(input, v) { - log.Printf("got %q returning %q", input, returnCased(v, casing)) return returnCased(v, casing) } } From 890218982ae9a8c27fc55e48ee8073cac96dd1e1 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Thu, 19 Oct 2023 16:14:09 +0200 Subject: [PATCH 5/5] tools/importer-rest-api-specs: cleanup --- .../parser/resourceids/generate_names.go | 3 +- .../components/parser/resourceids/models.go | 4 -- .../components/parser/resourceids/parser.go | 40 ------------------- 3 files changed, 1 insertion(+), 46 deletions(-) diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/generate_names.go b/tools/importer-rest-api-specs/components/parser/resourceids/generate_names.go index 0f73e287902..6aa35689472 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/generate_names.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/generate_names.go @@ -6,9 +6,8 @@ import ( "strings" "github.com/hashicorp/pandora/tools/importer-rest-api-specs/components/parser/cleanup" - "github.com/hashicorp/pandora/tools/sdk/resourcemanager" - "github.com/hashicorp/pandora/tools/importer-rest-api-specs/models" + "github.com/hashicorp/pandora/tools/sdk/resourcemanager" ) func (p *Parser) generateNamesForResourceIds(input []models.ParsedResourceId, uriToResourceId map[string]ParsedOperation) (*map[string]models.ParsedResourceId, error) { diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/models.go b/tools/importer-rest-api-specs/components/parser/resourceids/models.go index 2e162c2165a..1d4f2f9cb58 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/models.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/models.go @@ -28,10 +28,6 @@ type ParseResult struct { // object containing the parsed Resource ID. OperationIdsToParsedResourceIds map[string]ParsedOperation - //// OriginalUrisToResourceIDs is a mapping of the original URI to a ParsedOperation object - //// which allows mapping the original URI to the Normalized Resource ID once processed. - //OriginalUrisToResourceIDs map[string]ParsedOperation - // NamesToResourceIDs is a mapping of the ResourceID Names to the Parsed Resource ID objects NamesToResourceIDs map[string]models.ParsedResourceId diff --git a/tools/importer-rest-api-specs/components/parser/resourceids/parser.go b/tools/importer-rest-api-specs/components/parser/resourceids/parser.go index e5eb5583cc2..c986dc32afa 100644 --- a/tools/importer-rest-api-specs/components/parser/resourceids/parser.go +++ b/tools/importer-rest-api-specs/components/parser/resourceids/parser.go @@ -50,46 +50,6 @@ func (p *Parser) Parse() (*ParseResult, error) { NamesToResourceIDs: *namesToResourceIds, Constants: *distinctConstants, }, nil - - // ---------------------------------------------------------------------------------------------------- - // ---------------------------------------------------------------------------------------------------- - // ---------------------------------------------------------------------------------------------------- - - //p.logger.Trace("Parsing Resource IDs from Operations..") - //resourceIdsToSegments, err := p.parseSegmentsForEachOperation() - //if err != nil { - // return nil, fmt.Errorf("parsing Segments from Resource IDs: %+v", err) - //} - // - //p.logger.Trace("Identifying Distinct Resource IDs..") - //uniqueResourceIds := p.distinctResourceIds(*resourceIdsToSegments) - // - //p.logger.Trace("Detecting any Common Resource Ids") - //resourceIds := switchOutCommonResourceIDsAsNeeded(uniqueResourceIds) - // - //p.logger.Trace("Generating Names for Resource IDs..") - //namesToResourceIds, err := p.generateNamesForResourceIds(resourceIds, nil) - //if err != nil { - // return nil, fmt.Errorf("generating Names for Resource IDs: %+v", err) - //} - // - //p.logger.Trace("Mapping the Parsed Resource IDs into the originally-processed URIs..") - //originalUrisToResourceIds, err := p.mapProcessedResourceIdsToInputResourceIDs(*resourceIdsToSegments, *namesToResourceIds) - //if err != nil { - // return nil, fmt.Errorf("mapping Processed Resource IDs to the Input Resource IDs: %+v", err) - //} - // - //p.logger.Trace("Finding Distinct Constants for Resource IDs..") - //distinctConstants, err := p.findDistinctConstants(*namesToResourceIds) - //if err != nil { - // return nil, fmt.Errorf("finding Distinct Constants for Resource IDs: %+v", err) - //} - // - //return &ParseResult{ - // OriginalUrisToResourceIDs: *originalUrisToResourceIds, - // NamesToResourceIDs: *namesToResourceIds, - // Constants: *distinctConstants, - //}, nil } func (p *Parser) updateParsedOperationsWithProcessedResourceIds(operationIdsToSegments map[string]processedResourceId, namesToResourceIds map[string]models.ParsedResourceId) (*map[string]ParsedOperation, error) {