Merge pull request #4217 from hashicorp/refactor/data-api-to-use-repo…

…sitory

`tools/data-api-repository`: consolidating the reading/writing of data into `data-api-repository` / updated support for Common Types

api-definitions/handwritten-resource-manager/AADB2C/2021-04-01-preview/ApiVersionDefinition.json

@@ -5,5 +5,5 @@
  "resources": [
   "Tenants"
  ],
- "source": "handwritten"
+ "source": "HandWritten"
 }

api-definitions/handwritten-resource-manager/AADB2C/2021-04-01-preview/Tenants/Model-UpdateTenantProperties.json

@@ -1,5 +1,5 @@
 {
-  "name": "TenantProperties",
+  "name": "UpdateTenantProperties",
   "fields": [
     {
       "containsDiscriminatedTypeValue": false,

tools/data-api-differ/internal/dataapi/data_api_cmd.go

@@ -54,18 +54,11 @@ func (p *dataApiCmd) launchAndWait(ctx context.Context, client *v1.Client) error
 	log.Logger.Trace(fmt.Sprintf("Data API is launched at %q.", p.endpoint))
 
 	// then ensure it's accepting requests prior to hitting it (e.g. firewalls)
-	for attempts := 0; attempts < 30; attempts++ {
-		log.Logger.Trace(fmt.Sprintf("Checking the health of the Data API - attempt %d/30", attempts+1))
+	for attempts := 0; attempts < 50; attempts++ {
+		log.Logger.Trace(fmt.Sprintf("Checking the health of the Data API - attempt %d/50", attempts+1))
 
 		result, err := client.Health(ctx)
-		if err != nil {
-			if result != nil && result.HttpResponse != nil {
-				return fmt.Errorf("unexpected status code %d", result.HttpResponse.StatusCode)
-			}
-			return fmt.Errorf("connection failure")
-		}
-
-		if result.Available {
+		if result != nil && result.Available {
 			log.Logger.Trace("API available")
 			return nil
 		}

tools/data-api-differ/internal/differ/differ_api_versions_test.go

@@ -19,7 +19,7 @@ func TestDiff_APIVersionAdded_WithNestedDetails(t *testing.T) {
 		"2023-01-01": {
 			Resources: map[string]models.APIResource{
 				"Example": {
-					Constants: map[string]models.ConstantDetails{
+					Constants: map[string]models.SDKConstant{
 						"SomeConst": {
 							Type: models.StringSDKConstantType,
 							Values: map[string]string{

tools/data-api-differ/internal/differ/differ_operations_test.go

@@ -952,7 +952,7 @@ func TestDiff_OperationResponseObjectChanged(t *testing.T) {
 	updated := map[string]models.SDKOperation{
 		"First": {
 			ResponseObject: &models.SDKObjectDefinition{
-				Type:          models.ReferenceApiObjectDefinitionType,
+				Type:          models.ReferenceSDKObjectDefinitionType,
 				ReferenceName: pointer.To("SomeConstant"),
 			},
 		},

tools/data-api-repository/README.md

@@ -0,0 +1,10 @@
+## `./tools/data-api-sdk`
+
+This package provides a Repository which can be used to load/store API Definitions on-disk in a consistent manner.
+
+This is currently used in:
+
+* `./tools/data-api` - to load and serve the API Definitions, for use in other tooling.
+* `./tools/importer-rest-api-specs` - to remove any existing and then import all-new API Definitions for Azure Resource Manager.
+
+This package isn't intended to be used directly, and other tooling should interact with the Data API instead.

tools/data-api-repository/repository/directory_for_service.go

@@ -0,0 +1,57 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package repository
+
+import (
+	"fmt"
+	"path/filepath"
+
+	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
+)
+
+func (r *repositoryImpl) directoryForService(serviceName string, sourceDataOrigin sdkModels.SourceDataOrigin) (*string, error) {
+	// use the existing directory if we've parsed it
+	if dataSource, ok := r.availableDataSources[serviceName]; ok {
+		if dataSource.sourceDataOrigin != sourceDataOrigin {
+			return nil, fmt.Errorf("an existing Service named %q uses the SourceDataOrigin %q but found a duplicate for the SourceDataOrigin %q", serviceName, dataSource.sourceDataOrigin, sourceDataOrigin)
+		}
+
+		return &dataSource.workingDirectory, nil
+	}
+
+	// else fallback to the default ones
+	path, err := r.defaultDirectoryForSourceDataOrigin(sourceDataOrigin)
+	if err != nil {
+		return nil, err
+	}
+	serviceDirectory := filepath.Join(*path, serviceName)
+	return &serviceDirectory, nil
+}
+
+func (r *repositoryImpl) defaultDirectoryForSourceDataOrigin(sourceDataOrigin sdkModels.SourceDataOrigin) (*string, error) {
+	sourceDataOriginsToDefaultDirectories, ok := defaultDataDirectories[r.sourceDataType]
+	if !ok {
+		return nil, fmt.Errorf("missing a default directory for SourceDataType %q", string(r.sourceDataType))
+	}
+	defaultDirectory, ok := sourceDataOriginsToDefaultDirectories[sourceDataOrigin]
+	if !ok {
+		return nil, fmt.Errorf("missing a default directory for SourceDataOrigin %q within SourceDataType %q", sourceDataOrigin, string(r.sourceDataType))
+	}
+	dataDirectory := filepath.Join(r.workingDirectory, defaultDirectory)
+	return &dataDirectory, nil
+}
+
+var defaultDataDirectories = map[sdkModels.SourceDataType]map[sdkModels.SourceDataOrigin]string{
+	// NOTE: the ordering matters here, we want to load the imported data before any handwritten data
+	// This is to ensure that if the imported data contains a Discriminated Parent Type that we can add
+	// additional Implementations within the HandWritten Data which inherits from that, so this is automatically
+	// unmarshalled as required.
+	sdkModels.MicrosoftGraphSourceDataType: {
+		sdkModels.MicrosoftGraphMetaDataSourceDataOrigin: "microsoft-graph",
+	},
+	sdkModels.ResourceManagerSourceDataType: {
+		sdkModels.AzureRestAPISpecsSourceDataOrigin: "resource-manager",
+		sdkModels.HandWrittenSourceDataOrigin:       "handwritten-resource-manager",
+	},
+}

tools/data-api-repository/repository/get_all_services.go

@@ -0,0 +1,35 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package repository
+
+import (
+	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
+)
+
+// GetAllServices returns all the Services supported for this SourceDataType as a map of
+// Service Name (key) to Service (value).
+func (r *repositoryImpl) GetAllServices() (*map[string]sdkModels.Service, error) {
+	r.cacheLock.Lock()
+	defer r.cacheLock.Unlock()
+
+	output := make(map[string]sdkModels.Service)
+	for serviceName := range r.availableDataSources {
+		service, exists := r.cachedServices[serviceName]
+		if !exists {
+			// otherwise let's populate it in the cache
+			svc, err := r.loadService(serviceName)
+			if err != nil {
+				return nil, err
+			}
+			if svc == nil {
+				return nil, nil
+			}
+			r.cachedServices[serviceName] = *svc
+			service = *svc
+		}
+
+		output[serviceName] = service
+	}
+	return &output, nil
+}

tools/data-api-repository/repository/get_common_types.go

@@ -0,0 +1,18 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package repository
+
+import (
+	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
+)
+
+// GetCommonTypes returns all the Common Types (Constants and Models) for this SourceDataType
+// this returns a map of APIVersion (key) to CommonTypes (value).
+func (r *repositoryImpl) GetCommonTypes() (*map[string]sdkModels.CommonTypes, error) {
+	// The CommonTypes are loaded/cached when the Repository is initialized, as such we
+	// can just return those.
+	// When the Common Types are updated, the cache is invalidated/updated, so this should
+	// always be fresh.
+	return &r.cachedCommonTypes, nil
+}

tools/data-api-repository/repository/get_service.go

@@ -0,0 +1,30 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package repository
+
+import (
+	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
+)
+
+// GetService returns the specified Service by its `name` if it exists.
+// When the Service doesn't exist, `nil` will be returned.
+func (r *repositoryImpl) GetService(name string) (*sdkModels.Service, error) {
+	r.cacheLock.Lock()
+	defer r.cacheLock.Unlock()
+
+	service, exists := r.cachedServices[name]
+	if !exists {
+		// otherwise let's populate the cache
+		svc, err := r.loadService(name)
+		if err != nil {
+			return nil, err
+		}
+		if svc == nil {
+			return nil, nil
+		}
+		r.cachedServices[name] = *svc
+		service = *svc
+	}
+	return &service, nil
+}

tools/data-api-repository/repository/helpers_cache.go

@@ -0,0 +1,49 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package repository
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/go-hclog"
+	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
+)
+
+func (r *repositoryImpl) populateCacheInternal() error {
+	r.logger.Trace("Refreshing the cache of Available Services..")
+	availableDataSources, err := populateAvailableServicesCache(r.workingDirectory, r.sourceDataType, r.serviceNamesToLimitTo, r.logger)
+	if err != nil {
+		return fmt.Errorf("populating the Available Services: %+v", err)
+	}
+	r.availableDataSources = *availableDataSources
+
+	commonTypes, err := discoverCommonTypesWithin(r.workingDirectory, r.sourceDataType, r.logger)
+	if err != nil {
+		return fmt.Errorf("populating the Common Types: %+v", err)
+	}
+	r.cachedCommonTypes = *commonTypes
+
+	return nil
+}
+
+func populateAvailableServicesCache(workingDirectory string, sourceDataType sdkModels.SourceDataType, serviceNamesToLimitTo *[]string, logger hclog.Logger) (*map[string]availableService, error) {
+	availableDataSources, err := discoverAvailableSourceDataWithin(workingDirectory, sourceDataType, logger)
+	if err != nil {
+		return nil, fmt.Errorf("discovering the available data sources within %q: %+v", workingDirectory, err)
+	}
+
+	if serviceNamesToLimitTo != nil && len(*serviceNamesToLimitTo) > 0 {
+		filtered := make(map[string]availableService)
+
+		for _, serviceName := range *serviceNamesToLimitTo {
+			if v, ok := (*availableDataSources)[serviceName]; ok {
+				filtered[serviceName] = v
+			}
+		}
+
+		availableDataSources = &filtered
+	}
+
+	return availableDataSources, nil
+}