feat(api domain): extend options for Get requests

src/pkg/common/schemas/validation.json

@@ -362,9 +362,36 @@
                             "url": {
                                 "type": "string",
                                 "format": "uri"
+                            },
+                            "parameters": {
+                                "type": "object",
+                                "additionalProperties": { "type": "string"}
+                            },
+                            "options": {
+                                "$ref": "#/definitions/api-options"
                             }
                         }
-                    }
+                    },
+                    "required": ["name", "url"]
+                },
+                "options": {
+                    "$ref": "#/definitions/api-options"
+                }
+            },
+            "required": ["requests"]
+        },
+        "api-options": {
+            "type": "object",
+            "properties": {
+                "timeout": {
+                    "type": "string"
+                },
+                "proxy": {
+                    "type": "string"
+                },
+                "headers": {
+                    "type": "object",
+                    "additionalProperties": { "type": "string"}
                 }
             }
         },

src/pkg/common/types.go

@@ -10,6 +10,8 @@ import (
 	"github.com/defenseunicorns/go-oscal/src/pkg/uuid"
 	oscalValidation "github.com/defenseunicorns/go-oscal/src/pkg/validation"
 	oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2"
+	"sigs.k8s.io/yaml"
+
 	"github.com/defenseunicorns/lula/src/config"
 	"github.com/defenseunicorns/lula/src/pkg/common/schemas"
 	"github.com/defenseunicorns/lula/src/pkg/domains/api"
@@ -18,7 +20,6 @@ import (
 	"github.com/defenseunicorns/lula/src/pkg/providers/kyverno"
 	"github.com/defenseunicorns/lula/src/pkg/providers/opa"
 	"github.com/defenseunicorns/lula/src/types"
-	"sigs.k8s.io/yaml"
 )
 
 // Define base errors for validations
@@ -153,7 +154,8 @@ func (validation *Validation) ToLulaValidation(uuid string) (lulaValidation type
 	domain, err := GetDomain(validation.Domain)
 	if domain == nil {
 		return lulaValidation, fmt.Errorf("%w: %s", ErrInvalidDomain, validation.Domain.Type)
-	} else if err != nil {
+	}
+	if err != nil {
 		return lulaValidation, fmt.Errorf("%w: %v", ErrInvalidDomain, err)
 	}
 	lulaValidation.Domain = &domain

src/pkg/domains/api/api.go

@@ -1,57 +1,49 @@
 package api
 
 import (
-	"bytes"
-	"encoding/json"
+	"context"
 	"fmt"
-	"io"
-	"net/http"
 
 	"github.com/defenseunicorns/lula/src/types"
 )
 
-func MakeRequests(Requests []Request) (types.DomainResources, error) {
-	collection := make(map[string]interface{}, 0)
-
-	for _, request := range Requests {
-		transport := &http.Transport{}
-		client := &http.Client{Transport: transport}
-
-		resp, err := client.Get(request.URL)
-		if err != nil {
-			return nil, err
-		}
-		if resp.StatusCode != 200 {
-			return nil,
-				fmt.Errorf("expected status code 200 but got %d", resp.StatusCode)
-		}
-
-		defer resp.Body.Close()
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return nil, err
+func (a ApiDomain) makeRequests(ctx context.Context) (types.DomainResources, error) {
+	select {
+	case <-ctx.Done():
+		return nil, fmt.Errorf("canceled: %s", ctx.Err())
+	default:
+		collection := make(map[string]interface{}, 0)
+
+		// defaultOpts apply to all requests, but may be overridden by adding an
+		// options block to an individual request.
+		var defaultOpts *ApiOpts
+		if a.Spec.Options == nil {
+			// This isn't likely to be nil in real usage, since CreateApiDomain
+			// parses and mutates specs.
+			defaultOpts = new(ApiOpts)
+			defaultOpts.timeout = &defaultTimeout
+		} else {
+			defaultOpts = a.Spec.Options
 		}
 
-		contentType := resp.Header.Get("Content-Type")
-		if contentType == "application/json" {
-
-			var prettyBuff bytes.Buffer
-			err := json.Indent(&prettyBuff, body, "", "  ")
-			if err != nil {
-				return nil, err
+		// configure the default HTTP client using any top-level Options. Individual
+		// requests with overrides will get bespoke clients.
+		defaultClient := clientFromOpts(defaultOpts)
+
+		for _, request := range a.Spec.Requests {
+			var responseType interface{}
+			var err error
+			if request.Options == nil {
+				responseType, err = doHTTPReq(ctx, defaultClient, *request.reqURL, defaultOpts.Headers, request.reqParameters, responseType)
+			} else {
+				client := clientFromOpts(request.Options)
+				responseType, err = doHTTPReq(ctx, client, *request.reqURL, request.Options.Headers, request.reqParameters, responseType)
 			}
-			prettyJson := prettyBuff.String()
-
-			var tempData interface{}
-			err = json.Unmarshal([]byte(prettyJson), &tempData)
 			if err != nil {
 				return nil, err
 			}
-			collection[request.Name] = tempData
-
-		} else {
-			return nil, fmt.Errorf("content type %s is not supported", contentType)
+			collection[request.Name] = responseType
 		}
+		return collection, nil
 	}
-	return collection, nil
 }

src/pkg/domains/api/http_request.go

@@ -0,0 +1,73 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+func doHTTPReq[T any](ctx context.Context, client http.Client, url url.URL, headers map[string]string, queryParameters url.Values, respTy T) (T, error) {
+	// append any query parameters.
+	q := url.Query()
+
+	for k, v := range queryParameters {
+		// using Add instead of set incase the input URL already had a query encoded
+		q.Add(k, strings.Join(v, ","))
+	}
+	// set the query to the encoded parameters
+	url.RawQuery = q.Encode()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil)
+	if err != nil {
+		return respTy, err
+	}
+	// add each header to the request
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+
+	// do the thing
+	res, err := client.Do(req)
+	if err != nil {
+		return respTy, err
+	}
+
+	if res == nil {
+		return respTy, fmt.Errorf("error: calling %s returned empty response", url.Redacted())
+	}
+	defer res.Body.Close()
+
+	responseData, err := io.ReadAll(res.Body)
+	if err != nil {
+		return respTy, err
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return respTy, fmt.Errorf("expected status code 200 but got %d", res.StatusCode)
+	}
+
+	var responseObject T
+	err = json.Unmarshal(responseData, &responseObject)
+
+	if err != nil {
+		return respTy, fmt.Errorf("error unmarshaling response: %w", err)
+	}
+
+	return responseObject, nil
+}
+
+func clientFromOpts(opts *ApiOpts) http.Client {
+	transport := &http.Transport{}
+	if opts.proxyURL != nil {
+		transport.Proxy = http.ProxyURL(opts.proxyURL)
+	}
+	c := http.Client{Transport: transport}
+	if opts.timeout != nil {
+		c.Timeout = *opts.timeout
+	}
+	return c
+}

src/pkg/domains/api/spec.go

@@ -0,0 +1,90 @@
+package api
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"time"
+)
+
+var defaultTimeout = 30 * time.Second
+
+// validateAndMutateSpec validates the spec values and applies any defaults or
+// other mutations or normalizations necessary. The original values are not modified.
+// validateAndMutateSpec will validate the entire object and may return multiple
+// errors.
+func validateAndMutateSpec(spec *ApiSpec) (errs error) {
+	if spec == nil {
+		return errors.New("spec is required")
+	}
+	if len(spec.Requests) == 0 {
+		errs = errors.Join(errs, errors.New("some requests must be specified"))
+	}
+
+	if spec.Options == nil {
+		spec.Options = &ApiOpts{}
+	}
+	err := validateAndMutateOptions(spec.Options)
+	if err != nil {
+		errs = errors.Join(errs, err)
+	}
+
+	for i := range spec.Requests {
+		if spec.Requests[i].Name == "" {
+			errs = errors.Join(errs, errors.New("request name cannot be empty"))
+		}
+		if spec.Requests[i].URL == "" {
+			errs = errors.Join(errs, errors.New("request url cannot be empty"))
+		}
+		reqUrl, err := url.Parse(spec.Requests[i].URL)
+		if err != nil {
+			errs = errors.Join(errs, errors.New("invalid request url"))
+		} else {
+			spec.Requests[i].reqURL = reqUrl
+		}
+		if spec.Requests[i].Params != nil {
+			queryParameters := url.Values{}
+			for k, v := range spec.Requests[i].Params {
+				queryParameters.Add(k, v)
+			}
+			spec.Requests[i].reqParameters = queryParameters
+		}
+		if spec.Requests[i].Options != nil {
+			err = validateAndMutateOptions(spec.Requests[i].Options)
+			if err != nil {
+				errs = errors.Join(errs, err)
+			}
+		}
+	}
+
+	return errs
+}
+
+func validateAndMutateOptions(opts *ApiOpts) (errs error) {
+	if opts == nil {
+		return errors.New("opts cannot be nil")
+	}
+
+	if opts.Timeout != "" {
+		duration, err := time.ParseDuration(opts.Timeout)
+		if err != nil {
+			errs = errors.Join(errs, fmt.Errorf("invalid wait timeout string: %s", opts.Timeout))
+		}
+		opts.timeout = &duration
+	}
+
+	if opts.timeout == nil {
+		opts.timeout = &defaultTimeout
+	}
+
+	if opts.Proxy != "" {
+		proxyURL, err := url.Parse(opts.Proxy)
+		if err != nil {
+			// not logging the input URL in case it has embedded credentials
+			errs = errors.Join(errs, errors.New("invalid proxy string"))
+		}
+		opts.proxyURL = proxyURL
+	}
+
+	return errs
+}