Merge customizations for service s3

.changelog/51dc2b2349b545baa70246b6e1d54dd9.json

@@ -0,0 +1,9 @@
+{
+    "id": "51dc2b23-49b5-45ba-a702-46b6e1d54dd9",
+    "type": "feature",
+    "description": "Add S3Express support.",
+    "modules": [
+        "feature/s3/manager",
+        "service/s3"
+    ]
+}

Makefile

@@ -4,6 +4,7 @@ LINT_IGNORE_S3MANAGER_INPUT='feature/s3/manager/upload.go:.+struct field SSEKMSK
 # Names of these are tied to endpoint rules and they're internal so ignore them
 LINT_IGNORE_AWSRULESFN_ARN='internal/endpoints/awsrulesfn/arn.go'
 LINT_IGNORE_AWSRULESFN_PARTITION='internal/endpoints/awsrulesfn/partition.go'
+LINT_IGNORE_PRIVATE_METRICS='aws/middleware/private/metrics'
 
 UNIT_TEST_TAGS=
 BUILD_TAGS=-tags "example,codegen,integration,ec2env,perftest"
@@ -472,7 +473,8 @@ lint:
 	-e ${LINT_IGNORE_S3MANAGER_INPUT} \
 	-e ${LINTIGNORESINGLEFIGHT} \
 	-e ${LINT_IGNORE_AWSRULESFN_ARN} \
-	-e ${LINT_IGNORE_AWSRULESFN_PARTITION}`; \
+	-e ${LINT_IGNORE_AWSRULESFN_PARTITION} \
+	-e ${LINT_IGNORE_PRIVATE_METRICS}`; \
 	echo "$$dolint"; \
 	if [ "$$dolint" != "" ]; then exit 1; fi
 

aws/middleware/private/metrics/emf/emf.go

@@ -0,0 +1,90 @@
+// Package emf implements an EMF metrics publisher.
+//
+// This package is designated as private and is intended for use only by the
+// smithy client runtime. The exported API therein is not considered stable and
+// is subject to breaking changes without notice.
+package emf
+
+import (
+	"github.com/aws/aws-sdk-go-v2/aws/middleware/private/metrics"
+	"github.com/aws/aws-sdk-go-v2/internal/sdk"
+)
+
+const (
+	emfIdentifier        = "_aws"
+	timestampKey         = "Timestamp"
+	cloudWatchMetricsKey = "CloudWatchMetrics"
+	namespaceKey         = "Namespace"
+	dimensionsKey        = "Dimensions"
+	metricsKey           = "Metrics"
+)
+
+// Entry represents a log entry in the EMF format.
+type Entry struct {
+	namespace  string
+	serializer metrics.Serializer
+	metrics    []metric
+	dimensions [][]string
+	fields     map[string]interface{}
+}
+
+type metric struct {
+	Name string
+}
+
+// NewEntry creates a new Entry with the specified namespace and serializer.
+func NewEntry(namespace string, serializer metrics.Serializer) Entry {
+	return Entry{
+		namespace:  namespace,
+		serializer: serializer,
+		metrics:    []metric{},
+		dimensions: [][]string{{}},
+		fields:     map[string]interface{}{},
+	}
+}
+
+// Build constructs the EMF log entry as a JSON string.
+func (e *Entry) Build() (string, error) {
+
+	entry := map[string]interface{}{}
+
+	entry[emfIdentifier] = map[string]interface{}{
+		timestampKey: sdk.NowTime().UnixNano() / 1e6,
+		cloudWatchMetricsKey: []map[string]interface{}{
+			{
+				namespaceKey:  e.namespace,
+				dimensionsKey: e.dimensions,
+				metricsKey:    e.metrics,
+			},
+		},
+	}
+
+	for k, v := range e.fields {
+		entry[k] = v
+	}
+
+	jsonEntry, err := e.serializer.Serialize(entry)
+	if err != nil {
+		return "", err
+	}
+	return jsonEntry, nil
+}
+
+// AddDimension adds a CW Dimension to the EMF entry.
+func (e *Entry) AddDimension(key string, value string) {
+	// Dimensions are a list of lists. We only support a single list.
+	e.dimensions[0] = append(e.dimensions[0], key)
+	e.fields[key] = value
+}
+
+// AddMetric adds a CW Metric to the EMF entry.
+func (e *Entry) AddMetric(key string, value float64) {
+	e.metrics = append(e.metrics, metric{key})
+	e.fields[key] = value
+}
+
+// AddProperty adds a CW Property to the EMF entry.
+// Properties are not published as metrics, but they are available in logs and in CW insights.
+func (e *Entry) AddProperty(key string, value interface{}) {
+	e.fields[key] = value
+}

aws/middleware/private/metrics/emf/emf_test.go

@@ -0,0 +1,145 @@
+// This package is designated as private and is intended for use only by the
+// smithy client runtime. The exported API therein is not considered stable and
+// is subject to breaking changes without notice.
+
+package emf
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws/middleware/private/metrics"
+	"github.com/aws/aws-sdk-go-v2/internal/sdk"
+)
+
+type TestSerializerWithError struct{}
+
+func (TestSerializerWithError) Serialize(obj interface{}) (string, error) {
+	return "", fmt.Errorf("serialization error")
+}
+
+func TestCreateNewEntry(t *testing.T) {
+
+	sdk.NowTime = func() time.Time {
+		return time.Unix(1234, 0)
+	}
+
+	cases := map[string]struct {
+		Namespace     string
+		ExpectedEntry Entry
+	}{
+		"success": {
+			Namespace: "testNamespace",
+			ExpectedEntry: Entry{
+				namespace:  "testNamespace",
+				serializer: metrics.DefaultSerializer{},
+				metrics:    []metric{},
+				dimensions: [][]string{{}},
+				fields:     map[string]interface{}{},
+			},
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			actualEntry := NewEntry(c.Namespace, metrics.DefaultSerializer{})
+			if !reflect.DeepEqual(actualEntry, c.ExpectedEntry) {
+				t.Errorf("Entry contained unexpected values")
+			}
+		})
+	}
+}
+
+func TestBuild(t *testing.T) {
+
+	sdk.NowTime = func() time.Time {
+		return time.Unix(1234, 0)
+	}
+
+	cases := map[string]struct {
+		Namespace      string
+		Configure      func(entry *Entry)
+		Serializer     metrics.Serializer
+		ExpectedError  error
+		ExpectedResult string
+	}{
+		"completeEntry": {
+			Namespace:  "testNamespace",
+			Serializer: metrics.DefaultSerializer{},
+			Configure: func(entry *Entry) {
+				entry.AddMetric("testMetric1", 1)
+				entry.AddMetric("testMetric2", 2)
+				entry.AddDimension("testDimension1", "dim1")
+				entry.AddDimension("testDimension2", "dim2")
+				entry.AddProperty("testProperty1", "prop1")
+				entry.AddProperty("testProperty2", "prop2")
+			},
+			ExpectedError:  nil,
+			ExpectedResult: completeEntry,
+		},
+		"noMetrics": {
+			Namespace:  "testNamespace",
+			Serializer: metrics.DefaultSerializer{},
+			Configure: func(entry *Entry) {
+				entry.AddDimension("testDimension1", "dim1")
+				entry.AddDimension("testDimension2", "dim2")
+				entry.AddProperty("testProperty1", "prop1")
+				entry.AddProperty("testProperty2", "prop2")
+			},
+			ExpectedError:  nil,
+			ExpectedResult: noMetrics,
+		},
+		"noDimensions": {
+			Namespace:  "testNamespace",
+			Serializer: metrics.DefaultSerializer{},
+			Configure: func(entry *Entry) {
+				entry.AddMetric("testMetric1", 1)
+				entry.AddMetric("testMetric2", 2)
+				entry.AddProperty("testProperty1", "prop1")
+				entry.AddProperty("testProperty2", "prop2")
+			},
+			ExpectedError:  nil,
+			ExpectedResult: noDimensions,
+		},
+		"noProperties": {
+			Namespace:  "testNamespace",
+			Serializer: metrics.DefaultSerializer{},
+			Configure: func(entry *Entry) {
+				entry.AddMetric("testMetric1", 1)
+				entry.AddMetric("testMetric2", 2)
+				entry.AddDimension("testDimension1", "dim1")
+				entry.AddDimension("testDimension2", "dim2")
+			},
+			ExpectedError:  nil,
+			ExpectedResult: noProperties,
+		},
+		"serializationError": {
+			Namespace:  "testNamespace",
+			Serializer: TestSerializerWithError{},
+			Configure: func(entry *Entry) {
+			},
+			ExpectedError:  fmt.Errorf("serialization error"),
+			ExpectedResult: "",
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			entry := NewEntry(c.Namespace, c.Serializer)
+
+			c.Configure(&entry)
+
+			result, err := entry.Build()
+
+			if !reflect.DeepEqual(err, c.ExpectedError) {
+				t.Errorf("Unexpected error, should be '%s' but was '%s'", c.ExpectedError, err)
+			}
+
+			if !reflect.DeepEqual(result, c.ExpectedResult) {
+				t.Errorf("Unexpected result, should be '%s' but was '%s'", c.ExpectedResult, result)
+			}
+		})
+	}
+}

aws/middleware/private/metrics/emf/emf_test_data.go

@@ -0,0 +1,104 @@
+// This package is designated as private and is intended for use only by the
+// smithy client runtime. The exported API therein is not considered stable and
+// is subject to breaking changes without notice.
+
+package emf
+
+import "strings"
+
+func stripString(str string) string {
+	str = strings.Replace(str, " ", "", -1)
+	str = strings.Replace(str, "\t", "", -1)
+	str = strings.Replace(str, "\n", "", -1)
+	return str
+}
+
+var completeEntry = stripString(`
+{
+	"_aws": {
+		"CloudWatchMetrics": [{
+			"Dimensions": [
+				["testDimension1", "testDimension2"]
+			],
+			"Metrics": [{
+				"Name": "testMetric1"
+			}, {
+				"Name": "testMetric2"
+			}],
+			"Namespace": "testNamespace"
+		}],
+		"Timestamp": 1234000
+	},
+	"testDimension1": "dim1",
+	"testDimension2": "dim2",
+	"testMetric1": 1,
+	"testMetric2": 2,
+	"testProperty1": "prop1",
+	"testProperty2": "prop2"
+}
+`)
+
+var noMetrics = stripString(`
+{
+	"_aws": {
+		"CloudWatchMetrics": [{
+			"Dimensions": [
+				["testDimension1", "testDimension2"]
+			],
+			"Metrics": [],
+			"Namespace": "testNamespace"
+		}],
+		"Timestamp": 1234000
+	},
+	"testDimension1": "dim1",
+	"testDimension2": "dim2",
+	"testProperty1": "prop1",
+	"testProperty2": "prop2"
+}
+`)
+
+var noProperties = stripString(`
+{
+	"_aws": {
+		"CloudWatchMetrics": [{
+			"Dimensions": [
+				["testDimension1", "testDimension2"]
+			],
+			"Metrics": [{
+				"Name": "testMetric1"
+			}, {
+				"Name": "testMetric2"
+			}],
+			"Namespace": "testNamespace"
+		}],
+		"Timestamp": 1234000
+	},
+	"testDimension1": "dim1",
+	"testDimension2": "dim2",
+	"testMetric1": 1,
+	"testMetric2": 2
+}
+`)
+
+var noDimensions = stripString(`
+{
+	"_aws": {
+		"CloudWatchMetrics": [{
+			"Dimensions": [
+				[]
+			],
+			"Metrics": [{
+				"Name": "testMetric1"
+			}, {
+				"Name": "testMetric2"
+			}],
+			"Namespace": "testNamespace"
+		}],
+		"Timestamp": 1234000
+	},
+	"testMetric1": 1,
+	"testMetric2": 2,
+	"testProperty1": "prop1",
+	"testProperty2": "prop2"
+}
+`)