From bd7b575dafc6416806e0fb500eb3dd3496c04bfc Mon Sep 17 00:00:00 2001 From: Devin Humphreys Date: Fri, 24 Nov 2023 16:52:56 -0500 Subject: [PATCH] Update postman, default config, internal cleanup --- .github/README.md | 5 +- app/conf/config.go | 462 +++++++++---------- app/conf/config_test.go | 276 +++++------ app/conf/goaws.yaml | 12 +- app/conf/mock-data/mock-config.yaml | 2 +- app/gosqs/gosqs.go | 10 +- app/gosqs/gosqs_test.go | 26 +- app/gosqs/queue_attributes.go | 2 +- app/router/router.go | 6 +- app/servertest/server_test.go | 4 +- app/sqs_test.go | 2 +- postman/GoAWS Local.postman_environment.json | 135 ++++++ postman/README.md | 11 + 13 files changed, 550 insertions(+), 403 deletions(-) create mode 100644 postman/GoAWS Local.postman_environment.json create mode 100644 postman/README.md diff --git a/.github/README.md b/.github/README.md index 1620e4503..76d3cec5c 100644 --- a/.github/README.md +++ b/.github/README.md @@ -108,9 +108,12 @@ docker run \ ## Testing your installation +### Postman Environment: [LINK](..%2Fpostman%2FGoAWS%20Local.postman_environment.json) +### Postman Collection: [LINK](https://api.postman.com/collections/4714469-2b32c9da-aad4-4e9e-baee-6c11be6798a3?access_key=PMAT-01HG1KVFDXGGKH62KT141MBC0Z) + You can test that your installation is working correctly in one of two ways: - 1. Using the postman collection, use this [link to import it](https://www.getpostman.com/collections/091386eae8c70588348e). As well the Environment variable for the collection should be set as follows: URL = http://localhost:4100/. + 1. Using the postman collection, use this [link to import it](https://api.postman.com/collections/4714469-2b32c9da-aad4-4e9e-baee-6c11be6798a3?access_key=PMAT-01HG1KVFDXGGKH62KT141MBC0Z). As well the Environment variable for the collection should be set as follows: URL = http://localhost:4100/. 2. by using the AWS cli tools ([download link](http://docs.aws.amazon.com/cli/latest/userguide/installing.html)) here are some samples, you can refer to the [aws cli tools docs](http://docs.aws.amazon.com/cli/latest/reference/) for further information. diff --git a/app/conf/config.go b/app/conf/config.go index df8b3a859..176682ebe 100644 --- a/app/conf/config.go +++ b/app/conf/config.go @@ -1,249 +1,249 @@ package conf import ( - "encoding/json" - "fmt" - "io/fs" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - log "github.com/sirupsen/logrus" - - "github.com/Admiral-Piett/goaws/app" - "github.com/Admiral-Piett/goaws/app/common" - "github.com/ghodss/yaml" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/Admiral-Piett/goaws/app" + "github.com/Admiral-Piett/goaws/app/common" + "github.com/ghodss/yaml" ) var envs map[string]app.Environment func LoadYamlConfig(filename string, env string) []string { - ports := []string{"4100"} - - if filename == "" { - root, _ := filepath.Abs(".") - err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { - if "goaws.yaml" == d.Name() { - filename = path - } - return nil - }) - if err != nil || filename == "" { - log.Warn("Failure to find default config file") - return ports - } - } - log.Infof("Loading config file: %s", filename) - yamlFile, err := os.ReadFile(filename) - if err != nil { - return ports - } - - err = yaml.Unmarshal(yamlFile, &envs) - if err != nil { - log.Errorf("err: %v\n", err) - return ports - } - if env == "" { - env = "Local" - } - - if envs[env].Region == "" { - app.CurrentEnvironment.Region = "local" - } - - app.CurrentEnvironment = envs[env] - - if envs[env].Port != "" { - ports = []string{envs[env].Port} - } else if envs[env].SqsPort != "" && envs[env].SnsPort != "" { - ports = []string{envs[env].SqsPort, envs[env].SnsPort} - app.CurrentEnvironment.Port = envs[env].SqsPort - } - - common.LogMessages = false - common.LogFile = "./goaws_messages.log" - - if envs[env].LogToFile == true { - common.LogMessages = true - if envs[env].LogFile != "" { - common.LogFile = envs[env].LogFile - } - } - - if app.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout == 0 { - app.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout = 30 - } - - if app.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize == 0 { - app.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize = 262144 // 256K - } - - if app.CurrentEnvironment.AccountID == "" { - app.CurrentEnvironment.AccountID = "queue" - } - - if app.CurrentEnvironment.Host == "" { - app.CurrentEnvironment.Host = "localhost" - app.CurrentEnvironment.Port = "4100" - } - - app.SyncQueues.Lock() - app.SyncTopics.Lock() - for _, queue := range envs[env].Queues { - queueUrl := "http://" + app.CurrentEnvironment.Host + ":" + app.CurrentEnvironment.Port + - "/" + app.CurrentEnvironment.AccountID + "/" + queue.Name - if app.CurrentEnvironment.Region != "" { - queueUrl = "http://" + app.CurrentEnvironment.Region + "." + app.CurrentEnvironment.Host + ":" + - app.CurrentEnvironment.Port + "/" + app.CurrentEnvironment.AccountID + "/" + queue.Name - } - queueArn := "arn:aws:sqs:" + app.CurrentEnvironment.Region + ":" + app.CurrentEnvironment.AccountID + ":" + queue.Name - - if queue.ReceiveMessageWaitTimeSeconds == 0 { - queue.ReceiveMessageWaitTimeSeconds = app.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds - } - - if queue.MaximumMessageSize == 0 { - queue.MaximumMessageSize = app.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize - } - - if queue.VisibilityTimeout == 0 { - queue.VisibilityTimeout = app.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout - } - - app.SyncQueues.Queues[queue.Name] = &app.Queue{ - Name: queue.Name, - TimeoutSecs: queue.VisibilityTimeout, - Arn: queueArn, - URL: queueUrl, - ReceiveWaitTimeSecs: queue.ReceiveMessageWaitTimeSeconds, - MaximumMessageSize: queue.MaximumMessageSize, - IsFIFO: app.HasFIFOQueueName(queue.Name), - EnableDuplicates: app.CurrentEnvironment.EnableDuplicates, - Duplicates: make(map[string]time.Time), - } - } - - // loop one more time to create queue's RedrivePolicy and assign deadletter queues in case dead letter queue is defined first in the config - for _, queue := range envs[env].Queues { - q := app.SyncQueues.Queues[queue.Name] - if queue.RedrivePolicy != "" { - err := setQueueRedrivePolicy(app.SyncQueues.Queues, q, queue.RedrivePolicy) - if err != nil { - log.Errorf("err: %s", err) - return ports - } - } - - } - - for _, topic := range envs[env].Topics { - topicArn := "arn:aws:sns:" + app.CurrentEnvironment.Region + ":" + app.CurrentEnvironment.AccountID + ":" + topic.Name - - newTopic := &app.Topic{Name: topic.Name, Arn: topicArn} - newTopic.Subscriptions = make([]*app.Subscription, 0, 0) - - for _, subs := range topic.Subscriptions { - var newSub *app.Subscription - if strings.Contains(subs.Protocol, "http") { - newSub = createHttpSubscription(subs) - } else { - //Queue does not exist yet, create it. - newSub = createSqsSubscription(subs, topicArn) - } - if subs.FilterPolicy != "" { - filterPolicy := &app.FilterPolicy{} - err = json.Unmarshal([]byte(subs.FilterPolicy), filterPolicy) - if err != nil { - log.Errorf("err: %s", err) - return ports - } - newSub.FilterPolicy = filterPolicy - } - - newTopic.Subscriptions = append(newTopic.Subscriptions, newSub) - } - app.SyncTopics.Topics[topic.Name] = newTopic - } - - app.SyncQueues.Unlock() - app.SyncTopics.Unlock() - - return ports + ports := []string{"4100"} + + if filename == "" { + root, _ := filepath.Abs(".") + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if "goaws.yaml" == d.Name() { + filename = path + } + return nil + }) + if err != nil || filename == "" { + log.Warn("Failure to find default config file") + return ports + } + } + log.Infof("Loading config file: %s", filename) + yamlFile, err := os.ReadFile(filename) + if err != nil { + return ports + } + + err = yaml.Unmarshal(yamlFile, &envs) + if err != nil { + log.Errorf("err: %v\n", err) + return ports + } + if env == "" { + env = "Local" + } + + if envs[env].Region == "" { + app.CurrentEnvironment.Region = "local" + } + + app.CurrentEnvironment = envs[env] + + if envs[env].Port != "" { + ports = []string{envs[env].Port} + } else if envs[env].SqsPort != "" && envs[env].SnsPort != "" { + ports = []string{envs[env].SqsPort, envs[env].SnsPort} + app.CurrentEnvironment.Port = envs[env].SqsPort + } + + common.LogMessages = false + common.LogFile = "./goaws_messages.log" + + if envs[env].LogToFile == true { + common.LogMessages = true + if envs[env].LogFile != "" { + common.LogFile = envs[env].LogFile + } + } + + if app.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout == 0 { + app.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout = 30 + } + + if app.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize == 0 { + app.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize = 262144 // 256K + } + + if app.CurrentEnvironment.AccountID == "" { + app.CurrentEnvironment.AccountID = "queue" + } + + if app.CurrentEnvironment.Host == "" { + app.CurrentEnvironment.Host = "localhost" + app.CurrentEnvironment.Port = "4100" + } + + app.SyncQueues.Lock() + app.SyncTopics.Lock() + for _, queue := range envs[env].Queues { + queueUrl := "http://" + app.CurrentEnvironment.Host + ":" + app.CurrentEnvironment.Port + + "/" + app.CurrentEnvironment.AccountID + "/" + queue.Name + if app.CurrentEnvironment.Region != "" { + queueUrl = "http://" + app.CurrentEnvironment.Region + "." + app.CurrentEnvironment.Host + ":" + + app.CurrentEnvironment.Port + "/" + app.CurrentEnvironment.AccountID + "/" + queue.Name + } + queueArn := "arn:aws:sqs:" + app.CurrentEnvironment.Region + ":" + app.CurrentEnvironment.AccountID + ":" + queue.Name + + if queue.ReceiveMessageWaitTimeSeconds == 0 { + queue.ReceiveMessageWaitTimeSeconds = app.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds + } + + if queue.MaximumMessageSize == 0 { + queue.MaximumMessageSize = app.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize + } + + if queue.VisibilityTimeout == 0 { + queue.VisibilityTimeout = app.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout + } + + app.SyncQueues.Queues[queue.Name] = &app.Queue{ + Name: queue.Name, + TimeoutSecs: queue.VisibilityTimeout, + Arn: queueArn, + URL: queueUrl, + ReceiveWaitTimeSecs: queue.ReceiveMessageWaitTimeSeconds, + MaximumMessageSize: queue.MaximumMessageSize, + IsFIFO: app.HasFIFOQueueName(queue.Name), + EnableDuplicates: app.CurrentEnvironment.EnableDuplicates, + Duplicates: make(map[string]time.Time), + } + } + + // loop one more time to create queue's RedrivePolicy and assign deadletter queues in case dead letter queue is defined first in the config + for _, queue := range envs[env].Queues { + q := app.SyncQueues.Queues[queue.Name] + if queue.RedrivePolicy != "" { + err := setQueueRedrivePolicy(app.SyncQueues.Queues, q, queue.RedrivePolicy) + if err != nil { + log.Errorf("err: %s", err) + return ports + } + } + + } + + for _, topic := range envs[env].Topics { + topicArn := "arn:aws:sns:" + app.CurrentEnvironment.Region + ":" + app.CurrentEnvironment.AccountID + ":" + topic.Name + + newTopic := &app.Topic{Name: topic.Name, Arn: topicArn} + newTopic.Subscriptions = make([]*app.Subscription, 0, 0) + + for _, subs := range topic.Subscriptions { + var newSub *app.Subscription + if strings.Contains(subs.Protocol, "http") { + newSub = createHttpSubscription(subs) + } else { + //Queue does not exist yet, create it. + newSub = createSqsSubscription(subs, topicArn) + } + if subs.FilterPolicy != "" { + filterPolicy := &app.FilterPolicy{} + err = json.Unmarshal([]byte(subs.FilterPolicy), filterPolicy) + if err != nil { + log.Errorf("err: %s", err) + return ports + } + newSub.FilterPolicy = filterPolicy + } + + newTopic.Subscriptions = append(newTopic.Subscriptions, newSub) + } + app.SyncTopics.Topics[topic.Name] = newTopic + } + + app.SyncQueues.Unlock() + app.SyncTopics.Unlock() + + return ports } func createHttpSubscription(configSubscription app.EnvSubsciption) *app.Subscription { - newSub := &app.Subscription{EndPoint: configSubscription.EndPoint, Protocol: configSubscription.Protocol, TopicArn: configSubscription.TopicArn, Raw: configSubscription.Raw} - subArn, _ := common.NewUUID() - subArn = configSubscription.TopicArn + ":" + subArn - newSub.SubscriptionArn = subArn - return newSub + newSub := &app.Subscription{EndPoint: configSubscription.EndPoint, Protocol: configSubscription.Protocol, TopicArn: configSubscription.TopicArn, Raw: configSubscription.Raw} + subArn, _ := common.NewUUID() + subArn = configSubscription.TopicArn + ":" + subArn + newSub.SubscriptionArn = subArn + return newSub } func createSqsSubscription(configSubscription app.EnvSubsciption, topicArn string) *app.Subscription { - if _, ok := app.SyncQueues.Queues[configSubscription.QueueName]; !ok { - queueUrl := "http://" + app.CurrentEnvironment.Host + ":" + app.CurrentEnvironment.Port + - "/" + app.CurrentEnvironment.AccountID + "/" + configSubscription.QueueName - if app.CurrentEnvironment.Region != "" { - queueUrl = "http://" + app.CurrentEnvironment.Region + "." + app.CurrentEnvironment.Host + ":" + - app.CurrentEnvironment.Port + "/" + app.CurrentEnvironment.AccountID + "/" + configSubscription.QueueName - } - queueArn := "arn:aws:sqs:" + app.CurrentEnvironment.Region + ":" + app.CurrentEnvironment.AccountID + ":" + configSubscription.QueueName - app.SyncQueues.Queues[configSubscription.QueueName] = &app.Queue{ - Name: configSubscription.QueueName, - TimeoutSecs: app.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout, - Arn: queueArn, - URL: queueUrl, - ReceiveWaitTimeSecs: app.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds, - MaximumMessageSize: app.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize, - IsFIFO: app.HasFIFOQueueName(configSubscription.QueueName), - EnableDuplicates: app.CurrentEnvironment.EnableDuplicates, - Duplicates: make(map[string]time.Time), - } - } - qArn := app.SyncQueues.Queues[configSubscription.QueueName].Arn - newSub := &app.Subscription{EndPoint: qArn, Protocol: "sqs", TopicArn: topicArn, Raw: configSubscription.Raw} - subArn, _ := common.NewUUID() - subArn = topicArn + ":" + subArn - newSub.SubscriptionArn = subArn - return newSub + if _, ok := app.SyncQueues.Queues[configSubscription.QueueName]; !ok { + queueUrl := "http://" + app.CurrentEnvironment.Host + ":" + app.CurrentEnvironment.Port + + "/" + app.CurrentEnvironment.AccountID + "/" + configSubscription.QueueName + if app.CurrentEnvironment.Region != "" { + queueUrl = "http://" + app.CurrentEnvironment.Region + "." + app.CurrentEnvironment.Host + ":" + + app.CurrentEnvironment.Port + "/" + app.CurrentEnvironment.AccountID + "/" + configSubscription.QueueName + } + queueArn := "arn:aws:sqs:" + app.CurrentEnvironment.Region + ":" + app.CurrentEnvironment.AccountID + ":" + configSubscription.QueueName + app.SyncQueues.Queues[configSubscription.QueueName] = &app.Queue{ + Name: configSubscription.QueueName, + TimeoutSecs: app.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout, + Arn: queueArn, + URL: queueUrl, + ReceiveWaitTimeSecs: app.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds, + MaximumMessageSize: app.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize, + IsFIFO: app.HasFIFOQueueName(configSubscription.QueueName), + EnableDuplicates: app.CurrentEnvironment.EnableDuplicates, + Duplicates: make(map[string]time.Time), + } + } + qArn := app.SyncQueues.Queues[configSubscription.QueueName].Arn + newSub := &app.Subscription{EndPoint: qArn, Protocol: "sqs", TopicArn: topicArn, Raw: configSubscription.Raw} + subArn, _ := common.NewUUID() + subArn = topicArn + ":" + subArn + newSub.SubscriptionArn = subArn + return newSub } func setQueueRedrivePolicy(queues map[string]*app.Queue, q *app.Queue, strRedrivePolicy string) error { - // support both int and string maxReceiveCount (Amazon clients use string) - redrivePolicy1 := struct { - MaxReceiveCount int `json:"maxReceiveCount"` - DeadLetterTargetArn string `json:"deadLetterTargetArn"` - }{} - redrivePolicy2 := struct { - MaxReceiveCount string `json:"maxReceiveCount"` - DeadLetterTargetArn string `json:"deadLetterTargetArn"` - }{} - err1 := json.Unmarshal([]byte(strRedrivePolicy), &redrivePolicy1) - err2 := json.Unmarshal([]byte(strRedrivePolicy), &redrivePolicy2) - maxReceiveCount := redrivePolicy1.MaxReceiveCount - deadLetterQueueArn := redrivePolicy1.DeadLetterTargetArn - if err1 != nil && err2 != nil { - return fmt.Errorf("invalid json for queue redrive policy ") - } else if err1 != nil { - maxReceiveCount, _ = strconv.Atoi(redrivePolicy2.MaxReceiveCount) - deadLetterQueueArn = redrivePolicy2.DeadLetterTargetArn - } - - if (deadLetterQueueArn != "" && maxReceiveCount == 0) || - (deadLetterQueueArn == "" && maxReceiveCount != 0) { - return fmt.Errorf("invalid redrive policy values") - } - dlt := strings.Split(deadLetterQueueArn, ":") - deadLetterQueueName := dlt[len(dlt)-1] - deadLetterQueue, ok := queues[deadLetterQueueName] - if !ok { - return fmt.Errorf("deadletter queue not found") - } - q.DeadLetterQueue = deadLetterQueue - q.MaxReceiveCount = maxReceiveCount - - return nil + // support both int and string maxReceiveCount (Amazon clients use string) + redrivePolicy1 := struct { + MaxReceiveCount int `json:"maxReceiveCount"` + DeadLetterTargetArn string `json:"deadLetterTargetArn"` + }{} + redrivePolicy2 := struct { + MaxReceiveCount string `json:"maxReceiveCount"` + DeadLetterTargetArn string `json:"deadLetterTargetArn"` + }{} + err1 := json.Unmarshal([]byte(strRedrivePolicy), &redrivePolicy1) + err2 := json.Unmarshal([]byte(strRedrivePolicy), &redrivePolicy2) + maxReceiveCount := redrivePolicy1.MaxReceiveCount + deadLetterQueueArn := redrivePolicy1.DeadLetterTargetArn + if err1 != nil && err2 != nil { + return fmt.Errorf("invalid json for queue redrive policy ") + } else if err1 != nil { + maxReceiveCount, _ = strconv.Atoi(redrivePolicy2.MaxReceiveCount) + deadLetterQueueArn = redrivePolicy2.DeadLetterTargetArn + } + + if (deadLetterQueueArn != "" && maxReceiveCount == 0) || + (deadLetterQueueArn == "" && maxReceiveCount != 0) { + return fmt.Errorf("invalid redrive policy values") + } + dlt := strings.Split(deadLetterQueueArn, ":") + deadLetterQueueName := dlt[len(dlt)-1] + deadLetterQueue, ok := queues[deadLetterQueueName] + if !ok { + return fmt.Errorf("deadletter queue not found") + } + q.DeadLetterQueue = deadLetterQueue + q.MaxReceiveCount = maxReceiveCount + + return nil } diff --git a/app/conf/config_test.go b/app/conf/config_test.go index 2f6d8649b..f4aefe7cc 100644 --- a/app/conf/config_test.go +++ b/app/conf/config_test.go @@ -1,157 +1,159 @@ package conf import ( - "github.com/stretchr/testify/assert" - "testing" + "testing" - "github.com/Admiral-Piett/goaws/app" + "github.com/stretchr/testify/assert" + + "github.com/Admiral-Piett/goaws/app" ) func TestConfig_NoQueuesOrTopics(t *testing.T) { - env := "NoQueuesOrTopics" - port := LoadYamlConfig("./mock-data/mock-config.yaml", env) - if port[0] != "4100" { - t.Errorf("Expected port number 4200 but got %s\n", port) - } - - numQueues := len(envs[env].Queues) - if numQueues != 0 { - t.Errorf("Expected zero queues to be in the environment but got %d\n", numQueues) - } - numQueues = len(app.SyncQueues.Queues) - if numQueues != 0 { - t.Errorf("Expected zero queues to be in the sqs topics but got %d\n", numQueues) - } - - numTopics := len(envs[env].Topics) - if numTopics != 0 { - t.Errorf("Expected zero topics to be in the environment but got %d\n", numTopics) - } - numTopics = len(app.SyncTopics.Topics) - if numTopics != 0 { - t.Errorf("Expected zero topics to be in the sns topics but got %d\n", numTopics) - } + env := "NoQueuesOrTopics" + port := LoadYamlConfig("./mock-data/mock-config.yaml", env) + if port[0] != "4100" { + t.Errorf("Expected port number 4200 but got %s\n", port) + } + + numQueues := len(envs[env].Queues) + if numQueues != 0 { + t.Errorf("Expected zero queues to be in the environment but got %d\n", numQueues) + } + numQueues = len(app.SyncQueues.Queues) + if numQueues != 0 { + t.Errorf("Expected zero queues to be in the sqs topics but got %d\n", numQueues) + } + + numTopics := len(envs[env].Topics) + if numTopics != 0 { + t.Errorf("Expected zero topics to be in the environment but got %d\n", numTopics) + } + numTopics = len(app.SyncTopics.Topics) + if numTopics != 0 { + t.Errorf("Expected zero topics to be in the sns topics but got %d\n", numTopics) + } } func TestConfig_CreateQueuesTopicsAndSubscriptions(t *testing.T) { - env := "Local" - port := LoadYamlConfig("./mock-data/mock-config.yaml", env) - if port[0] != "4100" { - t.Errorf("Expected port number 4100 but got %s\n", port) - } - - numQueues := len(envs[env].Queues) - if numQueues != 4 { - t.Errorf("Expected three queues to be in the environment but got %d\n", numQueues) - } - numQueues = len(app.SyncQueues.Queues) - if numQueues != 6 { - t.Errorf("Expected five queues to be in the sqs topics but got %d\n", numQueues) - } - - numTopics := len(envs[env].Topics) - if numTopics != 2 { - t.Errorf("Expected two topics to be in the environment but got %d\n", numTopics) - } - numTopics = len(app.SyncTopics.Topics) - if numTopics != 2 { - t.Errorf("Expected two topics to be in the sns topics but got %d\n", numTopics) - } + env := "Local" + port := LoadYamlConfig("./mock-data/mock-config.yaml", env) + if port[0] != "4100" { + t.Errorf("Expected port number 4100 but got %s\n", port) + } + + numQueues := len(envs[env].Queues) + if numQueues != 4 { + t.Errorf("Expected three queues to be in the environment but got %d\n", numQueues) + } + numQueues = len(app.SyncQueues.Queues) + if numQueues != 6 { + t.Errorf("Expected five queues to be in the sqs topics but got %d\n", numQueues) + } + + numTopics := len(envs[env].Topics) + if numTopics != 2 { + t.Errorf("Expected two topics to be in the environment but got %d\n", numTopics) + } + numTopics = len(app.SyncTopics.Topics) + if numTopics != 2 { + t.Errorf("Expected two topics to be in the sns topics but got %d\n", numTopics) + } } func TestConfig_QueueAttributes(t *testing.T) { - env := "Local" - port := LoadYamlConfig("./mock-data/mock-config.yaml", env) - if port[0] != "4100" { - t.Errorf("Expected port number 4100 but got %s\n", port) - } - - receiveWaitTime := app.SyncQueues.Queues["local-queue1"].ReceiveWaitTimeSecs - if receiveWaitTime != 10 { - t.Errorf("Expected local-queue1 Queue to be configured with ReceiveMessageWaitTimeSeconds: 10 but got %d\n", receiveWaitTime) - } - timeoutSecs := app.SyncQueues.Queues["local-queue1"].TimeoutSecs - if timeoutSecs != 10 { - t.Errorf("Expected local-queue1 Queue to be configured with VisibilityTimeout: 10 but got %d\n", timeoutSecs) - } - maximumMessageSize := app.SyncQueues.Queues["local-queue1"].MaximumMessageSize - if maximumMessageSize != 1024 { - t.Errorf("Expected local-queue1 Queue to be configured with MaximumMessageSize: 1024 but got %d\n", maximumMessageSize) - } - - if app.SyncQueues.Queues["local-queue1"].DeadLetterQueue != nil { - t.Errorf("Expected local-queue1 Queue to be configured without redrive policy\n") - } - if app.SyncQueues.Queues["local-queue1"].MaxReceiveCount != 0 { - t.Errorf("Expected local-queue1 Queue to be configured without redrive policy and therefore MaxReceiveCount: 0 \n") - } - - maxReceiveCount := app.SyncQueues.Queues["local-queue3"].MaxReceiveCount - if maxReceiveCount != 100 { - t.Errorf("Expected local-queue2 Queue to be configured with MaxReceiveCount: 3 from RedrivePolicy but got %d\n", maxReceiveCount) - } - dlq := app.SyncQueues.Queues["local-queue3"].DeadLetterQueue - if dlq == nil { - t.Errorf("Expected local-queue3 to have one dead letter queue to redrive to\n") - } - if dlq.Name != "local-queue3-dlq" { - t.Errorf("Expected local-queue3 to have dead letter queue local-queue3-dlq but got %s\n", dlq.Name) - } - maximumMessageSize = app.SyncQueues.Queues["local-queue2"].MaximumMessageSize - if maximumMessageSize != 128 { - t.Errorf("Expected local-queue2 Queue to be configured with MaximumMessageSize: 128 but got %d\n", maximumMessageSize) - } - - timeoutSecs = app.SyncQueues.Queues["local-queue2"].TimeoutSecs - if timeoutSecs != 150 { - t.Errorf("Expected local-queue2 Queue to be configured with VisibilityTimeout: 150 but got %d\n", timeoutSecs) - } + env := "Local" + port := LoadYamlConfig("./mock-data/mock-config.yaml", env) + if port[0] != "4100" { + t.Errorf("Expected port number 4100 but got %s\n", port) + } + + receiveWaitTime := app.SyncQueues.Queues["local-queue1"].ReceiveWaitTimeSecs + if receiveWaitTime != 10 { + t.Errorf("Expected local-queue1 Queue to be configured with ReceiveMessageWaitTimeSeconds: 10 but got %d\n", receiveWaitTime) + } + timeoutSecs := app.SyncQueues.Queues["local-queue1"].TimeoutSecs + if timeoutSecs != 10 { + t.Errorf("Expected local-queue1 Queue to be configured with VisibilityTimeout: 10 but got %d\n", timeoutSecs) + } + maximumMessageSize := app.SyncQueues.Queues["local-queue1"].MaximumMessageSize + if maximumMessageSize != 1024 { + t.Errorf("Expected local-queue1 Queue to be configured with MaximumMessageSize: 1024 but got %d\n", maximumMessageSize) + } + + if app.SyncQueues.Queues["local-queue1"].DeadLetterQueue != nil { + t.Errorf("Expected local-queue1 Queue to be configured without redrive policy\n") + } + if app.SyncQueues.Queues["local-queue1"].MaxReceiveCount != 0 { + t.Errorf("Expected local-queue1 Queue to be configured without redrive policy and therefore MaxReceiveCount: 0 \n") + } + + maxReceiveCount := app.SyncQueues.Queues["local-queue3"].MaxReceiveCount + if maxReceiveCount != 100 { + t.Errorf("Expected local-queue2 Queue to be configured with MaxReceiveCount: 3 from RedrivePolicy but got %d\n", maxReceiveCount) + } + dlq := app.SyncQueues.Queues["local-queue3"].DeadLetterQueue + if dlq == nil { + t.Errorf("Expected local-queue3 to have one dead letter queue to redrive to\n") + } + if dlq.Name != "local-queue3-dlq" { + t.Errorf("Expected local-queue3 to have dead letter queue local-queue3-dlq but got %s\n", dlq.Name) + } + maximumMessageSize = app.SyncQueues.Queues["local-queue2"].MaximumMessageSize + if maximumMessageSize != 128 { + t.Errorf("Expected local-queue2 Queue to be configured with MaximumMessageSize: 128 but got %d\n", maximumMessageSize) + } + + timeoutSecs = app.SyncQueues.Queues["local-queue2"].TimeoutSecs + if timeoutSecs != 150 { + t.Errorf("Expected local-queue2 Queue to be configured with VisibilityTimeout: 150 but got %d\n", timeoutSecs) + } } func TestConfig_NoQueueAttributeDefaults(t *testing.T) { - env := "NoQueueAttributeDefaults" - LoadYamlConfig("./mock-data/mock-config.yaml", env) - - receiveWaitTime := app.SyncQueues.Queues["local-queue1"].ReceiveWaitTimeSecs - if receiveWaitTime != 0 { - t.Errorf("Expected local-queue1 Queue to be configured with ReceiveMessageWaitTimeSeconds: 0 but got %d\n", receiveWaitTime) - } - timeoutSecs := app.SyncQueues.Queues["local-queue1"].TimeoutSecs - if timeoutSecs != 30 { - t.Errorf("Expected local-queue1 Queue to be configured with VisibilityTimeout: 30 but got %d\n", timeoutSecs) - } - - receiveWaitTime = app.SyncQueues.Queues["local-queue2"].ReceiveWaitTimeSecs - if receiveWaitTime != 20 { - t.Errorf("Expected local-queue2 Queue to be configured with ReceiveMessageWaitTimeSeconds: 20 but got %d\n", receiveWaitTime) - } + env := "NoQueueAttributeDefaults" + LoadYamlConfig("./mock-data/mock-config.yaml", env) + + receiveWaitTime := app.SyncQueues.Queues["local-queue1"].ReceiveWaitTimeSecs + if receiveWaitTime != 0 { + t.Errorf("Expected local-queue1 Queue to be configured with ReceiveMessageWaitTimeSeconds: 0 but got %d\n", receiveWaitTime) + } + timeoutSecs := app.SyncQueues.Queues["local-queue1"].TimeoutSecs + if timeoutSecs != 30 { + t.Errorf("Expected local-queue1 Queue to be configured with VisibilityTimeout: 30 but got %d\n", timeoutSecs) + } + + receiveWaitTime = app.SyncQueues.Queues["local-queue2"].ReceiveWaitTimeSecs + if receiveWaitTime != 20 { + t.Errorf("Expected local-queue2 Queue to be configured with ReceiveMessageWaitTimeSeconds: 20 but got %d\n", receiveWaitTime) + } } func TestConfig_LoadYamlConfig_finds_default_config(t *testing.T) { - expectedQueues := []string{ - "local-queue1", - "local-queue2", - "local-queue3", - "local-queue4", - } - expectedTopics := []string{ - "local-topic1", - "sub-topic", - "local-topic2", - "my_topic", - } - - env := "Local" - LoadYamlConfig("", env) - - queues := app.SyncQueues.Queues - topics := app.SyncTopics.Topics - for _, expectedName := range expectedQueues { - _, ok := queues[expectedName] - assert.True(t, ok) - } - for _, expectedName := range expectedTopics { - _, ok := topics[expectedName] - assert.True(t, ok) - } + expectedQueues := []string{ + "local-queue1", + "local-queue2", + "local-queue3", + "local-queue3-dlq", + "local-queue4", + } + expectedTopics := []string{ + "local-topic1", + "local-topic2", + "local-topic3", + "local-topic4", + } + + env := "Local" + LoadYamlConfig("", env) + + queues := app.SyncQueues.Queues + topics := app.SyncTopics.Topics + for _, expectedName := range expectedQueues { + _, ok := queues[expectedName] + assert.True(t, ok) + } + for _, expectedName := range expectedTopics { + _, ok := topics[expectedName] + assert.True(t, ok) + } } diff --git a/app/conf/goaws.yaml b/app/conf/goaws.yaml index 9512fd3c7..a844fb10e 100755 --- a/app/conf/goaws.yaml +++ b/app/conf/goaws.yaml @@ -19,6 +19,9 @@ Local: # Environment name that can be passed on the - Name: local-queue1 # Queue name - Name: local-queue2 # Queue name ReceiveMessageWaitTimeSeconds: 20 # Queue receive message max wait time + - Name: local-queue3 # Queue name + RedrivePolicy: '{"maxReceiveCount": 100, "deadLetterTargetArn":"arn:aws:sqs:us-east-1:100010001000:local-queue3-dlq"}' + - Name: local-queue3-dlq # Queue name Topics: # List of topic to create at startup - Name: local-topic1 # Topic name - with some Subscriptions Subscriptions: # List of Subscriptions to create for this topic (queues will be created as required) @@ -27,18 +30,15 @@ Local: # Environment name that can be passed on the - QueueName: local-queue4 # Queue name Raw: true # Raw message delivery (true/false) #FilterPolicy: '{"foo": ["bar"]}' # Subscription's FilterPolicy, json object as a string - - Name: sub-topic # Topic name - with some Subscriptions - Subscriptions: # List of Subscriptions to create for this topic (queues will be created as required) - - QueueName: local-queue1 # Queue name - Raw: true # Raw message delivery (true/false) - Name: local-topic2 # Topic name - no Subscriptions - - Name: my_topic # Topic name - http subscription + - Name: local-topic3 # Topic name - http subscription Subscriptions: - Protocol: https EndPoint: https://enkrogwitfcgi.x.pipedream.net - TopicArn: arn:aws:sns:us-east-1:000000000000:my_topic + TopicArn: arn:aws:sns:us-east-1:100010001000:local-topic2 FilterPolicy: '{"event": ["my_event"]}' Raw: true + - Name: local-topic4 RandomLatency: # Parameters for introducing random latency into message queuing Min: 0 # Desired latency in milliseconds, if min and max are zero, no latency will be applied. Max: 0 # Desired latency in milliseconds diff --git a/app/conf/mock-data/mock-config.yaml b/app/conf/mock-data/mock-config.yaml index f0068ffa8..8f5dfe5e3 100644 --- a/app/conf/mock-data/mock-config.yaml +++ b/app/conf/mock-data/mock-config.yaml @@ -17,7 +17,7 @@ Local: # Environment name that can be passed on the MaximumMessageSize: 128 # Queue maximum message size (bytes) VisibilityTimeout: 150 # Queue visibility timeout - Name: local-queue3 # Queue name - RedrivePolicy: '{"maxReceiveCount": 100, "deadLetterTargetArn":"arn:aws:sqs:us-east-1:000000000000:local-queue3-dlq"}' + RedrivePolicy: '{"maxReceiveCount": 100, "deadLetterTargetArn":"arn:aws:sqs:us-east-1:100010001000:local-queue3-dlq"}' - Name: local-queue3-dlq # Queue name Topics: # List of topic to create at startup - Name: local-topic1 # Topic name - with some Subscriptions diff --git a/app/gosqs/gosqs.go b/app/gosqs/gosqs.go index 79ef356d5..63e1ee404 100644 --- a/app/gosqs/gosqs.go +++ b/app/gosqs/gosqs.go @@ -11,9 +11,9 @@ import ( log "github.com/sirupsen/logrus" - "github.com/gorilla/mux" "github.com/Admiral-Piett/goaws/app" "github.com/Admiral-Piett/goaws/app/common" + "github.com/gorilla/mux" ) func init() { @@ -302,10 +302,6 @@ func SendMessageBatch(w http.ResponseWriter, req *http.Request) { sendEntries[keyIndex-1].MessageGroupId = v[0] } - if keySegments[2] == "MessageGroupId" { - sendEntries[keyIndex-1].MessageGroupId = v[0] - } - if keySegments[2] == "MessageDeduplicationId" { sendEntries[keyIndex-1].MessageDeduplicationId = v[0] } @@ -826,7 +822,7 @@ func GetQueueAttributes(w http.ResponseWriter, req *http.Request) { // Retrieve FormValues required queueUrl := getQueueFromPath(req.FormValue("QueueUrl"), req.URL.String()) - attribute_names := map[string]bool {} + attribute_names := map[string]bool{} for field, value := range req.Form { if strings.HasPrefix(field, "AttributeName.") { @@ -846,7 +842,7 @@ func GetQueueAttributes(w http.ResponseWriter, req *http.Request) { } return false } - + queueName := "" if queueUrl == "" { vars := mux.Vars(req) diff --git a/app/gosqs/gosqs_test.go b/app/gosqs/gosqs_test.go index 7298b170e..b7b398be0 100644 --- a/app/gosqs/gosqs_test.go +++ b/app/gosqs/gosqs_test.go @@ -1983,15 +1983,14 @@ func TestGetQueueAttributes_GetAllAttributes(t *testing.T) { t.Fatalf("unexpected unmarshal error: %s", err) } - hasAttribute := func(attrs []app.Attribute, name string) bool { - for _, attr := range attrs { - if attr.Name == name { - return true - } + hasAttribute := func(attrs []app.Attribute, name string) bool { + for _, attr := range attrs { + if attr.Name == name { + return true } - return false } - + return false + } ok := hasAttribute(resp.Result.Attrs, "VisibilityTimeout") && hasAttribute(resp.Result.Attrs, "DelaySeconds") && @@ -2054,15 +2053,14 @@ func TestGetQueueAttributes_GetSelectedAttributes(t *testing.T) { t.Fatalf("unexpected unmarshal error: %s", err) } - hasAttribute := func(attrs []app.Attribute, name string) bool { - for _, attr := range attrs { - if attr.Name == name { - return true - } + hasAttribute := func(attrs []app.Attribute, name string) bool { + for _, attr := range attrs { + if attr.Name == name { + return true } - return false } - + return false + } ok := hasAttribute(resp.Result.Attrs, "ApproximateNumberOfMessages") && hasAttribute(resp.Result.Attrs, "ApproximateNumberOfMessagesNotVisible") diff --git a/app/gosqs/queue_attributes.go b/app/gosqs/queue_attributes.go index a3b65e546..312c7b0df 100644 --- a/app/gosqs/queue_attributes.go +++ b/app/gosqs/queue_attributes.go @@ -28,7 +28,7 @@ var ( // validateAndSetQueueAttributes applies the requested queue attributes to the given // queue. -// TODO Currently it only supports VisibilityTimeout, RedrivePolicy and ReceiveMessageWaitTimeSeconds attributes. +// TODO Currently it only supports VisibilityTimeout, MaximumMessageSize, DelaySeconds, RedrivePolicy and ReceiveMessageWaitTimeSeconds attributes. func validateAndSetQueueAttributes(q *app.Queue, u url.Values) error { attr := extractQueueAttributes(u) visibilityTimeout, _ := strconv.Atoi(attr["VisibilityTimeout"]) diff --git a/app/router/router.go b/app/router/router.go index 47766add0..699b6ab05 100644 --- a/app/router/router.go +++ b/app/router/router.go @@ -8,9 +8,9 @@ import ( "fmt" - "github.com/gorilla/mux" sns "github.com/Admiral-Piett/goaws/app/gosns" sqs "github.com/Admiral-Piett/goaws/app/gosqs" + "github.com/gorilla/mux" ) // New returns a new router @@ -48,13 +48,15 @@ var routingTable = map[string]http.HandlerFunc{ "CreateTopic": sns.CreateTopic, "DeleteTopic": sns.DeleteTopic, "Subscribe": sns.Subscribe, - "ConfirmSubscription": sns.ConfirmSubscription, "SetSubscriptionAttributes": sns.SetSubscriptionAttributes, "GetSubscriptionAttributes": sns.GetSubscriptionAttributes, "ListSubscriptionsByTopic": sns.ListSubscriptionsByTopic, "ListSubscriptions": sns.ListSubscriptions, "Unsubscribe": sns.Unsubscribe, "Publish": sns.Publish, + + // SNS Internal + "ConfirmSubscription": sns.ConfirmSubscription, } func health(w http.ResponseWriter, req *http.Request) { diff --git a/app/servertest/server_test.go b/app/servertest/server_test.go index fd670751b..48be85a01 100644 --- a/app/servertest/server_test.go +++ b/app/servertest/server_test.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "github.com/Admiral-Piett/goaws/app" + "github.com/Admiral-Piett/goaws/app/router" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -19,8 +21,6 @@ import ( "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" "github.com/gorilla/mux" - "github.com/Admiral-Piett/goaws/app" - "github.com/Admiral-Piett/goaws/app/router" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/app/sqs_test.go b/app/sqs_test.go index 73b31949c..da1cb984f 100644 --- a/app/sqs_test.go +++ b/app/sqs_test.go @@ -10,7 +10,7 @@ func TestMessage_IsReadyForReceipt(t *testing.T) { CurrentEnvironment.RandomLatency.Min = 100 CurrentEnvironment.RandomLatency.Max = 100 msg := Message{ - SentTime: time.Now(), + SentTime: time.Now(), } assert.False(t, msg.IsReadyForReceipt()) duration, _ := time.ParseDuration("105ms") diff --git a/postman/GoAWS Local.postman_environment.json b/postman/GoAWS Local.postman_environment.json new file mode 100644 index 000000000..9b37052c5 --- /dev/null +++ b/postman/GoAWS Local.postman_environment.json @@ -0,0 +1,135 @@ +{ + "id": "3bb94e17-cd88-4326-9217-0c0722d68310", + "name": "GoAWS Local", + "values": [ + { + "key": "port", + "value": "4100", + "type": "default", + "enabled": true + }, + { + "key": "url", + "value": "localhost:{{port}}", + "type": "default", + "enabled": true + }, + { + "key": "host", + "value": "goaws.com", + "type": "default", + "enabled": true + }, + { + "key": "account-id", + "value": "100010001000", + "type": "default", + "enabled": true + }, + { + "key": "region", + "value": "us-east-1", + "type": "default", + "enabled": true + }, + { + "key": "sqs-arn-base", + "value": "arn:aws:sqs:{{region}}:{{account-id}}", + "type": "default", + "enabled": true + }, + { + "key": "sqs-url-base", + "value": "http://{{region}}.{{host}}:{{port}}/{{account-id}}", + "type": "default", + "enabled": true + }, + { + "key": "sns-arn-base", + "value": "arn:aws:sns:{{region}}:{{account-id}}", + "type": "default", + "enabled": true + }, + { + "key": "queue1", + "value": "local-queue1", + "type": "default", + "enabled": true + }, + { + "key": "queue2", + "value": "local-queue2", + "type": "default", + "enabled": true + }, + { + "key": "queue3", + "value": "local-queue3", + "type": "default", + "enabled": true + }, + { + "key": "queue4", + "value": "local-queue4", + "type": "default", + "enabled": true + }, + { + "key": "queue5", + "value": "local-queue5", + "type": "default", + "enabled": true + }, + { + "key": "queue3-dead-letter", + "value": "local-queue3-dlq", + "type": "default", + "enabled": true + }, + { + "key": "topic1", + "value": "local-topic1", + "type": "default", + "enabled": true + }, + { + "key": "topic2", + "value": "local-topic2", + "type": "default", + "enabled": true + }, + { + "key": "topic3", + "value": "local-topic3", + "type": "default", + "enabled": true + }, + { + "key": "topic4", + "value": "local-topic4", + "type": "default", + "enabled": true + }, + { + "key": "new-queue", + "value": "new-queue", + "type": "default", + "enabled": true + }, + { + "key": "new-topic", + "value": "new-topic", + "type": "default", + "enabled": true + }, + { + "key": "basic-redrive-policy", + "value": "{\n \"maxReceiveCount\": \"10\",\n \"deadLetterTargetArn\": \"{{sqs-arn-base}}:{{queue3-dead-letter}}\"\n}", + "type": "default", + "enabled": true + } + ], + "_postman_variable_scope": "environment", + "_postman_exported_at": "2023-11-24T21:32:03.769Z", + "_postman_exported_using": "Postman/10.18.9" +} \ No newline at end of file diff --git a/postman/README.md b/postman/README.md new file mode 100644 index 000000000..96fff7173 --- /dev/null +++ b/postman/README.md @@ -0,0 +1,11 @@ +# Use Postman to exercise endpoints and create tests + +## Postman Environment +1. Click `import` in Postman's top-right hand corner +2. Drag and drop the `GoAWS Local.postman_environment.json` file in this directory. +3. Select `GoAWS Local` in the environment dropdown (top-right corner, may say `GoAWS Local` already or `No Environment`). + +## Postman Collection: [LINK](https://api.postman.com/collections/4714469-2b32c9da-aad4-4e9e-baee-6c11be6798a3?access_key=PMAT-01HG1KVFDXGGKH62KT141MBC0Z) +1. Click `import` in Postman's top-right hand corner +2. Paste the above link in Postman where is says `Paste cURL, raw text or URL...` +3. Access requests on the left hand side