Merge pull request #765 from nyaruka/minio

Support minio for local dev environment and CI

.github/workflows/ci.yml

@@ -21,7 +21,16 @@ jobs:
         ports:
           - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-
+      minio:
+        image: bitnami/minio:latest
+        env:
+          MINIO_ROOT_USER: root
+          MINIO_ROOT_PASSWORD: tembatemba
+          MINIO_DEFAULT_BUCKETS: temba-attachments,temba-logs
+        ports:
+          - 9000:9000     
+        options: --health-cmd "mc ready local" --health-interval 10s --health-timeout 5s --health-retries 5
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

backends/rapidpro/backend.go

@@ -45,9 +45,6 @@ const sentSetName = "msgs_sent_%s"
 // our timeout for backend operations
 const backendTimeout = time.Second * 20
 
-// storage directory (only used with file system storage)
-var storageDir = "_storage"
-
 var uuidRegex = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
 
 func init() {
@@ -169,31 +166,25 @@ func (b *backend) Start() error {
 		queue.StartDethrottler(b.redisPool, b.stopChan, b.waitGroup, msgQueueName)
 	}
 
-	// create our storage (S3 or file system)
-	if b.config.AWSAccessKeyID != "" || b.config.AWSUseCredChain {
-		s3config := &storage.S3Options{
-			AWSAccessKeyID:     b.config.AWSAccessKeyID,
-			AWSSecretAccessKey: b.config.AWSSecretAccessKey,
-			Region:             b.config.AWSRegion,
-			Endpoint:           b.config.S3Endpoint,
-			ForcePathStyle:     b.config.S3ForcePathStyle,
-			MaxRetries:         3,
-		}
-		s3Client, err := storage.NewS3Client(s3config)
-		if err != nil {
-			return err
-		}
-		b.attachmentStorage = storage.NewS3(s3Client, b.config.S3AttachmentsBucket, b.config.AWSRegion, s3.BucketCannedACLPublicRead, 32)
-		b.logStorage = storage.NewS3(s3Client, b.config.S3LogsBucket, b.config.AWSRegion, s3.BucketCannedACLPrivate, 32)
-	} else {
-		b.attachmentStorage = storage.NewFS(storageDir+"/attachments", 0766)
-		b.logStorage = storage.NewFS(storageDir+"/logs", 0766)
+	s3config := &storage.S3Options{
+		AWSAccessKeyID:     b.config.AWSAccessKeyID,
+		AWSSecretAccessKey: b.config.AWSSecretAccessKey,
+		Region:             b.config.AWSRegion,
+		Endpoint:           b.config.S3Endpoint,
+		ForcePathStyle:     b.config.S3ForcePathStyle,
+		MaxRetries:         3,
+	}
+	s3Client, err := storage.NewS3Client(s3config)
+	if err != nil {
+		return err
 	}
+	b.attachmentStorage = storage.NewS3(s3Client, b.config.S3AttachmentsBucket, b.config.AWSRegion, s3.BucketCannedACLPublicRead, 32)
+	b.logStorage = storage.NewS3(s3Client, b.config.S3LogsBucket, b.config.AWSRegion, s3.BucketCannedACLPrivate, 32)
 
 	// create and start channel caches...
-	b.channelsByUUID = cache.NewLocal[courier.ChannelUUID, *Channel](b.loadChannelByUUID, time.Minute)
+	b.channelsByUUID = cache.NewLocal(b.loadChannelByUUID, time.Minute)
 	b.channelsByUUID.Start()
-	b.channelsByAddr = cache.NewLocal[courier.ChannelAddress, *Channel](b.loadChannelByAddress, time.Minute)
+	b.channelsByAddr = cache.NewLocal(b.loadChannelByAddress, time.Minute)
 	b.channelsByAddr.Start()
 
 	// check our storages

backends/rapidpro/backend_test.go

@@ -42,12 +42,17 @@ func testConfig() *courier.Config {
 	config.DB = "postgres://courier_test:temba@localhost:5432/courier_test?sslmode=disable"
 	config.Redis = "redis://localhost:6379/0"
 	config.MediaDomain = "nyaruka.s3.com"
+
+	// configure S3 to use a local minio instance
+	config.AWSAccessKeyID = "root"
+	config.AWSSecretAccessKey = "tembatemba"
+	config.S3Endpoint = "http://localhost:9000"
+	config.S3ForcePathStyle = true
+
 	return config
 }
 
 func (ts *BackendTestSuite) SetupSuite() {
-	storageDir = "_test_storage"
-
 	// turn off logging
 	log.SetOutput(io.Discard)
 
@@ -82,10 +87,6 @@ func (ts *BackendTestSuite) SetupSuite() {
 func (ts *BackendTestSuite) TearDownSuite() {
 	ts.b.Stop()
 	ts.b.Cleanup()
-
-	if err := os.RemoveAll(storageDir); err != nil {
-		panic(err)
-	}
 }
 
 func (ts *BackendTestSuite) clearRedis() {
@@ -1086,7 +1087,7 @@ func (ts *BackendTestSuite) TestSaveAttachment() {
 
 	newURL, err := ts.b.SaveAttachment(ctx, knChannel, "image/jpeg", testJPG, "jpg")
 	ts.NoError(err)
-	ts.Equal("_test_storage/attachments/attachments/1/c00e/5d67/c00e5d67-c275-4389-aded-7d8b151cbd5b.jpg", newURL)
+	ts.Equal("https://temba-attachments.s3.us-east-1.amazonaws.com/attachments/1/c00e/5d67/c00e5d67-c275-4389-aded-7d8b151cbd5b.jpg", newURL)
 }
 
 func (ts *BackendTestSuite) TestWriteMsg() {
@@ -1215,7 +1216,7 @@ func (ts *BackendTestSuite) TestWriteMsgWithAttachments() {
 	// should have actually fetched and saved it to storage, with the correct content type
 	err = ts.b.WriteMsg(ctx, msg, clog)
 	ts.NoError(err)
-	ts.Equal([]string{"image/jpeg:_test_storage/attachments/attachments/1/9b95/5e36/9b955e36-ac16-4c6b-8ab6-9b9af5cd042a.jpg"}, msg.Attachments())
+	ts.Equal([]string{"image/jpeg:https://temba-attachments.s3.us-east-1.amazonaws.com/attachments/1/9b95/5e36/9b955e36-ac16-4c6b-8ab6-9b9af5cd042a.jpg"}, msg.Attachments())
 
 	// try an invalid embedded attachment
 	msg = ts.b.NewIncomingMsg(knChannel, urn, "invalid embedded attachment data", "", clog).(*Msg)

config.go

@@ -27,7 +27,6 @@ type Config struct {
 
 	AWSAccessKeyID     string `help:"access key ID to use for AWS services"`
 	AWSSecretAccessKey string `help:"secret access key to use for AWS services"`
-	AWSUseCredChain    bool   `help:"using the default AWS credential provider chain instead of values above"`
 	AWSRegion          string `help:"region to use for AWS services, e.g. us-east-1"`
 
 	S3Endpoint          string `help:"S3 service endpoint, e.g. https://s3.amazonaws.com"`
@@ -70,12 +69,11 @@ func NewDefaultConfig() *Config {
 
 		AWSAccessKeyID:     "",
 		AWSSecretAccessKey: "",
-		AWSUseCredChain:    false,
 		AWSRegion:          "us-east-1",
 
 		S3Endpoint:          "https://s3.amazonaws.com",
-		S3AttachmentsBucket: "courier-media",
-		S3LogsBucket:        "courier-logs",
+		S3AttachmentsBucket: "temba-attachments",
+		S3LogsBucket:        "temba-logs",
 		S3ForcePathStyle:    false,
 
 		FacebookApplicationSecret:    "missing_facebook_app_secret",