Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| package azureexporter | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "log/slog" | ||
| "os" | ||
|
|
||
| "github.com/Azure/azure-sdk-for-go/sdk/azidentity" | ||
| "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" | ||
| "github.com/thomaspoignant/go-feature-flag/exporter" | ||
| "github.com/thomaspoignant/go-feature-flag/exporter/fileexporter" | ||
| "github.com/thomaspoignant/go-feature-flag/utils/fflog" | ||
| ) | ||
|
|
||
| type Exporter struct { | ||
| // Container is the name of your Azure Blob Storage Container similar to Buckets in S3. | ||
| Container string | ||
|
|
||
| // Storage Account Name and Key | ||
| AccountName string | ||
| AccountKey string | ||
|
|
||
| // Format is the output format you want in your exported file. | ||
| // Available format are JSON, CSV, and Parquet. | ||
| // Default: JSON | ||
| Format string | ||
|
|
||
| // Path allows you to specify in which directory you want to export your data. | ||
| Path string | ||
|
|
||
| // Filename is the name of your output file | ||
| // You can use a templated config to define the name of your export files. | ||
| // Available replacement are {{ .Hostname}}, {{ .Timestamp}} and {{ .Format}} | ||
| // Default: "flag-variation-{{ .Hostname}}-{{ .Timestamp}}.{{ .Format}}" | ||
| Filename string | ||
|
|
||
| // CsvTemplate is used if your output format is CSV. | ||
| // This field will be ignored if you are using another format than CSV. | ||
| // You can decide which fields you want in your CSV line with a go-template syntax, | ||
| // please check exporter/feature_event.go to see what are the fields available. | ||
| // Default: | ||
| // {{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n | ||
| CsvTemplate string | ||
|
|
||
| // ParquetCompressionCodec is the parquet compression codec for better space efficiency. | ||
| // Available options https://github.com/apache/parquet-format/blob/master/Compression.md | ||
| // Default: SNAPPY | ||
| ParquetCompressionCodec string | ||
| } | ||
|
|
||
| func (f *Exporter) initializeAzureClient() (*azblob.Client, error) { | ||
| url := fmt.Sprintf("https://%s.blob.core.windows.net/", f.AccountName) | ||
|
|
||
| if f.AccountKey == "" { | ||
| cred, err := azidentity.NewDefaultAzureCredential(nil) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return azblob.NewClient(url, cred, nil) | ||
| } | ||
| cred, err := azblob.NewSharedKeyCredential(f.AccountName, f.AccountKey) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return azblob.NewClientWithSharedKeyCredential(url, cred, nil) | ||
| } | ||
|
|
||
| func (f *Exporter) Export(ctx context.Context, logger *fflog.FFLogger, featureEvents []exporter.FeatureEvent) error { | ||
| client, err := f.initializeAzureClient() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if f.Container == "" { | ||
| return fmt.Errorf("you should specify a container. %v is invalid", f.Container) | ||
| } | ||
|
|
||
| outputDir, err := os.MkdirTemp("", "go_feature_flag_AzureBlobStorage_export") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer func() { _ = os.Remove(outputDir) }() | ||
|
|
||
| fileExporter := fileexporter.Exporter{ | ||
| Format: f.Format, | ||
| OutputDir: outputDir, | ||
| Filename: f.Filename, | ||
| CsvTemplate: f.CsvTemplate, | ||
| ParquetCompressionCodec: f.ParquetCompressionCodec, | ||
| } | ||
| err = fileExporter.Export(ctx, logger, featureEvents) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| files, err := os.ReadDir(outputDir) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, file := range files { | ||
| fileName := file.Name() | ||
| of, err := os.Open(outputDir + fileName) | ||
| if err != nil { | ||
| logger.Error("[Azure Exporter] impossible to open file", slog.String("path", outputDir+"/"+fileName)) | ||
| continue | ||
| } | ||
| defer func() { _ = of.Close() }() | ||
|
|
||
| // prepend the path | ||
| source := fileName | ||
| if f.Path != "" { | ||
| source = f.Path + "/" + fileName | ||
| } | ||
|
|
||
| _, err = client.UploadFile(context.Background(), f.Container, source, of, nil) | ||
| if err != nil { | ||
| logger.Error("[Azure Exporter] failed to upload file", slog.String("path", outputDir+"/"+fileName)) | ||
| return err | ||
| } | ||
|
|
||
| logger.Info("[Azure Exporter] file uploaded.", slog.String("location", f.Container+"/"+fileName)) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (f *Exporter) IsBulk() bool { | ||
| return true | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package azureexporter_test | ||
|
|
||
| import ( | ||
| "context" | ||
| "log/slog" | ||
| "os" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| "github.com/thomaspoignant/go-feature-flag/exporter" | ||
| "github.com/thomaspoignant/go-feature-flag/exporter/azureexporter" | ||
| "github.com/thomaspoignant/go-feature-flag/utils/fflog" | ||
| ) | ||
|
|
||
| func TestAzureBlobStorage_Export(t *testing.T) { | ||
| hostname, _ := os.Hostname() | ||
| type fields struct { | ||
| Container string | ||
| AccountName string | ||
| AccountKey string | ||
| Format string | ||
| Path string | ||
| Filename string | ||
| CsvTemplate string | ||
| } | ||
|
|
||
| tests := []struct { | ||
| name string | ||
| fields fields | ||
| events []exporter.FeatureEvent | ||
| wantErr bool | ||
| expectedName string | ||
| }{ | ||
| { | ||
| name: "All default test", | ||
| fields: fields{ | ||
| Container: "test", | ||
| }, | ||
| events: []exporter.FeatureEvent{ | ||
| { | ||
| Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", | ||
| Variation: "Default", Value: "YO", Default: false, | ||
| }, | ||
| }, | ||
| expectedName: "^flag-variation-" + hostname + "-[0-9]*\\.json$", | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| f := azureexporter.Exporter{ | ||
| Container: tt.fields.Container, | ||
| AccountName: tt.fields.AccountName, | ||
| AccountKey: tt.fields.AccountKey, | ||
| Format: tt.fields.Format, | ||
| Path: tt.fields.Path, | ||
| Filename: tt.fields.Filename, | ||
| CsvTemplate: tt.fields.CsvTemplate, | ||
| } | ||
|
|
||
| err := f.Export(context.Background(), &fflog.FFLogger{LeveledLogger: slog.Default()}, tt.events) | ||
| if tt.wantErr { | ||
| assert.Error(t, err, "Export should error") | ||
| return | ||
| } | ||
| assert.NoError(t, err, "Export should not error") | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestAzureBlobStorage_IsBulk(t *testing.T) { | ||
| exporter := azureexporter.Exporter{} | ||
| assert.True(t, exporter.IsBulk(), "exporter is a bulk exporter") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.