Feature/delay render and error fallback

app.go

@@ -15,7 +15,6 @@ import (
 	"github.com/aws/aws-lambda-go/events"
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
 	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
@@ -24,39 +23,52 @@ import (
 	"github.com/kayac/go-katsubushi"
 	"github.com/mackerelio/mackerel-client-go"
 	"github.com/mashiike/grat"
-	"github.com/mashiike/ls3viewer"
 	"github.com/mashiike/prepalert/hclconfig"
 	"github.com/mashiike/queryrunner"
 	"github.com/mashiike/slogutils"
 )
 
 type App struct {
-	client    *mackerel.Client
-	auth      *hclconfig.AuthBlock
-	backend   *hclconfig.S3BackendBlock
-	rules     []*Rule
-	service   string
-	queueUrl  string
-	sqsClient *sqs.Client
-	uploader  *manager.Uploader
-	viewer    http.Handler
-	evalCtx   *hcl.EvalContext
+	mkrSvc              *MackerelService
+	backend             Backend
+	webhookClientID     string
+	webhookClientSecret string
+	rules               []*Rule
+	queueUrl            string
+	sqsClient           *sqs.Client
+	evalCtx             *hcl.EvalContext
 }
 
 func New(apikey string, cfg *hclconfig.Config) (*App, error) {
 	client := mackerel.NewClient(apikey)
+	svc := NewMackerelService(client)
+	awsCfg, err := config.LoadDefaultConfig(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("load aws default config:%w", err)
+	}
+
+	var backend Backend
+	switch {
+	case !cfg.Prepalert.S3Backend.IsEmpty():
+		s3Client := s3.NewFromConfig(awsCfg)
+		backend, err = NewS3Backend(s3Client, cfg.Prepalert.S3Backend, cfg.Prepalert.Auth)
+		if err != nil {
+			return nil, fmt.Errorf("initialize s3 backend:%w", err)
+		}
+	default:
+		backend = NewDiscardBackend()
+	}
+	slog.Info("setup backend", "backend", backend.String())
+
 	rules := make([]*Rule, 0, len(cfg.Rules))
 	for i, ruleBlock := range cfg.Rules {
-		rule, err := NewRule(client, ruleBlock)
+		rule, err := NewRule(svc, backend, ruleBlock, cfg.Prepalert.Service)
 		if err != nil {
 			return nil, fmt.Errorf("rules[%d]:%w", i, err)
 		}
 		rules = append(rules, rule)
 	}
-	awsCfg, err := config.LoadDefaultConfig(context.Background())
-	if err != nil {
-		return nil, fmt.Errorf("load aws default config:%w", err)
-	}
+
 	sqsClient := sqs.NewFromConfig(awsCfg)
 	slog.Info("try get sqs queue url", "sqs_queue_name", cfg.Prepalert.SQSQueueName)
 	output, err := sqsClient.GetQueueUrl(context.Background(), &sqs.GetQueueUrlInput{
@@ -66,39 +78,16 @@ func New(apikey string, cfg *hclconfig.Config) (*App, error) {
 		return nil, fmt.Errorf("can not get sqs queu url:%w", err)
 	}
 	app := &App{
-		client:    client,
-		auth:      cfg.Prepalert.Auth,
+		mkrSvc:    svc,
+		backend:   backend,
 		rules:     rules,
-		service:   cfg.Prepalert.Service,
 		sqsClient: sqsClient,
 		queueUrl:  *output.QueueUrl,
 		evalCtx:   cfg.EvalContext,
 	}
-	if backend := cfg.Prepalert.S3Backend; !backend.IsEmpty() {
-		slog.Info("enable s3 backend", "s3_backet_name", backend.BucketName)
-		app.backend = backend
-		s3Client := s3.NewFromConfig(awsCfg)
-		app.uploader = manager.NewUploader(s3Client)
-		viewerOptFns := []func(*ls3viewer.Options){
-			ls3viewer.WithBaseURL(backend.ViewerBaseURL.String()),
-		}
-		if app.EnableBasicAuth() && !backend.EnableGoogleAuth() {
-			viewerOptFns = append(viewerOptFns, ls3viewer.WithBasicAuth(app.auth.ClientID, app.auth.ClientSecret))
-		}
-		if backend.EnableGoogleAuth() {
-			viewerOptFns = append(viewerOptFns, ls3viewer.WithGoogleOIDC(
-				*backend.ViewerGoogleClientID,
-				*backend.ViewerGoogleClientSecret,
-				backend.ViewerSessionEncryptKey,
-				backend.Allowed,
-				backend.Denied,
-			))
-		}
-		h, err := ls3viewer.New(backend.BucketName, *backend.ObjectKeyPrefix, viewerOptFns...)
-		if err != nil {
-			return nil, fmt.Errorf("initialize ls3viewer:%w", err)
-		}
-		app.viewer = h
+	if !cfg.Prepalert.Auth.IsEmpty() {
+		app.webhookClientID = cfg.Prepalert.Auth.ClientID
+		app.webhookClientSecret = cfg.Prepalert.Auth.ClientSecret
 	}
 	return app, nil
 }
@@ -117,9 +106,6 @@ func (app *App) Run(ctx context.Context, opts *RunOptions) error {
 			slog.WarnContext(ctx, "mode webhook is deprecated. change to http")
 		}
 		slog.InfoContext(ctx, "run as http", "address", opts.Address, "prefix", opts.Prefix)
-		if app.EnableBasicAuth() {
-			slog.InfoContext(ctx, "with basec auth", "client_id", app.auth.ClientID)
-		}
 		ridge.RunWithContext(ctx, opts.Address, opts.Prefix, app)
 	case "worker":
 		slog.InfoContext(ctx, "run as worker", "batch_size", opts.BatchSize)
@@ -129,7 +115,7 @@ func (app *App) Run(ctx context.Context, opts *RunOptions) error {
 }
 
 func (app *App) Exec(ctx context.Context, alertID string) error {
-	body, err := app.NewWebhookBody(ctx, alertID)
+	body, err := app.mkrSvc.NewEmulatedWebhookBody(ctx, alertID)
 	if err != nil {
 		return err
 	}
@@ -172,12 +158,7 @@ func (app *App) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	r = r.WithContext(ctx)
 	slog.InfoContext(ctx, "accept HTTP request", "method", r.Method, "path", r.URL.Path)
 	if r.Method == http.MethodGet {
-		if !app.EnableBackend() {
-			slog.InfoContext(ctx, "backend is not enabled", "status", http.StatusMethodNotAllowed)
-			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
-			return
-		}
-		app.viewer.ServeHTTP(w, r)
+		app.backend.ServeHTTP(w, r)
 		return
 	}
 	if app.EnableBasicAuth() && !app.CheckBasicAuth(r) {
@@ -284,10 +265,8 @@ func (app *App) ProcessRules(ctx context.Context, body *WebhookBody) error {
 			continue
 		}
 		slog.InfoContext(ctx, "match rule", "rule", rule.Name())
-		ctxWithRuleName := slogutils.With(ctx, "rule_name", rule.Name())
-		slog.InfoContext(ctxWithRuleName, "match rule")
 		matchCount++
-		if err := app.ProcessRule(ctxWithRuleName, rule, body); err != nil {
+		if err := rule.Render(ctx, app.evalCtx.NewChild(), body); err != nil {
 			return fmt.Errorf("failed process Mackerel webhook body:%s: %w", rule.Name(), err)
 		}
 	}
@@ -296,19 +275,15 @@ func (app *App) ProcessRules(ctx context.Context, body *WebhookBody) error {
 }
 
 func (app *App) EnableBasicAuth() bool {
-	return !app.auth.IsEmpty()
-}
-
-func (app *App) EnableBackend() bool {
-	return !app.backend.IsEmpty()
+	return app.webhookClientID != "" && app.webhookClientSecret != ""
 }
 
 func (app *App) CheckBasicAuth(r *http.Request) bool {
 	clientID, clientSecret, ok := r.BasicAuth()
 	if !ok {
 		return false
 	}
-	return clientID == app.auth.ClientID && clientSecret == app.auth.ClientSecret
+	return clientID == app.webhookClientID && clientSecret == app.webhookClientSecret
 }
 
 func (app *App) WithQueryRunningContext(ctx context.Context, message *events.SQSMessage) context.Context {

backend.go

@@ -0,0 +1,121 @@
+package prepalert
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"path/filepath"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/mashiike/ls3viewer"
+	"github.com/mashiike/prepalert/hclconfig"
+	"github.com/zclconf/go-cty/cty"
+)
+
+//go:generate mockgen -source=$GOFILE -destination=./mock/mock_$GOFILE -package=mock
+
+type Backend interface {
+	http.Handler
+	fmt.Stringer
+	Upload(ctx context.Context, evalCtx *hcl.EvalContext, name string, body io.Reader) (string, bool, error)
+}
+
+type S3Backend struct {
+	cfg      *hclconfig.S3BackendBlock
+	uploader *manager.Uploader
+	h        http.Handler
+}
+
+func NewS3Backend(client manager.UploadAPIClient, cfg *hclconfig.S3BackendBlock, authCfg *hclconfig.AuthBlock) (*S3Backend, error) {
+	b := &S3Backend{
+		cfg:      cfg,
+		uploader: manager.NewUploader(client),
+	}
+	viewerOptFns := []func(*ls3viewer.Options){
+		ls3viewer.WithBaseURL(cfg.ViewerBaseURL.String()),
+	}
+	if !authCfg.IsEmpty() && !cfg.EnableGoogleAuth() {
+		viewerOptFns = append(viewerOptFns, ls3viewer.WithBasicAuth(authCfg.ClientID, authCfg.ClientSecret))
+	}
+	if cfg.EnableGoogleAuth() {
+		viewerOptFns = append(viewerOptFns, ls3viewer.WithGoogleOIDC(
+			*cfg.ViewerGoogleClientID,
+			*cfg.ViewerGoogleClientSecret,
+			cfg.ViewerSessionEncryptKey,
+			cfg.Allowed,
+			cfg.Denied,
+		))
+	}
+	h, err := ls3viewer.New(cfg.BucketName, *cfg.ObjectKeyPrefix, viewerOptFns...)
+	if err != nil {
+		return nil, fmt.Errorf("initialize ls3viewer:%w", err)
+	}
+	b.h = h
+	return b, nil
+}
+
+func (b *S3Backend) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	b.h.ServeHTTP(w, r)
+}
+
+func (b *S3Backend) String() string {
+	return fmt.Sprintf("s3_backend{location=s3://%s/%s}", b.cfg.BucketName, *b.cfg.ObjectKeyPrefix)
+}
+
+func (b *S3Backend) Upload(ctx context.Context, evalCtx *hcl.EvalContext, name string, body io.Reader) (string, bool, error) {
+	expr := *b.cfg.ObjectKeyTemplate
+	objectKeyTemplateValue, diags := expr.Value(evalCtx)
+	if diags.HasErrors() {
+		return "", false, fmt.Errorf("eval object key template: %w", diags)
+	}
+	if objectKeyTemplateValue.Type() != cty.String {
+		return "", false, errors.New("object key template is not string")
+	}
+	if !objectKeyTemplateValue.IsKnown() {
+		return "", false, errors.New("object key template is unknown")
+	}
+	objectKey := filepath.Join(*b.cfg.ObjectKeyPrefix, objectKeyTemplateValue.AsString(), fmt.Sprintf("%s.txt", name))
+	u := b.cfg.ViewerBaseURL.JoinPath(objectKeyTemplateValue.AsString(), fmt.Sprintf("%s.txt", name))
+	showDetailsURL := u.String()
+	slog.DebugContext(
+		ctx,
+		"try upload to backend",
+		"s3_url", fmt.Sprintf("s3://%s/%s", b.cfg.BucketName, objectKey),
+		"show_details_url", showDetailsURL,
+	)
+	output, err := b.uploader.Upload(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(b.cfg.BucketName),
+		Key:    aws.String(objectKey),
+		Body:   body,
+	})
+	if err != nil {
+		return "", false, fmt.Errorf("upload to backend failed: %w", err)
+	}
+	slog.InfoContext(ctx, "complete upload to backend", "s3_url", output.Location)
+	return showDetailsURL, true, nil
+}
+
+type DiscardBackend struct{}
+
+func NewDiscardBackend() *DiscardBackend {
+	return &DiscardBackend{}
+}
+
+func (b *DiscardBackend) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	slog.InfoContext(r.Context(), "backend is not enabled", "status", http.StatusMethodNotAllowed)
+	http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+}
+
+func (b *DiscardBackend) String() string {
+	return "discard_backend"
+}
+
+func (b *DiscardBackend) Upload(ctx context.Context, evalCtx *hcl.EvalContext, name string, body io.Reader) (string, bool, error) {
+	return "", false, nil
+}

go.mod

@@ -16,26 +16,30 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/handlename/ssmwrap v1.2.1
 	github.com/hashicorp/go-version v1.6.0
-	github.com/hashicorp/hcl/v2 v2.17.0
+	github.com/hashicorp/hcl/v2 v2.18.0
 	github.com/kayac/go-katsubushi v1.7.0
 	github.com/mackerelio/mackerel-client-go v0.26.0
 	github.com/manifoldco/promptui v0.9.0
 	github.com/mashiike/grat v0.0.0-20220831070259-6c4c03aba4d5
 	github.com/mashiike/hclconfig v0.8.0
+	github.com/mashiike/hclutil v0.1.0
 	github.com/mashiike/ls3viewer v0.2.0
 	github.com/mashiike/queryrunner v0.3.1
 	github.com/mashiike/slogutils v0.4.0
 	github.com/samber/lo v1.38.1
 	github.com/sebdah/goldie/v2 v2.5.3
 	github.com/stretchr/testify v1.8.1
-	github.com/zclconf/go-cty v1.13.2
+	github.com/zclconf/go-cty v1.14.0
+	go.uber.org/mock v0.2.0
+	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
 	golang.org/x/sync v0.2.0
 )
 
 require (
 	github.com/Songmu/retry v0.1.0 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/aws/aws-sdk-go v1.44.118 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.24 // indirect
@@ -81,12 +85,11 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/term v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect