From a34eb416c0053753649303ac176bbb24e3fd6c8d Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 12:52:11 +0900 Subject: [PATCH 01/19] Create GetInsightData for deploy frequency --- BUILD.bazel | 1 + pkg/app/api/grpcapi/web_api.go | 94 +++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index 27dfd1215a..483823b2c4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -51,6 +51,7 @@ genrule( # gazelle:exclude pkg/model/deployment.pb.validate.go # gazelle:exclude pkg/model/environment.pb.validate.go # gazelle:exclude pkg/model/event.pb.validate.go +# gazelle:exclude pkg/model/insight.pb.validate.go # gazelle:exclude pkg/model/logblock.pb.validate.go # gazelle:exclude pkg/model/piped.pb.validate.go # gazelle:exclude pkg/model/piped_stats.pb.validate.go diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index 80857efdeb..8be84b0e74 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1311,6 +1311,98 @@ func (a *WebAPI) ListAPIKeys(ctx context.Context, req *webservice.ListAPIKeysReq } // GetInsightData returns the accumulated insight data. -func (a *WebAPI) GetInsightData(_ context.Context, _ *webservice.GetInsightDataRequest) (*webservice.GetInsightDataResponse, error) { +func (a *WebAPI) GetInsightData(ctx context.Context, req *webservice.GetInsightDataRequest) (*webservice.GetInsightDataResponse, error) { + claims, err := rpcauth.ExtractClaims(ctx) + if err != nil { + a.logger.Error("failed to authenticate the current user", zap.Error(err)) + return nil, err + } + + switch req.MetricsKind { + case model.InsightMetricsKind_DEPLOYMENT_FREQUENCY: + return a.getInsightDataForDeployFrequency(ctx, claims.Role.ProjectId, req) + } return nil, status.Error(codes.Unimplemented, "") } + +// getInsightDataForDeployFrequency returns the accumulated insight data for deploy frequency. +func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectId string, req *webservice.GetInsightDataRequest) (*webservice.GetInsightDataResponse, error) { + counts := make([]*model.InsightDataPoint, req.DataPointCount) + + var movePoint func(time.Time, int) time.Time + var accumulateFrom time.Time + switch req.Step { + case model.InsightStep_DAILY: + movePoint = func(from time.Time, i int) time.Time { + return from.AddDate(0, 0, i) + } + accumulateFrom = time.Unix(req.RangeFrom, 0) + case model.InsightStep_WEEKLY: + movePoint = func(from time.Time, i int) time.Time { + return from.AddDate(0, 0, 7*i) + } + rangeFrom := time.Unix(req.RangeFrom, 0) + // Sunday in the week of rangeFrom + accumulateFrom = rangeFrom.AddDate(0, 0, -int(rangeFrom.Weekday())) + case model.InsightStep_MONTHLY: + movePoint = func(from time.Time, i int) time.Time { + return from.AddDate(0, i, 0) + } + rangeFrom := time.Unix(req.RangeFrom, 0) + accumulateFrom = time.Date(rangeFrom.Year(), rangeFrom.Month(), 1, 0, 0, 0, 0, time.Local) + case model.InsightStep_YEARLY: + movePoint = func(from time.Time, i int) time.Time { + return from.AddDate(i, 0, 0) + } + rangeFrom := time.Unix(req.RangeFrom, 0) + accumulateFrom = time.Date(rangeFrom.Year(), 1, 1, 0, 0, 0, 0, time.Local) + } + + for i := 0; i < int(req.DataPointCount); i++ { + target := movePoint(accumulateFrom, i) + + filters := []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: projectId, + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: movePoint(target, 1).Unix(), // target's finish time on unix time + }, + } + + if req.ApplicationId != "" { + filters = append(filters, datastore.ListFilter{ + Field: "ApplicationId", + Operator: "==", + Value: req.ApplicationId, + }) + } + + deployments, err := a.deploymentStore.ListDeployments(ctx, datastore.ListOptions{ + Filters: filters, + }) + if err != nil { + a.logger.Error("failed to get deployments", zap.Error(err)) + return nil, status.Error(codes.Internal, "Failed to get deployments") + } + + counts = append(counts, &model.InsightDataPoint{ + Timestamp: target.Unix(), + Value: float32(len(deployments)), + }) + } + + return &webservice.GetInsightDataResponse{ + UpdatedAt: time.Now().Unix(), + DataPoints: counts, + }, nil +} From 79393b0f08de60aca335374a82f91e1a0fb54447 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 12:52:29 +0900 Subject: [PATCH 02/19] Add test for getInsightDataForDeployFrequency --- pkg/app/api/grpcapi/web_api_test.go | 395 ++++++++++++++++++++++++++++ 1 file changed, 395 insertions(+) diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 58baa6ad30..72ca03f7ee 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -19,6 +19,7 @@ import ( "errors" "reflect" "testing" + "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -387,3 +388,397 @@ func TestValidatePipedBelongsToProject(t *testing.T) { }) } } + +func TestGetInsightDataForDeployFrequency(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tests := []struct { + name string + pipedID string + projectID string + pipedProjectCache cache.Cache + deploymentStore datastore.DeploymentStore + req *webservice.GetInsightDataRequest + wantErr bool + }{ + { + name: "valid with InsightStep_DAILY", + pipedID: "pipedID", + projectID: "projectID", + deploymentStore: func() datastore.DeploymentStore { + target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local) + targetNextDate := target.AddDate(0, 0, 1) + s := datastoretest.NewMockDeploymentStore(ctrl) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextDate.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + target = time.Date(2020, 1, 2, 0, 0, 0, 0, time.Local) + targetNextDate = target.AddDate(0, 0, 1) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextDate.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + return s + }(), + req: &webservice.GetInsightDataRequest{ + MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, + Step: model.InsightStep_DAILY, + RangeFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + DataPointCount: 2, + ApplicationId: "ApplicationId", + }, + wantErr: false, + }, + { + name: "valid with InsightStep_WEEKLY and rangeFrom is Sunday", + pipedID: "pipedID", + projectID: "projectID", + deploymentStore: func() datastore.DeploymentStore { + target := time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local) + targetNextWeek := target.AddDate(0, 0, 7) + s := datastoretest.NewMockDeploymentStore(ctrl) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextWeek.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local) + targetNextWeek = target.AddDate(0, 0, 7) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextWeek.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + return s + }(), + req: &webservice.GetInsightDataRequest{ + MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, + Step: model.InsightStep_WEEKLY, + RangeFrom: time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local).Unix(), // 2020/01/05 is Sunday + DataPointCount: 2, + ApplicationId: "ApplicationId", + }, + wantErr: false, + }, + { + name: "valid with InsightStep_WEEKLY and rangeFrom is not Sunday", + pipedID: "pipedID", + projectID: "projectID", + deploymentStore: func() datastore.DeploymentStore { + target := time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local) + targetNextWeek := target.AddDate(0, 0, 7) + s := datastoretest.NewMockDeploymentStore(ctrl) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextWeek.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local) + targetNextWeek = target.AddDate(0, 0, 7) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextWeek.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + return s + }(), + req: &webservice.GetInsightDataRequest{ + MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, + Step: model.InsightStep_WEEKLY, + RangeFrom: time.Date(2020, 1, 6, 0, 0, 0, 0, time.Local).Unix(), // 2020/01/06 is Monday + DataPointCount: 2, + ApplicationId: "ApplicationId", + }, + wantErr: false, + }, + { + name: "valid with InsightStep_MONTHLY", + pipedID: "pipedID", + projectID: "projectID", + deploymentStore: func() datastore.DeploymentStore { + target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local) + targetNextMonth := target.AddDate(0, 1, 0) + s := datastoretest.NewMockDeploymentStore(ctrl) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextMonth.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + target = time.Date(2020, 2, 1, 0, 0, 0, 0, time.Local) + targetNextMonth = target.AddDate(0, 1, 0) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextMonth.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + return s + }(), + req: &webservice.GetInsightDataRequest{ + MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, + Step: model.InsightStep_MONTHLY, + RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.Local).Unix(), + DataPointCount: 2, + ApplicationId: "ApplicationId", + }, + wantErr: false, + }, + { + name: "valid with InsightStep_YEARLY", + pipedID: "pipedID", + projectID: "projectID", + deploymentStore: func() datastore.DeploymentStore { + target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local) + targetNextYear := target.AddDate(1, 0, 0) + s := datastoretest.NewMockDeploymentStore(ctrl) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextYear.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + target = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) + targetNextYear = target.AddDate(1, 0, 0) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextYear.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }) + + return s + }(), + req: &webservice.GetInsightDataRequest{ + MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, + Step: model.InsightStep_YEARLY, + RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.Local).Unix(), + DataPointCount: 2, + ApplicationId: "ApplicationId", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + api := &WebAPI{ + pipedProjectCache: tt.pipedProjectCache, + deploymentStore: tt.deploymentStore, + } + _, err := api.getInsightDataForDeployFrequency(ctx, tt.projectID, tt.req) + assert.Equal(t, tt.wantErr, err != nil) + }) + } +} From 6d0a32adec200ff58934bb9ba437f893e04f2bcc Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 12:56:11 +0900 Subject: [PATCH 03/19] Rename projectId to projectID --- pkg/app/api/grpcapi/web_api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index 8be84b0e74..a06c885aa1 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1326,7 +1326,7 @@ func (a *WebAPI) GetInsightData(ctx context.Context, req *webservice.GetInsightD } // getInsightDataForDeployFrequency returns the accumulated insight data for deploy frequency. -func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectId string, req *webservice.GetInsightDataRequest) (*webservice.GetInsightDataResponse, error) { +func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID string, req *webservice.GetInsightDataRequest) (*webservice.GetInsightDataResponse, error) { counts := make([]*model.InsightDataPoint, req.DataPointCount) var movePoint func(time.Time, int) time.Time @@ -1365,7 +1365,7 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectId { Field: "ProjectId", Operator: "==", - Value: projectId, + Value: projectID, }, { Field: "CreatedAt", From 4344548f8cb058183671b181192e2f6d3fe11026 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 13:11:05 +0900 Subject: [PATCH 04/19] Add assert in test for response --- pkg/app/api/grpcapi/web_api.go | 4 +- pkg/app/api/grpcapi/web_api_test.go | 174 ++++++++++++++++++++++++++-- 2 files changed, 165 insertions(+), 13 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index a06c885aa1..da90d070d5 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1395,10 +1395,10 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID return nil, status.Error(codes.Internal, "Failed to get deployments") } - counts = append(counts, &model.InsightDataPoint{ + counts[i] = &model.InsightDataPoint{ Timestamp: target.Unix(), Value: float32(len(deployments)), - }) + } } return &webservice.GetInsightDataResponse{ diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 72ca03f7ee..061bfc9211 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -403,6 +403,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { pipedProjectCache cache.Cache deploymentStore datastore.DeploymentStore req *webservice.GetInsightDataRequest + res *webservice.GetInsightDataResponse wantErr bool }{ { @@ -437,7 +438,14 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + }, nil) target = time.Date(2020, 1, 2, 0, 0, 0, 0, time.Local) targetNextDate = target.AddDate(0, 0, 1) @@ -465,7 +473,17 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + { + Id: "id3", + }, + }, nil) return s }(), @@ -476,6 +494,19 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPointCount: 2, ApplicationId: "ApplicationId", }, + res: &webservice.GetInsightDataResponse{ + UpdatedAt: time.Now().Unix(), + DataPoints: []*model.InsightDataPoint{ + { + Value: 2, + Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + }, + { + Value: 3, + Timestamp: time.Date(2020, 1, 2, 0, 0, 0, 0, time.Local).Unix(), + }, + }, + }, wantErr: false, }, { @@ -510,7 +541,14 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + }, nil) target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local) targetNextWeek = target.AddDate(0, 0, 7) @@ -538,7 +576,17 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + { + Id: "id3", + }, + }, nil) return s }(), @@ -549,6 +597,19 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPointCount: 2, ApplicationId: "ApplicationId", }, + res: &webservice.GetInsightDataResponse{ + UpdatedAt: time.Now().Unix(), + DataPoints: []*model.InsightDataPoint{ + { + Value: 2, + Timestamp: time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local).Unix(), + }, + { + Value: 3, + Timestamp: time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local).Unix(), + }, + }, + }, wantErr: false, }, { @@ -583,7 +644,14 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + }, nil) target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local) targetNextWeek = target.AddDate(0, 0, 7) @@ -611,7 +679,17 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + { + Id: "id3", + }, + }, nil) return s }(), @@ -622,6 +700,19 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPointCount: 2, ApplicationId: "ApplicationId", }, + res: &webservice.GetInsightDataResponse{ + UpdatedAt: time.Now().Unix(), + DataPoints: []*model.InsightDataPoint{ + { + Value: 2, + Timestamp: time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local).Unix(), + }, + { + Value: 3, + Timestamp: time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local).Unix(), + }, + }, + }, wantErr: false, }, { @@ -656,7 +747,14 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + }, nil) target = time.Date(2020, 2, 1, 0, 0, 0, 0, time.Local) targetNextMonth = target.AddDate(0, 1, 0) @@ -684,7 +782,17 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + { + Id: "id3", + }, + }, nil) return s }(), @@ -695,6 +803,19 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPointCount: 2, ApplicationId: "ApplicationId", }, + res: &webservice.GetInsightDataResponse{ + UpdatedAt: time.Now().Unix(), + DataPoints: []*model.InsightDataPoint{ + { + Value: 2, + Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + }, + { + Value: 3, + Timestamp: time.Date(2020, 2, 1, 0, 0, 0, 0, time.Local).Unix(), + }, + }, + }, wantErr: false, }, { @@ -729,7 +850,14 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + }, nil) target = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) targetNextYear = target.AddDate(1, 0, 0) @@ -757,7 +885,17 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { Value: "ApplicationId", }, }, - }) + }).Return([]*model.Deployment{ + { + Id: "id1", + }, + { + Id: "id2", + }, + { + Id: "id3", + }, + }, nil) return s }(), @@ -768,6 +906,19 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPointCount: 2, ApplicationId: "ApplicationId", }, + res: &webservice.GetInsightDataResponse{ + UpdatedAt: time.Now().Unix(), + DataPoints: []*model.InsightDataPoint{ + { + Value: 2, + Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + }, + { + Value: 3, + Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + }, + }, + }, wantErr: false, }, } @@ -777,8 +928,9 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { pipedProjectCache: tt.pipedProjectCache, deploymentStore: tt.deploymentStore, } - _, err := api.getInsightDataForDeployFrequency(ctx, tt.projectID, tt.req) + res, err := api.getInsightDataForDeployFrequency(ctx, tt.projectID, tt.req) assert.Equal(t, tt.wantErr, err != nil) + assert.Equal(t, tt.res.DataPoints, res.DataPoints) }) } } From 54d2c7abd3a09c12e7cef50db0044b2e668439f6 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 13:43:13 +0900 Subject: [PATCH 05/19] Add test to check error pattern --- go.mod | 4 +- go.sum | 9 +++++ pkg/app/api/grpcapi/BUILD.bazel | 1 + pkg/app/api/grpcapi/web_api_test.go | 59 ++++++++++++++++++++++++++++- 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 289114f389..78b92f7d85 100644 --- a/go.mod +++ b/go.mod @@ -27,8 +27,8 @@ require ( github.com/stretchr/testify v1.5.1 go.mongodb.org/mongo-driver v1.4.0 go.uber.org/atomic v1.7.0 - go.uber.org/multierr v1.2.0 // indirect - go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0 + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de golang.org/x/net v0.0.0-20200822124328-c89045814202 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d diff --git a/go.sum b/go.sum index f53eb46d6e..91d076b06f 100644 --- a/go.sum +++ b/go.sum @@ -417,12 +417,19 @@ go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.2.0 h1:6I+W7f5VwC5SV9dNrZ3qXrDB9mD0dyGOi/ZJmYw03T4= go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0 h1:fRtzhL15Tocngn2WZQ91iA2apveqArjCX5b9Lp9O+eA= go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -606,6 +613,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/app/api/grpcapi/BUILD.bazel b/pkg/app/api/grpcapi/BUILD.bazel index 1d3f0b6d6b..bc761b677d 100644 --- a/pkg/app/api/grpcapi/BUILD.bazel +++ b/pkg/app/api/grpcapi/BUILD.bazel @@ -29,6 +29,7 @@ go_library( "@org_golang_google_grpc//codes:go_default_library", "@org_golang_google_grpc//status:go_default_library", "@org_uber_go_zap//:go_default_library", + "@org_uber_go_zap//zaptest/observer:go_default_library", ], ) diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 061bfc9211..ac3ece79f4 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -17,10 +17,14 @@ package grpcapi import ( "context" "errors" + "fmt" "reflect" "testing" "time" + "go.uber.org/zap" + zapobserver "go.uber.org/zap/zaptest/observer" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -921,16 +925,69 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { }, wantErr: false, }, + { + name: "return error when something wrong happen", + pipedID: "pipedID", + projectID: "projectID", + deploymentStore: func() datastore.DeploymentStore { + target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local) + targetNextYear := target.AddDate(1, 0, 0) + s := datastoretest.NewMockDeploymentStore(ctrl) + s.EXPECT(). + ListDeployments(gomock.Any(), datastore.ListOptions{ + Filters: []datastore.ListFilter{ + { + Field: "ProjectId", + Operator: "==", + Value: "projectID", + }, + { + Field: "CreatedAt", + Operator: ">=", + Value: target.Unix(), + }, + { + Field: "CreatedAt", + Operator: "<", + Value: targetNextYear.Unix(), + }, + { + Field: "ApplicationId", + Operator: "==", + Value: "ApplicationId", + }, + }, + }).Return([]*model.Deployment{}, fmt.Errorf("something wrong happens in ListDeployments")) + return s + }(), + req: &webservice.GetInsightDataRequest{ + MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, + Step: model.InsightStep_YEARLY, + RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.Local).Unix(), + DataPointCount: 2, + ApplicationId: "ApplicationId", + }, + res: nil, + wantErr: true, + }, } + + core, _ := zapobserver.New(zap.InfoLevel) + logger := zap.New(core) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + api := &WebAPI{ pipedProjectCache: tt.pipedProjectCache, deploymentStore: tt.deploymentStore, + logger: logger, } res, err := api.getInsightDataForDeployFrequency(ctx, tt.projectID, tt.req) assert.Equal(t, tt.wantErr, err != nil) - assert.Equal(t, tt.res.DataPoints, res.DataPoints) + if err == nil { + assert.Equal(t, tt.res.DataPoints, res.DataPoints) + } }) } } From 6c1965f87d1a39ab45546c8622ff5a060a780b44 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 16:32:08 +0900 Subject: [PATCH 06/19] Remove empty line --- pkg/app/api/grpcapi/web_api_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index ac3ece79f4..0a385d7413 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -977,7 +977,6 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - api := &WebAPI{ pipedProjectCache: tt.pipedProjectCache, deploymentStore: tt.deploymentStore, From 8681e872b1e5b0c866b03594a9116947c474f067 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 16:38:27 +0900 Subject: [PATCH 07/19] Fix to catch invalid step --- pkg/app/api/grpcapi/web_api.go | 2 ++ pkg/app/api/grpcapi/web_api_test.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index da90d070d5..284e646c9c 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1356,6 +1356,8 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID } rangeFrom := time.Unix(req.RangeFrom, 0) accumulateFrom = time.Date(rangeFrom.Year(), 1, 1, 0, 0, 0, 0, time.Local) + default: + return nil, status.Error(codes.InvalidArgument, "Invalid step") } for i := 0; i < int(req.DataPointCount); i++ { diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 0a385d7413..ef2937261f 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -926,7 +926,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { wantErr: false, }, { - name: "return error when something wrong happen", + name: "return error when something wrong happen on ListDeployments", pipedID: "pipedID", projectID: "projectID", deploymentStore: func() datastore.DeploymentStore { From 74cb3bd2ed1df45a0190f5a8d6d856b54ba59390 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 19:09:47 +0900 Subject: [PATCH 08/19] Fix to use zap.NewNop() --- go.mod | 4 ++-- go.sum | 9 --------- pkg/app/api/grpcapi/web_api_test.go | 6 +----- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 78b92f7d85..289114f389 100644 --- a/go.mod +++ b/go.mod @@ -27,8 +27,8 @@ require ( github.com/stretchr/testify v1.5.1 go.mongodb.org/mongo-driver v1.4.0 go.uber.org/atomic v1.7.0 - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.16.0 + go.uber.org/multierr v1.2.0 // indirect + go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de golang.org/x/net v0.0.0-20200822124328-c89045814202 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d diff --git a/go.sum b/go.sum index 91d076b06f..f53eb46d6e 100644 --- a/go.sum +++ b/go.sum @@ -417,19 +417,12 @@ go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.2.0 h1:6I+W7f5VwC5SV9dNrZ3qXrDB9mD0dyGOi/ZJmYw03T4= go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0 h1:fRtzhL15Tocngn2WZQ91iA2apveqArjCX5b9Lp9O+eA= go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -613,8 +606,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index ef2937261f..6ebdf94cf4 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -23,7 +23,6 @@ import ( "time" "go.uber.org/zap" - zapobserver "go.uber.org/zap/zaptest/observer" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -972,15 +971,12 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { }, } - core, _ := zapobserver.New(zap.InfoLevel) - logger := zap.New(core) - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { api := &WebAPI{ pipedProjectCache: tt.pipedProjectCache, deploymentStore: tt.deploymentStore, - logger: logger, + logger: zap.NewNop(), } res, err := api.getInsightDataForDeployFrequency(ctx, tt.projectID, tt.req) assert.Equal(t, tt.wantErr, err != nil) From efd3f8432296f8082f2686ee60cb6145e3e221d5 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 19:10:54 +0900 Subject: [PATCH 09/19] Rename accumulateFrom to start --- pkg/app/api/grpcapi/web_api.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index 284e646c9c..e74b4d1a18 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1330,38 +1330,38 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID counts := make([]*model.InsightDataPoint, req.DataPointCount) var movePoint func(time.Time, int) time.Time - var accumulateFrom time.Time + var start time.Time switch req.Step { case model.InsightStep_DAILY: movePoint = func(from time.Time, i int) time.Time { return from.AddDate(0, 0, i) } - accumulateFrom = time.Unix(req.RangeFrom, 0) + start = time.Unix(req.RangeFrom, 0) case model.InsightStep_WEEKLY: movePoint = func(from time.Time, i int) time.Time { return from.AddDate(0, 0, 7*i) } rangeFrom := time.Unix(req.RangeFrom, 0) // Sunday in the week of rangeFrom - accumulateFrom = rangeFrom.AddDate(0, 0, -int(rangeFrom.Weekday())) + start = rangeFrom.AddDate(0, 0, -int(rangeFrom.Weekday())) case model.InsightStep_MONTHLY: movePoint = func(from time.Time, i int) time.Time { return from.AddDate(0, i, 0) } rangeFrom := time.Unix(req.RangeFrom, 0) - accumulateFrom = time.Date(rangeFrom.Year(), rangeFrom.Month(), 1, 0, 0, 0, 0, time.Local) + start = time.Date(rangeFrom.Year(), rangeFrom.Month(), 1, 0, 0, 0, 0, time.Local) case model.InsightStep_YEARLY: movePoint = func(from time.Time, i int) time.Time { return from.AddDate(i, 0, 0) } rangeFrom := time.Unix(req.RangeFrom, 0) - accumulateFrom = time.Date(rangeFrom.Year(), 1, 1, 0, 0, 0, 0, time.Local) + start = time.Date(rangeFrom.Year(), 1, 1, 0, 0, 0, 0, time.Local) default: return nil, status.Error(codes.InvalidArgument, "Invalid step") } for i := 0; i < int(req.DataPointCount); i++ { - target := movePoint(accumulateFrom, i) + target := movePoint(start, i) filters := []datastore.ListFilter{ { From 79061bd6e020e5095a6928a1bdf86abec8198f5f Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 19:12:04 +0900 Subject: [PATCH 10/19] Fix to specify timezone UTC --- pkg/app/api/grpcapi/web_api.go | 4 +-- pkg/app/api/grpcapi/web_api_test.go | 54 ++++++++++++++--------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index e74b4d1a18..50dd2334c6 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1349,13 +1349,13 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID return from.AddDate(0, i, 0) } rangeFrom := time.Unix(req.RangeFrom, 0) - start = time.Date(rangeFrom.Year(), rangeFrom.Month(), 1, 0, 0, 0, 0, time.Local) + start = time.Date(rangeFrom.Year(), rangeFrom.Month(), 1, 0, 0, 0, 0, time.UTC) case model.InsightStep_YEARLY: movePoint = func(from time.Time, i int) time.Time { return from.AddDate(i, 0, 0) } rangeFrom := time.Unix(req.RangeFrom, 0) - start = time.Date(rangeFrom.Year(), 1, 1, 0, 0, 0, 0, time.Local) + start = time.Date(rangeFrom.Year(), 1, 1, 0, 0, 0, 0, time.UTC) default: return nil, status.Error(codes.InvalidArgument, "Invalid step") } diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 6ebdf94cf4..0d409aa8d8 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -414,7 +414,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { pipedID: "pipedID", projectID: "projectID", deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local) + target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) targetNextDate := target.AddDate(0, 0, 1) s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). @@ -450,7 +450,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { }, }, nil) - target = time.Date(2020, 1, 2, 0, 0, 0, 0, time.Local) + target = time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC) targetNextDate = target.AddDate(0, 0, 1) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ @@ -493,7 +493,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { req: &webservice.GetInsightDataRequest{ MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, Step: model.InsightStep_DAILY, - RangeFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + RangeFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), DataPointCount: 2, ApplicationId: "ApplicationId", }, @@ -502,11 +502,11 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPoints: []*model.InsightDataPoint{ { Value: 2, - Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), }, { Value: 3, - Timestamp: time.Date(2020, 1, 2, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC).Unix(), }, }, }, @@ -517,7 +517,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { pipedID: "pipedID", projectID: "projectID", deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local) + target := time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC) targetNextWeek := target.AddDate(0, 0, 7) s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). @@ -553,7 +553,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { }, }, nil) - target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local) + target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.UTC) targetNextWeek = target.AddDate(0, 0, 7) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ @@ -596,7 +596,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { req: &webservice.GetInsightDataRequest{ MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, Step: model.InsightStep_WEEKLY, - RangeFrom: time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local).Unix(), // 2020/01/05 is Sunday + RangeFrom: time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC).Unix(), // 2020/01/05 is Sunday DataPointCount: 2, ApplicationId: "ApplicationId", }, @@ -605,11 +605,11 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPoints: []*model.InsightDataPoint{ { Value: 2, - Timestamp: time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC).Unix(), }, { Value: 3, - Timestamp: time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2020, 1, 12, 0, 0, 0, 0, time.UTC).Unix(), }, }, }, @@ -620,7 +620,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { pipedID: "pipedID", projectID: "projectID", deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local) + target := time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC) targetNextWeek := target.AddDate(0, 0, 7) s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). @@ -656,7 +656,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { }, }, nil) - target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local) + target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.UTC) targetNextWeek = target.AddDate(0, 0, 7) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ @@ -699,7 +699,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { req: &webservice.GetInsightDataRequest{ MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, Step: model.InsightStep_WEEKLY, - RangeFrom: time.Date(2020, 1, 6, 0, 0, 0, 0, time.Local).Unix(), // 2020/01/06 is Monday + RangeFrom: time.Date(2020, 1, 6, 0, 0, 0, 0, time.UTC).Unix(), // 2020/01/06 is Monday DataPointCount: 2, ApplicationId: "ApplicationId", }, @@ -708,11 +708,11 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPoints: []*model.InsightDataPoint{ { Value: 2, - Timestamp: time.Date(2020, 1, 5, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC).Unix(), }, { Value: 3, - Timestamp: time.Date(2020, 1, 12, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2020, 1, 12, 0, 0, 0, 0, time.UTC).Unix(), }, }, }, @@ -723,7 +723,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { pipedID: "pipedID", projectID: "projectID", deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local) + target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) targetNextMonth := target.AddDate(0, 1, 0) s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). @@ -759,7 +759,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { }, }, nil) - target = time.Date(2020, 2, 1, 0, 0, 0, 0, time.Local) + target = time.Date(2020, 2, 1, 0, 0, 0, 0, time.UTC) targetNextMonth = target.AddDate(0, 1, 0) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ @@ -802,7 +802,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { req: &webservice.GetInsightDataRequest{ MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, Step: model.InsightStep_MONTHLY, - RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.Local).Unix(), + RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.UTC).Unix(), DataPointCount: 2, ApplicationId: "ApplicationId", }, @@ -811,11 +811,11 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPoints: []*model.InsightDataPoint{ { Value: 2, - Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), }, { Value: 3, - Timestamp: time.Date(2020, 2, 1, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2020, 2, 1, 0, 0, 0, 0, time.UTC).Unix(), }, }, }, @@ -826,7 +826,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { pipedID: "pipedID", projectID: "projectID", deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local) + target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) targetNextYear := target.AddDate(1, 0, 0) s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). @@ -862,7 +862,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { }, }, nil) - target = time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local) + target = time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC) targetNextYear = target.AddDate(1, 0, 0) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ @@ -905,7 +905,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { req: &webservice.GetInsightDataRequest{ MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, Step: model.InsightStep_YEARLY, - RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.Local).Unix(), + RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.UTC).Unix(), DataPointCount: 2, ApplicationId: "ApplicationId", }, @@ -914,11 +914,11 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { DataPoints: []*model.InsightDataPoint{ { Value: 2, - Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), }, { Value: 3, - Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.Local).Unix(), + Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), }, }, }, @@ -929,7 +929,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { pipedID: "pipedID", projectID: "projectID", deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.Local) + target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) targetNextYear := target.AddDate(1, 0, 0) s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). @@ -962,7 +962,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { req: &webservice.GetInsightDataRequest{ MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, Step: model.InsightStep_YEARLY, - RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.Local).Unix(), + RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.UTC).Unix(), DataPointCount: 2, ApplicationId: "ApplicationId", }, From 651c401d4650b7cedaf66af6d6537e495e13a360 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 19:15:57 +0900 Subject: [PATCH 11/19] Fix to reset to zero in weekly and daily --- pkg/app/api/grpcapi/web_api.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index 50dd2334c6..819f259cc5 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1336,14 +1336,16 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID movePoint = func(from time.Time, i int) time.Time { return from.AddDate(0, 0, i) } - start = time.Unix(req.RangeFrom, 0) + rangeFrom := time.Unix(req.RangeFrom, 0) + start = time.Date(rangeFrom.Year(), rangeFrom.Month(), rangeFrom.Day(), 0, 0, 0, 0, time.UTC) case model.InsightStep_WEEKLY: movePoint = func(from time.Time, i int) time.Time { return from.AddDate(0, 0, 7*i) } rangeFrom := time.Unix(req.RangeFrom, 0) // Sunday in the week of rangeFrom - start = rangeFrom.AddDate(0, 0, -int(rangeFrom.Weekday())) + sunday := rangeFrom.AddDate(0, 0, -int(rangeFrom.Weekday())) + start = time.Date(sunday.Year(), sunday.Month(), sunday.Day(), 0, 0, 0, 0, time.UTC) case model.InsightStep_MONTHLY: movePoint = func(from time.Time, i int) time.Time { return from.AddDate(0, i, 0) From 4939e2342458a1b12c8191d612aec1ac712adead Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 19:25:40 +0900 Subject: [PATCH 12/19] Add pageSize opts --- pkg/app/api/grpcapi/web_api.go | 25 ++++++++++++++++++------- pkg/app/api/grpcapi/web_api_test.go | 12 ++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index 819f259cc5..0aaf2816fd 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1391,17 +1391,28 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID }) } - deployments, err := a.deploymentStore.ListDeployments(ctx, datastore.ListOptions{ - Filters: filters, - }) - if err != nil { - a.logger.Error("failed to get deployments", zap.Error(err)) - return nil, status.Error(codes.Internal, "Failed to get deployments") + pageSize := 50 + count := 0 + for j := 0; ; j++ { + deployments, err := a.deploymentStore.ListDeployments(ctx, datastore.ListOptions{ + PageSize: pageSize, + Filters: filters, + }) + if err != nil { + a.logger.Error("failed to get deployments", zap.Error(err)) + return nil, status.Error(codes.Internal, "Failed to get deployments") + } + + count += len(deployments) + + if len(deployments) != 50 { + break + } } counts[i] = &model.InsightDataPoint{ Timestamp: target.Unix(), - Value: float32(len(deployments)), + Value: float32(count), } } diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 0d409aa8d8..ce403f238e 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -399,6 +399,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + PageSizeForListDeployments := 50 tests := []struct { name string pipedID string @@ -419,6 +420,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -454,6 +456,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { targetNextDate = target.AddDate(0, 0, 1) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -522,6 +525,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -557,6 +561,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { targetNextWeek = target.AddDate(0, 0, 7) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -625,6 +630,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -660,6 +666,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { targetNextWeek = target.AddDate(0, 0, 7) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -728,6 +735,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -763,6 +771,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { targetNextMonth = target.AddDate(0, 1, 0) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -831,6 +840,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -866,6 +876,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { targetNextYear = target.AddDate(1, 0, 0) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -934,6 +945,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ + PageSize: PageSizeForListDeployments, Filters: []datastore.ListFilter{ { Field: "ProjectId", From fbbd84c36ea3fba14464b8a95b565be745873a05 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 19:32:28 +0900 Subject: [PATCH 13/19] Support only daily to prevent heavy loading --- pkg/app/api/grpcapi/web_api.go | 20 -- pkg/app/api/grpcapi/web_api_test.go | 426 +--------------------------- 2 files changed, 3 insertions(+), 443 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index 0aaf2816fd..739c63cc75 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1338,26 +1338,6 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID } rangeFrom := time.Unix(req.RangeFrom, 0) start = time.Date(rangeFrom.Year(), rangeFrom.Month(), rangeFrom.Day(), 0, 0, 0, 0, time.UTC) - case model.InsightStep_WEEKLY: - movePoint = func(from time.Time, i int) time.Time { - return from.AddDate(0, 0, 7*i) - } - rangeFrom := time.Unix(req.RangeFrom, 0) - // Sunday in the week of rangeFrom - sunday := rangeFrom.AddDate(0, 0, -int(rangeFrom.Weekday())) - start = time.Date(sunday.Year(), sunday.Month(), sunday.Day(), 0, 0, 0, 0, time.UTC) - case model.InsightStep_MONTHLY: - movePoint = func(from time.Time, i int) time.Time { - return from.AddDate(0, i, 0) - } - rangeFrom := time.Unix(req.RangeFrom, 0) - start = time.Date(rangeFrom.Year(), rangeFrom.Month(), 1, 0, 0, 0, 0, time.UTC) - case model.InsightStep_YEARLY: - movePoint = func(from time.Time, i int) time.Time { - return from.AddDate(i, 0, 0) - } - rangeFrom := time.Unix(req.RangeFrom, 0) - start = time.Date(rangeFrom.Year(), 1, 1, 0, 0, 0, 0, time.UTC) default: return nil, status.Error(codes.InvalidArgument, "Invalid step") } diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index ce403f238e..0c29f51758 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -515,433 +515,13 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { }, wantErr: false, }, - { - name: "valid with InsightStep_WEEKLY and rangeFrom is Sunday", - pipedID: "pipedID", - projectID: "projectID", - deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC) - targetNextWeek := target.AddDate(0, 0, 7) - s := datastoretest.NewMockDeploymentStore(ctrl) - s.EXPECT(). - ListDeployments(gomock.Any(), datastore.ListOptions{ - PageSize: PageSizeForListDeployments, - Filters: []datastore.ListFilter{ - { - Field: "ProjectId", - Operator: "==", - Value: "projectID", - }, - { - Field: "CreatedAt", - Operator: ">=", - Value: target.Unix(), - }, - { - Field: "CreatedAt", - Operator: "<", - Value: targetNextWeek.Unix(), - }, - { - Field: "ApplicationId", - Operator: "==", - Value: "ApplicationId", - }, - }, - }).Return([]*model.Deployment{ - { - Id: "id1", - }, - { - Id: "id2", - }, - }, nil) - - target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.UTC) - targetNextWeek = target.AddDate(0, 0, 7) - s.EXPECT(). - ListDeployments(gomock.Any(), datastore.ListOptions{ - PageSize: PageSizeForListDeployments, - Filters: []datastore.ListFilter{ - { - Field: "ProjectId", - Operator: "==", - Value: "projectID", - }, - { - Field: "CreatedAt", - Operator: ">=", - Value: target.Unix(), - }, - { - Field: "CreatedAt", - Operator: "<", - Value: targetNextWeek.Unix(), - }, - { - Field: "ApplicationId", - Operator: "==", - Value: "ApplicationId", - }, - }, - }).Return([]*model.Deployment{ - { - Id: "id1", - }, - { - Id: "id2", - }, - { - Id: "id3", - }, - }, nil) - - return s - }(), - req: &webservice.GetInsightDataRequest{ - MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, - Step: model.InsightStep_WEEKLY, - RangeFrom: time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC).Unix(), // 2020/01/05 is Sunday - DataPointCount: 2, - ApplicationId: "ApplicationId", - }, - res: &webservice.GetInsightDataResponse{ - UpdatedAt: time.Now().Unix(), - DataPoints: []*model.InsightDataPoint{ - { - Value: 2, - Timestamp: time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC).Unix(), - }, - { - Value: 3, - Timestamp: time.Date(2020, 1, 12, 0, 0, 0, 0, time.UTC).Unix(), - }, - }, - }, - wantErr: false, - }, - { - name: "valid with InsightStep_WEEKLY and rangeFrom is not Sunday", - pipedID: "pipedID", - projectID: "projectID", - deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC) - targetNextWeek := target.AddDate(0, 0, 7) - s := datastoretest.NewMockDeploymentStore(ctrl) - s.EXPECT(). - ListDeployments(gomock.Any(), datastore.ListOptions{ - PageSize: PageSizeForListDeployments, - Filters: []datastore.ListFilter{ - { - Field: "ProjectId", - Operator: "==", - Value: "projectID", - }, - { - Field: "CreatedAt", - Operator: ">=", - Value: target.Unix(), - }, - { - Field: "CreatedAt", - Operator: "<", - Value: targetNextWeek.Unix(), - }, - { - Field: "ApplicationId", - Operator: "==", - Value: "ApplicationId", - }, - }, - }).Return([]*model.Deployment{ - { - Id: "id1", - }, - { - Id: "id2", - }, - }, nil) - - target = time.Date(2020, 1, 12, 0, 0, 0, 0, time.UTC) - targetNextWeek = target.AddDate(0, 0, 7) - s.EXPECT(). - ListDeployments(gomock.Any(), datastore.ListOptions{ - PageSize: PageSizeForListDeployments, - Filters: []datastore.ListFilter{ - { - Field: "ProjectId", - Operator: "==", - Value: "projectID", - }, - { - Field: "CreatedAt", - Operator: ">=", - Value: target.Unix(), - }, - { - Field: "CreatedAt", - Operator: "<", - Value: targetNextWeek.Unix(), - }, - { - Field: "ApplicationId", - Operator: "==", - Value: "ApplicationId", - }, - }, - }).Return([]*model.Deployment{ - { - Id: "id1", - }, - { - Id: "id2", - }, - { - Id: "id3", - }, - }, nil) - - return s - }(), - req: &webservice.GetInsightDataRequest{ - MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, - Step: model.InsightStep_WEEKLY, - RangeFrom: time.Date(2020, 1, 6, 0, 0, 0, 0, time.UTC).Unix(), // 2020/01/06 is Monday - DataPointCount: 2, - ApplicationId: "ApplicationId", - }, - res: &webservice.GetInsightDataResponse{ - UpdatedAt: time.Now().Unix(), - DataPoints: []*model.InsightDataPoint{ - { - Value: 2, - Timestamp: time.Date(2020, 1, 5, 0, 0, 0, 0, time.UTC).Unix(), - }, - { - Value: 3, - Timestamp: time.Date(2020, 1, 12, 0, 0, 0, 0, time.UTC).Unix(), - }, - }, - }, - wantErr: false, - }, - { - name: "valid with InsightStep_MONTHLY", - pipedID: "pipedID", - projectID: "projectID", - deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - targetNextMonth := target.AddDate(0, 1, 0) - s := datastoretest.NewMockDeploymentStore(ctrl) - s.EXPECT(). - ListDeployments(gomock.Any(), datastore.ListOptions{ - PageSize: PageSizeForListDeployments, - Filters: []datastore.ListFilter{ - { - Field: "ProjectId", - Operator: "==", - Value: "projectID", - }, - { - Field: "CreatedAt", - Operator: ">=", - Value: target.Unix(), - }, - { - Field: "CreatedAt", - Operator: "<", - Value: targetNextMonth.Unix(), - }, - { - Field: "ApplicationId", - Operator: "==", - Value: "ApplicationId", - }, - }, - }).Return([]*model.Deployment{ - { - Id: "id1", - }, - { - Id: "id2", - }, - }, nil) - - target = time.Date(2020, 2, 1, 0, 0, 0, 0, time.UTC) - targetNextMonth = target.AddDate(0, 1, 0) - s.EXPECT(). - ListDeployments(gomock.Any(), datastore.ListOptions{ - PageSize: PageSizeForListDeployments, - Filters: []datastore.ListFilter{ - { - Field: "ProjectId", - Operator: "==", - Value: "projectID", - }, - { - Field: "CreatedAt", - Operator: ">=", - Value: target.Unix(), - }, - { - Field: "CreatedAt", - Operator: "<", - Value: targetNextMonth.Unix(), - }, - { - Field: "ApplicationId", - Operator: "==", - Value: "ApplicationId", - }, - }, - }).Return([]*model.Deployment{ - { - Id: "id1", - }, - { - Id: "id2", - }, - { - Id: "id3", - }, - }, nil) - - return s - }(), - req: &webservice.GetInsightDataRequest{ - MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, - Step: model.InsightStep_MONTHLY, - RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.UTC).Unix(), - DataPointCount: 2, - ApplicationId: "ApplicationId", - }, - res: &webservice.GetInsightDataResponse{ - UpdatedAt: time.Now().Unix(), - DataPoints: []*model.InsightDataPoint{ - { - Value: 2, - Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), - }, - { - Value: 3, - Timestamp: time.Date(2020, 2, 1, 0, 0, 0, 0, time.UTC).Unix(), - }, - }, - }, - wantErr: false, - }, - { - name: "valid with InsightStep_YEARLY", - pipedID: "pipedID", - projectID: "projectID", - deploymentStore: func() datastore.DeploymentStore { - target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - targetNextYear := target.AddDate(1, 0, 0) - s := datastoretest.NewMockDeploymentStore(ctrl) - s.EXPECT(). - ListDeployments(gomock.Any(), datastore.ListOptions{ - PageSize: PageSizeForListDeployments, - Filters: []datastore.ListFilter{ - { - Field: "ProjectId", - Operator: "==", - Value: "projectID", - }, - { - Field: "CreatedAt", - Operator: ">=", - Value: target.Unix(), - }, - { - Field: "CreatedAt", - Operator: "<", - Value: targetNextYear.Unix(), - }, - { - Field: "ApplicationId", - Operator: "==", - Value: "ApplicationId", - }, - }, - }).Return([]*model.Deployment{ - { - Id: "id1", - }, - { - Id: "id2", - }, - }, nil) - - target = time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC) - targetNextYear = target.AddDate(1, 0, 0) - s.EXPECT(). - ListDeployments(gomock.Any(), datastore.ListOptions{ - PageSize: PageSizeForListDeployments, - Filters: []datastore.ListFilter{ - { - Field: "ProjectId", - Operator: "==", - Value: "projectID", - }, - { - Field: "CreatedAt", - Operator: ">=", - Value: target.Unix(), - }, - { - Field: "CreatedAt", - Operator: "<", - Value: targetNextYear.Unix(), - }, - { - Field: "ApplicationId", - Operator: "==", - Value: "ApplicationId", - }, - }, - }).Return([]*model.Deployment{ - { - Id: "id1", - }, - { - Id: "id2", - }, - { - Id: "id3", - }, - }, nil) - - return s - }(), - req: &webservice.GetInsightDataRequest{ - MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, - Step: model.InsightStep_YEARLY, - RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.UTC).Unix(), - DataPointCount: 2, - ApplicationId: "ApplicationId", - }, - res: &webservice.GetInsightDataResponse{ - UpdatedAt: time.Now().Unix(), - DataPoints: []*model.InsightDataPoint{ - { - Value: 2, - Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), - }, - { - Value: 3, - Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), - }, - }, - }, - wantErr: false, - }, { name: "return error when something wrong happen on ListDeployments", pipedID: "pipedID", projectID: "projectID", deploymentStore: func() datastore.DeploymentStore { target := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - targetNextYear := target.AddDate(1, 0, 0) + targetNextYear := target.AddDate(0, 0, 1) s := datastoretest.NewMockDeploymentStore(ctrl) s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ @@ -973,8 +553,8 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { }(), req: &webservice.GetInsightDataRequest{ MetricsKind: model.InsightMetricsKind_DEPLOYMENT_FREQUENCY, - Step: model.InsightStep_YEARLY, - RangeFrom: time.Date(2020, 1, 4, 0, 0, 0, 0, time.UTC).Unix(), + Step: model.InsightStep_DAILY, + RangeFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(), DataPointCount: 2, ApplicationId: "ApplicationId", }, From bb63a36e538f94bb6aa0ac57fdbb4234884b5b66 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 19:33:16 +0900 Subject: [PATCH 14/19] Remove unuse deps --- pkg/app/api/grpcapi/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/app/api/grpcapi/BUILD.bazel b/pkg/app/api/grpcapi/BUILD.bazel index bc761b677d..1d3f0b6d6b 100644 --- a/pkg/app/api/grpcapi/BUILD.bazel +++ b/pkg/app/api/grpcapi/BUILD.bazel @@ -29,7 +29,6 @@ go_library( "@org_golang_google_grpc//codes:go_default_library", "@org_golang_google_grpc//status:go_default_library", "@org_uber_go_zap//:go_default_library", - "@org_uber_go_zap//zaptest/observer:go_default_library", ], ) From 127d2a3a6d53ddc7b97b96f642693e592269d0a2 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Fri, 4 Dec 2020 20:08:03 +0900 Subject: [PATCH 15/19] Add comment for support --- pkg/app/api/grpcapi/web_api.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index 739c63cc75..48c699a26b 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1331,8 +1331,14 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID var movePoint func(time.Time, int) time.Time var start time.Time + // To prevent heavy loading + // - Support only daily + // - DataPointCount needs to be less than or equal to 7 switch req.Step { case model.InsightStep_DAILY: + if req.DataPointCount > 7 { + return nil, status.Error(codes.InvalidArgument, "DataPointCount needs to be less than or equal to 7") + } movePoint = func(from time.Time, i int) time.Time { return from.AddDate(0, 0, i) } From 33abb45bb48dba4bc46bb7b9d4d62b054cc6acd8 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Mon, 7 Dec 2020 06:50:13 +0900 Subject: [PATCH 16/19] Fix to specify page on ListDeployments option --- pkg/app/api/grpcapi/web_api.go | 1 + pkg/app/api/grpcapi/web_api_test.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index 48c699a26b..4a80dc4b02 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1382,6 +1382,7 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID for j := 0; ; j++ { deployments, err := a.deploymentStore.ListDeployments(ctx, datastore.ListOptions{ PageSize: pageSize, + Page: j + 1, Filters: filters, }) if err != nil { diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 0c29f51758..7312487d37 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -421,6 +421,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ PageSize: PageSizeForListDeployments, + Page: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -457,6 +458,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ PageSize: PageSizeForListDeployments, + Page: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -526,6 +528,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ PageSize: PageSizeForListDeployments, + Page: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", From e19da306bb5e1f5b2f29044ddb82fc487fd6307f Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Mon, 7 Dec 2020 06:54:59 +0900 Subject: [PATCH 17/19] Go fmted --- pkg/app/api/grpcapi/web_api_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 7312487d37..7032603dbc 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -421,7 +421,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ PageSize: PageSizeForListDeployments, - Page: 1, + Page: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -458,7 +458,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ PageSize: PageSizeForListDeployments, - Page: 1, + Page: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -528,7 +528,7 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ PageSize: PageSizeForListDeployments, - Page: 1, + Page: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", From 98c92e78c16c6723b7966f6be018994e07880ac5 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Mon, 7 Dec 2020 15:34:48 +0900 Subject: [PATCH 18/19] Fix not to get all data with page --- pkg/app/api/grpcapi/web_api.go | 27 +++++++++------------------ pkg/app/api/grpcapi/web_api_test.go | 3 --- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api.go b/pkg/app/api/grpcapi/web_api.go index 4a80dc4b02..23a9bb70d0 100644 --- a/pkg/app/api/grpcapi/web_api.go +++ b/pkg/app/api/grpcapi/web_api.go @@ -1326,6 +1326,7 @@ func (a *WebAPI) GetInsightData(ctx context.Context, req *webservice.GetInsightD } // getInsightDataForDeployFrequency returns the accumulated insight data for deploy frequency. +// This function is temporary implementation for front end. func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID string, req *webservice.GetInsightDataRequest) (*webservice.GetInsightDataResponse, error) { counts := make([]*model.InsightDataPoint, req.DataPointCount) @@ -1378,28 +1379,18 @@ func (a *WebAPI) getInsightDataForDeployFrequency(ctx context.Context, projectID } pageSize := 50 - count := 0 - for j := 0; ; j++ { - deployments, err := a.deploymentStore.ListDeployments(ctx, datastore.ListOptions{ - PageSize: pageSize, - Page: j + 1, - Filters: filters, - }) - if err != nil { - a.logger.Error("failed to get deployments", zap.Error(err)) - return nil, status.Error(codes.Internal, "Failed to get deployments") - } - - count += len(deployments) - - if len(deployments) != 50 { - break - } + deployments, err := a.deploymentStore.ListDeployments(ctx, datastore.ListOptions{ + PageSize: pageSize, + Filters: filters, + }) + if err != nil { + a.logger.Error("failed to get deployments", zap.Error(err)) + return nil, status.Error(codes.Internal, "Failed to get deployments") } counts[i] = &model.InsightDataPoint{ Timestamp: target.Unix(), - Value: float32(count), + Value: float32(len(deployments)), } } diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 7032603dbc..0c29f51758 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -421,7 +421,6 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ PageSize: PageSizeForListDeployments, - Page: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -458,7 +457,6 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ PageSize: PageSizeForListDeployments, - Page: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -528,7 +526,6 @@ func TestGetInsightDataForDeployFrequency(t *testing.T) { s.EXPECT(). ListDeployments(gomock.Any(), datastore.ListOptions{ PageSize: PageSizeForListDeployments, - Page: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", From 06e96954ab4e6eadb61a578f0ccd4dbc3e0deec0 Mon Sep 17 00:00:00 2001 From: sanposhiho Date: Mon, 7 Dec 2020 15:36:12 +0900 Subject: [PATCH 19/19] Remove empty line --- pkg/app/api/grpcapi/web_api_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/app/api/grpcapi/web_api_test.go b/pkg/app/api/grpcapi/web_api_test.go index 0c29f51758..60be953e35 100644 --- a/pkg/app/api/grpcapi/web_api_test.go +++ b/pkg/app/api/grpcapi/web_api_test.go @@ -22,10 +22,9 @@ import ( "testing" "time" - "go.uber.org/zap" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "go.uber.org/zap" "github.com/pipe-cd/pipe/pkg/app/api/service/webservice" "github.com/pipe-cd/pipe/pkg/cache"