Skip to content

Create insight collector #1287

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pipecd-bot merged 42 commits into from
Dec 22, 2020
Merged

Create insight collector #1287

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
0fc0b7d
Create insight collector
sanposhiho Dec 18, 2020
6a44216
Format unfmt-ed files
Dec 18, 2020
85e9a15
Fix by linter
sanposhiho Dec 18, 2020
dddcf8f
Merge branch 'create-insight-collector' of github.com:pipe-cd/pipe in…
sanposhiho Dec 18, 2020
26993d4
List Aplication with sort
sanposhiho Dec 18, 2020
eb9eeab
Run with cron
sanposhiho Dec 18, 2020
7014aba
Fix to get deployments from store once
sanposhiho Dec 18, 2020
f5fcc50
Fix to use split processes years and chunk
sanposhiho Dec 18, 2020
065356a
Fix to use timezone utc
sanposhiho Dec 18, 2020
b4f5d81
Add deps to BUILD.bazel
sanposhiho Dec 18, 2020
fd7d585
Fix import order and rm unuse option
sanposhiho Dec 21, 2020
185d231
Add control plane test
sanposhiho Dec 21, 2020
ca85ff5
Add error handling on insight collector
sanposhiho Dec 21, 2020
8d199dc
Format some points
sanposhiho Dec 21, 2020
21e55a4
Add function into interface
sanposhiho Dec 21, 2020
af2261e
Remove AppendDataPoint
sanposhiho Dec 21, 2020
6c3d89e
Use sorted list instead of map as updatedps
sanposhiho Dec 21, 2020
3e8419b
Add TODO comment
sanposhiho Dec 21, 2020
f32a508
Summarize page size in const valiable
sanposhiho Dec 21, 2020
b5dbab9
Fix todo comment
sanposhiho Dec 21, 2020
ff11d73
Add flag for disable insightcollector
sanposhiho Dec 21, 2020
9d3b3c1
Remove unuse func
sanposhiho Dec 21, 2020
77ff912
Fix error messages
sanposhiho Dec 21, 2020
5b352d1
Fix comment
sanposhiho Dec 21, 2020
08e8b14
Fix to use created at as fileter'
sanposhiho Dec 21, 2020
31c3187
Add option enable-insight-collector
sanposhiho Dec 21, 2020
5c288cd
Create function and split process
sanposhiho Dec 22, 2020
5065a06
Fix to process file not found pattern
sanposhiho Dec 22, 2020
c14aaec
Fix miss assignment
sanposhiho Dec 22, 2020
79f2b91
Use updatedAt instead of createdAt
sanposhiho Dec 22, 2020
be05090
Fix error handling
sanposhiho Dec 22, 2020
38c4be2
Fix testcase
sanposhiho Dec 22, 2020
dc6e7f1
Merge branch 'master' into create-insight-collector
sanposhiho Dec 22, 2020
c951140
Format unfmt-ed files
Dec 22, 2020
1945ac5
Add project level insight collector
sanposhiho Dec 22, 2020
a9e258c
Fix some variable name and comments
sanposhiho Dec 22, 2020
4641521
Merge branch 'create-insight-collector' of github.com:pipe-cd/pipe in…
sanposhiho Dec 22, 2020
d85c48b
Use ComplatedAt instead of UpdatedAt
sanposhiho Dec 22, 2020
ca698ba
Rename field
sanposhiho Dec 22, 2020
cdc6f9f
Add log message with duration
sanposhiho Dec 22, 2020
4c051d2
Fix to continue working with other chunk if fail
sanposhiho Dec 22, 2020
afde1f5
Ignore deleted application
sanposhiho Dec 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
296 changes: 0 additions & 296 deletions pkg/app/api/grpcapi/web_api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package grpcapi
import (
"context"
"errors"
"fmt"
"reflect"
"testing"
"time"
Expand Down Expand Up @@ -531,298 +530,3 @@ func TestCalculateInsightData(t *testing.T) {
})
}
}

func TestGetInsightDataForDeployFrequency(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

PageSizeForListDeployments := 50
tests := []struct {
name string
projectID string
applicationID string
targetRangeFrom time.Time
targetRangeTo time.Time
deploymentStore datastore.DeploymentStore
dataPoints *model.InsightDataPoint
wantErr bool
}{
{
name: "valid with InsightStep_DAILY",
projectID: "projectID",
applicationID: "ApplicationId",
targetRangeFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
targetRangeTo: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC),
deploymentStore: func() datastore.DeploymentStore {
s := datastoretest.NewMockDeploymentStore(ctrl)
s.EXPECT().
ListDeployments(gomock.Any(), datastore.ListOptions{
PageSize: PageSizeForListDeployments,
Filters: []datastore.ListFilter{
{
Field: "ProjectId",
Operator: "==",
Value: "projectID",
},
{
Field: "CreatedAt",
Operator: ">=",
Value: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "CreatedAt",
Operator: "<",
Value: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "ApplicationId",
Operator: "==",
Value: "ApplicationId",
},
},
}).Return([]*model.Deployment{
{
Id: "id1",
},
{
Id: "id2",
},
{
Id: "id3",
},
}, nil)
return s
}(),
dataPoints: &model.InsightDataPoint{
Value: 3,
Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(),
},
wantErr: false,
},
{
name: "return error when something wrong happen on ListDeployments",
projectID: "projectID",
applicationID: "ApplicationId",
targetRangeFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
targetRangeTo: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC),
deploymentStore: func() datastore.DeploymentStore {
s := datastoretest.NewMockDeploymentStore(ctrl)
s.EXPECT().
ListDeployments(gomock.Any(), datastore.ListOptions{
PageSize: PageSizeForListDeployments,
Filters: []datastore.ListFilter{
{
Field: "ProjectId",
Operator: "==",
Value: "projectID",
},
{
Field: "CreatedAt",
Operator: ">=",
Value: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "CreatedAt",
Operator: "<",
Value: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "ApplicationId",
Operator: "==",
Value: "ApplicationId",
},
},
}).Return([]*model.Deployment{}, fmt.Errorf("something wrong happens in ListDeployments"))
return s
}(),
wantErr: true,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
api := &WebAPI{
deploymentStore: tt.deploymentStore,
logger: zap.NewNop(),
}
value, err := api.getInsightDataForDeployFrequency(ctx, tt.projectID, tt.applicationID, tt.targetRangeFrom, tt.targetRangeTo)
assert.Equal(t, tt.wantErr, err != nil)
if err == nil {
assert.Equal(t, tt.dataPoints, value)
}
})
}
}
func TestGetInsightDataForChangeFailureRate(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

PageSizeForListDeployments := 50
tests := []struct {
name string
projectID string
applicationID string
targetRangeFrom time.Time
targetRangeTo time.Time
deploymentStore datastore.DeploymentStore
dataPoints *model.InsightDataPoint
wantErr bool
}{
{
name: "valid with InsightStep_DAILY",
projectID: "projectID",
applicationID: "ApplicationId",
targetRangeFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
targetRangeTo: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC),
deploymentStore: func() datastore.DeploymentStore {
s := datastoretest.NewMockDeploymentStore(ctrl)
s.EXPECT().
ListDeployments(gomock.Any(), datastore.ListOptions{
PageSize: PageSizeForListDeployments,
Filters: []datastore.ListFilter{
{
Field: "Status",
Operator: "==",
Value: model.DeploymentStatus_DEPLOYMENT_SUCCESS,
},
{
Field: "ProjectId",
Operator: "==",
Value: "projectID",
},
{
Field: "CreatedAt",
Operator: ">=",
Value: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "CreatedAt",
Operator: "<",
Value: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "ApplicationId",
Operator: "==",
Value: "ApplicationId",
},
},
}).Return([]*model.Deployment{
{
Id: "id1",
},
{
Id: "id2",
},
{
Id: "id3",
},
}, nil)

s.EXPECT().
ListDeployments(gomock.Any(), datastore.ListOptions{
PageSize: PageSizeForListDeployments,
Filters: []datastore.ListFilter{
{
Field: "Status",
Operator: "==",
Value: model.DeploymentStatus_DEPLOYMENT_FAILURE,
},
{
Field: "ProjectId",
Operator: "==",
Value: "projectID",
},
{
Field: "CreatedAt",
Operator: ">=",
Value: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "CreatedAt",
Operator: "<",
Value: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "ApplicationId",
Operator: "==",
Value: "ApplicationId",
},
},
}).Return([]*model.Deployment{
{
Id: "id1",
},
}, nil)
return s
}(),
dataPoints: &model.InsightDataPoint{
Value: 0.25,
Timestamp: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(),
},
wantErr: false,
},
{
name: "return error when something wrong happen on ListDeployments",
projectID: "projectID",
applicationID: "ApplicationId",
targetRangeFrom: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
targetRangeTo: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC),
deploymentStore: func() datastore.DeploymentStore {
s := datastoretest.NewMockDeploymentStore(ctrl)
s.EXPECT().
ListDeployments(gomock.Any(), datastore.ListOptions{
PageSize: PageSizeForListDeployments,
Filters: []datastore.ListFilter{
{
Field: "Status",
Operator: "==",
Value: model.DeploymentStatus_DEPLOYMENT_SUCCESS,
},
{
Field: "ProjectId",
Operator: "==",
Value: "projectID",
},
{
Field: "CreatedAt",
Operator: ">=",
Value: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "CreatedAt",
Operator: "<",
Value: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC).Unix(),
},
{
Field: "ApplicationId",
Operator: "==",
Value: "ApplicationId",
},
},
}).Return([]*model.Deployment{}, fmt.Errorf("something wrong happens in ListDeployments"))
return s
}(),
wantErr: true,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
api := &WebAPI{
deploymentStore: tt.deploymentStore,
logger: zap.NewNop(),
}
value, err := api.getInsightDataForChangeFailureRate(ctx, tt.projectID, tt.applicationID, tt.targetRangeFrom, tt.targetRangeTo)
assert.Equal(t, tt.wantErr, err != nil)
if err == nil {
assert.Equal(t, tt.dataPoints, value)
}
})
}
}
26 changes: 25 additions & 1 deletion pkg/app/ops/insightcollector/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,32 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")

go_library(
name = "go_default_library",
srcs = ["collector.go"],
importpath = "github.com/pipe-cd/pipe/pkg/app/ops/insightcollector",
visibility = ["//visibility:public"],
deps = [
"//pkg/datastore:go_default_library",
"//pkg/insightstore:go_default_library",
"//pkg/model:go_default_library",
"//pkg/filestore:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)

go_test(
name = "go_default_test",
size = "small",
srcs = [
"collector_test.go"
],
embed = [":go_default_library"],
deps = [
"//pkg/insightstore:go_default_library",
"//pkg/filestore/filestoretest:go_default_library",
"//pkg/datastore:go_default_library",
"//pkg/datastore/datastoretest:go_default_library",
"@com_github_golang_mock//gomock:go_default_library",
"@com_github_stretchr_testify//assert:go_default_library",
],
)
Loading