Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,28 @@ require (
cloud.google.com/go v0.65.0
cloud.google.com/go/firestore v1.2.0
cloud.google.com/go/storage v1.11.0
github.com/Azure/go-autorest/autorest v0.10.2 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46
github.com/aws/aws-sdk-go v1.36.21
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/envoyproxy/protoc-gen-validate v0.1.0
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/fsouza/fake-gcs-server v1.21.0
github.com/goccy/go-yaml v1.8.4
github.com/golang/mock v1.4.4
github.com/golang/protobuf v1.4.2
github.com/gomodule/redigo v2.0.0+incompatible
github.com/google/go-containerregistry v0.3.0
github.com/google/go-github/v29 v29.0.3
github.com/google/uuid v1.1.1
github.com/googleapis/gnostic v0.2.2 // indirect
github.com/hashicorp/golang-lru v0.5.3
github.com/klauspost/compress v1.10.11 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/minio/minio-go/v7 v7.0.5
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/onsi/ginkgo v1.12.0 // indirect
github.com/onsi/gomega v1.9.0 // indirect
github.com/prometheus/client_golang v1.6.0
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.9.1
Expand All @@ -36,9 +43,13 @@ require (
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3 // indirect
google.golang.org/api v0.31.0
google.golang.org/grpc v1.31.1
google.golang.org/protobuf v1.25.0
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/yaml.v2 v2.3.0 // indirect
istio.io/api v0.0.0-20200710191538-00b73d23c685
k8s.io/api v0.18.9
k8s.io/apimachinery v0.18.9
Expand Down
143 changes: 0 additions & 143 deletions go.sum

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions pkg/app/api/grpcapi/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@ func (a *API) RegisterEvent(ctx context.Context, req *apiservice.RegisterEventRe
Name: req.Name,
Data: req.Data,
Labels: req.Labels,
EventKey: model.MakeEventKey(req.Name, req.Labels),
ProjectId: key.ProjectId,
})
if errors.Is(err, datastore.ErrAlreadyExists) {
Expand Down
10 changes: 7 additions & 3 deletions pkg/app/api/grpcapi/piped_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -690,7 +690,7 @@ func (a *PipedAPI) GetLatestEvent(ctx context.Context, req *pipedservice.GetLate
return nil, err
}

// Try to fetch the most recently registered event.
// Try to fetch the most recently registered event that has the given parameters.
opts := datastore.ListOptions{
PageSize: 1,
Filters: []datastore.ListFilter{
Expand All @@ -704,11 +704,15 @@ func (a *PipedAPI) GetLatestEvent(ctx context.Context, req *pipedservice.GetLate
Operator: "==",
Value: req.Name,
},
// TODO: Enable to use labels to filter Events
{
Field: "EventKey",
Operator: "==",
Value: model.MakeEventKey(req.Name, req.Labels),
},
},
Orders: []datastore.Order{
{
Field: "UpdatedAt",
Field: "CreatedAt",
Direction: datastore.Desc,
},
},
Expand Down
1 change: 0 additions & 1 deletion pkg/app/piped/apistore/eventstore/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,4 @@ go_test(
size = "small",
srcs = ["store_test.go"],
embed = [":go_default_library"],
deps = ["@com_github_stretchr_testify//assert:go_default_library"],
)
29 changes: 2 additions & 27 deletions pkg/app/piped/apistore/eventstore/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ package eventstore
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -120,8 +118,7 @@ func (s *store) sync(ctx context.Context) error {
// Eliminate events that have duplicated key.
filtered := make(map[string]*model.Event, len(resp.Events))
for _, e := range resp.Events {
key := makeEventKey(e.Name, e.Labels)
filtered[key] = e
filtered[e.EventKey] = e
}
// Make the cache up-to-date.
s.mu.Lock()
Expand All @@ -144,7 +141,7 @@ func (s *store) Getter() Getter {
}

func (s *store) GetLatest(ctx context.Context, name string, labels map[string]string) (*model.Event, bool) {
key := makeEventKey(name, labels)
key := model.MakeEventKey(name, labels)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: This should be done before locking.

s.mu.RLock()
event, ok := s.latestEvents[key]
s.mu.RUnlock()
Expand All @@ -171,25 +168,3 @@ func (s *store) GetLatest(ctx context.Context, name string, labels map[string]st
s.latestEvents[key] = resp.Event
return resp.Event, true
}

// makeEventKey builds a unique identifier based on the given name and labels.
// It returns the exact same string as long as both are the same.
func makeEventKey(name string, labels map[string]string) string {
if len(labels) == 0 {
return name
}

var b strings.Builder
b.WriteString(name)

// Guarantee uniqueness by sorting by keys.
keys := make([]string, 0, len(labels))
for key := range labels {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
b.WriteString(fmt.Sprintf("/%s:%s", key, labels[key]))
}
return b.String()
}
48 changes: 0 additions & 48 deletions pkg/app/piped/apistore/eventstore/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,51 +13,3 @@
// limitations under the License.

package eventstore

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestMakeEventKey(t *testing.T) {
testcases := []struct {
testname string
name string
labels map[string]string
want string
}{
{
testname: "no name and labels given",
want: "",
},
{
testname: "no labels given",
name: "name1",
want: "name1",
},
{
testname: "no name given",
labels: map[string]string{
"key1": "value1",
},
want: "/key1:value1",
},
{
testname: "labels given",
name: "name1",
labels: map[string]string{
"key1": "value",
"key2": "value",
"key3": "value",
},
want: "name1/key1:value/key2:value/key3:value",
},
}
for _, tc := range testcases {
t.Run(tc.testname, func(t *testing.T) {
got := makeEventKey(tc.name, tc.labels)
assert.Equal(t, tc.want, got)
})
}
}
1 change: 1 addition & 0 deletions pkg/datastore/eventstore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func TestAddEvent(t *testing.T) {
Name: "name",
Data: "data",
ProjectId: "project",
EventKey: "82a3537ff0dbce7eec35d69edc3a189ee6f17d82f353a553f9aa96cb0be3ce89",
CreatedAt: 12345,
UpdatedAt: 12345,
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/model/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ go_library(
"deployment.go",
"docs.go",
"environment.go",
"event.go",
"filestore.go",
"image_name.go",
"imageprovider.go",
Expand All @@ -79,6 +80,7 @@ go_test(
srcs = [
"apikey_test.go",
"common_test.go",
"event_test.go",
"image_name_test.go",
"model_test.go",
"piped_test.go",
Expand Down
52 changes: 52 additions & 0 deletions pkg/model/event.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
// Copyright 2021 The PipeCD Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
)

var hashFunc = sha256.New

// MakeEventKey builds a fixed-length identifier based on the given name
// and labels. It returns the exact same string as long as both are the same.
func MakeEventKey(name string, labels map[string]string) string {
h := hashFunc()
if len(labels) == 0 {
h.Write([]byte(name))
return hex.EncodeToString(h.Sum(nil))
}

var b strings.Builder
b.WriteString(name)

// Guarantee uniqueness by sorting by keys.
keys := make([]string, 0, len(labels))
for key := range labels {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
b.WriteString(fmt.Sprintf("/%s:%s", key, labels[key]))
}

// Make a hash to be fixed length regardless of the length of the event key.
h.Write([]byte(b.String()))
return hex.EncodeToString(h.Sum(nil))
}
2 changes: 2 additions & 0 deletions pkg/model/event.proto
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ message Event {
// The key/value pairs that are attached to event.
// This is intended to be used to specify additional attributes of event.
map<string,string> labels = 5;
// A fixed-length identifier consists of its own name and labels.
string event_key = 6 [(validate.rules).string.min_len = 1];
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is the hash value that we talked about before?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My apologies, I totally forgot it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you. I've done added the hash just now.


// Unix time when the event was created.
int64 created_at = 14 [(validate.rules).int64.gt = 0];
Expand Down
93 changes: 93 additions & 0 deletions pkg/model/event_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
// Copyright 2021 The PipeCD Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestMakeEventKey(t *testing.T) {
testcases := []struct {
testname string
name1 string
labels1 map[string]string
name2 string
labels2 map[string]string
wantSame bool
}{
{
testname: "no name and labels given",
wantSame: true,
},
{
testname: "no labels given",
name1: "name",
name2: "name",
wantSame: true,
},
{
testname: "no name given",
labels1: map[string]string{
"key1": "value1",
},
labels2: map[string]string{
"key1": "value1",
},
wantSame: true,
},
{
testname: "the exact same labels given",
name1: "name",
labels1: map[string]string{
"key1": "value",
"key2": "value",
"key3": "value",
},
name2: "name",
labels2: map[string]string{
"key2": "value",
"key3": "value",
"key1": "value",
},
wantSame: true,
},
{
testname: "the sub match labels given",
name1: "name",
labels1: map[string]string{
"key1": "value",
"key2": "value",
"key3": "value",
},
name2: "name",
labels2: map[string]string{
"key1": "value",
},
wantSame: false,
},
}
for _, tc := range testcases {
t.Run(tc.testname, func(t *testing.T) {
// Check if we can get the exact same string.
got1 := MakeEventKey(tc.name1, tc.labels1)
got2 := MakeEventKey(tc.name2, tc.labels2)
assert.NotEmpty(t, got1)
assert.NotEmpty(t, got2)
assert.Equal(t, tc.wantSame, got1 == got2)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add the check of key length to ensure that they are not empty strings.

})
}
}
Loading