Skip to content

Remove redis from api dependencies #3326

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pipecd-bot merged 6 commits into from
Mar 1, 2022
Merged

Remove redis from api dependencies #3326

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9944830
remove redis from api dependencies
Hosshii Mar 1, 2022
03b68e3
Update pkg/app/server/grpcapi/piped_api.go
Hosshii Mar 1, 2022
af89f2e
Update pkg/app/server/grpcapi/piped_api.go
Hosshii Mar 1, 2022
501bb05
change arg, field order
Hosshii Mar 1, 2022
3a0a5d2
make gazelle
Hosshii Mar 1, 2022
6c9cd64
fix
Hosshii Mar 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/pipecd/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ go_library(
"//pkg/app/server/pipedverifier:go_default_library",
"//pkg/app/server/service/webservice:go_default_library",
"//pkg/app/server/stagelogstore:go_default_library",
"//pkg/app/server/unregisteredappstore:go_default_library",
"//pkg/cache/cachemetrics:go_default_library",
"//pkg/cache/rediscache:go_default_library",
"//pkg/cli:go_default_library",
Expand Down
7 changes: 5 additions & 2 deletions cmd/pipecd/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/pipe-cd/pipecd/pkg/app/server/pipedverifier"
"github.com/pipe-cd/pipecd/pkg/app/server/service/webservice"
"github.com/pipe-cd/pipecd/pkg/app/server/stagelogstore"
"github.com/pipe-cd/pipecd/pkg/app/server/unregisteredappstore"
"github.com/pipe-cd/pipecd/pkg/cache/cachemetrics"
"github.com/pipe-cd/pipecd/pkg/cache/rediscache"
"github.com/pipe-cd/pipecd/pkg/cli"
Expand Down Expand Up @@ -186,6 +187,7 @@ func (s *server) run(ctx context.Context, input cli.Input) error {
is := insightstore.NewStore(fs)
cmdOutputStore := commandoutputstore.NewStore(fs, input.Logger)
statCache := rediscache.NewHashCache(rd, defaultPipedStatHashKey)
unregisteredAppStore := unregisteredappstore.NewStore(rd, input.Logger)

// Start a gRPC server for handling PipedAPI requests.
{
Expand All @@ -198,7 +200,7 @@ func (s *server) run(ctx context.Context, input cli.Input) error {
datastore.NewPipedStore(ds, datastore.PipedCommander),
input.Logger,
)
service = grpcapi.NewPipedAPI(ctx, ds, cache, sls, alss, las, statCache, rd, cmdOutputStore, cfg.Address, input.Logger)
service = grpcapi.NewPipedAPI(ctx, ds, cache, sls, alss, las, statCache, cmdOutputStore, unregisteredAppStore, cfg.Address, input.Logger)
opts = []rpc.Option{
rpc.WithPort(s.pipedAPIPort),
rpc.WithGracePeriod(s.gracePeriod),
Expand Down Expand Up @@ -269,8 +271,9 @@ func (s *server) run(ctx context.Context, input cli.Input) error {
input.Logger.Error("failed to create a new JWT verifier", zap.Error(err))
return err
}
insightCache := rediscache.NewTTLCache(rd, 3*time.Hour)

service := grpcapi.NewWebAPI(ctx, ds, cache, sls, alss, is, statCache, rd, cfg.ProjectMap(), encryptDecrypter, input.Logger)
service := grpcapi.NewWebAPI(ctx, ds, cache, sls, alss, unregisteredAppStore, is, statCache, insightCache, cfg.ProjectMap(), encryptDecrypter, input.Logger)
opts := []rpc.Option{
rpc.WithPort(s.webAPIPort),
rpc.WithGracePeriod(s.gracePeriod),
Expand Down
3 changes: 1 addition & 2 deletions pkg/app/server/grpcapi/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,16 @@ go_library(
"//pkg/app/server/service/pipedservice:go_default_library",
"//pkg/app/server/service/webservice:go_default_library",
"//pkg/app/server/stagelogstore:go_default_library",
"//pkg/app/server/unregisteredappstore:go_default_library",
"//pkg/cache:go_default_library",
"//pkg/cache/memorycache:go_default_library",
"//pkg/cache/rediscache:go_default_library",
"//pkg/config:go_default_library",
"//pkg/crypto:go_default_library",
"//pkg/datastore:go_default_library",
"//pkg/filestore:go_default_library",
"//pkg/git:go_default_library",
"//pkg/insight/insightstore:go_default_library",
"//pkg/model:go_default_library",
"//pkg/redis:go_default_library",
"//pkg/rpc/rpcauth:go_default_library",
"@com_github_google_uuid//:go_default_library",
"@org_golang_google_grpc//:go_default_library",
Expand Down
23 changes: 6 additions & 17 deletions pkg/app/server/grpcapi/piped_api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
package grpcapi

import (
"bytes"
"context"
"encoding/gob"
"encoding/json"
"errors"
"fmt"
Expand All @@ -34,13 +32,12 @@ import (
"github.com/pipe-cd/pipecd/pkg/app/server/commandstore"
"github.com/pipe-cd/pipecd/pkg/app/server/service/pipedservice"
"github.com/pipe-cd/pipecd/pkg/app/server/stagelogstore"
"github.com/pipe-cd/pipecd/pkg/app/server/unregisteredappstore"
"github.com/pipe-cd/pipecd/pkg/cache"
"github.com/pipe-cd/pipecd/pkg/cache/memorycache"
"github.com/pipe-cd/pipecd/pkg/cache/rediscache"
"github.com/pipe-cd/pipecd/pkg/datastore"
"github.com/pipe-cd/pipecd/pkg/filestore"
"github.com/pipe-cd/pipecd/pkg/model"
"github.com/pipe-cd/pipecd/pkg/redis"
"github.com/pipe-cd/pipecd/pkg/rpc/rpcauth"
)

Expand Down Expand Up @@ -96,18 +93,18 @@ type PipedAPI struct {
analysisResultStore analysisresultstore.Store
commandStore commandstore.Store
commandOutputPutter commandOutputPutter
unregisteredAppStore unregisteredappstore.Store

appPipedCache cache.Cache
deploymentPipedCache cache.Cache
pipedStatCache cache.Cache
redis redis.Redis

webBaseURL string
logger *zap.Logger
}

// NewPipedAPI creates a new PipedAPI instance.
func NewPipedAPI(ctx context.Context, ds datastore.DataStore, sc cache.Cache, sls stagelogstore.Store, alss applicationlivestatestore.Store, las analysisresultstore.Store, hc cache.Cache, rd redis.Redis, cop commandOutputPutter, webBaseURL string, logger *zap.Logger) *PipedAPI {
func NewPipedAPI(ctx context.Context, ds datastore.DataStore, sc cache.Cache, sls stagelogstore.Store, alss applicationlivestatestore.Store, las analysisresultstore.Store, hc cache.Cache, cop commandOutputPutter, uas unregisteredappstore.Store, webBaseURL string, logger *zap.Logger) *PipedAPI {
w := datastore.PipedCommander
a := &PipedAPI{
applicationStore: datastore.NewApplicationStore(ds, w),
Expand All @@ -120,10 +117,10 @@ func NewPipedAPI(ctx context.Context, ds datastore.DataStore, sc cache.Cache, sl
analysisResultStore: las,
commandStore: commandstore.NewStore(w, ds, sc, logger),
commandOutputPutter: cop,
unregisteredAppStore: uas,
appPipedCache: memorycache.NewTTLCache(ctx, 24*time.Hour, 3*time.Hour),
deploymentPipedCache: memorycache.NewTTLCache(ctx, 24*time.Hour, 3*time.Hour),
pipedStatCache: hc,
redis: rd,
webBaseURL: webBaseURL,
logger: logger.Named("piped-api"),
}
Expand Down Expand Up @@ -905,16 +902,8 @@ func (a *PipedAPI) ReportUnregisteredApplicationConfigurations(ctx context.Conte
return nil, err
}

// Cache an encoded slice of *model.ApplicationInfo.
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
if err := enc.Encode(req.Applications); err != nil {
a.logger.Error("failed to encode the unregistered apps", zap.Error(err))
return nil, status.Error(codes.Internal, "failed to encode the unregistered apps")
}
key := makeUnregisteredAppsCacheKey(projectID)
c := rediscache.NewHashCache(a.redis, key)
if err := c.Put(pipedID, buf.Bytes()); err != nil {
err = a.unregisteredAppStore.PutApplications(projectID, pipedID, req.Applications)
if err != nil {
return nil, status.Error(codes.Internal, "failed to put the unregistered apps to the cache")
}

Expand Down
36 changes: 8 additions & 28 deletions pkg/app/server/grpcapi/web_api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
package grpcapi

import (
"bytes"
"context"
"encoding/gob"
"errors"
"fmt"
"sort"
Expand All @@ -34,15 +32,14 @@ import (
"github.com/pipe-cd/pipecd/pkg/app/server/commandstore"
"github.com/pipe-cd/pipecd/pkg/app/server/service/webservice"
"github.com/pipe-cd/pipecd/pkg/app/server/stagelogstore"
"github.com/pipe-cd/pipecd/pkg/app/server/unregisteredappstore"
"github.com/pipe-cd/pipecd/pkg/cache"
"github.com/pipe-cd/pipecd/pkg/cache/memorycache"
"github.com/pipe-cd/pipecd/pkg/cache/rediscache"
"github.com/pipe-cd/pipecd/pkg/config"
"github.com/pipe-cd/pipecd/pkg/datastore"
"github.com/pipe-cd/pipecd/pkg/filestore"
"github.com/pipe-cd/pipecd/pkg/insight/insightstore"
"github.com/pipe-cd/pipecd/pkg/model"
"github.com/pipe-cd/pipecd/pkg/redis"
"github.com/pipe-cd/pipecd/pkg/rpc/rpcauth"
)

Expand Down Expand Up @@ -114,14 +111,14 @@ type WebAPI struct {
applicationLiveStateStore applicationlivestatestore.Store
commandStore commandstore.Store
insightStore insightstore.Store
unregisteredAppStore unregisteredappstore.Store
encrypter encrypter

appProjectCache cache.Cache
deploymentProjectCache cache.Cache
pipedProjectCache cache.Cache
pipedStatCache cache.Cache
insightCache cache.Cache
redis redis.Redis

projectsInConfig map[string]config.ControlPlaneProject
logger *zap.Logger
Expand All @@ -134,9 +131,10 @@ func NewWebAPI(
sc cache.Cache,
sls stagelogstore.Store,
alss applicationlivestatestore.Store,
uas unregisteredappstore.Store,
is insightstore.Store,
psc cache.Cache,
rd redis.Redis,
ic cache.Cache,
projs map[string]config.ControlPlaneProject,
encrypter encrypter,
logger *zap.Logger,
Expand All @@ -154,14 +152,14 @@ func NewWebAPI(
applicationLiveStateStore: alss,
commandStore: commandstore.NewStore(w, ds, sc, logger),
insightStore: is,
unregisteredAppStore: uas,
projectsInConfig: projs,
encrypter: encrypter,
appProjectCache: memorycache.NewTTLCache(ctx, 24*time.Hour, 3*time.Hour),
deploymentProjectCache: memorycache.NewTTLCache(ctx, 24*time.Hour, 3*time.Hour),
pipedProjectCache: memorycache.NewTTLCache(ctx, 24*time.Hour, 3*time.Hour),
pipedStatCache: psc,
insightCache: rediscache.NewTTLCache(rd, 3*time.Hour),
redis: rd,
insightCache: ic,
logger: logger.Named("web-api"),
}
return a
Expand Down Expand Up @@ -410,11 +408,7 @@ func (a *WebAPI) ListUnregisteredApplications(ctx context.Context, _ *webservice
return nil, err
}

// Collect all apps that belong to the project.
key := makeUnregisteredAppsCacheKey(claims.Role.ProjectId)
c := rediscache.NewHashCache(a.redis, key)
// pipedToApps assumes to be a map["piped-id"][]byte(slice of *model.ApplicationInfo encoded by encoding/gob)
pipedToApps, err := c.GetAll()
allApps, err := a.unregisteredAppStore.ListApplications(ctx, claims.Role.ProjectId)
if errors.Is(err, cache.ErrNotFound) {
return &webservice.ListUnregisteredApplicationsResponse{}, nil
}
Expand All @@ -423,28 +417,14 @@ func (a *WebAPI) ListUnregisteredApplications(ctx context.Context, _ *webservice
return nil, status.Error(codes.Internal, "Failed to get unregistered apps")
}

// Integrate all apps cached for each Piped.
allApps := make([]*model.ApplicationInfo, 0)
for _, as := range pipedToApps {
b, ok := as.([]byte)
if !ok {
return nil, status.Error(codes.Internal, "Unexpected data cached")
}
dec := gob.NewDecoder(bytes.NewReader(b))
var apps []*model.ApplicationInfo
if err := dec.Decode(&apps); err != nil {
a.logger.Error("failed to decode the unregistered apps", zap.Error(err))
return nil, status.Error(codes.Internal, "failed to decode the unregistered apps")
}
allApps = append(allApps, apps...)
}
if len(allApps) == 0 {
return &webservice.ListUnregisteredApplicationsResponse{}, nil
}

sort.Slice(allApps, func(i, j int) bool {
return allApps[i].Path < allApps[j].Path
})

return &webservice.ListUnregisteredApplicationsResponse{
Applications: allApps,
}, nil
Expand Down
4 changes: 4 additions & 0 deletions pkg/model/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ go_library(
"deployment.go",
"deployment_chain.go",
"docs.go",
"environment.pb.go",
Copy link
Member

@nghialv nghialv Mar 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please remove this since it is no longer needed.
(I think you have to delete it at your local as well.)

"event.go",
"filestore.go",
"model.go",
Expand All @@ -69,7 +70,10 @@ go_library(
importpath = "github.com/pipe-cd/pipecd/pkg/model",
visibility = ["//visibility:public"],
deps = [
"@com_github_envoyproxy_protoc_gen_validate//validate:go_default_library",
"@org_golang_google_protobuf//proto:go_default_library",
"@org_golang_google_protobuf//reflect/protoreflect:go_default_library",
"@org_golang_google_protobuf//runtime/protoimpl:go_default_library",
"@org_golang_x_crypto//bcrypt:go_default_library",
"@org_golang_x_oauth2//:go_default_library",
"@org_golang_x_oauth2//github:go_default_library",
Expand Down