diff --git a/BUILD.bazel b/BUILD.bazel index 27dfd1215a..f46d78d351 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -43,6 +43,7 @@ genrule( # gazelle:exclude pkg/app/helloworld/service/service.pb.validate.go # gazelle:exclude pkg/app/api/service/webservice/service.pb.validate.go # gazelle:exclude pkg/app/api/service/pipedservice/service.pb.validate.go +# gazelle:exclude pkg/app/api/service/apiservice/service.pb.validate.go # gazelle:exclude pkg/model/apikey.pb.validate.go # gazelle:exclude pkg/model/application.pb.validate.go # gazelle:exclude pkg/model/application_live_state.pb.validate.go @@ -51,6 +52,7 @@ genrule( # gazelle:exclude pkg/model/deployment.pb.validate.go # gazelle:exclude pkg/model/environment.pb.validate.go # gazelle:exclude pkg/model/event.pb.validate.go +# gazelle:exclude pkg/model/insight.pb.validate.go # gazelle:exclude pkg/model/logblock.pb.validate.go # gazelle:exclude pkg/model/piped.pb.validate.go # gazelle:exclude pkg/model/piped_stats.pb.validate.go diff --git a/cmd/image.bzl b/cmd/image.bzl index 43d59d2979..f7f56f3a1c 100644 --- a/cmd/image.bzl +++ b/cmd/image.bzl @@ -2,6 +2,7 @@ def all_images(): cmds = { "piped": "piped", "pipecd": "pipecd", + "pipectl": "pipectl", "helloworld": "helloworld", } images = {} diff --git a/cmd/pipecd/BUILD.bazel b/cmd/pipecd/BUILD.bazel index cba832d612..fcfe558457 100644 --- a/cmd/pipecd/BUILD.bazel +++ b/cmd/pipecd/BUILD.bazel @@ -12,6 +12,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//pkg/admin:go_default_library", + "//pkg/app/api/apikeyverifier:go_default_library", "//pkg/app/api/applicationlivestatestore:go_default_library", "//pkg/app/api/authhandler:go_default_library", "//pkg/app/api/commandstore:go_default_library", diff --git a/cmd/pipecd/server.go b/cmd/pipecd/server.go index 9b8f08fce2..9f84f46975 100644 --- a/cmd/pipecd/server.go +++ b/cmd/pipecd/server.go @@ -29,6 +29,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/pipe-cd/pipe/pkg/admin" + "github.com/pipe-cd/pipe/pkg/app/api/apikeyverifier" "github.com/pipe-cd/pipe/pkg/app/api/applicationlivestatestore" "github.com/pipe-cd/pipe/pkg/app/api/authhandler" "github.com/pipe-cd/pipe/pkg/app/api/commandstore" @@ -65,6 +66,7 @@ type server struct { pipedAPIPort int webAPIPort int httpPort int + apiPort int adminPort int staticDir string cacheAddress string @@ -87,6 +89,7 @@ func NewServerCommand() *cobra.Command { pipedAPIPort: 9080, webAPIPort: 9081, httpPort: 9082, + apiPort: 9083, adminPort: 9085, staticDir: "pkg/app/web/public_files", cacheAddress: "cache:6379", @@ -101,6 +104,7 @@ func NewServerCommand() *cobra.Command { cmd.Flags().IntVar(&s.pipedAPIPort, "piped-api-port", s.pipedAPIPort, "The port number used to run a grpc server that serving serves incoming piped requests.") cmd.Flags().IntVar(&s.webAPIPort, "web-api-port", s.webAPIPort, "The port number used to run a grpc server that serves incoming web requests.") cmd.Flags().IntVar(&s.httpPort, "http-port", s.httpPort, "The port number used to run a http server that serves incoming http requests such as auth callbacks or webhook events.") + cmd.Flags().IntVar(&s.apiPort, "api-port", s.apiPort, "The port number used to run a grpc server for external apis.") cmd.Flags().IntVar(&s.adminPort, "admin-port", s.adminPort, "The port number used to run a HTTP server for admin tasks such as metrics, healthz.") cmd.Flags().StringVar(&s.staticDir, "static-dir", s.staticDir, "The directory where contains static assets.") cmd.Flags().StringVar(&s.cacheAddress, "cache-address", s.cacheAddress, "The address to cache service.") @@ -136,11 +140,6 @@ func (s *server) run(ctx context.Context, t cli.Telemetry) error { } t.Logger.Info("successfully loaded control-plane configuration") - var ( - pipedAPIServer *rpc.Server - webAPIServer *rpc.Server - ) - ds, err := createDatastore(ctx, cfg, t.Logger) if err != nil { t.Logger.Error("failed to create datastore", zap.Error(err)) @@ -204,9 +203,37 @@ func (s *server) run(ctx context.Context, t cli.Telemetry) error { opts = append(opts, rpc.WithGRPCReflection()) } - pipedAPIServer = rpc.NewServer(service, opts...) + server := rpc.NewServer(service, opts...) + group.Go(func() error { + return server.Run(ctx) + }) + } + + // Start a gRPC server for handling external API requests. + { + var ( + verifier = apikeyverifier.NewVerifier( + ctx, + datastore.NewAPIKeyStore(ds), + t.Logger, + ) + service = grpcapi.NewAPI(ds, cmds, t.Logger) + opts = []rpc.Option{ + rpc.WithPort(s.apiPort), + rpc.WithGracePeriod(s.gracePeriod), + rpc.WithLogger(t.Logger), + rpc.WithLogUnaryInterceptor(t.Logger), + rpc.WithAPIKeyAuthUnaryInterceptor(verifier, t.Logger), + rpc.WithRequestValidationUnaryInterceptor(), + } + ) + if s.tls { + opts = append(opts, rpc.WithTLS(s.certFile, s.keyFile)) + } + + server := rpc.NewServer(service, opts...) group.Go(func() error { - return pipedAPIServer.Run(ctx) + return server.Run(ctx) }) } @@ -239,9 +266,9 @@ func (s *server) run(ctx context.Context, t cli.Telemetry) error { opts = append(opts, rpc.WithGRPCReflection()) } - webAPIServer = rpc.NewServer(service, opts...) + server := rpc.NewServer(service, opts...) group.Go(func() error { - return webAPIServer.Run(ctx) + return server.Run(ctx) }) } diff --git a/cmd/pipectl/BUILD.bazel b/cmd/pipectl/BUILD.bazel new file mode 100644 index 0000000000..25c5d9a985 --- /dev/null +++ b/cmd/pipectl/BUILD.bazel @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("//bazel:image.bzl", "app_image") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/pipe-cd/pipe/cmd/pipectl", + visibility = ["//visibility:private"], + deps = ["//pkg/cli:go_default_library"], +) + +go_binary( + name = "pipectl", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +app_image( + name = "pipectl_app", + binary = ":pipectl", + repository = "pipectl", + visibility = ["//visibility:public"], +) diff --git a/cmd/pipectl/OWNERS b/cmd/pipectl/OWNERS new file mode 100644 index 0000000000..dba584ec27 --- /dev/null +++ b/cmd/pipectl/OWNERS @@ -0,0 +1,2 @@ +labels: + - area/pipectl diff --git a/cmd/pipectl/main.go b/cmd/pipectl/main.go new file mode 100644 index 0000000000..f68c461021 --- /dev/null +++ b/cmd/pipectl/main.go @@ -0,0 +1,34 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "log" + + "github.com/pipe-cd/pipe/pkg/cli" +) + +func main() { + app := cli.NewApp( + "pipectl", + "The command line tool for PipeCD.", + ) + + app.AddCommands() + + if err := app.Run(); err != nil { + log.Fatal(err) + } +} diff --git a/manifests/pipecd/templates/deployment.yaml b/manifests/pipecd/templates/deployment.yaml index 05f7ed7d4c..3a4e8cf3f3 100644 --- a/manifests/pipecd/templates/deployment.yaml +++ b/manifests/pipecd/templates/deployment.yaml @@ -108,6 +108,9 @@ spec: - name: http containerPort: 9082 protocol: TCP + - name: api + containerPort: 9083 + protocol: TCP - name: admin containerPort: 9085 protocol: TCP diff --git a/manifests/pipecd/templates/envoy-configmap.yaml b/manifests/pipecd/templates/envoy-configmap.yaml index c0fa54d98e..269c3bcf06 100644 --- a/manifests/pipecd/templates/envoy-configmap.yaml +++ b/manifests/pipecd/templates/envoy-configmap.yaml @@ -68,6 +68,11 @@ data: grpc: route: cluster: server-web-api + - match: + prefix: /pipe.api.service.apiservice.APIService/ + grpc: + route: + cluster: server-api - match: prefix: / route: @@ -111,6 +116,20 @@ data: socket_address: address: {{ include "pipecd.fullname" . }}-server port_value: 9081 + - name: server-api + http2_protocol_options: {} + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: server-api + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ include "pipecd.fullname" . }}-server + port_value: 9083 - name: server-http #http2_protocol_options: {} connect_timeout: 0.25s diff --git a/manifests/pipecd/templates/service.yaml b/manifests/pipecd/templates/service.yaml index 3a8a0a8bbe..e77d198d49 100644 --- a/manifests/pipecd/templates/service.yaml +++ b/manifests/pipecd/templates/service.yaml @@ -66,6 +66,9 @@ spec: - name: http port: 9082 targetPort: http + - name: api + port: 9083 + targetPort: api - name: admin port: 9085 targetPort: admin diff --git a/pkg/app/api/grpcapi/BUILD.bazel b/pkg/app/api/grpcapi/BUILD.bazel index 1d3f0b6d6b..4682943851 100644 --- a/pkg/app/api/grpcapi/BUILD.bazel +++ b/pkg/app/api/grpcapi/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "api.go", "deployment_config_templates.go", "piped_api.go", "web_api.go", @@ -13,6 +14,7 @@ go_library( deps = [ "//pkg/app/api/applicationlivestatestore:go_default_library", "//pkg/app/api/commandstore:go_default_library", + "//pkg/app/api/service/apiservice:go_default_library", "//pkg/app/api/service/pipedservice:go_default_library", "//pkg/app/api/service/webservice:go_default_library", "//pkg/app/api/stagelogstore:go_default_library", @@ -36,6 +38,7 @@ go_test( name = "go_default_test", size = "small", srcs = [ + "api_test.go", "piped_api_test.go", "web_api_test.go", ], diff --git a/pkg/app/api/grpcapi/api.go b/pkg/app/api/grpcapi/api.go new file mode 100644 index 0000000000..ecd7bbb0eb --- /dev/null +++ b/pkg/app/api/grpcapi/api.go @@ -0,0 +1,67 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpcapi + +import ( + "context" + + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/pipe-cd/pipe/pkg/app/api/commandstore" + "github.com/pipe-cd/pipe/pkg/app/api/service/apiservice" + "github.com/pipe-cd/pipe/pkg/datastore" +) + +// API implements the behaviors for the gRPC definitions of API. +type API struct { + applicationStore datastore.ApplicationStore + deploymentStore datastore.DeploymentStore + pipedStore datastore.PipedStore + commandStore commandstore.Store + + logger *zap.Logger +} + +// NewAPI creates a new API instance. +func NewAPI( + ds datastore.DataStore, + cmds commandstore.Store, + logger *zap.Logger, +) *API { + a := &API{ + applicationStore: datastore.NewApplicationStore(ds), + deploymentStore: datastore.NewDeploymentStore(ds), + pipedStore: datastore.NewPipedStore(ds), + commandStore: cmds, + logger: logger.Named("api"), + } + return a +} + +// Register registers all handling of this service into the specified gRPC server. +func (a *API) Register(server *grpc.Server) { + apiservice.RegisterAPIServiceServer(server, a) +} + +func (a *API) AddApplication(_ context.Context, _ *apiservice.AddApplicationRequest) (*apiservice.AddApplicationResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} + +func (a *API) SyncApplication(_ context.Context, _ *apiservice.SyncApplicationRequest) (*apiservice.SyncApplicationResponse, error) { + return nil, status.Error(codes.Unimplemented, "") +} diff --git a/pkg/app/api/grpcapi/api_test.go b/pkg/app/api/grpcapi/api_test.go new file mode 100644 index 0000000000..99a44504f1 --- /dev/null +++ b/pkg/app/api/grpcapi/api_test.go @@ -0,0 +1,15 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpcapi diff --git a/pkg/app/api/service/apiservice/BUILD.bazel b/pkg/app/api/service/apiservice/BUILD.bazel new file mode 100644 index 0000000000..99581bb2b7 --- /dev/null +++ b/pkg/app/api/service/apiservice/BUILD.bazel @@ -0,0 +1,37 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel:pgv_go_proto.bzl", "pgv_go_proto_library") + +proto_library( + name = "apiservice_proto", + srcs = ["service.proto"], + visibility = ["//visibility:public"], + # keep + deps = [ + "//pkg/model:model_proto", + "@com_github_envoyproxy_protoc_gen_validate//validate:validate_proto", + ], +) + +pgv_go_proto_library( + name = "apiservice_go_proto", + compilers = ["@io_bazel_rules_go//proto:go_grpc"], + importpath = "github.com/pipe-cd/pipe/pkg/app/api/service/apiservice", + proto = ":apiservice_proto", + visibility = ["//visibility:public"], + deps = [ + "//pkg/model:go_default_library", + ], +) + +go_library( + name = "go_default_library", + srcs = ["client.go"], + embed = [":apiservice_go_proto"], + importpath = "github.com/pipe-cd/pipe/pkg/app/api/service/apiservice", + visibility = ["//visibility:public"], + deps = [ + "//pkg/rpc/rpcclient:go_default_library", + "@org_golang_google_grpc//:go_default_library", + ], +) diff --git a/pkg/app/api/service/apiservice/client.go b/pkg/app/api/service/apiservice/client.go new file mode 100644 index 0000000000..ad2c581da3 --- /dev/null +++ b/pkg/app/api/service/apiservice/client.go @@ -0,0 +1,48 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiservice + +import ( + "context" + + "google.golang.org/grpc" + + "github.com/pipe-cd/pipe/pkg/rpc/rpcclient" +) + +type Client interface { + APIServiceClient + Close() error +} + +type client struct { + APIServiceClient + conn *grpc.ClientConn +} + +func NewClient(ctx context.Context, addr string, opts ...rpcclient.DialOption) (Client, error) { + conn, err := rpcclient.DialContext(ctx, addr, opts...) + if err != nil { + return nil, err + } + return &client{ + APIServiceClient: NewAPIServiceClient(conn), + conn: conn, + }, nil +} + +func (c *client) Close() error { + return c.conn.Close() +} diff --git a/pkg/app/api/service/apiservice/service.proto b/pkg/app/api/service/apiservice/service.proto new file mode 100644 index 0000000000..56274a39bb --- /dev/null +++ b/pkg/app/api/service/apiservice/service.proto @@ -0,0 +1,50 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package pipe.api.service.apiservice; +option go_package = "github.com/pipe-cd/pipe/pkg/app/api/service/apiservice"; + +import "validate/validate.proto"; +import "pkg/model/common.proto"; +import "pkg/model/application.proto"; + +// APIService contains all RPC definitions for external service, pipectl. +// All of these RPCs are authenticated by using API key. +service APIService { + rpc AddApplication(AddApplicationRequest) returns (AddApplicationResponse) {} + rpc SyncApplication(SyncApplicationRequest) returns (SyncApplicationResponse) {} +} + +message AddApplicationRequest { + string name = 1 [(validate.rules).string.min_len = 1]; + string env_id = 2 [(validate.rules).string.min_len = 1]; + string piped_id = 3 [(validate.rules).string.min_len = 1]; + model.ApplicationGitPath git_path = 4 [(validate.rules).message.required = true]; + model.ApplicationKind kind = 5 [(validate.rules).enum.defined_only = true]; + string cloud_provider = 6 [(validate.rules).string.min_len = 1]; +} + +message AddApplicationResponse { + string application_id = 1 [(validate.rules).string.min_len = 1]; +} + +message SyncApplicationRequest { + string application_id = 1 [(validate.rules).string.min_len = 1]; +} + +message SyncApplicationResponse { + string command_id = 1; +} diff --git a/pkg/datastore/apikey.go b/pkg/datastore/apikey.go index 0d153cafe5..662d12e17b 100644 --- a/pkg/datastore/apikey.go +++ b/pkg/datastore/apikey.go @@ -32,6 +32,7 @@ var ( type APIKeyStore interface { AddAPIKey(ctx context.Context, k *model.APIKey) error + GetAPIKey(ctx context.Context, id string) (*model.APIKey, error) DisableAPIKey(ctx context.Context, id, projectID string) error ListAPIKeys(ctx context.Context, opts ListOptions) ([]*model.APIKey, error) } @@ -64,6 +65,14 @@ func (s *apiKeyStore) AddAPIKey(ctx context.Context, k *model.APIKey) error { return s.ds.Create(ctx, apiKeyModelKind, k.Id, k) } +func (s *apiKeyStore) GetAPIKey(ctx context.Context, id string) (*model.APIKey, error) { + var entity model.APIKey + if err := s.ds.Get(ctx, apiKeyModelKind, id, &entity); err != nil { + return nil, err + } + return &entity, nil +} + func (s *apiKeyStore) ListAPIKeys(ctx context.Context, opts ListOptions) ([]*model.APIKey, error) { it, err := s.ds.Find(ctx, apiKeyModelKind, opts) if err != nil {