Directly use piped model to send to web client

pkg/app/api/api/fake_web_api.go

@@ -111,7 +111,7 @@ func (a *FakeWebAPI) DisablePiped(ctx context.Context, req *webservice.DisablePi
 
 func (a *FakeWebAPI) ListPipeds(ctx context.Context, req *webservice.ListPipedsRequest) (*webservice.ListPipedsResponse, error) {
 	now := time.Now()
-	pipeds := []*webservice.Piped{
+	pipeds := []*model.Piped{
 		{
 			Id:        "492220b1-c080-4781-9e55-7e278760e0ef",
 			Desc:      "piped for debug 1",
@@ -134,9 +134,9 @@ func (a *FakeWebAPI) ListPipeds(ctx context.Context, req *webservice.ListPipedsR
 		},
 	}
 	if req.WithStatus {
-		pipeds[0].Status = webservice.PipedConnectionStatus_PIPED_CONNECTION_ONLINE
-		pipeds[1].Status = webservice.PipedConnectionStatus_PIPED_CONNECTION_ONLINE
-		pipeds[2].Status = webservice.PipedConnectionStatus_PIPED_CONNECTION_OFFLINE
+		pipeds[0].Status = model.Piped_ONLINE
+		pipeds[1].Status = model.Piped_ONLINE
+		pipeds[2].Status = model.Piped_OFFLINE
 	}
 
 	return &webservice.ListPipedsResponse{
@@ -147,9 +147,10 @@ func (a *FakeWebAPI) ListPipeds(ctx context.Context, req *webservice.ListPipedsR
 func (a *FakeWebAPI) GetPiped(ctx context.Context, req *webservice.GetPipedRequest) (*webservice.GetPipedResponse, error) {
 	now := time.Now()
 	return &webservice.GetPipedResponse{
-		Piped: &webservice.Piped{
+		Piped: &model.Piped{
 			Id:        "492220b1-c080-4781-9e55-7e278760e0ef",
 			Desc:      "piped for debug 1",
+			KeyHash:   "redacted",
 			ProjectId: fakeProjectID,
 			Version:   "debug-version",
 			StartedAt: now.Add(-30 * time.Minute).Unix(),

pkg/app/api/api/piped_api.go

@@ -86,7 +86,9 @@ func (a *PipedAPI) ReportPipedMeta(ctx context.Context, req *pipedservice.Report
 	}
 
 	now := time.Now().Unix()
-	if err = a.pipedStore.UpdatePiped(ctx, pipedID, datastore.PipedMetadataUpdater(req.CloudProviders, req.Repositories, req.Version, now)); err != nil {
+	connStatus := model.Piped_ONLINE
+
+	if err = a.pipedStore.UpdatePiped(ctx, pipedID, datastore.PipedMetadataUpdater(req.CloudProviders, req.Repositories, connStatus, req.Version, now)); err != nil {
 		switch err {
 		case datastore.ErrNotFound:
 			return nil, status.Error(codes.InvalidArgument, "piped is not found")

pkg/app/api/api/web_api.go

@@ -138,6 +138,7 @@ func (a *WebAPI) RegisterPiped(ctx context.Context, req *webservice.RegisterPipe
 		Desc:      req.Desc,
 		KeyHash:   keyHash,
 		ProjectId: claims.Role.ProjectId,
+		Status:    model.Piped_OFFLINE,
 	}
 	err = a.pipedStore.AddPiped(ctx, &piped)
 	if errors.Is(err, datastore.ErrAlreadyExists) {
@@ -218,6 +219,7 @@ func (a *WebAPI) updatePiped(ctx context.Context, pipedID string, updater func(c
 	return nil
 }
 
+// TODO: Consider using piped-stats to decide piped connection status.
 func (a *WebAPI) ListPipeds(ctx context.Context, req *webservice.ListPipedsRequest) (*webservice.ListPipedsResponse, error) {
 	claims, err := rpcauth.ExtractClaims(ctx)
 	if err != nil {
@@ -242,22 +244,20 @@ func (a *WebAPI) ListPipeds(ctx context.Context, req *webservice.ListPipedsReque
 			})
 		}
 	}
+
 	pipeds, err := a.pipedStore.ListPipeds(ctx, opts)
 	if err != nil {
 		a.logger.Error("failed to get pipeds", zap.Error(err))
 		return nil, status.Error(codes.Internal, "failed to get pipeds")
 	}
-	// Returning a response that does not contain sensitive data like KeyHash is more safer.
-	webPipeds := make([]*webservice.Piped, len(pipeds))
+
+	// Redact all sensitive data inside piped message before sending to the client.
 	for i := range pipeds {
-		webPipeds[i] = webservice.MakePiped(pipeds[i])
-		if req.WithStatus {
-			// TODO: While reducing the number of query and get ping connection status from piped_stats
-			webPipeds[i].Status = webservice.PipedConnectionStatus_PIPED_CONNECTION_ONLINE
-		}
+		pipeds[i].RedactSensitiveData()
 	}
+
 	return &webservice.ListPipedsResponse{
-		Pipeds: webPipeds,
+		Pipeds: pipeds,
 	}, nil
 }
 
@@ -266,8 +266,12 @@ func (a *WebAPI) GetPiped(ctx context.Context, req *webservice.GetPipedRequest)
 	if err != nil {
 		return nil, err
 	}
+
+	// Redact all sensitive data inside piped message before sending to the client.
+	piped.RedactSensitiveData()
+
 	return &webservice.GetPipedResponse{
-		Piped: webservice.MakePiped(piped),
+		Piped: piped,
 	}, nil
 }
 

pkg/app/api/service/webservice/BUILD.bazel

@@ -31,7 +31,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "client.go",
-        "model.go",
+        "service.go",
         "service.pb.auth.go",
     ],
     embed = [":webservice_go_proto"],

pkg/app/api/service/webservice/model.go → pkg/app/api/service/webservice/service.go

@@ -14,29 +14,6 @@
 
 package webservice
 
-import (
-	"github.com/pipe-cd/pipe/pkg/model"
-)
-
-func MakePiped(input *model.Piped) *Piped {
-	if input == nil {
-		return nil
-	}
-	return &Piped{
-		Id:             input.Id,
-		Name:           input.Name,
-		Desc:           input.Desc,
-		ProjectId:      input.ProjectId,
-		Version:        input.Version,
-		StartedAt:      input.StartedAt,
-		CloudProviders: input.CloudProviders,
-		Repositories:   input.Repositories,
-		Disabled:       input.Disabled,
-		CreatedAt:      input.CreatedAt,
-		UpdatedAt:      input.UpdatedAt,
-	}
-}
-
 func (t *DeploymentConfigTemplate) HasLabel(label DeploymentConfigTemplateLabel) bool {
 	for _, l := range t.Labels {
 		if l == label {

pkg/app/api/service/webservice/service.proto

@@ -29,41 +29,6 @@ import "pkg/model/piped.proto";
 import "pkg/model/role.proto";
 import "google/protobuf/wrappers.proto";
 
-// Note: Does copy Piped message from pkg/model/piped.proto and exclude sensitive data because Piped message for web should not contain it.
-message Piped {
-    // The generated unique identifier.
-    string id = 1 [(validate.rules).string.min_len = 1];
-    // The name of the piped.
-    string name = 2 [(validate.rules).string.min_len = 1];
-    // The additional description about the piped.
-    string desc = 3;
-    // The ID of the project this environment belongs to.
-    string project_id = 4 [(validate.rules).string.min_len = 1];
-    // The latest connection status to piped.
-    PipedConnectionStatus status = 5 [(validate.rules).enum.defined_only = true];
-
-    // Currently running version.
-    string version = 7;
-    // Unix time when the piped is started up.
-    int64 started_at = 8;
-    // List of configured cloud providers.
-    repeated pipe.model.Piped.CloudProvider cloud_providers = 9;
-    // List of configured repositories.
-    repeated pipe.model.ApplicationGitRepository repositories = 10;
-
-    // Whether the piped is disabled or not.
-    bool disabled = 13;
-    // Unix time when the piped is created.
-    int64 created_at = 14 [(validate.rules).int64.gt = 0];
-    // Unix time of the last time when the piped is updated.
-    int64 updated_at = 15 [(validate.rules).int64.gt = 0];
-}
-
-enum PipedConnectionStatus {
-    PIPED_CONNECTION_ONLINE = 0;
-    PIPED_CONNECTION_OFFLINE = 1;
-}
-
 // WebService contains all RPC definitions for web client.
 // All of these RPCs are only called by web client and authenticated by using ID_TOKEN.
 service WebService {
@@ -167,15 +132,15 @@ message ListPipedsRequest {
 }
 
 message ListPipedsResponse {
-    repeated Piped pipeds = 1;
+    repeated model.Piped pipeds = 1;
 }
 
 message GetPipedRequest {
     string piped_id = 1 [(validate.rules).string.min_len = 1];
 }
 
 message GetPipedResponse {
-    Piped piped = 1;
+    model.Piped piped = 1;
 }
 
 message AddEnvironmentRequest {

pkg/app/web/src/__fixtures__/dummy-piped.ts

@@ -1,4 +1,4 @@
-import { Piped, PipedConnectionStatus } from "../modules/pipeds";
+import { Piped } from "../modules/pipeds";
 
 export const dummyPiped: Piped = {
   cloudProvidersList: [],
@@ -12,5 +12,5 @@ export const dummyPiped: Piped = {
   startedAt: 0,
   updatedAt: 0,
   version: "v0.1",
-  status: PipedConnectionStatus.PIPED_CONNECTION_ONLINE,
+  status: Piped_ONLINE,
 };

pkg/app/web/src/modules/pipeds.ts

@@ -3,7 +3,7 @@ import {
   createAsyncThunk,
   createEntityAdapter,
 } from "@reduxjs/toolkit";
-import { Piped as PipedModel } from "pipe/pkg/app/web/api_client/service_pb";
+import { Piped as PipedModel } from "pipe/pkg/app/web/model/piped_pb";
 import * as pipedsApi from "../api/piped";
 
 export type Piped = Required<PipedModel.AsObject>;
@@ -91,4 +91,3 @@ export const pipedsSlice = createSlice({
 });
 
 export const { clearRegisteredPipedInfo } = pipedsSlice.actions;
-export { PipedConnectionStatus } from "pipe/pkg/app/web/api_client/service_pb";

pkg/datastore/pipedstore.go

@@ -27,10 +27,11 @@ var (
 	pipedFactory = func() interface{} {
 		return &model.Piped{}
 	}
-	PipedMetadataUpdater = func(cloudProviders []*model.Piped_CloudProvider, repos []*model.ApplicationGitRepository, version string, startedAt int64) func(piped *model.Piped) error {
+	PipedMetadataUpdater = func(cloudProviders []*model.Piped_CloudProvider, repos []*model.ApplicationGitRepository, status model.Piped_ConnectionStatus, version string, startedAt int64) func(piped *model.Piped) error {
 		return func(piped *model.Piped) error {
 			piped.CloudProviders = cloudProviders
 			piped.Repositories = repos
+			piped.Status = status
 			piped.Version = version
 			piped.StartedAt = startedAt
 			return nil

pkg/model/piped.go

@@ -23,7 +23,8 @@ import (
 )
 
 const (
-	pipedKeyLength = 50
+	pipedKeyLength  = 50
+	redactedMessage = "redacted"
 )
 
 // GeneratePipedKey generates a new key for piped.
@@ -75,3 +76,7 @@ func generateRandomString(n int) string {
 
 	return *(*string)(unsafe.Pointer(&b))
 }
+
+func (p *Piped) RedactSensitiveData() {
+	p.KeyHash = redactedMessage
+}

pkg/model/piped.proto

@@ -26,6 +26,11 @@ message Piped {
         string type = 2 [(validate.rules).string.min_len = 1];
     }
 
+    enum ConnectionStatus {
+        ONLINE = 0;
+        OFFLINE = 1;
+    }
+
     // The generated unique identifier.
     string id = 1 [(validate.rules).string.min_len = 1];
     // The name of the piped.
@@ -46,6 +51,8 @@ message Piped {
     repeated CloudProvider cloud_providers = 9;
     // List of configured repositories.
     repeated ApplicationGitRepository repositories = 10;
+    // The latest connection status of piped.
+    ConnectionStatus status = 11 [(validate.rules).enum.defined_only = true];
 
     // Whether the piped is disabled or not.
     bool disabled = 13;