Skip to content

Commit

Permalink
Enable to execute shell on a build step for debugging
Browse files Browse the repository at this point in the history
Signed-off-by: Kohei Tokunaga <[email protected]>
  • Loading branch information
ktock committed Apr 20, 2022
1 parent 65f4948 commit 0e1cb07
Show file tree
Hide file tree
Showing 20 changed files with 1,396 additions and 332 deletions.
703 changes: 583 additions & 120 deletions api/services/control/control.pb.go

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions api/services/control/control.proto
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "google/protobuf/timestamp.proto";
import "github.com/moby/buildkit/solver/pb/ops.proto";
import "github.com/moby/buildkit/api/types/worker.proto";
import "github.com/moby/buildkit/frontend/gateway/pb/gateway.proto";

option (gogoproto.sizer_all) = true;
option (gogoproto.marshaler_all) = true;
Expand All @@ -19,8 +20,17 @@ service Control {
rpc Session(stream BytesMessage) returns (stream BytesMessage);
rpc ListWorkers(ListWorkersRequest) returns (ListWorkersResponse);
rpc Info(InfoRequest) returns (InfoResponse);

rpc DebugExecProcess(stream moby.buildkit.v1.frontend.ExecMessage) returns (stream moby.buildkit.v1.frontend.ExecMessage);
rpc DebugClose(DebugCloseRequest) returns (DebugCloseResponse);
}

message DebugCloseRequest {
string Ref = 1;
}

message DebugCloseResponse {}

message PruneRequest {
repeated string filter = 1;
bool all = 2;
Expand Down Expand Up @@ -62,6 +72,7 @@ message SolveRequest {
CacheOptions Cache = 8 [(gogoproto.nullable) = false];
repeated string Entitlements = 9 [(gogoproto.customtype) = "github.com/moby/buildkit/util/entitlements.Entitlement" ];
map<string, pb.Definition> FrontendInputs = 10;
bool Debug = 11;
}

message CacheOptions {
Expand Down
7 changes: 7 additions & 0 deletions client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,15 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"

"github.com/moby/buildkit/frontend/gateway/grpcclient"
gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
)

type Client struct {
conn *grpc.ClientConn
sessionDialer func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error)
execMsgs *grpcclient.MessageForwarder
}

type ClientOpt interface{}
Expand Down Expand Up @@ -150,6 +154,9 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error
c := &Client{
conn: conn,
sessionDialer: sessionDialer,
execMsgs: grpcclient.NewMessageForwarder(context.TODO(), func(ctx context.Context) (gatewaypb.LLBBridge_ExecProcessClient, error) {
return controlapi.NewControlClient(conn).DebugExecProcess(ctx)
}),
}

if tracerDelegate != nil {
Expand Down
33 changes: 33 additions & 0 deletions client/solve.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package client
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
Expand All @@ -27,6 +28,9 @@ import (
fstypes "github.com/tonistiigi/fsutil/types"
"go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"

gatewayclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/frontend/gateway/grpcclient"
)

type SolveOpt struct {
Expand All @@ -42,6 +46,7 @@ type SolveOpt struct {
AllowedEntitlements []entitlements.Entitlement
SharedSession *session.Session // TODO: refactor to better session syncing
SessionPreInitialized bool // TODO: refactor to better session syncing
Debug bool
}

type ExportEntry struct {
Expand Down Expand Up @@ -89,6 +94,9 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG

ref := identity.NewID()
eg, ctx := errgroup.WithContext(ctx)
if opt.Debug {
defer bklog.G(ctx).Infof("debug for build %q is enabled", ref)
}

statusContext, cancelStatus := context.WithCancel(context.Background())
defer cancelStatus()
Expand Down Expand Up @@ -216,6 +224,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
FrontendInputs: frontendInputs,
Cache: cacheOpt.options,
Entitlements: opt.AllowedEntitlements,
Debug: opt.Debug,
})
if err != nil {
return errors.Wrap(err, "failed to solve")
Expand Down Expand Up @@ -260,6 +269,10 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
if err != nil {
if err == io.EOF {
return nil
} else if (errors.Is(err, context.Canceled) || errors.Is(statusContext.Err(), context.Canceled)) && opt.Debug {
// We cancel watching status while the job hasn't been discarded for debugging. This situation isn't an error.
bklog.G(ctx).WithError(err).Info("canceled watching status")
return nil
}
return errors.Wrap(err, "failed to receive status")
}
Expand Down Expand Up @@ -332,6 +345,26 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
return res, nil
}

func (c *Client) DebugClose(ctx context.Context, ref string) error {
_, err := c.controlClient().DebugClose(ctx, &controlapi.DebugCloseRequest{
Ref: ref,
})
return err
}

func (c *Client) DebugExecProcess(ctx context.Context, ref, vtx string, req gatewayclient.StartRequest) (gatewayclient.ContainerProcess, error) {
// ensure message forwarder is started, only sets up stream first time called
if err := c.execMsgs.Start(); err != nil {
return nil, fmt.Errorf("failed to start execMsgs: %w", err)
}
if req.Attrs == nil {
req.Attrs = make(map[string]string)
}
req.Attrs["control.ref.debug"] = ref
req.Attrs["control.vtx.debug"] = vtx
return grpcclient.StartProcess(ctx, vtx, req, c.execMsgs)
}

func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) ([]filesync.SyncedDir, error) {
for _, d := range localDirs {
fi, err := os.Stat(d)
Expand Down
5 changes: 5 additions & 0 deletions cmd/buildctl/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ var buildCommand = cli.Command{
Name: "metadata-file",
Usage: "Output build metadata (e.g., image digest) to a file as JSON",
},
cli.BoolFlag{
Name: "debug-build",
Usage: "Allow debugging the build",
},
},
}

Expand Down Expand Up @@ -191,6 +195,7 @@ func buildAction(clicontext *cli.Context) error {
CacheImports: cacheImports,
Session: attachable,
AllowedEntitlements: allowed,
Debug: clicontext.Bool("debug-build"),
}

solveOpt.FrontendAttrs, err = build.ParseOpt(clicontext.StringSlice("opt"))
Expand Down
2 changes: 2 additions & 0 deletions cmd/buildctl/debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,7 @@ var debugCommand = cli.Command{
debug.DumpMetadataCommand,
debug.WorkersCommand,
debug.InfoCommand,
debugShellCommand,
debugCloseCommand,
},
}
72 changes: 72 additions & 0 deletions cmd/buildctl/debug_shell.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package main

import (
"context"
"fmt"
"os"

"github.com/containerd/console"
"github.com/moby/buildkit/client"
bccommon "github.com/moby/buildkit/cmd/buildctl/common"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/urfave/cli"
)

var debugShellCommand = cli.Command{
Name: "shell",
Usage: "exec shell in a vertex",
Action: debugShellAction,
}

var debugCloseCommand = cli.Command{
Name: "close",
Usage: "close a build job remaining for debugging",
Action: debugCloseAction,
}

func debugCloseAction(clicontext *cli.Context) error {
ref := clicontext.Args().Get(0)
if ref == "" {
return fmt.Errorf("ref must be specified")
}
c, err := bccommon.ResolveClient(clicontext)
if err != nil {
return err
}
ctx := clicontext.App.Metadata["context"].(context.Context)
return c.DebugClose(ctx, ref)
}

func debugShellAction(clicontext *cli.Context) error {
ref := clicontext.Args().Get(0)
vtx := clicontext.Args().Get(1)
if ref == "" || vtx == "" {
return fmt.Errorf("ref and vertex must be specified")
}
c, err := bccommon.ResolveClient(clicontext)
if err != nil {
return err
}
ctx := clicontext.App.Metadata["context"].(context.Context)
return execProcess(ctx, c, ref, vtx, gwclient.StartRequest{
Args: []string{"/bin/sh"},
Stdin: os.Stdin,
Stdout: os.Stderr,
Stderr: os.Stderr,
Tty: true,
})
}

func execProcess(ctx context.Context, c *client.Client, ref, vtx string, req gwclient.StartRequest) error {
con := console.Current()
defer con.Reset()
if err := con.SetRaw(); err != nil {
return err
}
p, err := c.DebugExecProcess(ctx, ref, vtx, req)
if err != nil {
return fmt.Errorf("failed to start process: %w", err)
}
resizeConsole(ctx, p, con)
return p.Wait()
}
13 changes: 13 additions & 0 deletions cmd/buildctl/exec_nolinux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
//go:build !linux
// +build !linux

package main

import (
"context"

"github.com/containerd/console"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
)

func resizeConsole(ctx context.Context, p gwclient.ContainerProcess, con console.Console) {}
32 changes: 32 additions & 0 deletions cmd/buildctl/exec_unix.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
//go:build linux
// +build linux

package main

import (
"context"
"os"
"os/signal"
"syscall"

"github.com/containerd/console"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
)

func resizeConsole(ctx context.Context, p gwclient.ContainerProcess, con console.Console) {
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGWINCH)
go func() {
for {
select {
case <-ch:
size, err := con.Size()
if err != nil {
continue
}
p.Resize(ctx, gwclient.WinSize{Cols: uint32(size.Width), Rows: uint32(size.Height)})
}
}
}()
ch <- syscall.SIGWINCH
}
Loading

0 comments on commit 0e1cb07

Please sign in to comment.