From 3fb5b821457445b4cef6ff1679556b451798ddd7 Mon Sep 17 00:00:00 2001 From: Mayank Bhatt Date: Mon, 21 Aug 2023 14:39:37 -0700 Subject: [PATCH] Use Envoy sidecar for guest and datastore file transfer. - vCenter has a local Envoy process exposed over a Unix socket dedicated for host connectivity. - During requests that involve traffic to ESX, when the envoy sidecar is in use (localhost:1080) in the VC client, leverage the envoy host gateway. - Avoid keeping the connection alive since we're only using it for one request to the host. (ie. no connection pool to the host(s) being maintained here) Testing: With a Go based service using localhost:1080, used iptables to block outbound traffic to 443 (and thereby prevent ESX access), verified that guest file transfer, datastore file upload continued to work. Added unit test to verify socket creation. I think it would make sense to enhance vcsim to support this host gateway socket, but from what I could tell, the existing vcsim tests for things like datastore upload only run vcsim with an ESX - I could not find anything in place to simulate both VC and ESX. --- guest/file_manager.go | 9 ++++ guest/toolbox/client.go | 9 ++++ internal/helpers.go | 61 ++++++++++++++++++++++++++ internal/helpers_test.go | 92 ++++++++++++++++++++++++++++++++++++++++ object/datastore.go | 29 +++++++++++-- 5 files changed, 197 insertions(+), 3 deletions(-) diff --git a/guest/file_manager.go b/guest/file_manager.go index 9dc0c194b..1141f54cc 100644 --- a/guest/file_manager.go +++ b/guest/file_manager.go @@ -168,6 +168,15 @@ func (m FileManager) TransferURL(ctx context.Context, u string) (*url.URL, error return turl, nil // won't matter if the VM was powered off since the call to InitiateFileTransfer will fail } + // VC supports the use of a Unix domain socket for guest file transfers. + if internal.UsingEnvoySidecar(m.c) { + // Rewrite the URL in the format unix:// + // Reciever must use a custom dialer. + // Nil check performed above, so Host is safe to access. + return internal.HostGatewayTransferURL(turl, *vm.Runtime.Host), nil + } + + // Determine host thumbprint, address etc. to be able to trust host. props := []string{ "name", "runtime.connectionState", diff --git a/guest/toolbox/client.go b/guest/toolbox/client.go index 9d492296c..38cc5946d 100644 --- a/guest/toolbox/client.go +++ b/guest/toolbox/client.go @@ -29,6 +29,7 @@ import ( "time" "github.com/vmware/govmomi/guest" + "github.com/vmware/govmomi/internal" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/mo" @@ -284,6 +285,10 @@ func (c *Client) Download(ctx context.Context, src string) (io.ReadCloser, int64 p := soap.DefaultDownload + if internal.UsingEnvoySidecar(c.ProcessManager.Client()) { + vc = internal.ClientWithEnvoyHostGateway(vc) + } + f, n, err := vc.Download(ctx, u, &p) if err != nil { return nil, n, err @@ -341,5 +346,9 @@ func (c *Client) Upload(ctx context.Context, src io.Reader, dst string, p soap.U return err } + if internal.UsingEnvoySidecar(c.ProcessManager.Client()) { + vc = internal.ClientWithEnvoyHostGateway(vc) + } + return vc.Client.Upload(ctx, src, u, &p) } diff --git a/internal/helpers.go b/internal/helpers.go index 6c2dda0d3..41e533fd7 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -17,15 +17,25 @@ limitations under the License. package internal import ( + "context" + "fmt" "net" + "net/http" + "net/url" "os" "path" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" ) +const ( + vCenterHostGatewaySocket = "/var/run/envoy-hgw/hgw-pipe" + vCenterHostGatewaySocketEnv = "VCENTER_ENVOY_HOST_GATEWAY" +) + // InventoryPath composed of entities by Name func InventoryPath(entities []mo.ManagedEntity) string { val := "/" @@ -78,3 +88,54 @@ func UsingEnvoySidecar(c *vim25.Client) bool { } return c.URL().Hostname() == envoySidecarHost && c.URL().Scheme == "http" && c.URL().Port() == envoySidecarPort } + +// ClientWithEnvoyHostGateway clones the provided soap.Client and returns a new +// one that uses a Unix socket to leverage vCenter's local Envoy host +// gateway. +// This should be used to construct clients that talk to ESX. +// This method returns a new *vim25.Client and does not modify the original input. +// This client disables HTTP keep alives and is intended for a single round +// trip. (eg. guest file transfer, datastore file transfer) +func ClientWithEnvoyHostGateway(vc *vim25.Client) *vim25.Client { + // Override the vim client with a new one that wraps a Unix socket transport. + // Using HTTP here so secure means nothing. + sc := soap.NewClient(vc.URL(), true) + // Clone the underlying HTTP transport, only replacing the dialer logic. + transport := sc.DefaultTransport().Clone() + hostGatewaySocketPath := os.Getenv(vCenterHostGatewaySocketEnv) + if hostGatewaySocketPath == "" { + hostGatewaySocketPath = vCenterHostGatewaySocket + } + transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", hostGatewaySocketPath) + } + // We use this client for a single request, so we don't require keepalives. + transport.DisableKeepAlives = true + sc.Client = http.Client{ + Transport: transport, + } + newVC := &vim25.Client{ + Client: sc, + } + return newVC +} + +// HostGatewayTransferURL rewrites the provided URL to be suitable for use +// with the Envoy host gateway on vCenter. +// It returns a copy of the provided URL with the host, scheme rewritten as needed. +// Receivers of such URLs must typically also use ClientWithEnvoyHostGateway to +// use the appropriate http.Transport to be able to make use of the host +// gateway. +// nil input yields an uninitialized struct. +func HostGatewayTransferURL(u *url.URL, hostMoref types.ManagedObjectReference) *url.URL { + if u == nil { + return &url.URL{} + } + // Make a copy of the provided URL. + turl := *u + turl.Host = "localhost" + turl.Scheme = "http" + oldPath := turl.Path + turl.Path = fmt.Sprintf("/hgw/%s%s", hostMoref.Value, oldPath) + return &turl +} diff --git a/internal/helpers_test.go b/internal/helpers_test.go index f811d1b8d..8afe2cbcb 100644 --- a/internal/helpers_test.go +++ b/internal/helpers_test.go @@ -14,9 +14,17 @@ limitations under the License. package internal_test import ( + "crypto/rand" + "encoding/hex" "fmt" + "io" + "net" + "net/http" "net/url" + "os" + "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" @@ -24,6 +32,7 @@ import ( "github.com/vmware/govmomi/simulator/esx" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" ) func TestHostSystemManagementIPs(t *testing.T) { @@ -57,3 +66,86 @@ func TestUsingVCEnvoySidecar(t *testing.T) { require.True(t, usingSidecar) }) } + +func TestClientUsingEnvoyHostGateway(t *testing.T) { + prefix := "hgw" + suffix := ".sock" + randBytes := make([]byte, 16) + _, err := rand.Read(randBytes) + require.NoError(t, err) + + testSocketPath := filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix) + + l, err := net.Listen("unix", testSocketPath) + require.NoError(t, err) + handler := &testHTTPServer{ + expectedURL: "http://localhost/foo", + response: "Hello, Unix socket!", + t: t, + } + server := http.Server{ + Handler: handler, + } + go server.Serve(l) + defer server.Close() + defer l.Close() + + // First make sure the test server works fine, since we're starting a goroutine. + unixDialer := func(proto, addr string) (conn net.Conn, err error) { + return net.Dial("unix", testSocketPath) + } + tr := &http.Transport{ + Dial: unixDialer, + } + client := &http.Client{Transport: tr} + + require.Eventually(t, func() bool { + _, err := client.Get(handler.expectedURL) + return err == nil + }, 15*time.Second, 1*time.Second, "Expected test HTTP server to be up") + + envVar := "VCENTER_ENVOY_HOST_GATEWAY" + oldValue := os.Getenv(envVar) + defer os.Setenv(envVar, oldValue) + os.Setenv(envVar, testSocketPath) + + // Build a new client using the test unix socket. + vc := &vim25.Client{Client: soap.NewClient(&url.URL{}, true)} + newClient := internal.ClientWithEnvoyHostGateway(vc) + + // An HTTP request made using the new client should hit the server listening on the Unix socket. + resp, err := newClient.Get(handler.expectedURL) + + // ...but should successfully connect to the Unix socket set up for testing. + require.NoError(t, err) + response, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + require.Equal(t, response, []byte(handler.response)) +} + +type testHTTPServer struct { + expectedURL string + response string + t *testing.T +} + +func (t *testHTTPServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + require.Equal(t.t, "/foo", req.URL.Path) + resp.Write([]byte(t.response)) +} + +func TestRewriteURLForHostGateway(t *testing.T) { + testURL, err := url.Parse("https://foo.bar/baz?query_param=1") + require.NoError(t, err) + + hostMoref := types.ManagedObjectReference{ + Type: "HostSystem", + Value: "host-123", + } + result := internal.HostGatewayTransferURL(testURL, hostMoref) + require.Equal(t, "localhost", result.Host) + require.Equal(t, "/hgw/host-123/baz", result.Path) + values := url.Values{"query_param": []string{"1"}} + require.Equal(t, values, result.Query()) +} diff --git a/object/datastore.go b/object/datastore.go index e2586a170..77d51262a 100644 --- a/object/datastore.go +++ b/object/datastore.go @@ -27,6 +27,7 @@ import ( "path" "strings" + "github.com/vmware/govmomi/internal" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/session" "github.com/vmware/govmomi/vim25" @@ -229,8 +230,18 @@ func (d Datastore) ServiceTicket(ctx context.Context, path string, method string delete(q, "dcPath") u.RawQuery = q.Encode() + // Now that we have a host selected, take a copy of the URL. + transferURL := *u + + if internal.UsingEnvoySidecar(d.Client()) { + // Rewrite the host URL to go through the Envoy sidecar on VC. + // Reciever must use a custom dialer. + u = internal.HostGatewayTransferURL(u, host.Reference()) + } + spec := types.SessionManagerHttpServiceRequestSpec{ - Url: u.String(), + // Use the original URL (without rewrites) for the session ticket. + Url: transferURL.String(), // See SessionManagerHttpServiceRequestSpecMethod enum Method: fmt.Sprintf("http%s%s", method[0:1], strings.ToLower(method[1:])), } @@ -303,7 +314,13 @@ func (d Datastore) UploadFile(ctx context.Context, file string, path string, par if err != nil { return err } - return d.Client().UploadFile(ctx, file, u, p) + vc := d.Client() + if internal.UsingEnvoySidecar(vc) { + // Override the vim client with a new one that wraps a Unix socket transport. + // Using HTTP here so secure means nothing. + vc = internal.ClientWithEnvoyHostGateway(vc) + } + return vc.UploadFile(ctx, file, u, p) } // Download via soap.Download with an http service ticket @@ -321,7 +338,13 @@ func (d Datastore) DownloadFile(ctx context.Context, path string, file string, p if err != nil { return err } - return d.Client().DownloadFile(ctx, file, u, p) + vc := d.Client() + if internal.UsingEnvoySidecar(vc) { + // Override the vim client with a new one that wraps a Unix socket transport. + // Using HTTP here so secure means nothing. + vc = internal.ClientWithEnvoyHostGateway(vc) + } + return vc.DownloadFile(ctx, file, u, p) } // AttachedHosts returns hosts that have this Datastore attached, accessible and writable.