Use Envoy sidecar for guest and datastore file transfer.

- vCenter has a local Envoy process exposed over a Unix socket dedicated
  for host connectivity.
- During requests that involve traffic to ESX, when the envoy sidecar is
  in use (localhost:1080) in the VC client, leverage the envoy host
  gateway.
- Avoid keeping the connection alive since we're only using it for one
  request to the host. (ie. no connection pool to the host(s) being
  maintained here)

Testing:

With a Go based service using localhost:1080, used iptables to block
outbound traffic to 443 (and thereby prevent ESX access), verified that
guest file transfer, datastore file upload continued to work.

Added unit test to verify socket creation.

I think it would make sense to enhance vcsim to support this host
gateway socket, but from what I could tell, the existing vcsim tests for
things like datastore upload only run vcsim with an ESX - I could not
find anything in place to simulate both VC and ESX.

guest/file_manager.go

@@ -168,6 +168,15 @@ func (m FileManager) TransferURL(ctx context.Context, u string) (*url.URL, error
 		return turl, nil // won't matter if the VM was powered off since the call to InitiateFileTransfer will fail
 	}
 
+	// VC supports the use of a Unix domain socket for guest file transfers.
+	if internal.UsingEnvoySidecar(m.c) {
+		// Rewrite the URL in the format unix://
+		// Reciever must use a custom dialer.
+		// Nil check performed above, so Host is safe to access.
+		return internal.HostGatewayTransferURL(turl, *vm.Runtime.Host), nil
+	}
+
+	// Determine host thumbprint, address etc. to be able to trust host.
 	props := []string{
 		"name",
 		"runtime.connectionState",

guest/toolbox/client.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/vmware/govmomi/guest"
+	"github.com/vmware/govmomi/internal"
 	"github.com/vmware/govmomi/property"
 	"github.com/vmware/govmomi/vim25"
 	"github.com/vmware/govmomi/vim25/mo"
@@ -284,6 +285,10 @@ func (c *Client) Download(ctx context.Context, src string) (io.ReadCloser, int64
 
 	p := soap.DefaultDownload
 
+	if internal.UsingEnvoySidecar(c.ProcessManager.Client()) {
+		vc = internal.ClientWithEnvoyHostGateway(vc)
+	}
+
 	f, n, err := vc.Download(ctx, u, &p)
 	if err != nil {
 		return nil, n, err
@@ -341,5 +346,9 @@ func (c *Client) Upload(ctx context.Context, src io.Reader, dst string, p soap.U
 		return err
 	}
 
+	if internal.UsingEnvoySidecar(c.ProcessManager.Client()) {
+		vc = internal.ClientWithEnvoyHostGateway(vc)
+	}
+
 	return vc.Client.Upload(ctx, src, u, &p)
 }

internal/helpers.go

@@ -17,15 +17,25 @@ limitations under the License.
 package internal
 
 import (
+	"context"
+	"fmt"
 	"net"
+	"net/http"
+	"net/url"
 	"os"
 	"path"
 
 	"github.com/vmware/govmomi/vim25"
 	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/soap"
 	"github.com/vmware/govmomi/vim25/types"
 )
 
+const (
+	vCenterHostGatewaySocket    = "/var/run/envoy-hgw/hgw-pipe"
+	vCenterHostGatewaySocketEnv = "VCENTER_ENVOY_HOST_GATEWAY"
+)
+
 // InventoryPath composed of entities by Name
 func InventoryPath(entities []mo.ManagedEntity) string {
 	val := "/"
@@ -78,3 +88,54 @@ func UsingEnvoySidecar(c *vim25.Client) bool {
 	}
 	return c.URL().Hostname() == envoySidecarHost && c.URL().Scheme == "http" && c.URL().Port() == envoySidecarPort
 }
+
+// ClientWithEnvoyHostGateway clones the provided soap.Client and returns a new
+// one that uses a Unix socket to leverage vCenter's local Envoy host
+// gateway.
+// This should be used to construct clients that talk to ESX.
+// This method returns a new *vim25.Client and does not modify the original input.
+// This client disables HTTP keep alives and is intended for a single round
+// trip. (eg. guest file transfer, datastore file transfer)
+func ClientWithEnvoyHostGateway(vc *vim25.Client) *vim25.Client {
+	// Override the vim client with a new one that wraps a Unix socket transport.
+	// Using HTTP here so secure means nothing.
+	sc := soap.NewClient(vc.URL(), true)
+	// Clone the underlying HTTP transport, only replacing the dialer logic.
+	transport := sc.DefaultTransport().Clone()
+	hostGatewaySocketPath := os.Getenv(vCenterHostGatewaySocketEnv)
+	if hostGatewaySocketPath == "" {
+		hostGatewaySocketPath = vCenterHostGatewaySocket
+	}
+	transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) {
+		return net.Dial("unix", hostGatewaySocketPath)
+	}
+	// We use this client for a single request, so we don't require keepalives.
+	transport.DisableKeepAlives = true
+	sc.Client = http.Client{
+		Transport: transport,
+	}
+	newVC := &vim25.Client{
+		Client: sc,
+	}
+	return newVC
+}
+
+// HostGatewayTransferURL rewrites the provided URL to be suitable for use
+// with the Envoy host gateway on vCenter.
+// It returns a copy of the provided URL with the host, scheme rewritten as needed.
+// Receivers of such URLs must typically also use ClientWithEnvoyHostGateway to
+// use the appropriate http.Transport to be able to make use of the host
+// gateway.
+// nil input yields an uninitialized struct.
+func HostGatewayTransferURL(u *url.URL, hostMoref types.ManagedObjectReference) *url.URL {
+	if u == nil {
+		return &url.URL{}
+	}
+	// Make a copy of the provided URL.
+	turl := *u
+	turl.Host = "localhost"
+	turl.Scheme = "http"
+	oldPath := turl.Path
+	turl.Path = fmt.Sprintf("/hgw/%s%s", hostMoref.Value, oldPath)
+	return &turl
+}

internal/helpers_test.go

@@ -14,16 +14,25 @@ limitations under the License.
 package internal_test
 
 import (
+	"crypto/rand"
+	"encoding/hex"
 	"fmt"
+	"io"
+	"net"
+	"net/http"
 	"net/url"
+	"os"
+	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/vmware/govmomi/internal"
 	"github.com/vmware/govmomi/simulator/esx"
 	"github.com/vmware/govmomi/vim25"
 	"github.com/vmware/govmomi/vim25/soap"
+	"github.com/vmware/govmomi/vim25/types"
 )
 
 func TestHostSystemManagementIPs(t *testing.T) {
@@ -57,3 +66,86 @@ func TestUsingVCEnvoySidecar(t *testing.T) {
 		require.True(t, usingSidecar)
 	})
 }
+
+func TestClientUsingEnvoyHostGateway(t *testing.T) {
+	prefix := "hgw"
+	suffix := ".sock"
+	randBytes := make([]byte, 16)
+	_, err := rand.Read(randBytes)
+	require.NoError(t, err)
+
+	testSocketPath := filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix)
+
+	l, err := net.Listen("unix", testSocketPath)
+	require.NoError(t, err)
+	handler := &testHTTPServer{
+		expectedURL: "http://localhost/foo",
+		response:    "Hello, Unix socket!",
+		t:           t,
+	}
+	server := http.Server{
+		Handler: handler,
+	}
+	go server.Serve(l)
+	defer server.Close()
+	defer l.Close()
+
+	// First make sure the test server works fine, since we're starting a goroutine.
+	unixDialer := func(proto, addr string) (conn net.Conn, err error) {
+		return net.Dial("unix", testSocketPath)
+	}
+	tr := &http.Transport{
+		Dial: unixDialer,
+	}
+	client := &http.Client{Transport: tr}
+
+	require.Eventually(t, func() bool {
+		_, err := client.Get(handler.expectedURL)
+		return err == nil
+	}, 15*time.Second, 1*time.Second, "Expected test HTTP server to be up")
+
+	envVar := "VCENTER_ENVOY_HOST_GATEWAY"
+	oldValue := os.Getenv(envVar)
+	defer os.Setenv(envVar, oldValue)
+	os.Setenv(envVar, testSocketPath)
+
+	// Build a new client using the test unix socket.
+	vc := &vim25.Client{Client: soap.NewClient(&url.URL{}, true)}
+	newClient := internal.ClientWithEnvoyHostGateway(vc)
+
+	// An HTTP request made using the new client should hit the server listening on the Unix socket.
+	resp, err := newClient.Get(handler.expectedURL)
+
+	// ...but should successfully connect to the Unix socket set up for testing.
+	require.NoError(t, err)
+	response, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	require.Equal(t, response, []byte(handler.response))
+}
+
+type testHTTPServer struct {
+	expectedURL string
+	response    string
+	t           *testing.T
+}
+
+func (t *testHTTPServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	require.Equal(t.t, "/foo", req.URL.Path)
+	resp.Write([]byte(t.response))
+}
+
+func TestRewriteURLForHostGateway(t *testing.T) {
+	testURL, err := url.Parse("https://foo.bar/baz?query_param=1")
+	require.NoError(t, err)
+
+	hostMoref := types.ManagedObjectReference{
+		Type:  "HostSystem",
+		Value: "host-123",
+	}
+	result := internal.HostGatewayTransferURL(testURL, hostMoref)
+	require.Equal(t, "localhost", result.Host)
+	require.Equal(t, "/hgw/host-123/baz", result.Path)
+	values := url.Values{"query_param": []string{"1"}}
+	require.Equal(t, values, result.Query())
+}

object/datastore.go

@@ -27,6 +27,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/vmware/govmomi/internal"
 	"github.com/vmware/govmomi/property"
 	"github.com/vmware/govmomi/session"
 	"github.com/vmware/govmomi/vim25"
@@ -229,8 +230,18 @@ func (d Datastore) ServiceTicket(ctx context.Context, path string, method string
 	delete(q, "dcPath")
 	u.RawQuery = q.Encode()
 
+	// Now that we have a host selected, take a copy of the URL.
+	transferURL := *u
+
+	if internal.UsingEnvoySidecar(d.Client()) {
+		// Rewrite the host URL to go through the Envoy sidecar on VC.
+		// Reciever must use a custom dialer.
+		u = internal.HostGatewayTransferURL(u, host.Reference())
+	}
+
 	spec := types.SessionManagerHttpServiceRequestSpec{
-		Url: u.String(),
+		// Use the original URL (without rewrites) for the session ticket.
+		Url: transferURL.String(),
 		// See SessionManagerHttpServiceRequestSpecMethod enum
 		Method: fmt.Sprintf("http%s%s", method[0:1], strings.ToLower(method[1:])),
 	}
@@ -303,7 +314,13 @@ func (d Datastore) UploadFile(ctx context.Context, file string, path string, par
 	if err != nil {
 		return err
 	}
-	return d.Client().UploadFile(ctx, file, u, p)
+	vc := d.Client()
+	if internal.UsingEnvoySidecar(vc) {
+		// Override the vim client with a new one that wraps a Unix socket transport.
+		// Using HTTP here so secure means nothing.
+		vc = internal.ClientWithEnvoyHostGateway(vc)
+	}
+	return vc.UploadFile(ctx, file, u, p)
 }
 
 // Download via soap.Download with an http service ticket
@@ -321,7 +338,13 @@ func (d Datastore) DownloadFile(ctx context.Context, path string, file string, p
 	if err != nil {
 		return err
 	}
-	return d.Client().DownloadFile(ctx, file, u, p)
+	vc := d.Client()
+	if internal.UsingEnvoySidecar(vc) {
+		// Override the vim client with a new one that wraps a Unix socket transport.
+		// Using HTTP here so secure means nothing.
+		vc = internal.ClientWithEnvoyHostGateway(vc)
+	}
+	return vc.DownloadFile(ctx, file, u, p)
 }
 
 // AttachedHosts returns hosts that have this Datastore attached, accessible and writable.