From f44640565fec7d31466614628086258b80ef79f3 Mon Sep 17 00:00:00 2001 From: Maria Shaldybin Date: Wed, 1 Dec 2021 23:33:16 +0000 Subject: [PATCH] Add TLS listener with app internal routes in SAN to Envoy Add another listener to Envoy that proxies port 61443 to 8080 inside of container. It serves the SSL certificate that contains SAN with all application internal routes. [#180173340](https://www.pivotaltracker.com/story/show/180173340) --- depot/containerstore/containerstore_test.go | 29 +- .../containerstorefakes/fake_cred_handler.go | 24 +- .../containerstorefakes/fake_proxymanager.go | 24 +- depot/containerstore/credmanager.go | 78 +++-- depot/containerstore/credmanager_test.go | 210 +++++++----- .../instance_identity_handler.go | 8 +- .../instance_identity_handler_test.go | 10 +- depot/containerstore/proxy_config_handler.go | 98 ++++-- .../proxy_config_handler_test.go | 304 ++++++++++++------ depot/containerstore/storenode.go | 29 +- resources.go | 46 +-- 11 files changed, 558 insertions(+), 302 deletions(-) diff --git a/depot/containerstore/containerstore_test.go b/depot/containerstore/containerstore_test.go index 07006d88..8107953d 100644 --- a/depot/containerstore/containerstore_test.go +++ b/depot/containerstore/containerstore_test.go @@ -1146,6 +1146,10 @@ var _ = Describe("Container Store", func() { AppPort: 8080, ProxyPort: 61001, }, + { + AppPort: 8080, + ProxyPort: 61443, + }, { AppPort: 9090, ProxyPort: 61002, @@ -1153,6 +1157,7 @@ var _ = Describe("Container Store", func() { }, []uint16{ 61001, 61002, + 61443, }, nil) containerStore = containerstore.New( @@ -1198,6 +1203,8 @@ var _ = Describe("Container Store", func() { info.MappedPorts = append(info.MappedPorts, garden.PortMapping{HostPort: 16002, ContainerPort: 61002}) case 61003: info.MappedPorts = append(info.MappedPorts, garden.PortMapping{HostPort: 16003, ContainerPort: 61003}) + case 61443: + info.MappedPorts = append(info.MappedPorts, garden.PortMapping{HostPort: 16004, ContainerPort: 61443}) default: return info, errors.New("failed-net-in") } @@ -1221,7 +1228,7 @@ var _ = Describe("Container Store", func() { Expect(err).NotTo(HaveOccurred()) containerSpec := gardenClient.CreateArgsForCall(0) - Expect(containerSpec.NetIn).To(HaveLen(4)) + Expect(containerSpec.NetIn).To(HaveLen(5)) Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ HostPort: 0, ContainerPort: 8080, })) @@ -1234,6 +1241,9 @@ var _ = Describe("Container Store", func() { Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ HostPort: 0, ContainerPort: 61002, })) + Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ + HostPort: 0, ContainerPort: 61443, + })) }) Context("when disabling unproxied port mappings", func() { @@ -1265,13 +1275,16 @@ var _ = Describe("Container Store", func() { Expect(err).NotTo(HaveOccurred()) containerSpec := gardenClient.CreateArgsForCall(0) - Expect(containerSpec.NetIn).To(HaveLen(2)) + Expect(containerSpec.NetIn).To(HaveLen(3)) Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ HostPort: 0, ContainerPort: 61001, })) Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ HostPort: 0, ContainerPort: 61002, })) + Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ + HostPort: 0, ContainerPort: 61443, + })) }) It("unproxied host ports are set to 0", func() { @@ -1283,6 +1296,11 @@ var _ = Describe("Container Store", func() { HostPort: 0, ContainerTLSProxyPort: 61001, HostTLSProxyPort: 16001, + }, executor.PortMapping{ + ContainerPort: 8080, + HostPort: 0, + ContainerTLSProxyPort: 61443, + HostTLSProxyPort: 16004, }, executor.PortMapping{ ContainerPort: 9090, HostPort: 0, @@ -1301,6 +1319,11 @@ var _ = Describe("Container Store", func() { HostPort: 16000, ContainerTLSProxyPort: 61001, HostTLSProxyPort: 16001, + }, executor.PortMapping{ + ContainerPort: 8080, + HostPort: 16000, + ContainerTLSProxyPort: 61443, + HostTLSProxyPort: 16004, }, executor.PortMapping{ ContainerPort: 9090, HostPort: 16004, @@ -1318,7 +1341,7 @@ var _ = Describe("Container Store", func() { Expect(containerStore.Run(logger, containerGuid)).NotTo(HaveOccurred()) Eventually(megatron.StepsRunnerCallCount).Should(Equal(1)) _, _, _, _, cfg := megatron.StepsRunnerArgsForCall(0) - Expect(cfg.ProxyTLSPorts).To(ConsistOf(uint16(61001), uint16(61002))) + Expect(cfg.ProxyTLSPorts).To(ConsistOf(uint16(61001), uint16(61002), uint16(61443))) }) It("bind mounts envoy", func() { diff --git a/depot/containerstore/containerstorefakes/fake_cred_handler.go b/depot/containerstore/containerstorefakes/fake_cred_handler.go index 69f8ea79..883fd11c 100644 --- a/depot/containerstore/containerstorefakes/fake_cred_handler.go +++ b/depot/containerstore/containerstorefakes/fake_cred_handler.go @@ -11,10 +11,10 @@ import ( ) type FakeCredentialHandler struct { - CloseStub func(containerstore.Credential, executor.Container) error + CloseStub func(containerstore.Credentials, executor.Container) error closeMutex sync.RWMutex closeArgsForCall []struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container } closeReturns struct { @@ -51,10 +51,10 @@ type FakeCredentialHandler struct { removeDirReturnsOnCall map[int]struct { result1 error } - UpdateStub func(containerstore.Credential, executor.Container) error + UpdateStub func(containerstore.Credentials, executor.Container) error updateMutex sync.RWMutex updateArgsForCall []struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container } updateReturns struct { @@ -67,11 +67,11 @@ type FakeCredentialHandler struct { invocationsMutex sync.RWMutex } -func (fake *FakeCredentialHandler) Close(arg1 containerstore.Credential, arg2 executor.Container) error { +func (fake *FakeCredentialHandler) Close(arg1 containerstore.Credentials, arg2 executor.Container) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] fake.closeArgsForCall = append(fake.closeArgsForCall, struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container }{arg1, arg2}) stub := fake.CloseStub @@ -93,13 +93,13 @@ func (fake *FakeCredentialHandler) CloseCallCount() int { return len(fake.closeArgsForCall) } -func (fake *FakeCredentialHandler) CloseCalls(stub func(containerstore.Credential, executor.Container) error) { +func (fake *FakeCredentialHandler) CloseCalls(stub func(containerstore.Credentials, executor.Container) error) { fake.closeMutex.Lock() defer fake.closeMutex.Unlock() fake.CloseStub = stub } -func (fake *FakeCredentialHandler) CloseArgsForCall(i int) (containerstore.Credential, executor.Container) { +func (fake *FakeCredentialHandler) CloseArgsForCall(i int) (containerstore.Credentials, executor.Container) { fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() argsForCall := fake.closeArgsForCall[i] @@ -259,11 +259,11 @@ func (fake *FakeCredentialHandler) RemoveDirReturnsOnCall(i int, result1 error) }{result1} } -func (fake *FakeCredentialHandler) Update(arg1 containerstore.Credential, arg2 executor.Container) error { +func (fake *FakeCredentialHandler) Update(arg1 containerstore.Credentials, arg2 executor.Container) error { fake.updateMutex.Lock() ret, specificReturn := fake.updateReturnsOnCall[len(fake.updateArgsForCall)] fake.updateArgsForCall = append(fake.updateArgsForCall, struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container }{arg1, arg2}) stub := fake.UpdateStub @@ -285,13 +285,13 @@ func (fake *FakeCredentialHandler) UpdateCallCount() int { return len(fake.updateArgsForCall) } -func (fake *FakeCredentialHandler) UpdateCalls(stub func(containerstore.Credential, executor.Container) error) { +func (fake *FakeCredentialHandler) UpdateCalls(stub func(containerstore.Credentials, executor.Container) error) { fake.updateMutex.Lock() defer fake.updateMutex.Unlock() fake.UpdateStub = stub } -func (fake *FakeCredentialHandler) UpdateArgsForCall(i int) (containerstore.Credential, executor.Container) { +func (fake *FakeCredentialHandler) UpdateArgsForCall(i int) (containerstore.Credentials, executor.Container) { fake.updateMutex.RLock() defer fake.updateMutex.RUnlock() argsForCall := fake.updateArgsForCall[i] diff --git a/depot/containerstore/containerstorefakes/fake_proxymanager.go b/depot/containerstore/containerstorefakes/fake_proxymanager.go index a29acc41..77046af5 100644 --- a/depot/containerstore/containerstorefakes/fake_proxymanager.go +++ b/depot/containerstore/containerstorefakes/fake_proxymanager.go @@ -11,10 +11,10 @@ import ( ) type FakeProxyManager struct { - CloseStub func(containerstore.Credential, executor.Container) error + CloseStub func(containerstore.Credentials, executor.Container) error closeMutex sync.RWMutex closeArgsForCall []struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container } closeReturns struct { @@ -67,10 +67,10 @@ type FakeProxyManager struct { removeDirReturnsOnCall map[int]struct { result1 error } - UpdateStub func(containerstore.Credential, executor.Container) error + UpdateStub func(containerstore.Credentials, executor.Container) error updateMutex sync.RWMutex updateArgsForCall []struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container } updateReturns struct { @@ -83,11 +83,11 @@ type FakeProxyManager struct { invocationsMutex sync.RWMutex } -func (fake *FakeProxyManager) Close(arg1 containerstore.Credential, arg2 executor.Container) error { +func (fake *FakeProxyManager) Close(arg1 containerstore.Credentials, arg2 executor.Container) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] fake.closeArgsForCall = append(fake.closeArgsForCall, struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container }{arg1, arg2}) stub := fake.CloseStub @@ -109,13 +109,13 @@ func (fake *FakeProxyManager) CloseCallCount() int { return len(fake.closeArgsForCall) } -func (fake *FakeProxyManager) CloseCalls(stub func(containerstore.Credential, executor.Container) error) { +func (fake *FakeProxyManager) CloseCalls(stub func(containerstore.Credentials, executor.Container) error) { fake.closeMutex.Lock() defer fake.closeMutex.Unlock() fake.CloseStub = stub } -func (fake *FakeProxyManager) CloseArgsForCall(i int) (containerstore.Credential, executor.Container) { +func (fake *FakeProxyManager) CloseArgsForCall(i int) (containerstore.Credentials, executor.Container) { fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() argsForCall := fake.closeArgsForCall[i] @@ -343,11 +343,11 @@ func (fake *FakeProxyManager) RemoveDirReturnsOnCall(i int, result1 error) { }{result1} } -func (fake *FakeProxyManager) Update(arg1 containerstore.Credential, arg2 executor.Container) error { +func (fake *FakeProxyManager) Update(arg1 containerstore.Credentials, arg2 executor.Container) error { fake.updateMutex.Lock() ret, specificReturn := fake.updateReturnsOnCall[len(fake.updateArgsForCall)] fake.updateArgsForCall = append(fake.updateArgsForCall, struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container }{arg1, arg2}) stub := fake.UpdateStub @@ -369,13 +369,13 @@ func (fake *FakeProxyManager) UpdateCallCount() int { return len(fake.updateArgsForCall) } -func (fake *FakeProxyManager) UpdateCalls(stub func(containerstore.Credential, executor.Container) error) { +func (fake *FakeProxyManager) UpdateCalls(stub func(containerstore.Credentials, executor.Container) error) { fake.updateMutex.Lock() defer fake.updateMutex.Unlock() fake.UpdateStub = stub } -func (fake *FakeProxyManager) UpdateArgsForCall(i int) (containerstore.Credential, executor.Container) { +func (fake *FakeProxyManager) UpdateArgsForCall(i int) (containerstore.Credentials, executor.Container) { fake.updateMutex.RLock() defer fake.updateMutex.RUnlock() argsForCall := fake.updateArgsForCall[i] diff --git a/depot/containerstore/credmanager.go b/depot/containerstore/credmanager.go index 746aa704..200648fc 100644 --- a/depot/containerstore/credmanager.go +++ b/depot/containerstore/credmanager.go @@ -21,6 +21,7 @@ import ( "code.cloudfoundry.org/executor" "code.cloudfoundry.org/garden" "code.cloudfoundry.org/lager" + "code.cloudfoundry.org/routing-info/internalroutes" ) const ( @@ -29,6 +30,11 @@ const ( CredCreationFailedCount = "CredCreationFailedCount" ) +type Credentials struct { + InstanceIdentityCredential Credential + C2CCredential Credential +} + type Credential struct { Cert string Key string @@ -85,12 +91,12 @@ type CredentialHandler interface { RemoveDir(logger lager.Logger, container executor.Container) error // Called periodically as new valid certificate/key pair are generated - Update(credentials Credential, container executor.Container) error + Update(credentials Credentials, container executor.Container) error // Called when the CredManager is preparing to exit. This is mainly to update // the EnvoyProxy with invalid certificates and prevent it from accepting // more incoming traffic from the gorouter - Close(invalidCredentials Credential, container executor.Container) error + Close(invalidCredentials Credentials, container executor.Container) error } func NewCredManager( @@ -242,11 +248,41 @@ const ( privateKeyPEMBlockType = "RSA PRIVATE KEY" ) -func (c *credManager) generateCreds(logger lager.Logger, container executor.Container, certGUID string) (Credential, error) { +func (c *credManager) generateCreds(logger lager.Logger, container executor.Container, certGUID string) (Credentials, error) { logger = logger.Session("generating-credentials") logger.Debug("starting") defer logger.Debug("complete") + ipForCert := container.InternalIP + if len(ipForCert) == 0 { + ipForCert = container.ExternalIP + } + + logger.Debug("generating-credentials-for-instance-identity") + idCred, err := c.generateCredForSAN(logger, + certificateSAN{IPAddress: ipForCert, OrganizationalUnits: container.CertificateProperties.OrganizationalUnit}, + certGUID, + ) + if err != nil { + return Credentials{}, err + } + + logger.Debug("generating-credentials-for-c2c") + c2cCred, err := c.generateCredForSAN(logger, + certificateSAN{InternalRoutes: container.InternalRoutes, OrganizationalUnits: container.CertificateProperties.OrganizationalUnit}, + certGUID, + ) + if err != nil { + return Credentials{}, err + } + + return Credentials{ + InstanceIdentityCredential: idCred, + C2CCredential: c2cCred, + }, nil +} + +func (c *credManager) generateCredForSAN(logger lager.Logger, certSAN certificateSAN, certGUID string) (Credential, error) { logger.Debug("generating-private-key") privateKey, err := rsa.GenerateKey(c.entropyReader, 2048) if err != nil { @@ -254,18 +290,12 @@ func (c *credManager) generateCreds(logger lager.Logger, container executor.Cont } logger.Debug("generated-private-key") - ipForCert := container.InternalIP - if len(ipForCert) == 0 { - ipForCert = container.ExternalIP - } - startValidity := c.clock.Now() - template := createCertificateTemplate(ipForCert, - certGUID, + template := createCertificateTemplate(certGUID, + certSAN, startValidity, startValidity.Add(c.validityPeriod), - container.CertificateProperties.OrganizationalUnit, ) logger.Debug("generating-serial-number") @@ -306,11 +336,10 @@ func (c *credManager) generateCreds(logger lager.Logger, container executor.Cont return Credential{}, err } - creds := Credential{ + return Credential{ Cert: certificateBuf.String(), Key: keyBuf.String(), - } - return creds, nil + }, nil } func pemEncode(bytes []byte, blockType string, writer io.Writer) error { @@ -321,21 +350,32 @@ func pemEncode(bytes []byte, blockType string, writer io.Writer) error { return pem.Encode(writer, block) } -func createCertificateTemplate(ipaddress, guid string, notBefore, notAfter time.Time, organizationalUnits []string) *x509.Certificate { +type certificateSAN struct { + IPAddress string + InternalRoutes internalroutes.InternalRoutes + OrganizationalUnits []string +} + +func createCertificateTemplate(guid string, certSAN certificateSAN, notBefore, notAfter time.Time) *x509.Certificate { var ipaddr []net.IP - if len(ipaddress) == 0 { + if len(certSAN.IPAddress) == 0 { ipaddr = []net.IP{} } else { - ipaddr = []net.IP{net.ParseIP(ipaddress)} + ipaddr = []net.IP{net.ParseIP(certSAN.IPAddress)} + } + dnsNames := []string{guid} + for _, route := range certSAN.InternalRoutes { + dnsNames = append(dnsNames, route.Hostname) } + return &x509.Certificate{ SerialNumber: big.NewInt(0), Subject: pkix.Name{ CommonName: guid, - OrganizationalUnit: organizationalUnits, + OrganizationalUnit: certSAN.OrganizationalUnits, }, IPAddresses: ipaddr, - DNSNames: []string{guid}, + DNSNames: dnsNames, NotBefore: notBefore, NotAfter: notAfter, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement, diff --git a/depot/containerstore/credmanager_test.go b/depot/containerstore/credmanager_test.go index 75b21c8d..00371aaf 100644 --- a/depot/containerstore/credmanager_test.go +++ b/depot/containerstore/credmanager_test.go @@ -21,6 +21,7 @@ import ( "code.cloudfoundry.org/garden" "code.cloudfoundry.org/lager" "code.cloudfoundry.org/lager/lagertest" + "code.cloudfoundry.org/routing-info/internalroutes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/tedsuo/ifrit" @@ -229,8 +230,14 @@ var _ = Describe("CredManager", func() { container = executor.Container{ Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), InternalIP: "127.0.0.1", - RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ - OrganizationalUnit: []string{"app:iamthelizardking"}}, + RunInfo: executor.RunInfo{ + InternalRoutes: internalroutes.InternalRoutes{ + {Hostname: "a.apps.internal"}, + {Hostname: "b.apps.internal"}, + }, + CertificateProperties: executor.CertificateProperties{ + OrganizationalUnit: []string{"app:iamthelizardking"}, + }, }, } }) @@ -311,26 +318,28 @@ var _ = Describe("CredManager", func() { Context("when the certificate is about to expire", func() { var ( - credBefore containerstore.Credential + credsBefore containerstore.Credentials ) JustBeforeEach(func() { Eventually(fakeCredHandler.UpdateCallCount).Should(Equal(1)) Eventually(containerProcess.Ready()).Should(BeClosed()) - credBefore, _ = fakeCredHandler.UpdateArgsForCall(0) + credsBefore, _ = fakeCredHandler.UpdateArgsForCall(0) }) testCredentialRotation := func(dur time.Duration) { callCount := fakeCredHandler.UpdateCallCount() var actualContainer executor.Container - credBefore, actualContainer = fakeCredHandler.UpdateArgsForCall(callCount - 1) + credsBefore, actualContainer = fakeCredHandler.UpdateArgsForCall(callCount - 1) Expect(actualContainer).To(Equal(container)) - certBefore, _ := parseCert(credBefore) - increment := certBefore.NotAfter.Add(-dur).Sub(clock.Now()) + idCertBefore, _ := parseCert(credsBefore.InstanceIdentityCredential) + c2cCertBefore, _ := parseCert(credsBefore.C2CCredential) + Expect(idCertBefore.NotAfter).To(Equal(c2cCertBefore.NotAfter)) + increment := idCertBefore.NotAfter.Add(-dur).Sub(clock.Now()) Expect(increment).To(BeNumerically(">", 0)) clock.WaitForWatcherAndIncrement(increment) @@ -339,19 +348,27 @@ var _ = Describe("CredManager", func() { cred, actualContainer := fakeCredHandler.UpdateArgsForCall(callCount) Expect(actualContainer).To(Equal(container)) - Expect(cred.Cert).NotTo(Equal(credBefore.Cert)) - Expect(cred.Key).NotTo(Equal(credBefore.Key)) + Expect(cred.InstanceIdentityCredential.Cert).NotTo(Equal(credsBefore.InstanceIdentityCredential.Cert)) + Expect(cred.InstanceIdentityCredential.Key).NotTo(Equal(credsBefore.InstanceIdentityCredential.Key)) + + Expect(cred.C2CCredential.Cert).NotTo(Equal(credsBefore.C2CCredential.Cert)) + Expect(cred.C2CCredential.Key).NotTo(Equal(credsBefore.C2CCredential.Key)) + + idCert, _ := parseCert(cred.InstanceIdentityCredential) + Expect(idCert.SerialNumber).NotTo(Equal(idCertBefore.SerialNumber)) - cert, _ := parseCert(cred) - Expect(cert.SerialNumber).NotTo(Equal(certBefore.SerialNumber)) + c2cCert, _ := parseCert(cred.InstanceIdentityCredential) + Expect(c2cCert.SerialNumber).NotTo(Equal(c2cCertBefore.SerialNumber)) } testNoCredentialRotation := func(dur time.Duration) { callCount := fakeCredHandler.UpdateCallCount() - credBefore, _ = fakeCredHandler.UpdateArgsForCall(callCount - 1) + credsBefore, _ = fakeCredHandler.UpdateArgsForCall(callCount - 1) - cert, _ := parseCert(credBefore) - increment := cert.NotAfter.Add(-dur).Sub(clock.Now()) + idCertBefore, _ := parseCert(credsBefore.InstanceIdentityCredential) + c2cCertBefore, _ := parseCert(credsBefore.C2CCredential) + Expect(idCertBefore.NotAfter).To(Equal(c2cCertBefore.NotAfter)) + increment := idCertBefore.NotAfter.Add(-dur).Sub(clock.Now()) Expect(increment).To(BeNumerically(">", 0)) clock.WaitForWatcherAndIncrement(increment) @@ -361,8 +378,8 @@ var _ = Describe("CredManager", func() { Context("when the handler returns an error", func() { JustBeforeEach(func() { fakeCredHandler.UpdateReturnsOnCall(1, errors.New("boooom!")) - certBefore, _ := parseCert(credBefore) - increment := certBefore.NotAfter.Sub(clock.Now()) + idCertBefore, _ := parseCert(credsBefore.InstanceIdentityCredential) + increment := idCertBefore.NotAfter.Sub(clock.Now()) clock.WaitForWatcherAndIncrement(increment) }) @@ -388,7 +405,7 @@ var _ = Describe("CredManager", func() { }) It("emits metrics on successful creation", func() { - cert, _ := parseCert(credBefore) + cert, _ := parseCert(credsBefore.InstanceIdentityCredential) increment := cert.NotAfter.Add(-5 * time.Second).Sub(clock.Now()) Expect(increment).To(BeNumerically(">", 0)) clock.WaitForWatcherAndIncrement(increment) @@ -449,52 +466,33 @@ var _ = Describe("CredManager", func() { rest []byte ) - JustBeforeEach(func() { - Eventually(containerProcess.Ready()).Should(BeClosed()) - - Eventually(fakeCredHandler.UpdateCallCount).Should(Equal(1)) - cred, actualContainer := fakeCredHandler.UpdateArgsForCall(0) - Expect(actualContainer).To(Equal(container)) - cert, rest = parseCert(cred) - }) - - It("has the container ip", func() { - ip := net.ParseIP(container.InternalIP) - Expect(cert.IPAddresses).To(ContainElement(ip.To4())) - }) - - It("has all required usages in the KU & EKU fields", func() { + testCertificateFields := func() { + By("has all required usages in the KU & EKU fields") Expect(cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageClientAuth)) Expect(cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageServerAuth)) Expect(cert.KeyUsage).To(Equal(x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement)) - }) - It("signed by the rep intermediate CA", func() { + By("signed by the rep intermediate CA") CaCertPool := x509.NewCertPool() CaCertPool.AddCert(CaCert) verifyOpts := x509.VerifyOptions{Roots: CaCertPool} Expect(cert.CheckSignatureFrom(CaCert)).To(Succeed()) _, err := cert.Verify(verifyOpts) Expect(err).NotTo(HaveOccurred()) - }) - It("common name should be set to the container guid", func() { + By("common name should be set to the container guid") Expect(cert.Subject.CommonName).To(Equal(container.Guid)) - }) - It("DNS SAN should be set to the container guid", func() { + By("DNS SAN should be set to the container guid") Expect(cert.DNSNames).To(ContainElement(container.Guid)) - }) - It("expires in after the configured validity period", func() { + By("expires in after the configured validity period") Expect(cert.NotAfter).To(Equal(clock.Now().Add(validityPeriod))) - }) - It("not before is set to current timestamp", func() { + By("not before is set to current timestamp") Expect(cert.NotBefore).To(Equal(clock.Now())) - }) - It("has the rep intermediate CA", func() { + By("has the rep intermediate CA") block, rest := pem.Decode(rest) Expect(block).NotTo(BeNil()) Expect(rest).To(BeEmpty()) @@ -502,53 +500,91 @@ var _ = Describe("CredManager", func() { certs, err := x509.ParseCertificates(block.Bytes) Expect(err).NotTo(HaveOccurred()) Expect(certs).To(HaveLen(1)) - cert = certs[0] - Expect(cert).To(Equal(CaCert)) - }) + Expect(certs[0]).To(Equal(CaCert)) - It("has the app guid in the subject's organizational units", func() { + By("has the app guid in the subject's organizational units") Expect(cert.Subject.OrganizationalUnit).To(ContainElement("app:iamthelizardking")) - }) + } - Context("when the container doesn't have an internal ip", func() { - Context("when the container has an external IP", func() { - BeforeEach(func() { - container = executor.Container{ - Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), - InternalIP: "", - ExternalIP: "54.23.123.234", - RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ - OrganizationalUnit: []string{"app:iamthelizardking"}}, - }, - } - }) + Describe("Instance identity certificate", func() { + JustBeforeEach(func() { + Eventually(containerProcess.Ready()).Should(BeClosed()) + + Eventually(fakeCredHandler.UpdateCallCount).Should(Equal(1)) + creds, actualContainer := fakeCredHandler.UpdateArgsForCall(0) + Expect(actualContainer).To(Equal(container)) + cert, rest = parseCert(creds.InstanceIdentityCredential) + }) - It("has the external ip", func() { - ip := net.ParseIP(container.ExternalIP) - Expect(cert.IPAddresses).To(ContainElement(ip.To4())) + Context("when the container doesn't have an internal ip", func() { + Context("when the container has an external IP", func() { + BeforeEach(func() { + container = executor.Container{ + Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), + InternalIP: "", + ExternalIP: "54.23.123.234", + RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ + OrganizationalUnit: []string{"app:iamthelizardking"}}, + }, + } + }) + + It("has the external ip", func() { + ip := net.ParseIP(container.ExternalIP) + Expect(cert.IPAddresses).To(ContainElement(ip.To4())) + }) + + It("does not have the empty internal ip", func() { + ip := net.ParseIP(container.InternalIP) + Expect(cert.IPAddresses).NotTo(ContainElement(ip.To4())) + }) }) - It("does not have the empty internal ip", func() { - ip := net.ParseIP(container.InternalIP) - Expect(cert.IPAddresses).NotTo(ContainElement(ip.To4())) + Context("when the container doesn't have an external ip", func() { + BeforeEach(func() { + container = executor.Container{ + Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), + InternalIP: "", + ExternalIP: "", + RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ + OrganizationalUnit: []string{"app:iamthelizardking"}}, + }, + } + }) + + It("has no SubjectAltName", func() { + Expect(cert.IPAddresses).To(BeEmpty()) + }) + }) }) - Context("when the container doesn't have an external ip", func() { - BeforeEach(func() { - container = executor.Container{ - Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), - InternalIP: "", - ExternalIP: "", - RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ - OrganizationalUnit: []string{"app:iamthelizardking"}}, - }, - } - }) - It("has no SubjectAltName", func() { - Expect(cert.IPAddresses).To(BeEmpty()) - }) + It("has the container ip", func() { + ip := net.ParseIP(container.InternalIP) + Expect(cert.IPAddresses).To(ContainElement(ip.To4())) + }) + + It("has the correct fields", func() { + testCertificateFields() + }) + }) + + Describe("C2C certificate", func() { + JustBeforeEach(func() { + Eventually(containerProcess.Ready()).Should(BeClosed()) + Eventually(fakeCredHandler.UpdateCallCount).Should(Equal(1)) + creds, actualContainer := fakeCredHandler.UpdateArgsForCall(0) + Expect(actualContainer).To(Equal(container)) + cert, rest = parseCert(creds.C2CCredential) + }) + + It("has the internal routes in SAN", func() { + Expect(cert.DNSNames).To(ConsistOf(container.Guid, "a.apps.internal", "b.apps.internal")) + }) + + It("has the correct fields", func() { + testCertificateFields() }) }) }) @@ -568,16 +604,24 @@ var _ = Describe("CredManager", func() { It("Generates an invalid cert and sends the invalid cert on the cred channel", func() { Eventually(fakeCredHandler.CloseCallCount).Should(Equal(1)) - cred, actualContainer := fakeCredHandler.CloseArgsForCall(0) + creds, actualContainer := fakeCredHandler.CloseArgsForCall(0) Expect(actualContainer).To(Equal(container)) - block, _ := pem.Decode([]byte(cred.Cert)) + block, _ := pem.Decode([]byte(creds.InstanceIdentityCredential.Cert)) Expect(block).NotTo(BeNil()) certs, err := x509.ParseCertificates(block.Bytes) Expect(err).NotTo(HaveOccurred()) cert := certs[0] Expect(cert.Subject.CommonName).To(BeEmpty()) + + block, _ = pem.Decode([]byte(creds.C2CCredential.Cert)) + Expect(block).NotTo(BeNil()) + certs, err = x509.ParseCertificates(block.Bytes) + Expect(err).NotTo(HaveOccurred()) + cert = certs[0] + + Expect(cert.Subject.CommonName).To(BeEmpty()) }) }) }) diff --git a/depot/containerstore/instance_identity_handler.go b/depot/containerstore/instance_identity_handler.go index 24e6dfc7..4f9c481d 100644 --- a/depot/containerstore/instance_identity_handler.go +++ b/depot/containerstore/instance_identity_handler.go @@ -49,7 +49,7 @@ func (h *InstanceIdentityHandler) RemoveDir(logger lager.Logger, container execu return os.RemoveAll(filepath.Join(h.credDir, container.Guid)) } -func (h *InstanceIdentityHandler) Update(cred Credential, container executor.Container) error { +func (h *InstanceIdentityHandler) Update(creds Credentials, container executor.Container) error { instanceKeyPath := filepath.Join(h.credDir, container.Guid, "instance.key") tmpInstanceKeyPath := instanceKeyPath + ".tmp" certificatePath := filepath.Join(h.credDir, container.Guid, "instance.crt") @@ -67,12 +67,12 @@ func (h *InstanceIdentityHandler) Update(cred Credential, container executor.Con } defer certificate.Close() - _, err = certificate.Write([]byte(cred.Cert)) + _, err = certificate.Write([]byte(creds.InstanceIdentityCredential.Cert)) if err != nil { return err } - _, err = instanceKey.Write([]byte(cred.Key)) + _, err = instanceKey.Write([]byte(creds.InstanceIdentityCredential.Key)) if err != nil { return err } @@ -100,6 +100,6 @@ func (h *InstanceIdentityHandler) Update(cred Credential, container executor.Con return nil } -func (h *InstanceIdentityHandler) Close(cred Credential, container executor.Container) error { +func (h *InstanceIdentityHandler) Close(creds Credentials, container executor.Container) error { return nil } diff --git a/depot/containerstore/instance_identity_handler_test.go b/depot/containerstore/instance_identity_handler_test.go index c51e5aec..d5593f1b 100644 --- a/depot/containerstore/instance_identity_handler_test.go +++ b/depot/containerstore/instance_identity_handler_test.go @@ -83,7 +83,7 @@ var _ = Describe("InstanceIdentityHandler", func() { }) It("puts private key into container directory", func() { - err := handler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := handler.Update(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "cert", Key: "key"}}, container) Expect(err).NotTo(HaveOccurred()) certPath := filepath.Join(tmpdir, "some-guid") @@ -97,7 +97,7 @@ var _ = Describe("InstanceIdentityHandler", func() { }) It("puts the certificate into container directory", func() { - err := handler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := handler.Update(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "cert", Key: "key"}}, container) Expect(err).NotTo(HaveOccurred()) certPath := filepath.Join(tmpdir, "some-guid") @@ -115,12 +115,12 @@ var _ = Describe("InstanceIdentityHandler", func() { BeforeEach(func() { _, _, err := handler.CreateDir(logger, container) Expect(err).To(BeNil()) - err = handler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err = handler.Update(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "cert", Key: "key"}}, container) Expect(err).NotTo(HaveOccurred()) }) It("does not put private key into container directory", func() { - err := handler.Close(containerstore.Credential{Cert: "invalid-cert", Key: "invalid-key"}, container) + err := handler.Close(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "invalid-cert", Key: "invalid-key"}}, container) Expect(err).NotTo(HaveOccurred()) certPath := filepath.Join(tmpdir, "some-guid") @@ -134,7 +134,7 @@ var _ = Describe("InstanceIdentityHandler", func() { }) It("does not put the certificate into container directory", func() { - err := handler.Close(containerstore.Credential{Cert: "invalid-cert", Key: "invalid-key"}, container) + err := handler.Close(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "invalid-cert", Key: "invalid-key"}}, container) Expect(err).NotTo(HaveOccurred()) certPath := filepath.Join(tmpdir, "some-guid") diff --git a/depot/containerstore/proxy_config_handler.go b/depot/containerstore/proxy_config_handler.go index 09707a3c..fb42fdbe 100644 --- a/depot/containerstore/proxy_config_handler.go +++ b/depot/containerstore/proxy_config_handler.go @@ -38,9 +38,10 @@ import ( ) const ( - StartProxyPort = 61001 - EndProxyPort = 65534 - C2CTLSPort = 61443 + StartProxyPort = 61001 + EndProxyPort = 65534 + DefaultHTTPPort = 8080 + C2CTLSPort = 61443 TimeOut = 250000000 @@ -49,6 +50,11 @@ const ( AdsClusterName = "pilot-ads" AdminAccessLog = os.DevNull + + InstanceIdentityCertAndKeyFilename = "sds-id-cert-and-key.yaml" + InstanceIdentityValidationContextFilename = "sds-id-validation-context.yaml" + C2CCertAndKeyFilename = "sds-c2c-cert-and-key.yaml" + ContainerEnvoyConfigDir = "/etc/cf-assets/envoy_config" ) var ( @@ -97,10 +103,10 @@ func (p *NoopProxyConfigHandler) CreateDir(logger lager.Logger, container execut func (p *NoopProxyConfigHandler) RemoveDir(logger lager.Logger, container executor.Container) error { return nil } -func (p *NoopProxyConfigHandler) Update(credentials Credential, container executor.Container) error { +func (p *NoopProxyConfigHandler) Update(credentials Credentials, container executor.Container) error { return nil } -func (p *NoopProxyConfigHandler) Close(invalidCredentials Credential, container executor.Container) error { +func (p *NoopProxyConfigHandler) Close(invalidCredentials Credentials, container executor.Container) error { return nil } @@ -186,6 +192,14 @@ func (p *ProxyConfigHandler) ProxyPorts(logger lager.Logger, container *executor ProxyPort: port, }) + if containerPorts[portCount] == DefaultHTTPPort { + proxyPortMapping = append(proxyPortMapping, executor.ProxyPortMapping{ + AppPort: containerPorts[portCount], + ProxyPort: C2CTLSPort, + }) + extraPorts = append(extraPorts, C2CTLSPort) + } + portCount++ } @@ -208,7 +222,7 @@ func (p *ProxyConfigHandler) CreateDir(logger lager.Logger, container executor.C { Origin: garden.BindMountOriginHost, SrcPath: proxyConfigDir, - DstPath: "/etc/cf-assets/envoy_config", + DstPath: ContainerEnvoyConfigDir, }, } @@ -230,7 +244,7 @@ func (p *ProxyConfigHandler) RemoveDir(logger lager.Logger, container executor.C return os.RemoveAll(proxyConfigDir) } -func (p *ProxyConfigHandler) Update(credentials Credential, container executor.Container) error { +func (p *ProxyConfigHandler) Update(credentials Credentials, container executor.Container) error { if !container.EnableContainerProxy { return nil } @@ -238,7 +252,7 @@ func (p *ProxyConfigHandler) Update(credentials Credential, container executor.C return p.writeConfig(credentials, container) } -func (p *ProxyConfigHandler) Close(invalidCredentials Credential, container executor.Container) error { +func (p *ProxyConfigHandler) Close(invalidCredentials Credentials, container executor.Container) error { if !container.EnableContainerProxy { return nil } @@ -252,10 +266,11 @@ func (p *ProxyConfigHandler) Close(invalidCredentials Credential, container exec return nil } -func (p *ProxyConfigHandler) writeConfig(credentials Credential, container executor.Container) error { +func (p *ProxyConfigHandler) writeConfig(credentials Credentials, container executor.Container) error { proxyConfigPath := filepath.Join(p.containerProxyConfigPath, container.Guid, "envoy.yaml") - sdsServerCertAndKeyPath := filepath.Join(p.containerProxyConfigPath, container.Guid, "sds-server-cert-and-key.yaml") - sdsServerValidationContextPath := filepath.Join(p.containerProxyConfigPath, container.Guid, "sds-server-validation-context.yaml") + sdsIDCertAndKeyPath := filepath.Join(p.containerProxyConfigPath, container.Guid, InstanceIdentityCertAndKeyFilename) + sdsC2CCertAndKeyPath := filepath.Join(p.containerProxyConfigPath, container.Guid, C2CCertAndKeyFilename) + sdsIDValidationContextPath := filepath.Join(p.containerProxyConfigPath, container.Guid, InstanceIdentityValidationContextFilename) adminPort, err := getAvailablePort(container.Ports) if err != nil { @@ -278,22 +293,28 @@ func (p *ProxyConfigHandler) writeConfig(credentials Credential, container execu return err } - sdsServerCertAndKey := generateSDSCertAndKey(container, credentials) - err = writeDiscoveryResponseYAML(sdsServerCertAndKey, sdsServerCertAndKeyPath) + sdsIDCertAndKey := generateSDSCertAndKey("id-cert-and-key", credentials.InstanceIdentityCredential) + err = writeDiscoveryResponseYAML(sdsIDCertAndKey, sdsIDCertAndKeyPath) if err != nil { return err } - sdsServerValidationContext, err := generateSDSCAResource( + sdsC2CCertAndKey := generateSDSCertAndKey("c2c-cert-and-key", credentials.C2CCredential) + err = writeDiscoveryResponseYAML(sdsC2CCertAndKey, sdsC2CCertAndKeyPath) + if err != nil { + return err + } + + sdsIDValidationContext, err := generateSDSCAResource( container, - credentials, + credentials.InstanceIdentityCredential, p.containerProxyTrustedCACerts, p.containerProxyVerifySubjectAltName, ) if err != nil { return err } - err = writeDiscoveryResponseYAML(sdsServerValidationContext, sdsServerValidationContextPath) + err = writeDiscoveryResponseYAML(sdsIDValidationContext, sdsIDValidationContextPath) if err != nil { return err } @@ -322,8 +343,14 @@ func generateProxyConfig( http2Enabled bool, ) (*envoy_bootstrap.Bootstrap, error) { clusters := []*envoy_cluster.Cluster{} - for index, portMap := range container.Ports { - clusterName := fmt.Sprintf("%d-service-cluster", index) + + uniqueContainerPorts := map[uint16]struct{}{} + for _, portMap := range container.Ports { + uniqueContainerPorts[portMap.ContainerPort] = struct{}{} + } + + for containerPort := range uniqueContainerPorts { + clusterName := fmt.Sprintf("service-cluster-%d", containerPort) clusters = append(clusters, &envoy_cluster.Cluster{ Name: clusterName, ClusterDiscoveryType: &envoy_cluster.Cluster_Type{Type: envoy_cluster.Cluster_STATIC}, @@ -334,7 +361,7 @@ func generateProxyConfig( LbEndpoints: []*envoy_endpoint.LbEndpoint{{ HostIdentifier: &envoy_endpoint.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint.Endpoint{ - Address: envoyAddr(container.InternalIP, portMap.ContainerPort), + Address: envoyAddr(container.InternalIP, containerPort), }, }, }}, @@ -461,12 +488,12 @@ func writeProxyConfig(proxyConfig *envoy_bootstrap.Bootstrap, path string) error func generateListeners(container executor.Container, requireClientCerts, http2Enabled bool) ([]*envoy_listener.Listener, error) { listeners := []*envoy_listener.Listener{} - for index, portMap := range container.Ports { + for _, portMap := range container.Ports { filterName := TcpProxy - clusterName := fmt.Sprintf("%d-service-cluster", index) + clusterName := fmt.Sprintf("service-cluster-%d", portMap.ContainerPort) filterConfig, err := ptypes.MarshalAny(&envoy_tcp_proxy.TcpProxy{ - StatPrefix: fmt.Sprintf("%d-stats", index), + StatPrefix: fmt.Sprintf("stats-%d-%d", portMap.ContainerPort, portMap.ContainerTLSProxyPort), ClusterSpecifier: &envoy_tcp_proxy.TcpProxy_Cluster{ Cluster: clusterName, }, @@ -475,15 +502,21 @@ func generateListeners(container executor.Container, requireClientCerts, http2En return nil, err } + sdsServerCertAndKeyFile := InstanceIdentityCertAndKeyFilename + sdsServerSecretName := "id-cert-and-key" + if portMap.ContainerTLSProxyPort == C2CTLSPort { + sdsServerCertAndKeyFile = C2CCertAndKeyFilename + sdsServerSecretName = "c2c-cert-and-key" + } + tlsContext := &envoy_tls.DownstreamTlsContext{ - RequireClientCertificate: &wrappers.BoolValue{Value: requireClientCerts}, CommonTlsContext: &envoy_tls.CommonTlsContext{ TlsCertificateSdsSecretConfigs: []*envoy_tls.SdsSecretConfig{ { - Name: "server-cert-and-key", + Name: sdsServerSecretName, SdsConfig: &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Path{ - Path: "/etc/cf-assets/envoy_config/sds-server-cert-and-key.yaml", + Path: filepath.Join(ContainerEnvoyConfigDir, sdsServerCertAndKeyFile), }, }, }, @@ -498,13 +531,14 @@ func generateListeners(container executor.Container, requireClientCerts, http2En tlsContext.CommonTlsContext.AlpnProtocols = AlpnProtocols } - if requireClientCerts { + if requireClientCerts && portMap.ContainerTLSProxyPort != C2CTLSPort { + tlsContext.RequireClientCertificate = &wrappers.BoolValue{Value: requireClientCerts} tlsContext.CommonTlsContext.ValidationContextType = &envoy_tls.CommonTlsContext_ValidationContextSdsSecretConfig{ ValidationContextSdsSecretConfig: &envoy_tls.SdsSecretConfig{ - Name: "server-validation-context", + Name: "id-validation-context", SdsConfig: &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Path{ - Path: "/etc/cf-assets/envoy_config/sds-server-validation-context.yaml", + Path: filepath.Join(ContainerEnvoyConfigDir, InstanceIdentityValidationContextFilename), }, }, }, @@ -516,7 +550,7 @@ func generateListeners(container executor.Container, requireClientCerts, http2En return nil, err } - listenerName := fmt.Sprintf("listener-%d", portMap.ContainerPort) + listenerName := fmt.Sprintf("listener-%d-%d", portMap.ContainerPort, portMap.ContainerTLSProxyPort) listener := &envoy_listener.Listener{ Name: listenerName, Address: envoyAddr("0.0.0.0", portMap.ContainerTLSProxyPort), @@ -545,9 +579,9 @@ func generateListeners(container executor.Container, requireClientCerts, http2En return listeners, nil } -func generateSDSCertAndKey(container executor.Container, creds Credential) proto.Message { +func generateSDSCertAndKey(name string, creds Credential) proto.Message { return &envoy_tls.Secret{ - Name: "server-cert-and-key", + Name: name, Type: &envoy_tls.Secret_TlsCertificate{ TlsCertificate: &envoy_tls.TlsCertificate{ CertificateChain: &envoy_core.DataSource{ @@ -579,7 +613,7 @@ func generateSDSCAResource(container executor.Container, creds Credential, trust } return &envoy_tls.Secret{ - Name: "server-validation-context", + Name: "id-validation-context", Type: &envoy_tls.Secret_ValidationContext{ ValidationContext: &envoy_tls.CertificateValidationContext{ TrustedCa: &envoy_core.DataSource{ diff --git a/depot/containerstore/proxy_config_handler_test.go b/depot/containerstore/proxy_config_handler_test.go index 61fd4ac4..4e0c6488 100644 --- a/depot/containerstore/proxy_config_handler_test.go +++ b/depot/containerstore/proxy_config_handler_test.go @@ -50,8 +50,9 @@ var _ = Describe("ProxyConfigHandler", func() { configPath string rotatingCredChan chan containerstore.Credential container executor.Container - sdsServerCertAndKeyFile string - sdsServerValidationContextFile string + sdsIDCertAndKeyFile string + sdsC2CCertAndKeyFile string + sdsIDValidationContextFile string proxyConfigFile string proxyConfigHandler *containerstore.ProxyConfigHandler reloadDuration time.Duration @@ -61,6 +62,7 @@ var _ = Describe("ProxyConfigHandler", func() { containerProxyRequireClientCerts bool adsServers []string http2Enabled bool + credentials containerstore.Credentials ) BeforeEach(func() { @@ -85,8 +87,9 @@ var _ = Describe("ProxyConfigHandler", func() { configPath = filepath.Join(proxyConfigDir, container.Guid) proxyConfigFile = filepath.Join(configPath, "envoy.yaml") - sdsServerCertAndKeyFile = filepath.Join(configPath, "sds-server-cert-and-key.yaml") - sdsServerValidationContextFile = filepath.Join(configPath, "sds-server-validation-context.yaml") + sdsIDCertAndKeyFile = filepath.Join(configPath, "sds-id-cert-and-key.yaml") + sdsIDValidationContextFile = filepath.Join(configPath, "sds-id-validation-context.yaml") + sdsC2CCertAndKeyFile = filepath.Join(configPath, "sds-c2c-cert-and-key.yaml") logger = lagertest.NewTestLogger("proxymanager") @@ -103,6 +106,17 @@ var _ = Describe("ProxyConfigHandler", func() { "10.255.217.3:15010", } http2Enabled = true + cert, key, _ := generateCertAndKey() + credentials = containerstore.Credentials{ + InstanceIdentityCredential: containerstore.Credential{ + Cert: cert, + Key: key, + }, + C2CCredential: containerstore.Credential{ + Cert: cert, + Key: key, + }, + } }) JustBeforeEach(func() { @@ -155,7 +169,7 @@ var _ = Describe("ProxyConfigHandler", func() { Describe("Close", func() { JustBeforeEach(func() { Expect(proxyConfigFile).NotTo(BeAnExistingFile()) - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) }) @@ -167,7 +181,7 @@ var _ = Describe("ProxyConfigHandler", func() { Describe("Update", func() { JustBeforeEach(func() { Expect(proxyConfigFile).NotTo(BeAnExistingFile()) - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) }) @@ -260,12 +274,7 @@ var _ = Describe("ProxyConfigHandler", func() { Describe("ProxyPorts", func() { BeforeEach(func() { container.Ports = []executor.PortMapping{ - { - ContainerPort: 8080, - }, - { - ContainerPort: 9090, - }, + {ContainerPort: 8080}, } }) @@ -282,21 +291,60 @@ var _ = Describe("ProxyConfigHandler", func() { }) }) - It("each port gets an equivalent extra proxy port", func() { - ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) - Expect(err).NotTo(HaveOccurred()) - Expect(ports).To(ConsistOf([]executor.ProxyPortMapping{ - { - AppPort: 8080, - ProxyPort: 61001, - }, - { - AppPort: 9090, - ProxyPort: 61002, - }, - })) + Context("when there is a default HTTP port (8080)", func() { + BeforeEach(func() { + container.Ports = []executor.PortMapping{ + {ContainerPort: 8080}, + {ContainerPort: 9090}, + } + }) + + It("each port gets an equivalent extra proxy port and additional 61443 port for 8080", func() { + ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) + Expect(err).NotTo(HaveOccurred()) + Expect(ports).To(ConsistOf([]executor.ProxyPortMapping{ + { + AppPort: 8080, + ProxyPort: 61001, + }, + { + AppPort: 8080, + ProxyPort: 61443, + }, + { + AppPort: 9090, + ProxyPort: 61002, + }, + })) + + Expect(extraPorts).To(ConsistOf([]uint16{61001, 61002, 61443})) + }) + }) + + Context("when there is no default HTTP port (8080)", func() { + BeforeEach(func() { + container.Ports = []executor.PortMapping{ + {ContainerPort: 7070}, + {ContainerPort: 9090}, + } + }) - Expect(extraPorts).To(ConsistOf([]uint16{61001, 61002})) + It("each port gets an equivalent extra proxy port", func() { + ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) + Expect(err).NotTo(HaveOccurred()) + Expect(ports).To(ConsistOf([]executor.ProxyPortMapping{ + { + AppPort: 7070, + ProxyPort: 61001, + }, + { + AppPort: 9090, + ProxyPort: 61002, + }, + })) + + Expect(extraPorts).To(ConsistOf([]uint16{61001, 61002})) + }) }) Context("when the requested ports are in the 6100n range", func() { @@ -373,6 +421,10 @@ var _ = Describe("ProxyConfigHandler", func() { ContainerPort: 8080, ContainerTLSProxyPort: 61001, }, + { + ContainerPort: 8080, + ContainerTLSProxyPort: 61443, + }, } containerProxyRequireClientCerts = true @@ -384,7 +436,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("does not write a envoy config file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "", Key: ""}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Expect(proxyConfigFile).NotTo(BeAnExistingFile()) @@ -397,7 +449,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates a proxy config without tls_context", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) @@ -416,7 +468,7 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Clusters).To(HaveLen(2)) expectedCluster{ - name: "0-service-cluster", + name: "service-cluster-8080", hosts: []*envoy_core.Address{envoyAddr("10.0.0.1", 8080)}, maxConnections: math.MaxUint32, }.check(proxyConfig.StaticResources.Clusters[0]) @@ -430,25 +482,40 @@ var _ = Describe("ProxyConfigHandler", func() { }}.check(adsCluster) Expect(adsCluster.Http2ProtocolOptions).To(Equal(&envoy_core.Http2ProtocolOptions{})) - Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(1)) + Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(2)) expectedListener{ - name: "listener-8080", + name: "listener-8080-61001", listenPort: 61001, - statPrefix: "0-stats", - clusterName: "0-service-cluster", + statPrefix: "stats-8080-61001", + clusterName: "service-cluster-8080", requireClientCertificate: false, alpnProtocols: []string{"h2,http/1.1"}, - }.check(proxyConfig.StaticResources.Listeners[0]) + }.check(proxyConfig.StaticResources.Listeners[0], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) + expectedListener{ + name: "listener-8080-61443", + listenPort: 61443, + statPrefix: "stats-8080-61443", + clusterName: "service-cluster-8080", + requireClientCertificate: false, + alpnProtocols: []string{"h2,http/1.1"}, + }.check(proxyConfig.StaticResources.Listeners[1], + "c2c-cert-and-key", + "/etc/cf-assets/envoy_config/sds-c2c-cert-and-key.yaml", + ) }) }) - It("creates appropriate sds-server-cert-and-key.yaml configuration file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + It("creates appropriate sds-id-validation-context.yaml configuration file", func() { + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Eventually(sdsServerCertAndKeyFile).Should(BeAnExistingFile()) + Eventually(sdsIDCertAndKeyFile).Should(BeAnExistingFile()) + Eventually(sdsC2CCertAndKeyFile).Should(BeAnExistingFile()) var sdsCertificateDiscoveryResponse envoy_discovery.DiscoveryResponse - Expect(yamlFileToProto(sdsServerCertAndKeyFile, &sdsCertificateDiscoveryResponse)).To(Succeed()) + Expect(yamlFileToProto(sdsIDCertAndKeyFile, &sdsCertificateDiscoveryResponse)).To(Succeed()) Expect(sdsCertificateDiscoveryResponse.VersionInfo).To(Equal("0")) Expect(sdsCertificateDiscoveryResponse.Resources).To(HaveLen(1)) @@ -456,17 +523,17 @@ var _ = Describe("ProxyConfigHandler", func() { var secret envoy_tls.Secret Expect(ptypes.UnmarshalAny(sdsCertificateDiscoveryResponse.Resources[0], &secret)).To(Succeed()) - Expect(secret.Name).To(Equal("server-cert-and-key")) + Expect(secret.Name).To(Equal("id-cert-and-key")) Expect(secret.Type).To(Equal(&envoy_tls.Secret_TlsCertificate{ TlsCertificate: &envoy_tls.TlsCertificate{ CertificateChain: &envoy_core.DataSource{ Specifier: &envoy_core.DataSource_InlineString{ - InlineString: "cert", + InlineString: credentials.InstanceIdentityCredential.Cert, }, }, PrivateKey: &envoy_core.DataSource{ Specifier: &envoy_core.DataSource_InlineString{ - InlineString: "key", + InlineString: credentials.InstanceIdentityCredential.Key, }, }, }, @@ -490,22 +557,31 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(fileWatcher.Close()).To(Succeed()) }) - It("should ensure the sds-server-cert-and-key.yaml file is recreated when updating the config", func() { - Expect(ioutil.WriteFile(sdsServerCertAndKeyFile, []byte("old-content"), 0666)).To(Succeed()) - Expect(fileWatcher.Add(sdsServerCertAndKeyFile)).To(Succeed()) - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + It("should ensure the sds-id-cert-and-key.yaml file is recreated when updating the config", func() { + Expect(ioutil.WriteFile(sdsIDCertAndKeyFile, []byte("old-content"), 0666)).To(Succeed()) + Expect(fileWatcher.Add(sdsIDCertAndKeyFile)).To(Succeed()) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsServerCertAndKeyFile, Op: fsnotify.Remove}))) + Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsIDCertAndKeyFile, Op: fsnotify.Remove}))) }) - It("should ensure the sds-server-validation-context.yaml file is recreated when updating the config", func() { - Expect(ioutil.WriteFile(sdsServerValidationContextFile, []byte("old-content"), 0666)).To(Succeed()) - Expect(fileWatcher.Add(sdsServerValidationContextFile)).To(Succeed()) - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + It("should ensure the sds-c2c-cert-and-key.yaml file is recreated when updating the config", func() { + Expect(ioutil.WriteFile(sdsC2CCertAndKeyFile, []byte("old-content"), 0666)).To(Succeed()) + Expect(fileWatcher.Add(sdsC2CCertAndKeyFile)).To(Succeed()) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsServerValidationContextFile, Op: fsnotify.Remove}))) + Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsC2CCertAndKeyFile, Op: fsnotify.Remove}))) + }) + + It("should ensure the sds-id-validation-context.yaml file is recreated when updating the config", func() { + Expect(ioutil.WriteFile(sdsIDValidationContextFile, []byte("old-content"), 0666)).To(Succeed()) + Expect(fileWatcher.Add(sdsIDValidationContextFile)).To(Succeed()) + err := proxyConfigHandler.Update(credentials, container) + Expect(err).NotTo(HaveOccurred()) + + Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsIDValidationContextFile, Op: fsnotify.Remove}))) }) }) @@ -520,13 +596,13 @@ var _ = Describe("ProxyConfigHandler", func() { containerProxyVerifySubjectAltName = []string{"valid-alt-name-1", "valid-alt-name-2"} }) - It("creates appropriate sds-server-validation-context.yaml configuration file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + It("creates appropriate sds-id-validation-context.yaml configuration file", func() { + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Eventually(sdsServerValidationContextFile).Should(BeAnExistingFile()) + Eventually(sdsIDValidationContextFile).Should(BeAnExistingFile()) var sdsDiscoveryResponse envoy_discovery.DiscoveryResponse - Expect(yamlFileToProto(sdsServerValidationContextFile, &sdsDiscoveryResponse)).To(Succeed()) + Expect(yamlFileToProto(sdsIDValidationContextFile, &sdsDiscoveryResponse)).To(Succeed()) Expect(sdsDiscoveryResponse.VersionInfo).To(Equal("0")) Expect(sdsDiscoveryResponse.Resources).To(HaveLen(1)) @@ -534,7 +610,7 @@ var _ = Describe("ProxyConfigHandler", func() { var secret envoy_tls.Secret Expect(ptypes.UnmarshalAny(sdsDiscoveryResponse.Resources[0], &secret)).To(Succeed()) - Expect(secret.Name).To(Equal("server-validation-context")) + Expect(secret.Name).To(Equal("id-validation-context")) Expect(secret.Type).To(Equal(&envoy_tls.Secret_ValidationContext{ ValidationContext: &envoy_tls.CertificateValidationContext{ TrustedCa: &envoy_core.DataSource{ @@ -552,7 +628,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates the appropriate proxy config at start", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) @@ -571,7 +647,7 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Clusters).To(HaveLen(2)) expectedCluster{ - name: "0-service-cluster", + name: "service-cluster-8080", hosts: []*envoy_core.Address{envoyAddr("10.0.0.1", 8080)}, maxConnections: math.MaxUint32, }.check(proxyConfig.StaticResources.Clusters[0]) @@ -585,15 +661,29 @@ var _ = Describe("ProxyConfigHandler", func() { }}.check(adsCluster) Expect(adsCluster.Http2ProtocolOptions).To(Equal(&envoy_core.Http2ProtocolOptions{})) - Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(1)) + Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(2)) expectedListener{ - name: "listener-8080", + name: "listener-8080-61001", listenPort: 61001, - statPrefix: "0-stats", - clusterName: "0-service-cluster", + statPrefix: "stats-8080-61001", + clusterName: "service-cluster-8080", requireClientCertificate: true, alpnProtocols: []string{"h2,http/1.1"}, - }.check(proxyConfig.StaticResources.Listeners[0]) + }.check(proxyConfig.StaticResources.Listeners[0], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) + expectedListener{ + name: "listener-8080-61443", + listenPort: 61443, + statPrefix: "stats-8080-61443", + clusterName: "service-cluster-8080", + requireClientCertificate: false, + alpnProtocols: []string{"h2,http/1.1"}, + }.check(proxyConfig.StaticResources.Listeners[1], + "c2c-cert-and-key", + "/etc/cf-assets/envoy_config/sds-c2c-cert-and-key.yaml", + ) adsConfigSource := &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Ads{ @@ -625,7 +715,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates a proxy config without the ads config", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) @@ -634,7 +724,7 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Clusters).To(HaveLen(1)) cluster := proxyConfig.StaticResources.Clusters[0] - Expect(cluster.Name).To(Equal("0-service-cluster")) + Expect(cluster.Name).To(Equal("service-cluster-8080")) Expect(proxyConfig.DynamicResources).To(BeNil()) }) @@ -648,7 +738,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("returns an error", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).To(MatchError("ads server address is invalid: malformed")) }) }) @@ -661,7 +751,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("returns an error", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).To(MatchError("ads server address is invalid: 0.0.0.0:mal")) }) }) @@ -672,22 +762,36 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates a proxy config without ALPN for listeners", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) var proxyConfig envoy_bootstrap.Bootstrap Expect(yamlFileToProto(proxyConfigFile, &proxyConfig)).To(Succeed()) - Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(1)) + Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(2)) expectedListener{ - name: "listener-8080", + name: "listener-8080-61001", listenPort: 61001, - statPrefix: "0-stats", - clusterName: "0-service-cluster", + statPrefix: "stats-8080-61001", + clusterName: "service-cluster-8080", requireClientCertificate: true, alpnProtocols: nil, - }.check(proxyConfig.StaticResources.Listeners[0]) + }.check(proxyConfig.StaticResources.Listeners[0], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) + expectedListener{ + name: "listener-8080-61443", + listenPort: 61443, + statPrefix: "stats-8080-61443", + clusterName: "service-cluster-8080", + requireClientCertificate: false, + alpnProtocols: nil, + }.check(proxyConfig.StaticResources.Listeners[1], + "c2c-cert-and-key", + "/etc/cf-assets/envoy_config/sds-c2c-cert-and-key.yaml", + ) }) }) @@ -706,7 +810,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates the appropriate proxy config file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) @@ -725,13 +829,13 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Clusters).To(HaveLen(3)) expectedCluster{ - name: "0-service-cluster", + name: "service-cluster-8080", hosts: []*envoy_core.Address{envoyAddr("10.0.0.1", 8080)}, maxConnections: math.MaxUint32, }.check(proxyConfig.StaticResources.Clusters[0]) expectedCluster{ - name: "1-service-cluster", + name: "service-cluster-2222", hosts: []*envoy_core.Address{envoyAddr("10.0.0.1", 2222)}, maxConnections: math.MaxUint32, }.check(proxyConfig.StaticResources.Clusters[1]) @@ -748,22 +852,28 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(2)) expectedListener{ - name: "listener-8080", + name: "listener-8080-61001", listenPort: 61001, - statPrefix: "0-stats", - clusterName: "0-service-cluster", + statPrefix: "stats-8080-61001", + clusterName: "service-cluster-8080", requireClientCertificate: true, alpnProtocols: []string{"h2,http/1.1"}, - }.check(proxyConfig.StaticResources.Listeners[0]) + }.check(proxyConfig.StaticResources.Listeners[0], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) expectedListener{ - name: "listener-2222", + name: "listener-2222-61002", listenPort: 61002, - statPrefix: "1-stats", - clusterName: "1-service-cluster", + statPrefix: "stats-2222-61002", + clusterName: "service-cluster-2222", requireClientCertificate: true, alpnProtocols: []string{"h2,http/1.1"}, - }.check(proxyConfig.StaticResources.Listeners[1]) + }.check(proxyConfig.StaticResources.Listeners[1], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) }) Context("when no ports are left", func() { @@ -780,7 +890,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("returns an error", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).To(Equal(containerstore.ErrNoPortsAvailable)) }) }) @@ -788,10 +898,6 @@ var _ = Describe("ProxyConfigHandler", func() { }) Describe("Close", func() { - var ( - cert, key string - ) - BeforeEach(func() { container.Ports = []executor.PortMapping{ { @@ -799,8 +905,6 @@ var _ = Describe("ProxyConfigHandler", func() { ContainerTLSProxyPort: 61001, }, } - - cert, key, _ = generateCertAndKey() }) JustBeforeEach(func() { @@ -811,7 +915,7 @@ var _ = Describe("ProxyConfigHandler", func() { It("returns after the configured reload duration", func() { ch := make(chan struct{}) go func() { - proxyConfigHandler.Close(containerstore.Credential{Cert: cert, Key: key}, container) + proxyConfigHandler.Close(credentials, container) close(ch) }() @@ -826,7 +930,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("does not write a envoy config file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "", Key: ""}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Expect(proxyConfigFile).NotTo(BeAnExistingFile()) @@ -836,7 +940,7 @@ var _ = Describe("ProxyConfigHandler", func() { ch := make(chan struct{}) go func() { defer GinkgoRecover() - proxyConfigHandler.Close(containerstore.Credential{Cert: cert, Key: key}, container) + proxyConfigHandler.Close(credentials, container) close(ch) }() @@ -921,7 +1025,7 @@ type expectedListener struct { alpnProtocols []string } -func (l expectedListener) check(listener *envoy_listener.Listener) { +func (l expectedListener) check(listener *envoy_listener.Listener, sdsSecretName string, sdsFileName string) { Expect(listener.Name).To(Equal(l.name)) Expect(listener.Address).To(Equal(envoyAddr("0.0.0.0", l.listenPort))) Expect(listener.FilterChains).To(HaveLen(1)) @@ -941,14 +1045,13 @@ func (l expectedListener) check(listener *envoy_listener.Listener) { Expect(ptypes.UnmarshalAny(filterChain.TransportSocket.GetTypedConfig(), &downstreamTlsContext)).To(Succeed()) Expect(filterChain.TransportSocket.Name).To(Equal(l.name)) - Expect(downstreamTlsContext.RequireClientCertificate.Value).To(Equal(l.requireClientCertificate)) Expect(downstreamTlsContext.CommonTlsContext.AlpnProtocols).To(Equal(l.alpnProtocols)) Expect(downstreamTlsContext.CommonTlsContext.TlsCertificateSdsSecretConfigs).To(ConsistOf( &envoy_tls.SdsSecretConfig{ - Name: "server-cert-and-key", + Name: sdsSecretName, SdsConfig: &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Path{ - Path: "/etc/cf-assets/envoy_config/sds-server-cert-and-key.yaml", + Path: sdsFileName, }, }, }, @@ -958,12 +1061,13 @@ func (l expectedListener) check(listener *envoy_listener.Listener) { })) if l.requireClientCertificate { + Expect(downstreamTlsContext.RequireClientCertificate.Value).To(BeTrue()) Expect(downstreamTlsContext.CommonTlsContext.ValidationContextType).To(Equal(&envoy_tls.CommonTlsContext_ValidationContextSdsSecretConfig{ ValidationContextSdsSecretConfig: &envoy_tls.SdsSecretConfig{ - Name: "server-validation-context", + Name: "id-validation-context", SdsConfig: &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Path{ - Path: "/etc/cf-assets/envoy_config/sds-server-validation-context.yaml", + Path: "/etc/cf-assets/envoy_config/sds-id-validation-context.yaml", }, }, }, diff --git a/depot/containerstore/storenode.go b/depot/containerstore/storenode.go index 78b98568..56bdbd74 100644 --- a/depot/containerstore/storenode.go +++ b/depot/containerstore/storenode.go @@ -427,9 +427,9 @@ func (n *storeNode) portMappingFromContainerInfo( proxyPorts := make(map[uint16]struct{}) // construct a map from unproxied to proxied port - appToProxyPortMappings := make(map[uint16]uint16) + appToProxyPortMappings := make(map[uint16][]uint16) for _, mapping := range proxyToAppPort { - appToProxyPortMappings[mapping.AppPort] = mapping.ProxyPort + appToProxyPortMappings[mapping.AppPort] = append(appToProxyPortMappings[mapping.AppPort], mapping.ProxyPort) proxyPorts[mapping.ProxyPort] = struct{}{} } @@ -456,14 +456,23 @@ func (n *storeNode) portMappingFromContainerInfo( } hostPort := containerToHostPortMappings[appPort] - proxyContainerPort := appToProxyPortMappings[appPort] - proxyHostPort := containerToHostPortMappings[proxyContainerPort] - ports = append(ports, executor.PortMapping{ - HostPort: hostPort, - ContainerPort: appPort, - ContainerTLSProxyPort: proxyContainerPort, - HostTLSProxyPort: proxyHostPort, - }) + proxyContainerPorts := appToProxyPortMappings[appPort] + if len(proxyContainerPorts) > 0 { + for _, proxyContainerPort := range proxyContainerPorts { + proxyHostPort := containerToHostPortMappings[proxyContainerPort] + ports = append(ports, executor.PortMapping{ + HostPort: hostPort, + ContainerPort: appPort, + ContainerTLSProxyPort: proxyContainerPort, + HostTLSProxyPort: proxyHostPort, + }) + } + } else { + ports = append(ports, executor.PortMapping{ + HostPort: hostPort, + ContainerPort: appPort, + }) + } } return ports diff --git a/resources.go b/resources.go index c5cb0199..26dc28eb 100644 --- a/resources.go +++ b/resources.go @@ -5,6 +5,7 @@ import ( "time" "code.cloudfoundry.org/bbs/models" + "code.cloudfoundry.org/routing-info/internalroutes" ) const ContainerOwnerProperty = "executor:owner" @@ -165,28 +166,29 @@ type Sidecar struct { } type RunInfo struct { - RootFSPath string `json:"rootfs"` - CPUWeight uint `json:"cpu_weight"` - Ports []PortMapping `json:"ports"` - LogConfig LogConfig `json:"log_config"` - MetricsConfig MetricsConfig `json:"metrics_config"` - StartTimeoutMs uint `json:"start_timeout_ms"` - Privileged bool `json:"privileged"` - CachedDependencies []CachedDependency `json:"cached_dependencies"` - Setup *models.Action `json:"setup"` - Action *models.Action `json:"run"` - Monitor *models.Action `json:"monitor"` - CheckDefinition *models.CheckDefinition `json:"check_definition"` - EgressRules []*models.SecurityGroupRule `json:"egress_rules,omitempty"` - Env []EnvironmentVariable `json:"env,omitempty"` - TrustedSystemCertificatesPath string `json:"trusted_system_certificates_path,omitempty"` - VolumeMounts []VolumeMount `json:"volume_mounts"` - Network *Network `json:"network,omitempty"` - CertificateProperties CertificateProperties `json:"certificate_properties"` - ImageUsername string `json:"image_username"` - ImagePassword string `json:"image_password"` - EnableContainerProxy bool `json:"enable_container_proxy"` - Sidecars []Sidecar `json:"sidecars"` + RootFSPath string `json:"rootfs"` + CPUWeight uint `json:"cpu_weight"` + Ports []PortMapping `json:"ports"` + InternalRoutes internalroutes.InternalRoutes `json:"internal_routes"` + LogConfig LogConfig `json:"log_config"` + MetricsConfig MetricsConfig `json:"metrics_config"` + StartTimeoutMs uint `json:"start_timeout_ms"` + Privileged bool `json:"privileged"` + CachedDependencies []CachedDependency `json:"cached_dependencies"` + Setup *models.Action `json:"setup"` + Action *models.Action `json:"run"` + Monitor *models.Action `json:"monitor"` + CheckDefinition *models.CheckDefinition `json:"check_definition"` + EgressRules []*models.SecurityGroupRule `json:"egress_rules,omitempty"` + Env []EnvironmentVariable `json:"env,omitempty"` + TrustedSystemCertificatesPath string `json:"trusted_system_certificates_path,omitempty"` + VolumeMounts []VolumeMount `json:"volume_mounts"` + Network *Network `json:"network,omitempty"` + CertificateProperties CertificateProperties `json:"certificate_properties"` + ImageUsername string `json:"image_username"` + ImagePassword string `json:"image_password"` + EnableContainerProxy bool `json:"enable_container_proxy"` + Sidecars []Sidecar `json:"sidecars"` } type BindMountMode uint8