diff --git a/depot/containerstore/containerstore_test.go b/depot/containerstore/containerstore_test.go index 07006d88..8107953d 100644 --- a/depot/containerstore/containerstore_test.go +++ b/depot/containerstore/containerstore_test.go @@ -1146,6 +1146,10 @@ var _ = Describe("Container Store", func() { AppPort: 8080, ProxyPort: 61001, }, + { + AppPort: 8080, + ProxyPort: 61443, + }, { AppPort: 9090, ProxyPort: 61002, @@ -1153,6 +1157,7 @@ var _ = Describe("Container Store", func() { }, []uint16{ 61001, 61002, + 61443, }, nil) containerStore = containerstore.New( @@ -1198,6 +1203,8 @@ var _ = Describe("Container Store", func() { info.MappedPorts = append(info.MappedPorts, garden.PortMapping{HostPort: 16002, ContainerPort: 61002}) case 61003: info.MappedPorts = append(info.MappedPorts, garden.PortMapping{HostPort: 16003, ContainerPort: 61003}) + case 61443: + info.MappedPorts = append(info.MappedPorts, garden.PortMapping{HostPort: 16004, ContainerPort: 61443}) default: return info, errors.New("failed-net-in") } @@ -1221,7 +1228,7 @@ var _ = Describe("Container Store", func() { Expect(err).NotTo(HaveOccurred()) containerSpec := gardenClient.CreateArgsForCall(0) - Expect(containerSpec.NetIn).To(HaveLen(4)) + Expect(containerSpec.NetIn).To(HaveLen(5)) Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ HostPort: 0, ContainerPort: 8080, })) @@ -1234,6 +1241,9 @@ var _ = Describe("Container Store", func() { Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ HostPort: 0, ContainerPort: 61002, })) + Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ + HostPort: 0, ContainerPort: 61443, + })) }) Context("when disabling unproxied port mappings", func() { @@ -1265,13 +1275,16 @@ var _ = Describe("Container Store", func() { Expect(err).NotTo(HaveOccurred()) containerSpec := gardenClient.CreateArgsForCall(0) - Expect(containerSpec.NetIn).To(HaveLen(2)) + Expect(containerSpec.NetIn).To(HaveLen(3)) Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ HostPort: 0, ContainerPort: 61001, })) Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ HostPort: 0, ContainerPort: 61002, })) + Expect(containerSpec.NetIn).To(ContainElement(garden.NetIn{ + HostPort: 0, ContainerPort: 61443, + })) }) It("unproxied host ports are set to 0", func() { @@ -1283,6 +1296,11 @@ var _ = Describe("Container Store", func() { HostPort: 0, ContainerTLSProxyPort: 61001, HostTLSProxyPort: 16001, + }, executor.PortMapping{ + ContainerPort: 8080, + HostPort: 0, + ContainerTLSProxyPort: 61443, + HostTLSProxyPort: 16004, }, executor.PortMapping{ ContainerPort: 9090, HostPort: 0, @@ -1301,6 +1319,11 @@ var _ = Describe("Container Store", func() { HostPort: 16000, ContainerTLSProxyPort: 61001, HostTLSProxyPort: 16001, + }, executor.PortMapping{ + ContainerPort: 8080, + HostPort: 16000, + ContainerTLSProxyPort: 61443, + HostTLSProxyPort: 16004, }, executor.PortMapping{ ContainerPort: 9090, HostPort: 16004, @@ -1318,7 +1341,7 @@ var _ = Describe("Container Store", func() { Expect(containerStore.Run(logger, containerGuid)).NotTo(HaveOccurred()) Eventually(megatron.StepsRunnerCallCount).Should(Equal(1)) _, _, _, _, cfg := megatron.StepsRunnerArgsForCall(0) - Expect(cfg.ProxyTLSPorts).To(ConsistOf(uint16(61001), uint16(61002))) + Expect(cfg.ProxyTLSPorts).To(ConsistOf(uint16(61001), uint16(61002), uint16(61443))) }) It("bind mounts envoy", func() { diff --git a/depot/containerstore/containerstorefakes/fake_cred_handler.go b/depot/containerstore/containerstorefakes/fake_cred_handler.go index 69f8ea79..883fd11c 100644 --- a/depot/containerstore/containerstorefakes/fake_cred_handler.go +++ b/depot/containerstore/containerstorefakes/fake_cred_handler.go @@ -11,10 +11,10 @@ import ( ) type FakeCredentialHandler struct { - CloseStub func(containerstore.Credential, executor.Container) error + CloseStub func(containerstore.Credentials, executor.Container) error closeMutex sync.RWMutex closeArgsForCall []struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container } closeReturns struct { @@ -51,10 +51,10 @@ type FakeCredentialHandler struct { removeDirReturnsOnCall map[int]struct { result1 error } - UpdateStub func(containerstore.Credential, executor.Container) error + UpdateStub func(containerstore.Credentials, executor.Container) error updateMutex sync.RWMutex updateArgsForCall []struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container } updateReturns struct { @@ -67,11 +67,11 @@ type FakeCredentialHandler struct { invocationsMutex sync.RWMutex } -func (fake *FakeCredentialHandler) Close(arg1 containerstore.Credential, arg2 executor.Container) error { +func (fake *FakeCredentialHandler) Close(arg1 containerstore.Credentials, arg2 executor.Container) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] fake.closeArgsForCall = append(fake.closeArgsForCall, struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container }{arg1, arg2}) stub := fake.CloseStub @@ -93,13 +93,13 @@ func (fake *FakeCredentialHandler) CloseCallCount() int { return len(fake.closeArgsForCall) } -func (fake *FakeCredentialHandler) CloseCalls(stub func(containerstore.Credential, executor.Container) error) { +func (fake *FakeCredentialHandler) CloseCalls(stub func(containerstore.Credentials, executor.Container) error) { fake.closeMutex.Lock() defer fake.closeMutex.Unlock() fake.CloseStub = stub } -func (fake *FakeCredentialHandler) CloseArgsForCall(i int) (containerstore.Credential, executor.Container) { +func (fake *FakeCredentialHandler) CloseArgsForCall(i int) (containerstore.Credentials, executor.Container) { fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() argsForCall := fake.closeArgsForCall[i] @@ -259,11 +259,11 @@ func (fake *FakeCredentialHandler) RemoveDirReturnsOnCall(i int, result1 error) }{result1} } -func (fake *FakeCredentialHandler) Update(arg1 containerstore.Credential, arg2 executor.Container) error { +func (fake *FakeCredentialHandler) Update(arg1 containerstore.Credentials, arg2 executor.Container) error { fake.updateMutex.Lock() ret, specificReturn := fake.updateReturnsOnCall[len(fake.updateArgsForCall)] fake.updateArgsForCall = append(fake.updateArgsForCall, struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container }{arg1, arg2}) stub := fake.UpdateStub @@ -285,13 +285,13 @@ func (fake *FakeCredentialHandler) UpdateCallCount() int { return len(fake.updateArgsForCall) } -func (fake *FakeCredentialHandler) UpdateCalls(stub func(containerstore.Credential, executor.Container) error) { +func (fake *FakeCredentialHandler) UpdateCalls(stub func(containerstore.Credentials, executor.Container) error) { fake.updateMutex.Lock() defer fake.updateMutex.Unlock() fake.UpdateStub = stub } -func (fake *FakeCredentialHandler) UpdateArgsForCall(i int) (containerstore.Credential, executor.Container) { +func (fake *FakeCredentialHandler) UpdateArgsForCall(i int) (containerstore.Credentials, executor.Container) { fake.updateMutex.RLock() defer fake.updateMutex.RUnlock() argsForCall := fake.updateArgsForCall[i] diff --git a/depot/containerstore/containerstorefakes/fake_proxymanager.go b/depot/containerstore/containerstorefakes/fake_proxymanager.go index a29acc41..77046af5 100644 --- a/depot/containerstore/containerstorefakes/fake_proxymanager.go +++ b/depot/containerstore/containerstorefakes/fake_proxymanager.go @@ -11,10 +11,10 @@ import ( ) type FakeProxyManager struct { - CloseStub func(containerstore.Credential, executor.Container) error + CloseStub func(containerstore.Credentials, executor.Container) error closeMutex sync.RWMutex closeArgsForCall []struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container } closeReturns struct { @@ -67,10 +67,10 @@ type FakeProxyManager struct { removeDirReturnsOnCall map[int]struct { result1 error } - UpdateStub func(containerstore.Credential, executor.Container) error + UpdateStub func(containerstore.Credentials, executor.Container) error updateMutex sync.RWMutex updateArgsForCall []struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container } updateReturns struct { @@ -83,11 +83,11 @@ type FakeProxyManager struct { invocationsMutex sync.RWMutex } -func (fake *FakeProxyManager) Close(arg1 containerstore.Credential, arg2 executor.Container) error { +func (fake *FakeProxyManager) Close(arg1 containerstore.Credentials, arg2 executor.Container) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] fake.closeArgsForCall = append(fake.closeArgsForCall, struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container }{arg1, arg2}) stub := fake.CloseStub @@ -109,13 +109,13 @@ func (fake *FakeProxyManager) CloseCallCount() int { return len(fake.closeArgsForCall) } -func (fake *FakeProxyManager) CloseCalls(stub func(containerstore.Credential, executor.Container) error) { +func (fake *FakeProxyManager) CloseCalls(stub func(containerstore.Credentials, executor.Container) error) { fake.closeMutex.Lock() defer fake.closeMutex.Unlock() fake.CloseStub = stub } -func (fake *FakeProxyManager) CloseArgsForCall(i int) (containerstore.Credential, executor.Container) { +func (fake *FakeProxyManager) CloseArgsForCall(i int) (containerstore.Credentials, executor.Container) { fake.closeMutex.RLock() defer fake.closeMutex.RUnlock() argsForCall := fake.closeArgsForCall[i] @@ -343,11 +343,11 @@ func (fake *FakeProxyManager) RemoveDirReturnsOnCall(i int, result1 error) { }{result1} } -func (fake *FakeProxyManager) Update(arg1 containerstore.Credential, arg2 executor.Container) error { +func (fake *FakeProxyManager) Update(arg1 containerstore.Credentials, arg2 executor.Container) error { fake.updateMutex.Lock() ret, specificReturn := fake.updateReturnsOnCall[len(fake.updateArgsForCall)] fake.updateArgsForCall = append(fake.updateArgsForCall, struct { - arg1 containerstore.Credential + arg1 containerstore.Credentials arg2 executor.Container }{arg1, arg2}) stub := fake.UpdateStub @@ -369,13 +369,13 @@ func (fake *FakeProxyManager) UpdateCallCount() int { return len(fake.updateArgsForCall) } -func (fake *FakeProxyManager) UpdateCalls(stub func(containerstore.Credential, executor.Container) error) { +func (fake *FakeProxyManager) UpdateCalls(stub func(containerstore.Credentials, executor.Container) error) { fake.updateMutex.Lock() defer fake.updateMutex.Unlock() fake.UpdateStub = stub } -func (fake *FakeProxyManager) UpdateArgsForCall(i int) (containerstore.Credential, executor.Container) { +func (fake *FakeProxyManager) UpdateArgsForCall(i int) (containerstore.Credentials, executor.Container) { fake.updateMutex.RLock() defer fake.updateMutex.RUnlock() argsForCall := fake.updateArgsForCall[i] diff --git a/depot/containerstore/credmanager.go b/depot/containerstore/credmanager.go index 746aa704..200648fc 100644 --- a/depot/containerstore/credmanager.go +++ b/depot/containerstore/credmanager.go @@ -21,6 +21,7 @@ import ( "code.cloudfoundry.org/executor" "code.cloudfoundry.org/garden" "code.cloudfoundry.org/lager" + "code.cloudfoundry.org/routing-info/internalroutes" ) const ( @@ -29,6 +30,11 @@ const ( CredCreationFailedCount = "CredCreationFailedCount" ) +type Credentials struct { + InstanceIdentityCredential Credential + C2CCredential Credential +} + type Credential struct { Cert string Key string @@ -85,12 +91,12 @@ type CredentialHandler interface { RemoveDir(logger lager.Logger, container executor.Container) error // Called periodically as new valid certificate/key pair are generated - Update(credentials Credential, container executor.Container) error + Update(credentials Credentials, container executor.Container) error // Called when the CredManager is preparing to exit. This is mainly to update // the EnvoyProxy with invalid certificates and prevent it from accepting // more incoming traffic from the gorouter - Close(invalidCredentials Credential, container executor.Container) error + Close(invalidCredentials Credentials, container executor.Container) error } func NewCredManager( @@ -242,11 +248,41 @@ const ( privateKeyPEMBlockType = "RSA PRIVATE KEY" ) -func (c *credManager) generateCreds(logger lager.Logger, container executor.Container, certGUID string) (Credential, error) { +func (c *credManager) generateCreds(logger lager.Logger, container executor.Container, certGUID string) (Credentials, error) { logger = logger.Session("generating-credentials") logger.Debug("starting") defer logger.Debug("complete") + ipForCert := container.InternalIP + if len(ipForCert) == 0 { + ipForCert = container.ExternalIP + } + + logger.Debug("generating-credentials-for-instance-identity") + idCred, err := c.generateCredForSAN(logger, + certificateSAN{IPAddress: ipForCert, OrganizationalUnits: container.CertificateProperties.OrganizationalUnit}, + certGUID, + ) + if err != nil { + return Credentials{}, err + } + + logger.Debug("generating-credentials-for-c2c") + c2cCred, err := c.generateCredForSAN(logger, + certificateSAN{InternalRoutes: container.InternalRoutes, OrganizationalUnits: container.CertificateProperties.OrganizationalUnit}, + certGUID, + ) + if err != nil { + return Credentials{}, err + } + + return Credentials{ + InstanceIdentityCredential: idCred, + C2CCredential: c2cCred, + }, nil +} + +func (c *credManager) generateCredForSAN(logger lager.Logger, certSAN certificateSAN, certGUID string) (Credential, error) { logger.Debug("generating-private-key") privateKey, err := rsa.GenerateKey(c.entropyReader, 2048) if err != nil { @@ -254,18 +290,12 @@ func (c *credManager) generateCreds(logger lager.Logger, container executor.Cont } logger.Debug("generated-private-key") - ipForCert := container.InternalIP - if len(ipForCert) == 0 { - ipForCert = container.ExternalIP - } - startValidity := c.clock.Now() - template := createCertificateTemplate(ipForCert, - certGUID, + template := createCertificateTemplate(certGUID, + certSAN, startValidity, startValidity.Add(c.validityPeriod), - container.CertificateProperties.OrganizationalUnit, ) logger.Debug("generating-serial-number") @@ -306,11 +336,10 @@ func (c *credManager) generateCreds(logger lager.Logger, container executor.Cont return Credential{}, err } - creds := Credential{ + return Credential{ Cert: certificateBuf.String(), Key: keyBuf.String(), - } - return creds, nil + }, nil } func pemEncode(bytes []byte, blockType string, writer io.Writer) error { @@ -321,21 +350,32 @@ func pemEncode(bytes []byte, blockType string, writer io.Writer) error { return pem.Encode(writer, block) } -func createCertificateTemplate(ipaddress, guid string, notBefore, notAfter time.Time, organizationalUnits []string) *x509.Certificate { +type certificateSAN struct { + IPAddress string + InternalRoutes internalroutes.InternalRoutes + OrganizationalUnits []string +} + +func createCertificateTemplate(guid string, certSAN certificateSAN, notBefore, notAfter time.Time) *x509.Certificate { var ipaddr []net.IP - if len(ipaddress) == 0 { + if len(certSAN.IPAddress) == 0 { ipaddr = []net.IP{} } else { - ipaddr = []net.IP{net.ParseIP(ipaddress)} + ipaddr = []net.IP{net.ParseIP(certSAN.IPAddress)} + } + dnsNames := []string{guid} + for _, route := range certSAN.InternalRoutes { + dnsNames = append(dnsNames, route.Hostname) } + return &x509.Certificate{ SerialNumber: big.NewInt(0), Subject: pkix.Name{ CommonName: guid, - OrganizationalUnit: organizationalUnits, + OrganizationalUnit: certSAN.OrganizationalUnits, }, IPAddresses: ipaddr, - DNSNames: []string{guid}, + DNSNames: dnsNames, NotBefore: notBefore, NotAfter: notAfter, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement, diff --git a/depot/containerstore/credmanager_test.go b/depot/containerstore/credmanager_test.go index 75b21c8d..00371aaf 100644 --- a/depot/containerstore/credmanager_test.go +++ b/depot/containerstore/credmanager_test.go @@ -21,6 +21,7 @@ import ( "code.cloudfoundry.org/garden" "code.cloudfoundry.org/lager" "code.cloudfoundry.org/lager/lagertest" + "code.cloudfoundry.org/routing-info/internalroutes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/tedsuo/ifrit" @@ -229,8 +230,14 @@ var _ = Describe("CredManager", func() { container = executor.Container{ Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), InternalIP: "127.0.0.1", - RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ - OrganizationalUnit: []string{"app:iamthelizardking"}}, + RunInfo: executor.RunInfo{ + InternalRoutes: internalroutes.InternalRoutes{ + {Hostname: "a.apps.internal"}, + {Hostname: "b.apps.internal"}, + }, + CertificateProperties: executor.CertificateProperties{ + OrganizationalUnit: []string{"app:iamthelizardking"}, + }, }, } }) @@ -311,26 +318,28 @@ var _ = Describe("CredManager", func() { Context("when the certificate is about to expire", func() { var ( - credBefore containerstore.Credential + credsBefore containerstore.Credentials ) JustBeforeEach(func() { Eventually(fakeCredHandler.UpdateCallCount).Should(Equal(1)) Eventually(containerProcess.Ready()).Should(BeClosed()) - credBefore, _ = fakeCredHandler.UpdateArgsForCall(0) + credsBefore, _ = fakeCredHandler.UpdateArgsForCall(0) }) testCredentialRotation := func(dur time.Duration) { callCount := fakeCredHandler.UpdateCallCount() var actualContainer executor.Container - credBefore, actualContainer = fakeCredHandler.UpdateArgsForCall(callCount - 1) + credsBefore, actualContainer = fakeCredHandler.UpdateArgsForCall(callCount - 1) Expect(actualContainer).To(Equal(container)) - certBefore, _ := parseCert(credBefore) - increment := certBefore.NotAfter.Add(-dur).Sub(clock.Now()) + idCertBefore, _ := parseCert(credsBefore.InstanceIdentityCredential) + c2cCertBefore, _ := parseCert(credsBefore.C2CCredential) + Expect(idCertBefore.NotAfter).To(Equal(c2cCertBefore.NotAfter)) + increment := idCertBefore.NotAfter.Add(-dur).Sub(clock.Now()) Expect(increment).To(BeNumerically(">", 0)) clock.WaitForWatcherAndIncrement(increment) @@ -339,19 +348,27 @@ var _ = Describe("CredManager", func() { cred, actualContainer := fakeCredHandler.UpdateArgsForCall(callCount) Expect(actualContainer).To(Equal(container)) - Expect(cred.Cert).NotTo(Equal(credBefore.Cert)) - Expect(cred.Key).NotTo(Equal(credBefore.Key)) + Expect(cred.InstanceIdentityCredential.Cert).NotTo(Equal(credsBefore.InstanceIdentityCredential.Cert)) + Expect(cred.InstanceIdentityCredential.Key).NotTo(Equal(credsBefore.InstanceIdentityCredential.Key)) + + Expect(cred.C2CCredential.Cert).NotTo(Equal(credsBefore.C2CCredential.Cert)) + Expect(cred.C2CCredential.Key).NotTo(Equal(credsBefore.C2CCredential.Key)) + + idCert, _ := parseCert(cred.InstanceIdentityCredential) + Expect(idCert.SerialNumber).NotTo(Equal(idCertBefore.SerialNumber)) - cert, _ := parseCert(cred) - Expect(cert.SerialNumber).NotTo(Equal(certBefore.SerialNumber)) + c2cCert, _ := parseCert(cred.InstanceIdentityCredential) + Expect(c2cCert.SerialNumber).NotTo(Equal(c2cCertBefore.SerialNumber)) } testNoCredentialRotation := func(dur time.Duration) { callCount := fakeCredHandler.UpdateCallCount() - credBefore, _ = fakeCredHandler.UpdateArgsForCall(callCount - 1) + credsBefore, _ = fakeCredHandler.UpdateArgsForCall(callCount - 1) - cert, _ := parseCert(credBefore) - increment := cert.NotAfter.Add(-dur).Sub(clock.Now()) + idCertBefore, _ := parseCert(credsBefore.InstanceIdentityCredential) + c2cCertBefore, _ := parseCert(credsBefore.C2CCredential) + Expect(idCertBefore.NotAfter).To(Equal(c2cCertBefore.NotAfter)) + increment := idCertBefore.NotAfter.Add(-dur).Sub(clock.Now()) Expect(increment).To(BeNumerically(">", 0)) clock.WaitForWatcherAndIncrement(increment) @@ -361,8 +378,8 @@ var _ = Describe("CredManager", func() { Context("when the handler returns an error", func() { JustBeforeEach(func() { fakeCredHandler.UpdateReturnsOnCall(1, errors.New("boooom!")) - certBefore, _ := parseCert(credBefore) - increment := certBefore.NotAfter.Sub(clock.Now()) + idCertBefore, _ := parseCert(credsBefore.InstanceIdentityCredential) + increment := idCertBefore.NotAfter.Sub(clock.Now()) clock.WaitForWatcherAndIncrement(increment) }) @@ -388,7 +405,7 @@ var _ = Describe("CredManager", func() { }) It("emits metrics on successful creation", func() { - cert, _ := parseCert(credBefore) + cert, _ := parseCert(credsBefore.InstanceIdentityCredential) increment := cert.NotAfter.Add(-5 * time.Second).Sub(clock.Now()) Expect(increment).To(BeNumerically(">", 0)) clock.WaitForWatcherAndIncrement(increment) @@ -449,52 +466,33 @@ var _ = Describe("CredManager", func() { rest []byte ) - JustBeforeEach(func() { - Eventually(containerProcess.Ready()).Should(BeClosed()) - - Eventually(fakeCredHandler.UpdateCallCount).Should(Equal(1)) - cred, actualContainer := fakeCredHandler.UpdateArgsForCall(0) - Expect(actualContainer).To(Equal(container)) - cert, rest = parseCert(cred) - }) - - It("has the container ip", func() { - ip := net.ParseIP(container.InternalIP) - Expect(cert.IPAddresses).To(ContainElement(ip.To4())) - }) - - It("has all required usages in the KU & EKU fields", func() { + testCertificateFields := func() { + By("has all required usages in the KU & EKU fields") Expect(cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageClientAuth)) Expect(cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageServerAuth)) Expect(cert.KeyUsage).To(Equal(x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement)) - }) - It("signed by the rep intermediate CA", func() { + By("signed by the rep intermediate CA") CaCertPool := x509.NewCertPool() CaCertPool.AddCert(CaCert) verifyOpts := x509.VerifyOptions{Roots: CaCertPool} Expect(cert.CheckSignatureFrom(CaCert)).To(Succeed()) _, err := cert.Verify(verifyOpts) Expect(err).NotTo(HaveOccurred()) - }) - It("common name should be set to the container guid", func() { + By("common name should be set to the container guid") Expect(cert.Subject.CommonName).To(Equal(container.Guid)) - }) - It("DNS SAN should be set to the container guid", func() { + By("DNS SAN should be set to the container guid") Expect(cert.DNSNames).To(ContainElement(container.Guid)) - }) - It("expires in after the configured validity period", func() { + By("expires in after the configured validity period") Expect(cert.NotAfter).To(Equal(clock.Now().Add(validityPeriod))) - }) - It("not before is set to current timestamp", func() { + By("not before is set to current timestamp") Expect(cert.NotBefore).To(Equal(clock.Now())) - }) - It("has the rep intermediate CA", func() { + By("has the rep intermediate CA") block, rest := pem.Decode(rest) Expect(block).NotTo(BeNil()) Expect(rest).To(BeEmpty()) @@ -502,53 +500,91 @@ var _ = Describe("CredManager", func() { certs, err := x509.ParseCertificates(block.Bytes) Expect(err).NotTo(HaveOccurred()) Expect(certs).To(HaveLen(1)) - cert = certs[0] - Expect(cert).To(Equal(CaCert)) - }) + Expect(certs[0]).To(Equal(CaCert)) - It("has the app guid in the subject's organizational units", func() { + By("has the app guid in the subject's organizational units") Expect(cert.Subject.OrganizationalUnit).To(ContainElement("app:iamthelizardking")) - }) + } - Context("when the container doesn't have an internal ip", func() { - Context("when the container has an external IP", func() { - BeforeEach(func() { - container = executor.Container{ - Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), - InternalIP: "", - ExternalIP: "54.23.123.234", - RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ - OrganizationalUnit: []string{"app:iamthelizardking"}}, - }, - } - }) + Describe("Instance identity certificate", func() { + JustBeforeEach(func() { + Eventually(containerProcess.Ready()).Should(BeClosed()) + + Eventually(fakeCredHandler.UpdateCallCount).Should(Equal(1)) + creds, actualContainer := fakeCredHandler.UpdateArgsForCall(0) + Expect(actualContainer).To(Equal(container)) + cert, rest = parseCert(creds.InstanceIdentityCredential) + }) - It("has the external ip", func() { - ip := net.ParseIP(container.ExternalIP) - Expect(cert.IPAddresses).To(ContainElement(ip.To4())) + Context("when the container doesn't have an internal ip", func() { + Context("when the container has an external IP", func() { + BeforeEach(func() { + container = executor.Container{ + Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), + InternalIP: "", + ExternalIP: "54.23.123.234", + RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ + OrganizationalUnit: []string{"app:iamthelizardking"}}, + }, + } + }) + + It("has the external ip", func() { + ip := net.ParseIP(container.ExternalIP) + Expect(cert.IPAddresses).To(ContainElement(ip.To4())) + }) + + It("does not have the empty internal ip", func() { + ip := net.ParseIP(container.InternalIP) + Expect(cert.IPAddresses).NotTo(ContainElement(ip.To4())) + }) }) - It("does not have the empty internal ip", func() { - ip := net.ParseIP(container.InternalIP) - Expect(cert.IPAddresses).NotTo(ContainElement(ip.To4())) + Context("when the container doesn't have an external ip", func() { + BeforeEach(func() { + container = executor.Container{ + Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), + InternalIP: "", + ExternalIP: "", + RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ + OrganizationalUnit: []string{"app:iamthelizardking"}}, + }, + } + }) + + It("has no SubjectAltName", func() { + Expect(cert.IPAddresses).To(BeEmpty()) + }) + }) }) - Context("when the container doesn't have an external ip", func() { - BeforeEach(func() { - container = executor.Container{ - Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelNode()), - InternalIP: "", - ExternalIP: "", - RunInfo: executor.RunInfo{CertificateProperties: executor.CertificateProperties{ - OrganizationalUnit: []string{"app:iamthelizardking"}}, - }, - } - }) - It("has no SubjectAltName", func() { - Expect(cert.IPAddresses).To(BeEmpty()) - }) + It("has the container ip", func() { + ip := net.ParseIP(container.InternalIP) + Expect(cert.IPAddresses).To(ContainElement(ip.To4())) + }) + + It("has the correct fields", func() { + testCertificateFields() + }) + }) + + Describe("C2C certificate", func() { + JustBeforeEach(func() { + Eventually(containerProcess.Ready()).Should(BeClosed()) + Eventually(fakeCredHandler.UpdateCallCount).Should(Equal(1)) + creds, actualContainer := fakeCredHandler.UpdateArgsForCall(0) + Expect(actualContainer).To(Equal(container)) + cert, rest = parseCert(creds.C2CCredential) + }) + + It("has the internal routes in SAN", func() { + Expect(cert.DNSNames).To(ConsistOf(container.Guid, "a.apps.internal", "b.apps.internal")) + }) + + It("has the correct fields", func() { + testCertificateFields() }) }) }) @@ -568,16 +604,24 @@ var _ = Describe("CredManager", func() { It("Generates an invalid cert and sends the invalid cert on the cred channel", func() { Eventually(fakeCredHandler.CloseCallCount).Should(Equal(1)) - cred, actualContainer := fakeCredHandler.CloseArgsForCall(0) + creds, actualContainer := fakeCredHandler.CloseArgsForCall(0) Expect(actualContainer).To(Equal(container)) - block, _ := pem.Decode([]byte(cred.Cert)) + block, _ := pem.Decode([]byte(creds.InstanceIdentityCredential.Cert)) Expect(block).NotTo(BeNil()) certs, err := x509.ParseCertificates(block.Bytes) Expect(err).NotTo(HaveOccurred()) cert := certs[0] Expect(cert.Subject.CommonName).To(BeEmpty()) + + block, _ = pem.Decode([]byte(creds.C2CCredential.Cert)) + Expect(block).NotTo(BeNil()) + certs, err = x509.ParseCertificates(block.Bytes) + Expect(err).NotTo(HaveOccurred()) + cert = certs[0] + + Expect(cert.Subject.CommonName).To(BeEmpty()) }) }) }) diff --git a/depot/containerstore/instance_identity_handler.go b/depot/containerstore/instance_identity_handler.go index 24e6dfc7..4f9c481d 100644 --- a/depot/containerstore/instance_identity_handler.go +++ b/depot/containerstore/instance_identity_handler.go @@ -49,7 +49,7 @@ func (h *InstanceIdentityHandler) RemoveDir(logger lager.Logger, container execu return os.RemoveAll(filepath.Join(h.credDir, container.Guid)) } -func (h *InstanceIdentityHandler) Update(cred Credential, container executor.Container) error { +func (h *InstanceIdentityHandler) Update(creds Credentials, container executor.Container) error { instanceKeyPath := filepath.Join(h.credDir, container.Guid, "instance.key") tmpInstanceKeyPath := instanceKeyPath + ".tmp" certificatePath := filepath.Join(h.credDir, container.Guid, "instance.crt") @@ -67,12 +67,12 @@ func (h *InstanceIdentityHandler) Update(cred Credential, container executor.Con } defer certificate.Close() - _, err = certificate.Write([]byte(cred.Cert)) + _, err = certificate.Write([]byte(creds.InstanceIdentityCredential.Cert)) if err != nil { return err } - _, err = instanceKey.Write([]byte(cred.Key)) + _, err = instanceKey.Write([]byte(creds.InstanceIdentityCredential.Key)) if err != nil { return err } @@ -100,6 +100,6 @@ func (h *InstanceIdentityHandler) Update(cred Credential, container executor.Con return nil } -func (h *InstanceIdentityHandler) Close(cred Credential, container executor.Container) error { +func (h *InstanceIdentityHandler) Close(creds Credentials, container executor.Container) error { return nil } diff --git a/depot/containerstore/instance_identity_handler_test.go b/depot/containerstore/instance_identity_handler_test.go index c51e5aec..d5593f1b 100644 --- a/depot/containerstore/instance_identity_handler_test.go +++ b/depot/containerstore/instance_identity_handler_test.go @@ -83,7 +83,7 @@ var _ = Describe("InstanceIdentityHandler", func() { }) It("puts private key into container directory", func() { - err := handler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := handler.Update(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "cert", Key: "key"}}, container) Expect(err).NotTo(HaveOccurred()) certPath := filepath.Join(tmpdir, "some-guid") @@ -97,7 +97,7 @@ var _ = Describe("InstanceIdentityHandler", func() { }) It("puts the certificate into container directory", func() { - err := handler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := handler.Update(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "cert", Key: "key"}}, container) Expect(err).NotTo(HaveOccurred()) certPath := filepath.Join(tmpdir, "some-guid") @@ -115,12 +115,12 @@ var _ = Describe("InstanceIdentityHandler", func() { BeforeEach(func() { _, _, err := handler.CreateDir(logger, container) Expect(err).To(BeNil()) - err = handler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err = handler.Update(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "cert", Key: "key"}}, container) Expect(err).NotTo(HaveOccurred()) }) It("does not put private key into container directory", func() { - err := handler.Close(containerstore.Credential{Cert: "invalid-cert", Key: "invalid-key"}, container) + err := handler.Close(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "invalid-cert", Key: "invalid-key"}}, container) Expect(err).NotTo(HaveOccurred()) certPath := filepath.Join(tmpdir, "some-guid") @@ -134,7 +134,7 @@ var _ = Describe("InstanceIdentityHandler", func() { }) It("does not put the certificate into container directory", func() { - err := handler.Close(containerstore.Credential{Cert: "invalid-cert", Key: "invalid-key"}, container) + err := handler.Close(containerstore.Credentials{InstanceIdentityCredential: containerstore.Credential{Cert: "invalid-cert", Key: "invalid-key"}}, container) Expect(err).NotTo(HaveOccurred()) certPath := filepath.Join(tmpdir, "some-guid") diff --git a/depot/containerstore/proxy_config_handler.go b/depot/containerstore/proxy_config_handler.go index 09707a3c..fb42fdbe 100644 --- a/depot/containerstore/proxy_config_handler.go +++ b/depot/containerstore/proxy_config_handler.go @@ -38,9 +38,10 @@ import ( ) const ( - StartProxyPort = 61001 - EndProxyPort = 65534 - C2CTLSPort = 61443 + StartProxyPort = 61001 + EndProxyPort = 65534 + DefaultHTTPPort = 8080 + C2CTLSPort = 61443 TimeOut = 250000000 @@ -49,6 +50,11 @@ const ( AdsClusterName = "pilot-ads" AdminAccessLog = os.DevNull + + InstanceIdentityCertAndKeyFilename = "sds-id-cert-and-key.yaml" + InstanceIdentityValidationContextFilename = "sds-id-validation-context.yaml" + C2CCertAndKeyFilename = "sds-c2c-cert-and-key.yaml" + ContainerEnvoyConfigDir = "/etc/cf-assets/envoy_config" ) var ( @@ -97,10 +103,10 @@ func (p *NoopProxyConfigHandler) CreateDir(logger lager.Logger, container execut func (p *NoopProxyConfigHandler) RemoveDir(logger lager.Logger, container executor.Container) error { return nil } -func (p *NoopProxyConfigHandler) Update(credentials Credential, container executor.Container) error { +func (p *NoopProxyConfigHandler) Update(credentials Credentials, container executor.Container) error { return nil } -func (p *NoopProxyConfigHandler) Close(invalidCredentials Credential, container executor.Container) error { +func (p *NoopProxyConfigHandler) Close(invalidCredentials Credentials, container executor.Container) error { return nil } @@ -186,6 +192,14 @@ func (p *ProxyConfigHandler) ProxyPorts(logger lager.Logger, container *executor ProxyPort: port, }) + if containerPorts[portCount] == DefaultHTTPPort { + proxyPortMapping = append(proxyPortMapping, executor.ProxyPortMapping{ + AppPort: containerPorts[portCount], + ProxyPort: C2CTLSPort, + }) + extraPorts = append(extraPorts, C2CTLSPort) + } + portCount++ } @@ -208,7 +222,7 @@ func (p *ProxyConfigHandler) CreateDir(logger lager.Logger, container executor.C { Origin: garden.BindMountOriginHost, SrcPath: proxyConfigDir, - DstPath: "/etc/cf-assets/envoy_config", + DstPath: ContainerEnvoyConfigDir, }, } @@ -230,7 +244,7 @@ func (p *ProxyConfigHandler) RemoveDir(logger lager.Logger, container executor.C return os.RemoveAll(proxyConfigDir) } -func (p *ProxyConfigHandler) Update(credentials Credential, container executor.Container) error { +func (p *ProxyConfigHandler) Update(credentials Credentials, container executor.Container) error { if !container.EnableContainerProxy { return nil } @@ -238,7 +252,7 @@ func (p *ProxyConfigHandler) Update(credentials Credential, container executor.C return p.writeConfig(credentials, container) } -func (p *ProxyConfigHandler) Close(invalidCredentials Credential, container executor.Container) error { +func (p *ProxyConfigHandler) Close(invalidCredentials Credentials, container executor.Container) error { if !container.EnableContainerProxy { return nil } @@ -252,10 +266,11 @@ func (p *ProxyConfigHandler) Close(invalidCredentials Credential, container exec return nil } -func (p *ProxyConfigHandler) writeConfig(credentials Credential, container executor.Container) error { +func (p *ProxyConfigHandler) writeConfig(credentials Credentials, container executor.Container) error { proxyConfigPath := filepath.Join(p.containerProxyConfigPath, container.Guid, "envoy.yaml") - sdsServerCertAndKeyPath := filepath.Join(p.containerProxyConfigPath, container.Guid, "sds-server-cert-and-key.yaml") - sdsServerValidationContextPath := filepath.Join(p.containerProxyConfigPath, container.Guid, "sds-server-validation-context.yaml") + sdsIDCertAndKeyPath := filepath.Join(p.containerProxyConfigPath, container.Guid, InstanceIdentityCertAndKeyFilename) + sdsC2CCertAndKeyPath := filepath.Join(p.containerProxyConfigPath, container.Guid, C2CCertAndKeyFilename) + sdsIDValidationContextPath := filepath.Join(p.containerProxyConfigPath, container.Guid, InstanceIdentityValidationContextFilename) adminPort, err := getAvailablePort(container.Ports) if err != nil { @@ -278,22 +293,28 @@ func (p *ProxyConfigHandler) writeConfig(credentials Credential, container execu return err } - sdsServerCertAndKey := generateSDSCertAndKey(container, credentials) - err = writeDiscoveryResponseYAML(sdsServerCertAndKey, sdsServerCertAndKeyPath) + sdsIDCertAndKey := generateSDSCertAndKey("id-cert-and-key", credentials.InstanceIdentityCredential) + err = writeDiscoveryResponseYAML(sdsIDCertAndKey, sdsIDCertAndKeyPath) if err != nil { return err } - sdsServerValidationContext, err := generateSDSCAResource( + sdsC2CCertAndKey := generateSDSCertAndKey("c2c-cert-and-key", credentials.C2CCredential) + err = writeDiscoveryResponseYAML(sdsC2CCertAndKey, sdsC2CCertAndKeyPath) + if err != nil { + return err + } + + sdsIDValidationContext, err := generateSDSCAResource( container, - credentials, + credentials.InstanceIdentityCredential, p.containerProxyTrustedCACerts, p.containerProxyVerifySubjectAltName, ) if err != nil { return err } - err = writeDiscoveryResponseYAML(sdsServerValidationContext, sdsServerValidationContextPath) + err = writeDiscoveryResponseYAML(sdsIDValidationContext, sdsIDValidationContextPath) if err != nil { return err } @@ -322,8 +343,14 @@ func generateProxyConfig( http2Enabled bool, ) (*envoy_bootstrap.Bootstrap, error) { clusters := []*envoy_cluster.Cluster{} - for index, portMap := range container.Ports { - clusterName := fmt.Sprintf("%d-service-cluster", index) + + uniqueContainerPorts := map[uint16]struct{}{} + for _, portMap := range container.Ports { + uniqueContainerPorts[portMap.ContainerPort] = struct{}{} + } + + for containerPort := range uniqueContainerPorts { + clusterName := fmt.Sprintf("service-cluster-%d", containerPort) clusters = append(clusters, &envoy_cluster.Cluster{ Name: clusterName, ClusterDiscoveryType: &envoy_cluster.Cluster_Type{Type: envoy_cluster.Cluster_STATIC}, @@ -334,7 +361,7 @@ func generateProxyConfig( LbEndpoints: []*envoy_endpoint.LbEndpoint{{ HostIdentifier: &envoy_endpoint.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint.Endpoint{ - Address: envoyAddr(container.InternalIP, portMap.ContainerPort), + Address: envoyAddr(container.InternalIP, containerPort), }, }, }}, @@ -461,12 +488,12 @@ func writeProxyConfig(proxyConfig *envoy_bootstrap.Bootstrap, path string) error func generateListeners(container executor.Container, requireClientCerts, http2Enabled bool) ([]*envoy_listener.Listener, error) { listeners := []*envoy_listener.Listener{} - for index, portMap := range container.Ports { + for _, portMap := range container.Ports { filterName := TcpProxy - clusterName := fmt.Sprintf("%d-service-cluster", index) + clusterName := fmt.Sprintf("service-cluster-%d", portMap.ContainerPort) filterConfig, err := ptypes.MarshalAny(&envoy_tcp_proxy.TcpProxy{ - StatPrefix: fmt.Sprintf("%d-stats", index), + StatPrefix: fmt.Sprintf("stats-%d-%d", portMap.ContainerPort, portMap.ContainerTLSProxyPort), ClusterSpecifier: &envoy_tcp_proxy.TcpProxy_Cluster{ Cluster: clusterName, }, @@ -475,15 +502,21 @@ func generateListeners(container executor.Container, requireClientCerts, http2En return nil, err } + sdsServerCertAndKeyFile := InstanceIdentityCertAndKeyFilename + sdsServerSecretName := "id-cert-and-key" + if portMap.ContainerTLSProxyPort == C2CTLSPort { + sdsServerCertAndKeyFile = C2CCertAndKeyFilename + sdsServerSecretName = "c2c-cert-and-key" + } + tlsContext := &envoy_tls.DownstreamTlsContext{ - RequireClientCertificate: &wrappers.BoolValue{Value: requireClientCerts}, CommonTlsContext: &envoy_tls.CommonTlsContext{ TlsCertificateSdsSecretConfigs: []*envoy_tls.SdsSecretConfig{ { - Name: "server-cert-and-key", + Name: sdsServerSecretName, SdsConfig: &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Path{ - Path: "/etc/cf-assets/envoy_config/sds-server-cert-and-key.yaml", + Path: filepath.Join(ContainerEnvoyConfigDir, sdsServerCertAndKeyFile), }, }, }, @@ -498,13 +531,14 @@ func generateListeners(container executor.Container, requireClientCerts, http2En tlsContext.CommonTlsContext.AlpnProtocols = AlpnProtocols } - if requireClientCerts { + if requireClientCerts && portMap.ContainerTLSProxyPort != C2CTLSPort { + tlsContext.RequireClientCertificate = &wrappers.BoolValue{Value: requireClientCerts} tlsContext.CommonTlsContext.ValidationContextType = &envoy_tls.CommonTlsContext_ValidationContextSdsSecretConfig{ ValidationContextSdsSecretConfig: &envoy_tls.SdsSecretConfig{ - Name: "server-validation-context", + Name: "id-validation-context", SdsConfig: &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Path{ - Path: "/etc/cf-assets/envoy_config/sds-server-validation-context.yaml", + Path: filepath.Join(ContainerEnvoyConfigDir, InstanceIdentityValidationContextFilename), }, }, }, @@ -516,7 +550,7 @@ func generateListeners(container executor.Container, requireClientCerts, http2En return nil, err } - listenerName := fmt.Sprintf("listener-%d", portMap.ContainerPort) + listenerName := fmt.Sprintf("listener-%d-%d", portMap.ContainerPort, portMap.ContainerTLSProxyPort) listener := &envoy_listener.Listener{ Name: listenerName, Address: envoyAddr("0.0.0.0", portMap.ContainerTLSProxyPort), @@ -545,9 +579,9 @@ func generateListeners(container executor.Container, requireClientCerts, http2En return listeners, nil } -func generateSDSCertAndKey(container executor.Container, creds Credential) proto.Message { +func generateSDSCertAndKey(name string, creds Credential) proto.Message { return &envoy_tls.Secret{ - Name: "server-cert-and-key", + Name: name, Type: &envoy_tls.Secret_TlsCertificate{ TlsCertificate: &envoy_tls.TlsCertificate{ CertificateChain: &envoy_core.DataSource{ @@ -579,7 +613,7 @@ func generateSDSCAResource(container executor.Container, creds Credential, trust } return &envoy_tls.Secret{ - Name: "server-validation-context", + Name: "id-validation-context", Type: &envoy_tls.Secret_ValidationContext{ ValidationContext: &envoy_tls.CertificateValidationContext{ TrustedCa: &envoy_core.DataSource{ diff --git a/depot/containerstore/proxy_config_handler_test.go b/depot/containerstore/proxy_config_handler_test.go index 61fd4ac4..4e0c6488 100644 --- a/depot/containerstore/proxy_config_handler_test.go +++ b/depot/containerstore/proxy_config_handler_test.go @@ -50,8 +50,9 @@ var _ = Describe("ProxyConfigHandler", func() { configPath string rotatingCredChan chan containerstore.Credential container executor.Container - sdsServerCertAndKeyFile string - sdsServerValidationContextFile string + sdsIDCertAndKeyFile string + sdsC2CCertAndKeyFile string + sdsIDValidationContextFile string proxyConfigFile string proxyConfigHandler *containerstore.ProxyConfigHandler reloadDuration time.Duration @@ -61,6 +62,7 @@ var _ = Describe("ProxyConfigHandler", func() { containerProxyRequireClientCerts bool adsServers []string http2Enabled bool + credentials containerstore.Credentials ) BeforeEach(func() { @@ -85,8 +87,9 @@ var _ = Describe("ProxyConfigHandler", func() { configPath = filepath.Join(proxyConfigDir, container.Guid) proxyConfigFile = filepath.Join(configPath, "envoy.yaml") - sdsServerCertAndKeyFile = filepath.Join(configPath, "sds-server-cert-and-key.yaml") - sdsServerValidationContextFile = filepath.Join(configPath, "sds-server-validation-context.yaml") + sdsIDCertAndKeyFile = filepath.Join(configPath, "sds-id-cert-and-key.yaml") + sdsIDValidationContextFile = filepath.Join(configPath, "sds-id-validation-context.yaml") + sdsC2CCertAndKeyFile = filepath.Join(configPath, "sds-c2c-cert-and-key.yaml") logger = lagertest.NewTestLogger("proxymanager") @@ -103,6 +106,17 @@ var _ = Describe("ProxyConfigHandler", func() { "10.255.217.3:15010", } http2Enabled = true + cert, key, _ := generateCertAndKey() + credentials = containerstore.Credentials{ + InstanceIdentityCredential: containerstore.Credential{ + Cert: cert, + Key: key, + }, + C2CCredential: containerstore.Credential{ + Cert: cert, + Key: key, + }, + } }) JustBeforeEach(func() { @@ -155,7 +169,7 @@ var _ = Describe("ProxyConfigHandler", func() { Describe("Close", func() { JustBeforeEach(func() { Expect(proxyConfigFile).NotTo(BeAnExistingFile()) - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) }) @@ -167,7 +181,7 @@ var _ = Describe("ProxyConfigHandler", func() { Describe("Update", func() { JustBeforeEach(func() { Expect(proxyConfigFile).NotTo(BeAnExistingFile()) - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) }) @@ -260,12 +274,7 @@ var _ = Describe("ProxyConfigHandler", func() { Describe("ProxyPorts", func() { BeforeEach(func() { container.Ports = []executor.PortMapping{ - { - ContainerPort: 8080, - }, - { - ContainerPort: 9090, - }, + {ContainerPort: 8080}, } }) @@ -282,21 +291,60 @@ var _ = Describe("ProxyConfigHandler", func() { }) }) - It("each port gets an equivalent extra proxy port", func() { - ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) - Expect(err).NotTo(HaveOccurred()) - Expect(ports).To(ConsistOf([]executor.ProxyPortMapping{ - { - AppPort: 8080, - ProxyPort: 61001, - }, - { - AppPort: 9090, - ProxyPort: 61002, - }, - })) + Context("when there is a default HTTP port (8080)", func() { + BeforeEach(func() { + container.Ports = []executor.PortMapping{ + {ContainerPort: 8080}, + {ContainerPort: 9090}, + } + }) + + It("each port gets an equivalent extra proxy port and additional 61443 port for 8080", func() { + ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) + Expect(err).NotTo(HaveOccurred()) + Expect(ports).To(ConsistOf([]executor.ProxyPortMapping{ + { + AppPort: 8080, + ProxyPort: 61001, + }, + { + AppPort: 8080, + ProxyPort: 61443, + }, + { + AppPort: 9090, + ProxyPort: 61002, + }, + })) + + Expect(extraPorts).To(ConsistOf([]uint16{61001, 61002, 61443})) + }) + }) + + Context("when there is no default HTTP port (8080)", func() { + BeforeEach(func() { + container.Ports = []executor.PortMapping{ + {ContainerPort: 7070}, + {ContainerPort: 9090}, + } + }) - Expect(extraPorts).To(ConsistOf([]uint16{61001, 61002})) + It("each port gets an equivalent extra proxy port", func() { + ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) + Expect(err).NotTo(HaveOccurred()) + Expect(ports).To(ConsistOf([]executor.ProxyPortMapping{ + { + AppPort: 7070, + ProxyPort: 61001, + }, + { + AppPort: 9090, + ProxyPort: 61002, + }, + })) + + Expect(extraPorts).To(ConsistOf([]uint16{61001, 61002})) + }) }) Context("when the requested ports are in the 6100n range", func() { @@ -373,6 +421,10 @@ var _ = Describe("ProxyConfigHandler", func() { ContainerPort: 8080, ContainerTLSProxyPort: 61001, }, + { + ContainerPort: 8080, + ContainerTLSProxyPort: 61443, + }, } containerProxyRequireClientCerts = true @@ -384,7 +436,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("does not write a envoy config file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "", Key: ""}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Expect(proxyConfigFile).NotTo(BeAnExistingFile()) @@ -397,7 +449,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates a proxy config without tls_context", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) @@ -416,7 +468,7 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Clusters).To(HaveLen(2)) expectedCluster{ - name: "0-service-cluster", + name: "service-cluster-8080", hosts: []*envoy_core.Address{envoyAddr("10.0.0.1", 8080)}, maxConnections: math.MaxUint32, }.check(proxyConfig.StaticResources.Clusters[0]) @@ -430,25 +482,40 @@ var _ = Describe("ProxyConfigHandler", func() { }}.check(adsCluster) Expect(adsCluster.Http2ProtocolOptions).To(Equal(&envoy_core.Http2ProtocolOptions{})) - Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(1)) + Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(2)) expectedListener{ - name: "listener-8080", + name: "listener-8080-61001", listenPort: 61001, - statPrefix: "0-stats", - clusterName: "0-service-cluster", + statPrefix: "stats-8080-61001", + clusterName: "service-cluster-8080", requireClientCertificate: false, alpnProtocols: []string{"h2,http/1.1"}, - }.check(proxyConfig.StaticResources.Listeners[0]) + }.check(proxyConfig.StaticResources.Listeners[0], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) + expectedListener{ + name: "listener-8080-61443", + listenPort: 61443, + statPrefix: "stats-8080-61443", + clusterName: "service-cluster-8080", + requireClientCertificate: false, + alpnProtocols: []string{"h2,http/1.1"}, + }.check(proxyConfig.StaticResources.Listeners[1], + "c2c-cert-and-key", + "/etc/cf-assets/envoy_config/sds-c2c-cert-and-key.yaml", + ) }) }) - It("creates appropriate sds-server-cert-and-key.yaml configuration file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + It("creates appropriate sds-id-validation-context.yaml configuration file", func() { + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Eventually(sdsServerCertAndKeyFile).Should(BeAnExistingFile()) + Eventually(sdsIDCertAndKeyFile).Should(BeAnExistingFile()) + Eventually(sdsC2CCertAndKeyFile).Should(BeAnExistingFile()) var sdsCertificateDiscoveryResponse envoy_discovery.DiscoveryResponse - Expect(yamlFileToProto(sdsServerCertAndKeyFile, &sdsCertificateDiscoveryResponse)).To(Succeed()) + Expect(yamlFileToProto(sdsIDCertAndKeyFile, &sdsCertificateDiscoveryResponse)).To(Succeed()) Expect(sdsCertificateDiscoveryResponse.VersionInfo).To(Equal("0")) Expect(sdsCertificateDiscoveryResponse.Resources).To(HaveLen(1)) @@ -456,17 +523,17 @@ var _ = Describe("ProxyConfigHandler", func() { var secret envoy_tls.Secret Expect(ptypes.UnmarshalAny(sdsCertificateDiscoveryResponse.Resources[0], &secret)).To(Succeed()) - Expect(secret.Name).To(Equal("server-cert-and-key")) + Expect(secret.Name).To(Equal("id-cert-and-key")) Expect(secret.Type).To(Equal(&envoy_tls.Secret_TlsCertificate{ TlsCertificate: &envoy_tls.TlsCertificate{ CertificateChain: &envoy_core.DataSource{ Specifier: &envoy_core.DataSource_InlineString{ - InlineString: "cert", + InlineString: credentials.InstanceIdentityCredential.Cert, }, }, PrivateKey: &envoy_core.DataSource{ Specifier: &envoy_core.DataSource_InlineString{ - InlineString: "key", + InlineString: credentials.InstanceIdentityCredential.Key, }, }, }, @@ -490,22 +557,31 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(fileWatcher.Close()).To(Succeed()) }) - It("should ensure the sds-server-cert-and-key.yaml file is recreated when updating the config", func() { - Expect(ioutil.WriteFile(sdsServerCertAndKeyFile, []byte("old-content"), 0666)).To(Succeed()) - Expect(fileWatcher.Add(sdsServerCertAndKeyFile)).To(Succeed()) - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + It("should ensure the sds-id-cert-and-key.yaml file is recreated when updating the config", func() { + Expect(ioutil.WriteFile(sdsIDCertAndKeyFile, []byte("old-content"), 0666)).To(Succeed()) + Expect(fileWatcher.Add(sdsIDCertAndKeyFile)).To(Succeed()) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsServerCertAndKeyFile, Op: fsnotify.Remove}))) + Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsIDCertAndKeyFile, Op: fsnotify.Remove}))) }) - It("should ensure the sds-server-validation-context.yaml file is recreated when updating the config", func() { - Expect(ioutil.WriteFile(sdsServerValidationContextFile, []byte("old-content"), 0666)).To(Succeed()) - Expect(fileWatcher.Add(sdsServerValidationContextFile)).To(Succeed()) - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + It("should ensure the sds-c2c-cert-and-key.yaml file is recreated when updating the config", func() { + Expect(ioutil.WriteFile(sdsC2CCertAndKeyFile, []byte("old-content"), 0666)).To(Succeed()) + Expect(fileWatcher.Add(sdsC2CCertAndKeyFile)).To(Succeed()) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsServerValidationContextFile, Op: fsnotify.Remove}))) + Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsC2CCertAndKeyFile, Op: fsnotify.Remove}))) + }) + + It("should ensure the sds-id-validation-context.yaml file is recreated when updating the config", func() { + Expect(ioutil.WriteFile(sdsIDValidationContextFile, []byte("old-content"), 0666)).To(Succeed()) + Expect(fileWatcher.Add(sdsIDValidationContextFile)).To(Succeed()) + err := proxyConfigHandler.Update(credentials, container) + Expect(err).NotTo(HaveOccurred()) + + Eventually(fileWatcher.Events).Should(Receive(Equal(fsnotify.Event{Name: sdsIDValidationContextFile, Op: fsnotify.Remove}))) }) }) @@ -520,13 +596,13 @@ var _ = Describe("ProxyConfigHandler", func() { containerProxyVerifySubjectAltName = []string{"valid-alt-name-1", "valid-alt-name-2"} }) - It("creates appropriate sds-server-validation-context.yaml configuration file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + It("creates appropriate sds-id-validation-context.yaml configuration file", func() { + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Eventually(sdsServerValidationContextFile).Should(BeAnExistingFile()) + Eventually(sdsIDValidationContextFile).Should(BeAnExistingFile()) var sdsDiscoveryResponse envoy_discovery.DiscoveryResponse - Expect(yamlFileToProto(sdsServerValidationContextFile, &sdsDiscoveryResponse)).To(Succeed()) + Expect(yamlFileToProto(sdsIDValidationContextFile, &sdsDiscoveryResponse)).To(Succeed()) Expect(sdsDiscoveryResponse.VersionInfo).To(Equal("0")) Expect(sdsDiscoveryResponse.Resources).To(HaveLen(1)) @@ -534,7 +610,7 @@ var _ = Describe("ProxyConfigHandler", func() { var secret envoy_tls.Secret Expect(ptypes.UnmarshalAny(sdsDiscoveryResponse.Resources[0], &secret)).To(Succeed()) - Expect(secret.Name).To(Equal("server-validation-context")) + Expect(secret.Name).To(Equal("id-validation-context")) Expect(secret.Type).To(Equal(&envoy_tls.Secret_ValidationContext{ ValidationContext: &envoy_tls.CertificateValidationContext{ TrustedCa: &envoy_core.DataSource{ @@ -552,7 +628,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates the appropriate proxy config at start", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) @@ -571,7 +647,7 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Clusters).To(HaveLen(2)) expectedCluster{ - name: "0-service-cluster", + name: "service-cluster-8080", hosts: []*envoy_core.Address{envoyAddr("10.0.0.1", 8080)}, maxConnections: math.MaxUint32, }.check(proxyConfig.StaticResources.Clusters[0]) @@ -585,15 +661,29 @@ var _ = Describe("ProxyConfigHandler", func() { }}.check(adsCluster) Expect(adsCluster.Http2ProtocolOptions).To(Equal(&envoy_core.Http2ProtocolOptions{})) - Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(1)) + Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(2)) expectedListener{ - name: "listener-8080", + name: "listener-8080-61001", listenPort: 61001, - statPrefix: "0-stats", - clusterName: "0-service-cluster", + statPrefix: "stats-8080-61001", + clusterName: "service-cluster-8080", requireClientCertificate: true, alpnProtocols: []string{"h2,http/1.1"}, - }.check(proxyConfig.StaticResources.Listeners[0]) + }.check(proxyConfig.StaticResources.Listeners[0], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) + expectedListener{ + name: "listener-8080-61443", + listenPort: 61443, + statPrefix: "stats-8080-61443", + clusterName: "service-cluster-8080", + requireClientCertificate: false, + alpnProtocols: []string{"h2,http/1.1"}, + }.check(proxyConfig.StaticResources.Listeners[1], + "c2c-cert-and-key", + "/etc/cf-assets/envoy_config/sds-c2c-cert-and-key.yaml", + ) adsConfigSource := &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Ads{ @@ -625,7 +715,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates a proxy config without the ads config", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) @@ -634,7 +724,7 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Clusters).To(HaveLen(1)) cluster := proxyConfig.StaticResources.Clusters[0] - Expect(cluster.Name).To(Equal("0-service-cluster")) + Expect(cluster.Name).To(Equal("service-cluster-8080")) Expect(proxyConfig.DynamicResources).To(BeNil()) }) @@ -648,7 +738,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("returns an error", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).To(MatchError("ads server address is invalid: malformed")) }) }) @@ -661,7 +751,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("returns an error", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).To(MatchError("ads server address is invalid: 0.0.0.0:mal")) }) }) @@ -672,22 +762,36 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates a proxy config without ALPN for listeners", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) var proxyConfig envoy_bootstrap.Bootstrap Expect(yamlFileToProto(proxyConfigFile, &proxyConfig)).To(Succeed()) - Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(1)) + Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(2)) expectedListener{ - name: "listener-8080", + name: "listener-8080-61001", listenPort: 61001, - statPrefix: "0-stats", - clusterName: "0-service-cluster", + statPrefix: "stats-8080-61001", + clusterName: "service-cluster-8080", requireClientCertificate: true, alpnProtocols: nil, - }.check(proxyConfig.StaticResources.Listeners[0]) + }.check(proxyConfig.StaticResources.Listeners[0], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) + expectedListener{ + name: "listener-8080-61443", + listenPort: 61443, + statPrefix: "stats-8080-61443", + clusterName: "service-cluster-8080", + requireClientCertificate: false, + alpnProtocols: nil, + }.check(proxyConfig.StaticResources.Listeners[1], + "c2c-cert-and-key", + "/etc/cf-assets/envoy_config/sds-c2c-cert-and-key.yaml", + ) }) }) @@ -706,7 +810,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("creates the appropriate proxy config file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Eventually(proxyConfigFile).Should(BeAnExistingFile()) @@ -725,13 +829,13 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Clusters).To(HaveLen(3)) expectedCluster{ - name: "0-service-cluster", + name: "service-cluster-8080", hosts: []*envoy_core.Address{envoyAddr("10.0.0.1", 8080)}, maxConnections: math.MaxUint32, }.check(proxyConfig.StaticResources.Clusters[0]) expectedCluster{ - name: "1-service-cluster", + name: "service-cluster-2222", hosts: []*envoy_core.Address{envoyAddr("10.0.0.1", 2222)}, maxConnections: math.MaxUint32, }.check(proxyConfig.StaticResources.Clusters[1]) @@ -748,22 +852,28 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(proxyConfig.StaticResources.Listeners).To(HaveLen(2)) expectedListener{ - name: "listener-8080", + name: "listener-8080-61001", listenPort: 61001, - statPrefix: "0-stats", - clusterName: "0-service-cluster", + statPrefix: "stats-8080-61001", + clusterName: "service-cluster-8080", requireClientCertificate: true, alpnProtocols: []string{"h2,http/1.1"}, - }.check(proxyConfig.StaticResources.Listeners[0]) + }.check(proxyConfig.StaticResources.Listeners[0], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) expectedListener{ - name: "listener-2222", + name: "listener-2222-61002", listenPort: 61002, - statPrefix: "1-stats", - clusterName: "1-service-cluster", + statPrefix: "stats-2222-61002", + clusterName: "service-cluster-2222", requireClientCertificate: true, alpnProtocols: []string{"h2,http/1.1"}, - }.check(proxyConfig.StaticResources.Listeners[1]) + }.check(proxyConfig.StaticResources.Listeners[1], + "id-cert-and-key", + "/etc/cf-assets/envoy_config/sds-id-cert-and-key.yaml", + ) }) Context("when no ports are left", func() { @@ -780,7 +890,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("returns an error", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "cert", Key: "key"}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).To(Equal(containerstore.ErrNoPortsAvailable)) }) }) @@ -788,10 +898,6 @@ var _ = Describe("ProxyConfigHandler", func() { }) Describe("Close", func() { - var ( - cert, key string - ) - BeforeEach(func() { container.Ports = []executor.PortMapping{ { @@ -799,8 +905,6 @@ var _ = Describe("ProxyConfigHandler", func() { ContainerTLSProxyPort: 61001, }, } - - cert, key, _ = generateCertAndKey() }) JustBeforeEach(func() { @@ -811,7 +915,7 @@ var _ = Describe("ProxyConfigHandler", func() { It("returns after the configured reload duration", func() { ch := make(chan struct{}) go func() { - proxyConfigHandler.Close(containerstore.Credential{Cert: cert, Key: key}, container) + proxyConfigHandler.Close(credentials, container) close(ch) }() @@ -826,7 +930,7 @@ var _ = Describe("ProxyConfigHandler", func() { }) It("does not write a envoy config file", func() { - err := proxyConfigHandler.Update(containerstore.Credential{Cert: "", Key: ""}, container) + err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) Expect(proxyConfigFile).NotTo(BeAnExistingFile()) @@ -836,7 +940,7 @@ var _ = Describe("ProxyConfigHandler", func() { ch := make(chan struct{}) go func() { defer GinkgoRecover() - proxyConfigHandler.Close(containerstore.Credential{Cert: cert, Key: key}, container) + proxyConfigHandler.Close(credentials, container) close(ch) }() @@ -921,7 +1025,7 @@ type expectedListener struct { alpnProtocols []string } -func (l expectedListener) check(listener *envoy_listener.Listener) { +func (l expectedListener) check(listener *envoy_listener.Listener, sdsSecretName string, sdsFileName string) { Expect(listener.Name).To(Equal(l.name)) Expect(listener.Address).To(Equal(envoyAddr("0.0.0.0", l.listenPort))) Expect(listener.FilterChains).To(HaveLen(1)) @@ -941,14 +1045,13 @@ func (l expectedListener) check(listener *envoy_listener.Listener) { Expect(ptypes.UnmarshalAny(filterChain.TransportSocket.GetTypedConfig(), &downstreamTlsContext)).To(Succeed()) Expect(filterChain.TransportSocket.Name).To(Equal(l.name)) - Expect(downstreamTlsContext.RequireClientCertificate.Value).To(Equal(l.requireClientCertificate)) Expect(downstreamTlsContext.CommonTlsContext.AlpnProtocols).To(Equal(l.alpnProtocols)) Expect(downstreamTlsContext.CommonTlsContext.TlsCertificateSdsSecretConfigs).To(ConsistOf( &envoy_tls.SdsSecretConfig{ - Name: "server-cert-and-key", + Name: sdsSecretName, SdsConfig: &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Path{ - Path: "/etc/cf-assets/envoy_config/sds-server-cert-and-key.yaml", + Path: sdsFileName, }, }, }, @@ -958,12 +1061,13 @@ func (l expectedListener) check(listener *envoy_listener.Listener) { })) if l.requireClientCertificate { + Expect(downstreamTlsContext.RequireClientCertificate.Value).To(BeTrue()) Expect(downstreamTlsContext.CommonTlsContext.ValidationContextType).To(Equal(&envoy_tls.CommonTlsContext_ValidationContextSdsSecretConfig{ ValidationContextSdsSecretConfig: &envoy_tls.SdsSecretConfig{ - Name: "server-validation-context", + Name: "id-validation-context", SdsConfig: &envoy_core.ConfigSource{ ConfigSourceSpecifier: &envoy_core.ConfigSource_Path{ - Path: "/etc/cf-assets/envoy_config/sds-server-validation-context.yaml", + Path: "/etc/cf-assets/envoy_config/sds-id-validation-context.yaml", }, }, }, diff --git a/depot/containerstore/storenode.go b/depot/containerstore/storenode.go index 78b98568..56bdbd74 100644 --- a/depot/containerstore/storenode.go +++ b/depot/containerstore/storenode.go @@ -427,9 +427,9 @@ func (n *storeNode) portMappingFromContainerInfo( proxyPorts := make(map[uint16]struct{}) // construct a map from unproxied to proxied port - appToProxyPortMappings := make(map[uint16]uint16) + appToProxyPortMappings := make(map[uint16][]uint16) for _, mapping := range proxyToAppPort { - appToProxyPortMappings[mapping.AppPort] = mapping.ProxyPort + appToProxyPortMappings[mapping.AppPort] = append(appToProxyPortMappings[mapping.AppPort], mapping.ProxyPort) proxyPorts[mapping.ProxyPort] = struct{}{} } @@ -456,14 +456,23 @@ func (n *storeNode) portMappingFromContainerInfo( } hostPort := containerToHostPortMappings[appPort] - proxyContainerPort := appToProxyPortMappings[appPort] - proxyHostPort := containerToHostPortMappings[proxyContainerPort] - ports = append(ports, executor.PortMapping{ - HostPort: hostPort, - ContainerPort: appPort, - ContainerTLSProxyPort: proxyContainerPort, - HostTLSProxyPort: proxyHostPort, - }) + proxyContainerPorts := appToProxyPortMappings[appPort] + if len(proxyContainerPorts) > 0 { + for _, proxyContainerPort := range proxyContainerPorts { + proxyHostPort := containerToHostPortMappings[proxyContainerPort] + ports = append(ports, executor.PortMapping{ + HostPort: hostPort, + ContainerPort: appPort, + ContainerTLSProxyPort: proxyContainerPort, + HostTLSProxyPort: proxyHostPort, + }) + } + } else { + ports = append(ports, executor.PortMapping{ + HostPort: hostPort, + ContainerPort: appPort, + }) + } } return ports diff --git a/resources.go b/resources.go index c5cb0199..26dc28eb 100644 --- a/resources.go +++ b/resources.go @@ -5,6 +5,7 @@ import ( "time" "code.cloudfoundry.org/bbs/models" + "code.cloudfoundry.org/routing-info/internalroutes" ) const ContainerOwnerProperty = "executor:owner" @@ -165,28 +166,29 @@ type Sidecar struct { } type RunInfo struct { - RootFSPath string `json:"rootfs"` - CPUWeight uint `json:"cpu_weight"` - Ports []PortMapping `json:"ports"` - LogConfig LogConfig `json:"log_config"` - MetricsConfig MetricsConfig `json:"metrics_config"` - StartTimeoutMs uint `json:"start_timeout_ms"` - Privileged bool `json:"privileged"` - CachedDependencies []CachedDependency `json:"cached_dependencies"` - Setup *models.Action `json:"setup"` - Action *models.Action `json:"run"` - Monitor *models.Action `json:"monitor"` - CheckDefinition *models.CheckDefinition `json:"check_definition"` - EgressRules []*models.SecurityGroupRule `json:"egress_rules,omitempty"` - Env []EnvironmentVariable `json:"env,omitempty"` - TrustedSystemCertificatesPath string `json:"trusted_system_certificates_path,omitempty"` - VolumeMounts []VolumeMount `json:"volume_mounts"` - Network *Network `json:"network,omitempty"` - CertificateProperties CertificateProperties `json:"certificate_properties"` - ImageUsername string `json:"image_username"` - ImagePassword string `json:"image_password"` - EnableContainerProxy bool `json:"enable_container_proxy"` - Sidecars []Sidecar `json:"sidecars"` + RootFSPath string `json:"rootfs"` + CPUWeight uint `json:"cpu_weight"` + Ports []PortMapping `json:"ports"` + InternalRoutes internalroutes.InternalRoutes `json:"internal_routes"` + LogConfig LogConfig `json:"log_config"` + MetricsConfig MetricsConfig `json:"metrics_config"` + StartTimeoutMs uint `json:"start_timeout_ms"` + Privileged bool `json:"privileged"` + CachedDependencies []CachedDependency `json:"cached_dependencies"` + Setup *models.Action `json:"setup"` + Action *models.Action `json:"run"` + Monitor *models.Action `json:"monitor"` + CheckDefinition *models.CheckDefinition `json:"check_definition"` + EgressRules []*models.SecurityGroupRule `json:"egress_rules,omitempty"` + Env []EnvironmentVariable `json:"env,omitempty"` + TrustedSystemCertificatesPath string `json:"trusted_system_certificates_path,omitempty"` + VolumeMounts []VolumeMount `json:"volume_mounts"` + Network *Network `json:"network,omitempty"` + CertificateProperties CertificateProperties `json:"certificate_properties"` + ImageUsername string `json:"image_username"` + ImagePassword string `json:"image_password"` + EnableContainerProxy bool `json:"enable_container_proxy"` + Sidecars []Sidecar `json:"sidecars"` } type BindMountMode uint8