Run ci with fips check (#1691)

* run ci with fips check

* disable custom certificate path in fips mode

* prevent disabling certificate verification in fips mode

* fix goimports

* remove fips disabled checks from non-fips tests

* Update Makefile

Co-authored-by: Tim Rühsen <[email protected]>

* Revert "Update Makefile"

This reverts commit e585f00.

* Update Makefile

Co-authored-by: Tim Rühsen <[email protected]>

---------

Co-authored-by: Tim Rühsen <[email protected]>

.github/workflows/ci.yml

@@ -51,6 +51,21 @@ jobs:
       - name: Unit tests
         run: make test
 
+  test-fips:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          # TODO switch to go.mod once we update to 1.24+
+          go-version: 1.24
+          cache: true
+      - env:
+          GOFIPS140: "latest"
+          GODEBUG: "fips141=only"
+        run: make test-fips
+
   check-update-modules:
     runs-on: ubuntu-latest
     timeout-minutes: 30

Makefile

@@ -36,9 +36,11 @@ check-vet:
 docker-test:
 	scripts/docker-compose-testing run -T --rm go-agent-tests make test
 
-.PHONY: test
+.PHONY: test-fips test
+test-fips: ARGS=-tags=requirefips
+test-fips: test
 test:
-	@for dir in $(shell scripts/moduledirs.sh); do (cd $$dir && go test -race -v -timeout=$(TEST_TIMEOUT) ./...) || exit $$?; done
+	@for dir in $(shell scripts/moduledirs.sh); do (cd "$$dir" && go test -race -v -timeout=$(TEST_TIMEOUT) $(ARGS) ./...) || exit $$?; done
 
 .PHONY: coverage
 coverage:

transport/crypto.go

@@ -0,0 +1,97 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !requirefips
+
+package transport // import "go.elastic.co/apm/v2/transport"
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+
+	"go.elastic.co/apm/v2/internal/configutil"
+)
+
+const envVerifyServerCert = "ELASTIC_APM_VERIFY_SERVER_CERT"
+
+func checkVerifyServerCert() (bool, error) {
+	return configutil.ParseBoolEnv(envVerifyServerCert, true)
+}
+
+func addCertPath(tlsClientConfig *tls.Config) error {
+	if serverCertPath := os.Getenv(envServerCert); serverCertPath != "" {
+		serverCert, err := loadCertificate(serverCertPath)
+		if err != nil {
+			return errors.Wrapf(err, "failed to load certificate from %s", serverCertPath)
+		}
+		// Disable standard verification, we'll check that the
+		// server supplies the exact certificate provided.
+		tlsClientConfig.InsecureSkipVerify = true
+		tlsClientConfig.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+			return verifyPeerCertificate(rawCerts, serverCert)
+		}
+	}
+	if serverCACertPath := os.Getenv(envServerCACert); serverCACertPath != "" {
+		rootCAs := x509.NewCertPool()
+		additionalCerts, err := os.ReadFile(serverCACertPath)
+		if err != nil {
+			return errors.Wrapf(err, "failed to load root CA file from %s", serverCACertPath)
+		}
+		if !rootCAs.AppendCertsFromPEM(additionalCerts) {
+			return fmt.Errorf("failed to load CA certs from %s", serverCACertPath)
+		}
+		tlsClientConfig.RootCAs = rootCAs
+	}
+
+	return nil
+}
+
+func loadCertificate(path string) (*x509.Certificate, error) {
+	pemBytes, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	for {
+		var certBlock *pem.Block
+		certBlock, pemBytes = pem.Decode(pemBytes)
+		if certBlock == nil {
+			return nil, errors.New("missing or invalid certificate")
+		}
+		if certBlock.Type == "CERTIFICATE" {
+			return x509.ParseCertificate(certBlock.Bytes)
+		}
+	}
+}
+
+func verifyPeerCertificate(rawCerts [][]byte, trusted *x509.Certificate) error {
+	if len(rawCerts) == 0 {
+		return errors.New("missing leaf certificate")
+	}
+	cert, err := x509.ParseCertificate(rawCerts[0])
+	if err != nil {
+		return errors.Wrap(err, "failed to parse certificate from server")
+	}
+	if !cert.Equal(trusted) {
+		return errors.New("failed to verify server certificate")
+	}
+	return nil
+}

transport/crypto_fips.go

@@ -0,0 +1,30 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build requirefips
+
+package transport // import "go.elastic.co/apm/v2/transport"
+
+import "crypto/tls"
+
+func checkVerifyServerCert() (bool, error) {
+	return true, nil
+}
+
+func addCertPath(tlsClientConfig *tls.Config) error {
+	return nil
+}

transport/crypto_test.go

@@ -0,0 +1,216 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//go:build !requirefips
+
+package transport_test
+
+import (
+	"context"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"go.elastic.co/apm/v2/transport"
+)
+
+func TestHTTPTransportEnvVerifyServerCert(t *testing.T) {
+	var h recordingHandler
+	server := httptest.NewTLSServer(&h)
+	defer server.Close()
+	defer patchEnv("ELASTIC_APM_SERVER_URL", server.URL)()
+	defer patchEnv("ELASTIC_APM_VERIFY_SERVER_CERT", "false")()
+
+	transport, err := transport.NewHTTPTransport(transport.HTTPTransportOptions{})
+	assert.NoError(t, err)
+
+	assert.NotNil(t, transport.Client)
+	assert.IsType(t, &http.Transport{}, transport.Client.Transport)
+	httpTransport := transport.Client.Transport.(*http.Transport)
+	assert.NotNil(t, httpTransport.TLSClientConfig)
+	assert.True(t, httpTransport.TLSClientConfig.InsecureSkipVerify)
+
+	err = transport.SendStream(context.Background(), strings.NewReader(""))
+	assert.NoError(t, err)
+}
+
+func TestHTTPTransportServerFailover(t *testing.T) {
+	defer patchEnv("ELASTIC_APM_VERIFY_SERVER_CERT", "false")()
+
+	var hosts []string
+	errorHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		hosts = append(hosts, req.Host)
+		http.Error(w, "error-message", http.StatusInternalServerError)
+	})
+	server1 := httptest.NewServer(errorHandler)
+	defer server1.Close()
+	server2 := httptest.NewTLSServer(errorHandler)
+	defer server2.Close()
+
+	transport, err := transport.NewHTTPTransport(transport.HTTPTransportOptions{})
+	require.NoError(t, err)
+	err = transport.SetServerURL(mustParseURL(server1.URL), mustParseURL(server2.URL))
+	require.NoError(t, err)
+
+	for i := 0; i < 4; i++ {
+		err := transport.SendStream(context.Background(), strings.NewReader(""))
+		assert.EqualError(t, err, "request failed with 500 Internal Server Error: error-message")
+	}
+	assert.Len(t, hosts, 4)
+
+	// Each time SendStream returns an error, the transport should switch
+	// to the next URL in the list. The list is shuffled so we only compare
+	// the output values to each other, rather than to the original input.
+	assert.NotEqual(t, hosts[0], hosts[1])
+	assert.Equal(t, hosts[0], hosts[2])
+	assert.Equal(t, hosts[1], hosts[3])
+}
+
+func TestHTTPTransportServerCert(t *testing.T) {
+	var h recordingHandler
+	server := httptest.NewUnstartedServer(&h)
+	server.Config.ErrorLog = log.New(ioutil.Discard, "", 0)
+	server.StartTLS()
+	defer server.Close()
+	defer patchEnv("ELASTIC_APM_SERVER_URL", server.URL)()
+
+	p := strings.NewReader("")
+
+	newTransport := func() *transport.HTTPTransport {
+		transport, err := transport.NewHTTPTransport(transport.HTTPTransportOptions{})
+		require.NoError(t, err)
+		return transport
+	}
+
+	// SendStream should fail, because we haven't told the client about
+	// the server certificate, nor disabled certificate verification.
+	transport := newTransport()
+	err := transport.SendStream(context.Background(), p)
+	assert.Error(t, err)
+
+	// Set a certificate that doesn't match, SendStream should still fail.
+	defer patchEnv("ELASTIC_APM_SERVER_CERT", "./testdata/cert.pem")()
+	transport = newTransport()
+	err = transport.SendStream(context.Background(), p)
+	assert.Error(t, err)
+
+	f, err := ioutil.TempFile("", "apm-test")
+	require.NoError(t, err)
+	defer os.Remove(f.Name())
+	defer f.Close()
+	defer patchEnv("ELASTIC_APM_SERVER_CERT", f.Name())()
+
+	// Reconfigure the transport so that it knows about the
+	// server certificate. We avoid using server.Client here, as
+	// it is not available in older versions of Go.
+	err = pem.Encode(f, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: server.TLS.Certificates[0].Certificate[0],
+	})
+	require.NoError(t, err)
+
+	transport = newTransport()
+	err = transport.SendStream(context.Background(), p)
+	assert.NoError(t, err)
+}
+
+func TestHTTPTransportServerCertInvalid(t *testing.T) {
+	f, err := ioutil.TempFile("", "apm-test")
+	require.NoError(t, err)
+	defer os.Remove(f.Name())
+	defer f.Close()
+	defer patchEnv("ELASTIC_APM_SERVER_CERT", f.Name())()
+
+	fmt.Fprintln(f, `
+-----BEGIN GARBAGE-----
+garbage
+-----END GARBAGE-----
+`[1:])
+
+	_, err = transport.NewHTTPTransport(transport.HTTPTransportOptions{})
+	assert.EqualError(t, err, fmt.Sprintf("failed to load certificate from %s: missing or invalid certificate", f.Name()))
+}
+
+func TestHTTPTransportCACert(t *testing.T) {
+	var h recordingHandler
+	server := httptest.NewUnstartedServer(&h)
+	server.Config.ErrorLog = log.New(ioutil.Discard, "", 0)
+	server.StartTLS()
+	defer server.Close()
+	defer patchEnv("ELASTIC_APM_SERVER_URL", server.URL)()
+
+	p := strings.NewReader("")
+
+	// SendStream should fail, because we haven't told the client about
+	// the server certificate, nor disabled certificate verification.
+	trans, err := transport.NewHTTPTransport(transport.HTTPTransportOptions{})
+	assert.NoError(t, err)
+	assert.NotNil(t, trans)
+	err = trans.SendStream(context.Background(), p)
+	assert.Error(t, err)
+
+	// Set the env var to a file that doesn't exist, should get an error
+	defer patchEnv("ELASTIC_APM_SERVER_CA_CERT_FILE", "./testdata/file_that_doesnt_exist.pem")()
+	trans, err = transport.NewHTTPTransport(transport.HTTPTransportOptions{})
+	assert.Error(t, err)
+	assert.Nil(t, trans)
+
+	// Set the env var to a file that has no cert, should get an error
+	f, err := ioutil.TempFile("", "apm-test-1")
+	require.NoError(t, err)
+	defer os.Remove(f.Name())
+	defer f.Close()
+	defer patchEnv("ELASTIC_APM_SERVER_CA_CERT_FILE", f.Name())()
+	trans, err = transport.NewHTTPTransport(transport.HTTPTransportOptions{})
+	assert.Error(t, err)
+	assert.Nil(t, trans)
+
+	// Set a certificate that doesn't match, SendStream should still fail
+	defer patchEnv("ELASTIC_APM_SERVER_CA_CERT_FILE", "./testdata/cert.pem")()
+	trans, err = transport.NewHTTPTransport(transport.HTTPTransportOptions{})
+	assert.NoError(t, err)
+	assert.NotNil(t, trans)
+	err = trans.SendStream(context.Background(), p)
+	assert.Error(t, err)
+
+	f, err = ioutil.TempFile("", "apm-test-2")
+	require.NoError(t, err)
+	defer os.Remove(f.Name())
+	defer f.Close()
+	defer patchEnv("ELASTIC_APM_SERVER_CA_CERT_FILE", f.Name())()
+
+	err = pem.Encode(f, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: server.TLS.Certificates[0].Certificate[0],
+	})
+	require.NoError(t, err)
+
+	trans, err = transport.NewHTTPTransport(transport.HTTPTransportOptions{})
+	assert.NoError(t, err)
+	assert.NotNil(t, trans)
+	err = trans.SendStream(context.Background(), p)
+	assert.NoError(t, err)
+}