Enable map key sorting in utils.FastMarshal

lib/services/local/trust.go

@@ -16,10 +16,12 @@ package local
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/lib/backend"
 	"github.com/gravitational/teleport/lib/services"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/gravitational/trace"
 )
@@ -91,6 +93,8 @@ func (s *CA) UpsertCertAuthority(ca types.CertAuthority) error {
 	return nil
 }
 
+const compareAndSwapFixExample = "tctl get %s/%s/%s --with-secrets > ca.yaml && tctl create -f ca.yaml && rm ca.yaml"
+
 // CompareAndSwapCertAuthority updates the cert authority value
 // if the existing value matches existing parameter, returns nil if succeeds,
 // trace.CompareFailed otherwise.
@@ -121,6 +125,10 @@ func (s *CA) CompareAndSwapCertAuthority(new, existing types.CertAuthority) erro
 	_, err = s.CompareAndSwap(context.TODO(), existingItem, newItem)
 	if err != nil {
 		if trace.IsCompareFailed(err) {
+			if len(existing.GetMetadata().Labels) >= 2 {
+				exampleCmd := fmt.Sprintf(compareAndSwapFixExample, existing.GetKind(), existing.GetSubKind(), existing.GetName())
+				log.Warnf("comparison failed on certificate authority with multiple labels; if this occurs consistently, try re-saving the resource: %s", exampleCmd)
+			}
 			return trace.CompareFailed("cluster %v settings have been updated, try again", new.GetName())
 		}
 		return trace.Wrap(err)

lib/utils/jsontools.go

@@ -70,10 +70,19 @@ func FastUnmarshal(data []byte, v interface{}) error {
 	return nil
 }
 
+// SafeConfig uses jsoniter's ConfigFastest settings but enables map key
+// sorting to ensure CompareAndSwap checks consistently succeed.
+var SafeConfig = jsoniter.Config{
+	EscapeHTML:                    false,
+	MarshalFloatWith6Digits:       true, // will lose precision
+	ObjectFieldMustBeSimpleString: true, // do not unescape object field
+	SortMapKeys:                   true,
+}.Froze()
+
 // FastMarshal uses the json-iterator library for fast JSON marshalling.
 // Note, this function unmarshals floats with 6 digits precision.
 func FastMarshal(v interface{}) ([]byte, error) {
-	data, err := jsoniter.ConfigFastest.Marshal(v)
+	data, err := SafeConfig.Marshal(v)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}

lib/utils/jsontools_test.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2022 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package utils
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestMarshalMapConsistency ensures serialized byte comparisons succeed
+// after multiple serialize/deserialize round trips. Some JSON marshalling
+// backends don't sort map keys for performance reasons, which can make
+// operations that depend on the byte ordering fail (e.g. CompareAndSwap).
+func TestMarshalMapConsistency(t *testing.T) {
+	value := map[string]string{
+		"teleport.dev/foo": "1234",
+		"teleport.dev/bar": "5678",
+	}
+
+	compareTo, err := FastMarshal(value)
+	require.NoError(t, err)
+
+	for i := 0; i < 100; i++ {
+		roundTrip := make(map[string]string)
+		err := FastUnmarshal(compareTo, &roundTrip)
+		require.NoError(t, err)
+
+		val, err := FastMarshal(roundTrip)
+		require.NoError(t, err)
+
+		require.Truef(t, bytes.Equal(val, compareTo), "maps must serialize consistently (attempt %d)", i)
+	}
+}