feat(txt-registry): deprecate legacy txt-format

.editorconfig

@@ -20,3 +20,7 @@ insert_final_newline = true
 [{Makefile,go.mod,go.sum,*.go}]
 indent_style = tab
 indent_size = 4
+
+[*.py]
+indent_style = space
+indent_size = 4

.pre-commit-config.yaml

@@ -23,5 +23,6 @@ repos:
     rev: v0.44.0
     hooks:
     - id: markdownlint
+      args: ["--fix"]
 
 minimum_pre_commit_version: !!str 3.2

controller/execute.go

@@ -392,7 +392,7 @@ func selectRegistry(cfg *externaldns.Config, p provider.Provider) (registry.Regi
 	case "noop":
 		r, err = registry.NewNoopRegistry(p)
 	case "txt":
-		r, err = registry.NewTXTRegistry(p, cfg.TXTPrefix, cfg.TXTSuffix, cfg.TXTOwnerID, cfg.TXTCacheInterval, cfg.TXTWildcardReplacement, cfg.ManagedDNSRecordTypes, cfg.ExcludeDNSRecordTypes, cfg.TXTEncryptEnabled, []byte(cfg.TXTEncryptAESKey), cfg.TXTNewFormatOnly)
+		r, err = registry.NewTXTRegistry(p, cfg.TXTPrefix, cfg.TXTSuffix, cfg.TXTOwnerID, cfg.TXTCacheInterval, cfg.TXTWildcardReplacement, cfg.ManagedDNSRecordTypes, cfg.ExcludeDNSRecordTypes, cfg.TXTEncryptEnabled, []byte(cfg.TXTEncryptAESKey))
 	case "aws-sd":
 		r, err = registry.NewAWSSDRegistry(p, cfg.TXTOwnerID)
 	default:

controller/execute_test.go

@@ -83,7 +83,6 @@ func TestSelectRegistry(t *testing.T) {
 				TXTWildcardReplacement: "wildcard",
 				ManagedDNSRecordTypes:  []string{"A", "CNAME"},
 				ExcludeDNSRecordTypes:  []string{"TXT"},
-				TXTNewFormatOnly:       true,
 			},
 			provider: &MockProvider{},
 			wantErr:  false,

docs/flags.md

@@ -164,7 +164,6 @@
 | `--txt-wildcard-replacement=""` | When using the TXT registry, a custom string that's used instead of an asterisk for TXT records corresponding to wildcard DNS records (optional) |
 | `--[no-]txt-encrypt-enabled` | When using the TXT registry, set if TXT records should be encrypted before stored (default: disabled) |
 | `--txt-encrypt-aes-key=""` | When using the TXT registry, set TXT record decryption and encryption 32 byte aes key (required when --txt-encrypt=true) |
-| `--[no-]txt-new-format-only` | When using the TXT registry, only use new format records which include record type information (e.g., prefix: 'a-'). Reduces number of TXT records (default: disabled) |
 | `--dynamodb-region=""` | When using the DynamoDB registry, the AWS region of the DynamoDB table (optional) |
 | `--dynamodb-table="external-dns"` | When using the DynamoDB registry, the name of the DynamoDB table (default: "external-dns") |
 | `--txt-cache-interval=0s` | The interval between cache synchronizations in duration format (default: disabled) |

docs/registry/txt.md

@@ -3,8 +3,51 @@
 The TXT registry is the default registry.
 It stores DNS record metadata in TXT records, using the same provider.
 
+If you plan to manage apex domains with external-dns whilst using a txt registry, you should ensure when using --txt-prefix that you specify the record type substitution and that it ends in a period (**.**). The record should be created under the same domain as the apex record being managed, i.e. --txt-prefix=someprefix-%{record_type}.
+
+> Note: `--txt-prefix` and `--txt-suffix` contribute to the 63-byte maximum record length. To avoid errors, use them only if absolutely required and keep them as short as possible.
+
 ## Record Format Options
 
+### For version `v0.18+`
+
+The TXT registry supports single format for storing DNS record metadata:
+
+- Creates a TXT record with record type information (e.g., 'a-' prefix for A records)
+
+The TXT registry would try to guarantee a consistency in between providers and sources, if provider supports the behaviour.
+
+If you are dealing with APEX domains, like `example.com` and TXT records are failing to be created for managed record types specified by `--managed-record-types`, consider following options:
+
+1. TXT record with prefix based on requirements. Example `--txt-prefix="%{record_type}-abc-"` or `--txt-prefix="%{record_type}.abc-"`
+2. TXT record with suffix based on requirements. Example `--txt-suffix="-abc-%{record_type}"` or `--txt-suffix="-abc.%{record_type}."`
+
+If configured `--txt-prefix="%{record_type}-abc-"` for apex domain `ex.com` the expected result is
+
+|              Name              |  TYPE   |
+|:------------------------------:|:-------:|
+| `cname-a-abc-nginx-v2.ex.com.` |  `TXT`  |
+|       `nginx-v2.ex.com.`       | `CNAME` |
+
+If configured `--txt-suffix="-abc.%{record_type}"` for apex domain `ex.com` the expected result is
+
+|              Name              |  TYPE   |
+|:------------------------------:|:-------:|
+| `cname-nginx-v2-abc.a.ex.com.` |  `TXT`  |
+|      `nginx-v3.ex.com..`       | `CNAME` |
+
+### Manually Cleanup Legacy TXT Records
+
+> While deleting registry TXT records won't cause downtime, a well-thought-out migration and cleanup plan is crucial.
+
+Occasionally, it may be necessary to remove outdated TXT records from your registry.
+
+An example script for AWS can be found in [scripts/aws-cleanup-legacy-txt-records.py](../../scripts/aws-cleanup-legacy-txt-records.py) with instructions on how to run it.
+The script performs targeted deletion of TXT records that include `ResourceRecords` matching the `heritage=external-dns,external-dns/owner=default` or similar pattern.
+In the event of unintended deletion of all TXT records managed by `external-dns`, `external-dns` will initiate a full DNS record regeneration, along with`TXT` and `non-TXT` records. Just be aware, this operation's duration is directly proportional to the DNS estate size."
+
+### For version `v0.16.0 & v0.16.1`
+
 The TXT registry supports two formats for storing DNS record metadata:
 
 - Legacy format: Creates a TXT record without record type information
@@ -31,14 +74,14 @@ The `--txt-new-format-only` flag should be used in addition to your existing ext
 
 ### Migration to New Format Only
 
+> Note: `external-dns` will not automatically remove legacy format records when switching to new-format-only mode. You'll need to clean up the old records manually if desired.
+
 When transitioning from dual-format to new-format-only records:
 
 - Ensure all your `external-dns` instances support the new format
 - Enable the `--txt-new-format-only` flag on your external-dns instances
 Manually clean up any existing legacy format TXT records from your DNS provider
 
-Note: `external-dns` will not automatically remove legacy format records when switching to new-format-only mode. You'll need to clean up the old records manually if desired.
-
 ## Prefixes and Suffixes
 
 In order to avoid having the registry TXT records collide with

pkg/apis/externaldns/types.go

@@ -145,7 +145,6 @@ type Config struct {
 	TXTSuffix                                     string
 	TXTEncryptEnabled                             bool
 	TXTEncryptAESKey                              string `secure:"yes"`
-	TXTNewFormatOnly                              bool
 	Interval                                      time.Duration
 	MinEventSyncInterval                          time.Duration
 	Once                                          bool
@@ -365,7 +364,6 @@ var defaultConfig = &Config{
 	TXTCacheInterval:             0,
 	TXTEncryptAESKey:             "",
 	TXTEncryptEnabled:            false,
-	TXTNewFormatOnly:             false,
 	TXTOwnerID:                   "default",
 	TXTPrefix:                    "",
 	TXTSuffix:                    "",
@@ -622,7 +620,6 @@ func App(cfg *Config) *kingpin.Application {
 	app.Flag("txt-wildcard-replacement", "When using the TXT registry, a custom string that's used instead of an asterisk for TXT records corresponding to wildcard DNS records (optional)").Default(defaultConfig.TXTWildcardReplacement).StringVar(&cfg.TXTWildcardReplacement)
 	app.Flag("txt-encrypt-enabled", "When using the TXT registry, set if TXT records should be encrypted before stored (default: disabled)").BoolVar(&cfg.TXTEncryptEnabled)
 	app.Flag("txt-encrypt-aes-key", "When using the TXT registry, set TXT record decryption and encryption 32 byte aes key (required when --txt-encrypt=true)").Default(defaultConfig.TXTEncryptAESKey).StringVar(&cfg.TXTEncryptAESKey)
-	app.Flag("txt-new-format-only", "When using the TXT registry, only use new format records which include record type information (e.g., prefix: 'a-'). Reduces number of TXT records (default: disabled)").BoolVar(&cfg.TXTNewFormatOnly)
 	app.Flag("dynamodb-region", "When using the DynamoDB registry, the AWS region of the DynamoDB table (optional)").Default(cfg.AWSDynamoDBRegion).StringVar(&cfg.AWSDynamoDBRegion)
 	app.Flag("dynamodb-table", "When using the DynamoDB registry, the name of the DynamoDB table (default: \"external-dns\")").Default(defaultConfig.AWSDynamoDBTable).StringVar(&cfg.AWSDynamoDBTable)
 

pkg/apis/externaldns/types_test.go

@@ -101,7 +101,6 @@ var (
 		TXTOwnerID:                                    "default",
 		TXTPrefix:                                     "",
 		TXTCacheInterval:                              0,
-		TXTNewFormatOnly:                              false,
 		Interval:                                      time.Minute,
 		MinEventSyncInterval:                          5 * time.Second,
 		Once:                                          false,
@@ -213,7 +212,6 @@ var (
 		TXTOwnerID:                                    "owner-1",
 		TXTPrefix:                                     "associated-txt-record",
 		TXTCacheInterval:                              12 * time.Hour,
-		TXTNewFormatOnly:                              true,
 		Interval:                                      10 * time.Minute,
 		MinEventSyncInterval:                          50 * time.Second,
 		Once:                                          true,
@@ -357,7 +355,6 @@ func TestParseFlags(t *testing.T) {
 				"--txt-owner-id=owner-1",
 				"--txt-prefix=associated-txt-record",
 				"--txt-cache-interval=12h",
-				"--txt-new-format-only",
 				"--dynamodb-table=custom-table",
 				"--interval=10m",
 				"--min-event-sync-interval=50s",

registry/txt.go

@@ -58,8 +58,6 @@ type TXTRegistry struct {
 	// encrypt text records
 	txtEncryptEnabled bool
 	txtEncryptAESKey  []byte
-
-	newFormatOnly bool
 }
 
 // NewTXTRegistry returns a new TXTRegistry object. When newFormatOnly is true, it will only
@@ -68,8 +66,7 @@ type TXTRegistry struct {
 func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID string,
 	cacheInterval time.Duration, txtWildcardReplacement string,
 	managedRecordTypes, excludeRecordTypes []string,
-	txtEncryptEnabled bool, txtEncryptAESKey []byte,
-	newFormatOnly bool) (*TXTRegistry, error) {
+	txtEncryptEnabled bool, txtEncryptAESKey []byte) (*TXTRegistry, error) {
 	if ownerID == "" {
 		return nil, errors.New("owner id cannot be empty")
 	}
@@ -103,7 +100,6 @@ func NewTXTRegistry(provider provider.Provider, txtPrefix, txtSuffix, ownerID st
 		excludeRecordTypes:  excludeRecordTypes,
 		txtEncryptEnabled:   txtEncryptEnabled,
 		txtEncryptAESKey:    txtEncryptAESKey,
-		newFormatOnly:       newFormatOnly,
 	}, nil
 }
 
@@ -236,25 +232,13 @@ func (im *TXTRegistry) Records(ctx context.Context) ([]*endpoint.Endpoint, error
 func (im *TXTRegistry) generateTXTRecord(r *endpoint.Endpoint) []*endpoint.Endpoint {
 	endpoints := make([]*endpoint.Endpoint, 0)
 
-	// Create legacy format record by default unless newFormatOnly is true
-	if !im.newFormatOnly && !im.txtEncryptEnabled && !im.mapper.recordTypeInAffix() && r.RecordType != endpoint.RecordTypeAAAA {
-		// old TXT record format
-		txt := endpoint.NewEndpoint(im.mapper.toTXTName(r.DNSName), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey))
-		if txt != nil {
-			txt.WithSetIdentifier(r.SetIdentifier)
-			txt.Labels[endpoint.OwnedRecordLabelKey] = r.DNSName
-			txt.ProviderSpecific = r.ProviderSpecific
-			endpoints = append(endpoints, txt)
-		}
-	}
-
 	// Always create new format record
 	recordType := r.RecordType
 	// AWS Alias records are encoded as type "cname"
 	if isAlias, found := r.GetProviderSpecificProperty("alias"); found && isAlias == "true" && recordType == endpoint.RecordTypeA {
 		recordType = endpoint.RecordTypeCNAME
 	}
-	txtNew := endpoint.NewEndpoint(im.mapper.toNewTXTName(r.DNSName, recordType), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey))
+	txtNew := endpoint.NewEndpoint(im.mapper.toTXTName(r.DNSName, recordType), endpoint.RecordTypeTXT, r.Labels.Serialize(true, im.txtEncryptEnabled, im.txtEncryptAESKey))
 	if txtNew != nil {
 		txtNew.WithSetIdentifier(r.SetIdentifier)
 		txtNew.Labels[endpoint.OwnedRecordLabelKey] = r.DNSName
@@ -336,8 +320,7 @@ func (im *TXTRegistry) AdjustEndpoints(endpoints []*endpoint.Endpoint) ([]*endpo
 
 type nameMapper interface {
 	toEndpointName(string) (endpointName string, recordType string)
-	toTXTName(string) string
-	toNewTXTName(string, string) string
+	toTXTName(string, string) string
 	recordTypeInAffix() bool
 }
 
@@ -437,22 +420,6 @@ func (pr affixNameMapper) toEndpointName(txtDNSName string) (endpointName string
 	return "", ""
 }
 
-func (pr affixNameMapper) toTXTName(endpointDNSName string) string {
-	DNSName := strings.SplitN(endpointDNSName, ".", 2)
-
-	prefix := pr.dropAffixTemplate(pr.prefix)
-	suffix := pr.dropAffixTemplate(pr.suffix)
-	// If specified, replace a leading asterisk in the generated txt record name with some other string
-	if pr.wildcardReplacement != "" && DNSName[0] == "*" {
-		DNSName[0] = pr.wildcardReplacement
-	}
-
-	if len(DNSName) < 2 {
-		return prefix + DNSName[0] + suffix
-	}
-	return prefix + DNSName[0] + suffix + "." + DNSName[1]
-}
-
 func (pr affixNameMapper) recordTypeInAffix() bool {
 	if strings.Contains(pr.prefix, recordTemplate) {
 		return true
@@ -470,7 +437,7 @@ func (pr affixNameMapper) normalizeAffixTemplate(afix, recordType string) string
 	return afix
 }
 
-func (pr affixNameMapper) toNewTXTName(endpointDNSName, recordType string) string {
+func (pr affixNameMapper) toTXTName(endpointDNSName, recordType string) string {
 	DNSName := strings.SplitN(endpointDNSName, ".", 2)
 	recordType = strings.ToLower(recordType)
 	recordT := recordType + "-"

registry/txt_encryption_test.go

@@ -61,7 +61,7 @@ func TestNewTXTRegistryEncryptionConfig(t *testing.T) {
 		},
 	}
 	for _, test := range tests {
-		actual, err := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "", []string{}, []string{}, test.encEnabled, test.aesKeyRaw, false)
+		actual, err := NewTXTRegistry(p, "txt.", "", "owner", time.Hour, "", []string{}, []string{}, test.encEnabled, test.aesKeyRaw)
 		if test.errorExpected {
 			require.Error(t, err)
 		} else {
@@ -107,7 +107,7 @@ func TestGenerateTXTGenerateTextRecordEncryptionWihDecryption(t *testing.T) {
 		for _, k := range withEncryptionKeys {
 			t.Run(fmt.Sprintf("key '%s' with decrypted result '%s'", k, test.decrypted), func(t *testing.T) {
 				key := []byte(k)
-				r, err := NewTXTRegistry(p, "", "", "owner", time.Minute, "", []string{}, []string{}, true, key, false)
+				r, err := NewTXTRegistry(p, "", "", "owner", time.Minute, "", []string{}, []string{}, true, key)
 				assert.NoError(t, err, "Error creating TXT registry")
 				txtRecords := r.generateTXTRecord(test.record)
 				assert.Len(t, txtRecords, len(test.record.Targets))
@@ -144,7 +144,7 @@ func TestApplyRecordsWithEncryption(t *testing.T) {
 
 	key := []byte("ZPitL0NGVQBZbTD6DwXJzD8RiStSazzYXQsdUowLURY=")
 
-	r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, key, false)
+	r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, key)
 
 	_ = r.ApplyChanges(ctx, &plan.Changes{
 		Create: []*endpoint.Endpoint{
@@ -202,7 +202,7 @@ func TestApplyRecordsWithEncryptionKeyChanged(t *testing.T) {
 	}
 
 	for _, key := range withEncryptionKeys {
-		r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, []byte(key), false)
+		r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, []byte(key))
 		_ = r.ApplyChanges(ctx, &plan.Changes{
 			Create: []*endpoint.Endpoint{
 				newEndpointWithOwner("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner"),
@@ -232,7 +232,7 @@ func TestApplyRecordsOnEncryptionKeyChangeWithKeyIdLabel(t *testing.T) {
 	}
 
 	for i, key := range withEncryptionKeys {
-		r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, []byte(key), false)
+		r, _ := NewTXTRegistry(p, "", "", "owner", time.Hour, "", []string{}, []string{}, true, []byte(key))
 		keyId := fmt.Sprintf("key-id-%d", i)
 		changes := []*endpoint.Endpoint{
 			newEndpointWithOwnerAndOwnedRecordWithKeyIDLabel("new-record-1.test-zone.example.org", "new-loadbalancer-1.lb.com", endpoint.RecordTypeCNAME, "owner", "", keyId),