[Fleet] Add support for section headers, divider toggling, and reorganization

code/go/internal/validator/semantic/validate_sections.go

@@ -0,0 +1,146 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"fmt"
+	"io/fs"
+	"path"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+// ValidateSections validates sections definitions in manifests.
+// It checks that:
+// - section names are unique within each scope
+// - vars that reference a section via the `section` attribute name a section
+// defined in the `sections` list at the same scope level
+func ValidateSections(fsys fspath.FS) specerrors.ValidationErrors {
+	d, err := fs.ReadFile(fsys, "manifest.yml")
+	if err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("failed to read file \"%s\": %w", fsys.Path("manifest.yml"), err)}
+	}
+
+	var manifest sectionsManifest
+	if err := yaml.Unmarshal(d, &manifest); err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to parse manifest: %w", fsys.Path("manifest.yml"), err)}
+	}
+
+	errs := validateSectionsManifest(fsys.Path("manifest.yml"), manifest)
+
+	// Validate data stream manifests.
+	dataStreams, err := listDataStreams(fsys)
+	if err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("failed to list data streams: %w", err)}
+	}
+	for _, ds := range dataStreams {
+		errs = append(errs, validateDataStreamSections(fsys, path.Join("data_stream", ds, "manifest.yml"))...)
+	}
+
+	return errs
+}
+
+type sectionsVar struct {
+	Name    string `yaml:"name"`
+	Section string `yaml:"section"`
+}
+
+type manifestSection struct {
+	Name string `yaml:"name"`
+}
+
+type sectionsManifest struct {
+	Sections        []manifestSection `yaml:"sections"`
+	Vars            []sectionsVar     `yaml:"vars"`
+	PolicyTemplates []struct {
+		Sections []manifestSection `yaml:"sections"`
+		Vars     []sectionsVar     `yaml:"vars"`
+		Inputs   []struct {
+			Sections []manifestSection `yaml:"sections"`
+			Vars     []sectionsVar     `yaml:"vars"`
+		} `yaml:"inputs"`
+	} `yaml:"policy_templates"`
+}
+
+type sectionsDataStreamStream struct {
+	Title    string            `yaml:"title"`
+	Input    string            `yaml:"input"`
+	Sections []manifestSection `yaml:"sections"`
+	Vars     []sectionsVar     `yaml:"vars"`
+}
+
+type sectionsDataStreamManifest struct {
+	Streams []sectionsDataStreamStream `yaml:"streams"`
+}
+
+func validateSectionsManifest(filePath string, manifest sectionsManifest) specerrors.ValidationErrors {
+	var errs specerrors.ValidationErrors
+
+	errs = append(errs, validateSectionsScope(filePath, "package root", manifest.Sections, manifest.Vars)...)
+
+	for _, pt := range manifest.PolicyTemplates {
+		errs = append(errs, validateSectionsScope(filePath, "policy template", pt.Sections, pt.Vars)...)
+		for _, input := range pt.Inputs {
+			errs = append(errs, validateSectionsScope(filePath, "input", input.Sections, input.Vars)...)
+		}
+	}
+
+	return errs
+}
+
+func validateDataStreamSections(fsys fspath.FS, filePath string) specerrors.ValidationErrors {
+	d, err := fs.ReadFile(fsys, filePath)
+	if err != nil {
+		// File might not exist, which is fine.
+		return nil
+	}
+
+	var manifest sectionsDataStreamManifest
+	if err := yaml.Unmarshal(d, &manifest); err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to parse manifest: %w", fsys.Path(filePath), err)}
+	}
+
+	var errs specerrors.ValidationErrors
+	for i, stream := range manifest.Streams {
+		streamID := stream.Title
+		if streamID == "" {
+			streamID = stream.Input
+		}
+		if streamID == "" {
+			streamID = fmt.Sprintf("stream[%d]", i)
+		}
+		errs = append(errs, validateSectionsScope(fsys.Path(filePath), fmt.Sprintf("stream %q", streamID), stream.Sections, stream.Vars)...)
+	}
+
+	return errs
+}
+
+func validateSectionsScope(filePath, scope string, sections []manifestSection, vars []sectionsVar) specerrors.ValidationErrors {
+	var errs specerrors.ValidationErrors
+
+	// Build set of defined section names, checking for duplicates.
+	sectionNames := make(map[string]bool)
+	for _, s := range sections {
+		if sectionNames[s.Name] {
+			errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: duplicate section name %q in %s", filePath, s.Name, scope))
+		}
+		sectionNames[s.Name] = true
+	}
+
+	// Verify that each var's section attribute references a defined section.
+	for _, v := range vars {
+		if v.Section == "" {
+			continue
+		}
+		if !sectionNames[v.Section] {
+			errs = append(errs, specerrors.NewStructuredErrorf("file \"%s\" is invalid: var %q references undefined section %q in %s", filePath, v.Name, v.Section, scope))
+		}
+	}
+
+	return errs
+}

code/go/internal/validator/semantic/validate_sections_test.go

@@ -0,0 +1,241 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestValidateSectionsManifest(t *testing.T) {
+	cases := []struct {
+		title    string
+		manifest string
+		errors   []string
+	}{
+		{
+			title: "valid: sections defined, vars reference them",
+			manifest: `
+sections:
+  - name: auth_section
+    title: Authentication
+vars:
+  - name: username
+    section: auth_section
+  - name: password
+    section: auth_section
+`,
+		},
+		{
+			title: "valid: no sections and no section references",
+			manifest: `
+vars:
+  - name: username
+  - name: password
+`,
+		},
+		{
+			title: "valid: sections defined but no vars reference them",
+			manifest: `
+sections:
+  - name: auth_section
+    title: Authentication
+vars:
+  - name: username
+`,
+		},
+		{
+			title: "valid: some vars reference sections, others do not",
+			manifest: `
+sections:
+  - name: auth_section
+    title: Authentication
+vars:
+  - name: region
+  - name: username
+    section: auth_section
+`,
+		},
+		{
+			title: "invalid: var references undefined section",
+			manifest: `
+vars:
+  - name: username
+    section: auth_section
+`,
+			errors: []string{
+				`file "manifest.yml" is invalid: var "username" references undefined section "auth_section" in package root`,
+			},
+		},
+		{
+			title: "invalid: duplicate section name",
+			manifest: `
+sections:
+  - name: auth_section
+    title: Authentication
+  - name: auth_section
+    title: Authentication (duplicate)
+vars:
+  - name: username
+    section: auth_section
+`,
+			errors: []string{
+				`file "manifest.yml" is invalid: duplicate section name "auth_section" in package root`,
+			},
+		},
+		{
+			title: "valid: sections scoped to policy template",
+			manifest: `
+policy_templates:
+  - sections:
+      - name: auth_section
+        title: Authentication
+    vars:
+      - name: username
+        section: auth_section
+`,
+		},
+		{
+			title: "invalid: policy template var references section not defined at that level",
+			manifest: `
+sections:
+  - name: auth_section
+    title: Authentication
+policy_templates:
+  - vars:
+      - name: username
+        section: auth_section
+`,
+			errors: []string{
+				`file "manifest.yml" is invalid: var "username" references undefined section "auth_section" in policy template`,
+			},
+		},
+		{
+			title: "valid: sections scoped to input",
+			manifest: `
+policy_templates:
+  - inputs:
+      - sections:
+          - name: auth_section
+            title: Authentication
+        vars:
+          - name: username
+            section: auth_section
+`,
+		},
+		{
+			title: "invalid: input var references section not defined at that level",
+			manifest: `
+policy_templates:
+  - inputs:
+      - vars:
+          - name: username
+            section: auth_section
+`,
+			errors: []string{
+				`file "manifest.yml" is invalid: var "username" references undefined section "auth_section" in input`,
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			var manifest sectionsManifest
+			err := yaml.Unmarshal([]byte(c.manifest), &manifest)
+			require.NoError(t, err)
+
+			errors := validateSectionsManifest("manifest.yml", manifest)
+			assert.Len(t, errors, len(c.errors))
+			for _, err := range errors {
+				assert.Contains(t, c.errors, err.Error())
+			}
+		})
+	}
+}
+
+func TestValidateSectionsScope(t *testing.T) {
+	cases := []struct {
+		title    string
+		sections []manifestSection
+		vars     []sectionsVar
+		errors   []string
+	}{
+		{
+			title: "valid: all section references resolve",
+			sections: []manifestSection{
+				{Name: "auth_section"},
+				{Name: "advanced_section"},
+			},
+			vars: []sectionsVar{
+				{Name: "username", Section: "auth_section"},
+				{Name: "timeout", Section: "advanced_section"},
+				{Name: "region"},
+			},
+		},
+		{
+			title: "valid: no vars with section attributes",
+			sections: []manifestSection{
+				{Name: "auth_section"},
+			},
+			vars: []sectionsVar{
+				{Name: "username"},
+			},
+		},
+		{
+			title:    "valid: empty sections and vars",
+			sections: nil,
+			vars:     nil,
+		},
+		{
+			title:    "invalid: var references undefined section",
+			sections: nil,
+			vars: []sectionsVar{
+				{Name: "username", Section: "missing_section"},
+			},
+			errors: []string{
+				`file "test.yml" is invalid: var "username" references undefined section "missing_section" in package root`,
+			},
+		},
+		{
+			title: "invalid: duplicate section name",
+			sections: []manifestSection{
+				{Name: "auth_section"},
+				{Name: "auth_section"},
+			},
+			vars: nil,
+			errors: []string{
+				`file "test.yml" is invalid: duplicate section name "auth_section" in package root`,
+			},
+		},
+		{
+			title: "invalid: multiple vars reference undefined sections",
+			sections: []manifestSection{
+				{Name: "auth_section"},
+			},
+			vars: []sectionsVar{
+				{Name: "username", Section: "auth_section"},
+				{Name: "api_key", Section: "missing_section"},
+				{Name: "token", Section: "another_missing"},
+			},
+			errors: []string{
+				`file "test.yml" is invalid: var "api_key" references undefined section "missing_section" in package root`,
+				`file "test.yml" is invalid: var "token" references undefined section "another_missing" in package root`,
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			errors := validateSectionsScope("test.yml", "package root", c.sections, c.vars)
+			assert.Len(t, errors, len(c.errors))
+			for _, err := range errors {
+				assert.Contains(t, c.errors, err.Error())
+			}
+		})
+	}
+}

code/go/internal/validator/spec.go

@@ -225,6 +225,7 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules
 		{fn: semantic.ValidateCapabilitiesRequired, since: semver.MustParse("2.10.0")}, // capabilities definition was added in spec version 2.10.0
 		{fn: semantic.ValidateRequiredVarGroups},
 		{fn: semantic.ValidateVarGroups, since: semver.MustParse("3.6.0")},
+		{fn: semantic.ValidateSections},
 		{fn: semantic.ValidateDocsStructure},
 		{fn: semantic.ValidateDeploymentModes, types: []string{"integration"}},
 		{fn: semantic.ValidateDurationVariables, since: semver.MustParse("3.5.0")},