Adjust property names for package dependencies to original proposal

code/go/internal/validator/semantic/validate_package_references.go

@@ -85,7 +85,7 @@ func extractPackageNamesFromRequires(packages interface{}) []requiredPackage {
 			continue
 		}
 
-		name, ok := pkgMap["name"].(string)
+		name, ok := pkgMap["package"].(string)
 		if !ok {
 			continue
 		}

code/go/internal/validator/semantic/validate_package_references_test.go

@@ -24,7 +24,7 @@ format_version: 3.6.0
 type: integration
 requires:
   input:
-    - name: filelog_otel
+    - package: filelog_otel
       version: "1.0.0"
 policy_templates:
   - name: apache
@@ -47,7 +47,7 @@ format_version: 3.6.0
 type: integration
 requires:
   input:
-    - name: filelog_otel
+    - package: filelog_otel
       version: "1.0.0"
 policy_templates:
   - name: apache
@@ -72,7 +72,7 @@ format_version: 3.6.0
 type: integration
 requires:
   content:
-    - name: apache_otel
+    - package: apache_otel
       version: "^1.0.0"
 policy_templates:
   - name: apache
@@ -97,7 +97,7 @@ format_version: 3.6.0
 type: integration
 requires:
   input:
-    - name: filelog_otel
+    - package: filelog_otel
       version: "1.0.0"
 `), 0o644)
 		require.NoError(t, err)
@@ -127,7 +127,7 @@ format_version: 3.6.0
 type: integration
 requires:
   input:
-    - name: filelog_otel
+    - package: filelog_otel
       version: "1.0.0"
 `), 0o644)
 		require.NoError(t, err)
@@ -159,7 +159,7 @@ format_version: 3.6.0
 type: integration
 requires:
   content:
-    - name: security_rules
+    - package: security_rules
       version: "^1.0.0"
 `), 0o644)
 		require.NoError(t, err)
@@ -212,7 +212,7 @@ format_version: 3.6.0
 type: integration
 requires:
   input:
-    - name: valid_package
+    - package: valid_package
       version: "1.0.0"
 policy_templates:
   - name: apache

code/go/internal/validator/semantic/validate_test_package_requirements.go

@@ -6,6 +6,8 @@ package semantic
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/Masterminds/semver/v3"
 
@@ -51,7 +53,7 @@ func getRequiredPackagesWithConstraints(manifest pkgpath.File) (map[string]strin
 	if err == nil && inputPackages != nil {
 		if pkgArray, ok := inputPackages.([]interface{}); ok {
 			for i := 0; i < len(pkgArray); i++ {
-				name, err := manifest.Values(fmt.Sprintf("$.requires.input[%d].name", i))
+				name, err := manifest.Values(fmt.Sprintf("$.requires.input[%d].package", i))
 				if err != nil || name == nil {
 					continue
 				}
@@ -73,7 +75,7 @@ func getRequiredPackagesWithConstraints(manifest pkgpath.File) (map[string]strin
 	if err == nil && contentPackages != nil {
 		if pkgArray, ok := contentPackages.([]interface{}); ok {
 			for i := 0; i < len(pkgArray); i++ {
-				name, err := manifest.Values(fmt.Sprintf("$.requires.content[%d].name", i))
+				name, err := manifest.Values(fmt.Sprintf("$.requires.content[%d].package", i))
 				if err != nil || name == nil {
 					continue
 				}
@@ -114,14 +116,21 @@ func validateIntegrationTestRequirements(fsys fspath.FS, requiredPackages map[st
 					if reqMap, ok := req.(map[string]interface{}); ok {
 						pkgName, _ := reqMap["package"].(string)
 						version, _ := reqMap["version"].(string)
-
-						if pkgName == "" || version == "" {
+						if pkgName != "" && version != "" {
+							err := validateTestRequirementPackageVersion(fsys.Path("_dev/test/config.yml"), testType, idx, pkgName, version, requiredPackages)
+							if err != nil {
+								errs = append(errs, err)
+							}
 							continue
 						}
 
-						err := validateTestRequirement(fsys.Path("_dev/test/config.yml"), testType, idx, pkgName, version, requiredPackages)
-						if err != nil {
-							errs = append(errs, err)
+						source, _ := reqMap["source"].(string)
+						if source != "" {
+							err := validateTestRequirementSource(fsys.Path("_dev/test/config.yml"), source)
+							if err != nil {
+								errs = append(errs, err)
+							}
+							continue
 						}
 					}
 				}
@@ -133,31 +142,47 @@ func validateIntegrationTestRequirements(fsys fspath.FS, requiredPackages map[st
 }
 
 func validateDataStreamTestRequirements(fsys fspath.FS, requiredPackages map[string]string) specerrors.ValidationErrors {
-	testConfigs, err := pkgpath.Files(fsys, "data_stream/*/_dev/test/*/config.yml")
-	if err != nil {
-		return nil
+	var errs specerrors.ValidationErrors
+
+	patterns := []string{
+		"data_stream/*/_dev/test/system/test-*-config.yml",
+		"data_stream/*/_dev/test/policy/test-*.yml",
+		"data_stream/*/_dev/test/static/test-*-config.yml",
 	}
 
-	var errs specerrors.ValidationErrors
-	for _, config := range testConfigs {
-		requires, err := config.Values("$.requires")
-		if err != nil || requires == nil {
+	for _, pattern := range patterns {
+		testConfigs, err := pkgpath.Files(fsys, pattern)
+		if err != nil {
 			continue
 		}
 
-		if reqArray, ok := requires.([]interface{}); ok {
-			for idx, req := range reqArray {
-				if reqMap, ok := req.(map[string]interface{}); ok {
-					pkgName, _ := reqMap["package"].(string)
-					version, _ := reqMap["version"].(string)
+		for _, config := range testConfigs {
+			requires, err := config.Values("$.requires")
+			if err != nil || requires == nil {
+				continue
+			}
 
-					if pkgName == "" || version == "" {
-						continue
-					}
+			if reqArray, ok := requires.([]interface{}); ok {
+				for idx, req := range reqArray {
+					if reqMap, ok := req.(map[string]interface{}); ok {
+						pkgName, _ := reqMap["package"].(string)
+						version, _ := reqMap["version"].(string)
+						if pkgName != "" && version != "" {
+							err := validateTestRequirementPackageVersion(fsys.Path(config.Path()), "", idx, pkgName, version, requiredPackages)
+							if err != nil {
+								errs = append(errs, err)
+							}
+							continue
+						}
 
-					err := validateTestRequirement(fsys.Path(config.Path()), "", idx, pkgName, version, requiredPackages)
-					if err != nil {
-						errs = append(errs, err)
+						source, _ := reqMap["source"].(string)
+						if source != "" {
+							err := validateTestRequirementSource(fsys.Path(config.Path()), source)
+							if err != nil {
+								errs = append(errs, err)
+							}
+							continue
+						}
 					}
 				}
 			}
@@ -167,7 +192,7 @@ func validateDataStreamTestRequirements(fsys fspath.FS, requiredPackages map[str
 	return errs
 }
 
-func validateTestRequirement(configPath, testType string, idx int, pkgName, version string, requiredPackages map[string]string) *specerrors.StructuredError {
+func validateTestRequirementPackageVersion(configPath, testType string, idx int, pkgName, version string, requiredPackages map[string]string) *specerrors.StructuredError {
 	constraint, exists := requiredPackages[pkgName]
 	if !exists {
 		location := fmt.Sprintf("requires[%d]", idx)
@@ -216,3 +241,22 @@ func validateTestRequirement(configPath, testType string, idx int, pkgName, vers
 
 	return nil
 }
+
+// validateTestRequirementSource checks if the relative path exists. This could be done with "format: relative-path" in
+// the spec, but this format checker only works with relative files inside the package. In this case the source package
+// is going to be outside the current package.
+func validateTestRequirementSource(configFile, source string) *specerrors.StructuredError {
+	cleanSource := filepath.Clean(filepath.FromSlash(source))
+	if filepath.IsAbs(cleanSource) {
+		return specerrors.NewStructuredErrorf(
+			"file \"%s\" is invalid: source path to required package \"%s\" must be relative",
+			configFile, source)
+	}
+	targetPath := filepath.Join(filepath.Dir(configFile), filepath.FromSlash(source))
+	if _, err := os.Stat(targetPath); err != nil {
+		return specerrors.NewStructuredErrorf(
+			"file \"%s\" is invalid: source path to required package \"%s\" does not exist",
+			configFile, source)
+	}
+	return nil
+}

code/go/internal/validator/semantic/validate_test_package_requirements_test.go

@@ -20,6 +20,7 @@ func TestValidateTestPackageRequirements(t *testing.T) {
 		manifest       string
 		testConfig     string
 		testConfigPath string
+		sourceDirs     []string
 		expectError    bool
 		errorContains  string
 	}{
@@ -28,7 +29,7 @@ func TestValidateTestPackageRequirements(t *testing.T) {
 format_version: 3.6.0
 requires:
   input:
-    - name: sql_input
+    - package: sql_input
       version: ^1.2.0`,
 			testConfig: `system:
   requires:
@@ -42,20 +43,20 @@ requires:
 format_version: 3.6.0
 requires:
   content:
-    - name: logs_package
+    - package: logs_package
       version: ~1.0.0`,
 			testConfig: `requires:
   - package: logs_package
     version: 1.0.3`,
-			testConfigPath: "data_stream/example/_dev/test/system/config.yml",
+			testConfigPath: "data_stream/example/_dev/test/system/test-default-config.yml",
 			expectError:    false,
 		},
 		"package_not_in_manifest": {
 			manifest: `name: test
 format_version: 3.6.0
 requires:
   input:
-    - name: sql_input
+    - package: sql_input
       version: ^1.2.0`,
 			testConfig: `policy:
   requires:
@@ -70,7 +71,7 @@ requires:
 format_version: 3.6.0
 requires:
   input:
-    - name: sql_input
+    - package: sql_input
       version: ^2.0.0`,
 			testConfig: `system:
   requires:
@@ -85,7 +86,7 @@ requires:
 format_version: 3.6.0
 requires:
   input:
-    - name: sql_input
+    - package: sql_input
       version: ^1.0.0`,
 			testConfig: `system:
   requires:
@@ -100,10 +101,10 @@ requires:
 format_version: 3.6.0
 requires:
   input:
-    - name: pkg1
+    - package: pkg1
       version: ^1.0.0
   content:
-    - name: pkg2
+    - package: pkg2
       version: ~2.0.0`,
 			testConfig: `system:
   requires:
@@ -121,7 +122,7 @@ policy:
 format_version: 3.6.0
 requires:
   input:
-    - name: sql_input
+    - package: sql_input
       version: ^1.0.0`,
 			testConfig: `system:
   skip:
@@ -141,6 +142,46 @@ format_version: 3.6.0`,
 			expectError:    true,
 			errorContains:  "sql_input\" which is not listed in manifest requires",
 		},
+		"valid_source_path_requirement": {
+			manifest: `name: test
+format_version: 3.6.0`,
+			testConfig: `system:
+  requires:
+    - source: ../my_input_package`,
+			testConfigPath: "_dev/test/config.yml",
+			// source "../my_input_package" is relative to _dev/test/, resolves to _dev/my_input_package
+			sourceDirs:  []string{"_dev/my_input_package"},
+			expectError: false,
+		},
+		"invalid_source_path_requirement": {
+			manifest: `name: test
+format_version: 3.6.0`,
+			testConfig: `system:
+  requires:
+    - source: ../nonexistent_package`,
+			testConfigPath: "_dev/test/config.yml",
+			expectError:    true,
+			errorContains:  `source path to required package "../nonexistent_package" does not exist`,
+		},
+		"valid_datastream_source_path_requirement": {
+			manifest: `name: test
+format_version: 3.6.0`,
+			testConfig: `requires:
+  - source: ../my_content_package`,
+			testConfigPath: "data_stream/example/_dev/test/system/test-default-config.yml",
+			// source "../my_content_package" is relative to .../system/, resolves to .../test/my_content_package
+			sourceDirs:  []string{"data_stream/example/_dev/test/my_content_package"},
+			expectError: false,
+		},
+		"invalid_datastream_source_path_requirement": {
+			manifest: `name: test
+format_version: 3.6.0`,
+			testConfig: `requires:
+  - source: ../nonexistent_package`,
+			testConfigPath: "data_stream/example/_dev/test/system/test-default-config.yml",
+			expectError:    true,
+			errorContains:  `source path to required package "../nonexistent_package" does not exist`,
+		},
 	}
 
 	for name, tc := range tests {
@@ -151,6 +192,12 @@ format_version: 3.6.0`,
 			err := os.WriteFile(filepath.Join(pkgRoot, "manifest.yml"), []byte(tc.manifest), 0644)
 			require.NoError(t, err)
 
+			// Create source directories referenced by source entries
+			for _, sourceDir := range tc.sourceDirs {
+				err = os.MkdirAll(filepath.Join(pkgRoot, sourceDir), 0755)
+				require.NoError(t, err)
+			}
+
 			// Create test config in appropriate path
 			testConfigFullPath := filepath.Join(pkgRoot, tc.testConfigPath)
 			err = os.MkdirAll(filepath.Dir(testConfigFullPath), 0755)

code/go/pkg/validator/validator_test.go

@@ -244,7 +244,7 @@ func TestValidateFile(t *testing.T) {
 		"bad_requires": {
 			"manifest.yml",
 			[]string{
-				`field requires.content.0.name: Does not match pattern '^[a-z0-9_]+$'`,
+				`field requires.content.0.package: Does not match pattern '^[a-z0-9_]+$'`,
 				`field requires.input.0: version is required`,
 				`field requires.input.1.version: version "^1.0.0" for package "filelog_otel" must be a valid semantic version, constraints are not allowed`,
 			},