fix(evaluator): load globbed dataset files in a deterministic order

e2e/test_evaluator_plugin.py

@@ -476,7 +476,7 @@ def test_fileset_fragment_and_glob_datasets(evaluator_sdk: NeMoPlatform) -> None
 
         cases = {
             "specific file": (f"{workspace}/{fileset_name}#part-a.json", [1.0, 0.0]),
-            "glob": (f"{workspace}/{fileset_name}#part-*.json", [1.0, 1.0, 0.0]),
+            "glob": (f"{workspace}/{fileset_name}#part-*.json", [1.0, 0.0, 1.0]),
         }
         for label, (reference, expected_scores) in cases.items():
             job = evaluator_sdk.evaluator.submit(

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/datasets/loader.py

@@ -163,7 +163,7 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]:
         pattern: Optional explicit file name or glob pattern.
 
     Returns:
-        List of discovered files.
+        List of discovered files, sorted by path.
 
     Raises:
         DatasetLoadError: If files cannot be found or selected paths are invalid.
@@ -172,7 +172,7 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]:
         raise DatasetLoadError(f"Dataset directory not found: {base_path}")
 
     if pattern is None:
-        files = [f for f in base_path.rglob("*") if f.is_file()]
+        files = sorted(f for f in base_path.rglob("*") if f.is_file())
         if not files:
             raise DatasetLoadError(f"No files found in {base_path}")
         return files
@@ -184,7 +184,7 @@ def discover_files(base_path: Path, pattern: str | None) -> list[Path]:
         return [file_path]
 
     if is_glob_pattern(pattern):
-        files = list(base_path.glob(pattern))
+        files = sorted(base_path.glob(pattern))
         if not files:
             raise DatasetLoadError(f"No files found matching pattern '{pattern}' in {base_path}")
         return [f for f in files if f.is_file()]

packages/nemo_evaluator_sdk/tests/execution/test_metric_execution.py

@@ -520,6 +520,26 @@ def test_glob_pattern_discovers_files(self, tmp_path: Path):
 
         assert sorted(discover_files(tmp_path / "splits", "**/*.jsonl")) == sorted([train_path, validation_path])
 
+    def test_glob_pattern_returns_files_in_sorted_order(self, tmp_path: Path):
+        for name in ("part-c.json", "part-a.json", "part-b.json"):
+            (tmp_path / name).touch()
+
+        assert discover_files(tmp_path, "part-*.json") == [
+            tmp_path / "part-a.json",
+            tmp_path / "part-b.json",
+            tmp_path / "part-c.json",
+        ]
+
+    def test_directory_without_a_pattern_returns_files_in_sorted_order(self, tmp_path: Path):
+        for name in ("c.jsonl", "a.jsonl", "b.jsonl"):
+            (tmp_path / name).touch()
+
+        assert discover_files(tmp_path, None) == [
+            tmp_path / "a.jsonl",
+            tmp_path / "b.jsonl",
+            tmp_path / "c.jsonl",
+        ]
+
 
 class TestIsCompletionsEndpoint:
     @pytest.mark.parametrize(