diff --git a/benchmarks/longmemeval_bench.py b/benchmarks/longmemeval_bench.py index 2cb9836a42..9ef08e7f4f 100644 --- a/benchmarks/longmemeval_bench.py +++ b/benchmarks/longmemeval_bench.py @@ -2917,7 +2917,7 @@ def _load_or_create_split(split_file: str, data: list, dev_size: int = 50, seed: split_path = Path(split_file) if split_path.exists(): - with open(split_path) as f: + with open(split_path, encoding="utf-8") as f: return json.load(f) # Create new split @@ -2927,9 +2927,9 @@ def _load_or_create_split(split_file: str, data: list, dev_size: int = 50, seed: dev_ids = all_ids[:dev_size] held_out_ids = all_ids[dev_size:] split = {"dev": dev_ids, "held_out": held_out_ids, "seed": seed, "dev_size": dev_size} - with open(split_path, "w") as f: + with open(split_path, "w", encoding="utf-8") as f: json.dump(split, f, indent=2) - print(f" Created new split: {len(dev_ids)} dev / {len(held_out_ids)} held-out → {split_path}") + print(f" Created new split: {len(dev_ids)} dev / {len(held_out_ids)} held-out -> {split_path}") return split @@ -2957,7 +2957,7 @@ def run_benchmark( split_subset: "dev" (50 questions for tuning) or "held_out" (450 for final evaluation). None = run all questions. """ - with open(data_file) as f: + with open(data_file, encoding="utf-8") as f: data = json.load(f) # Apply train/test split filter before limit/skip @@ -2966,7 +2966,7 @@ def run_benchmark( subset_ids = set(split[split_subset]) before = len(data) data = [entry for entry in data if entry["question_id"] in subset_ids] - print(f" Split filter ({split_subset}): {before} → {len(data)} questions") + print(f" Split filter ({split_subset}): {before} -> {len(data)} questions") if limit > 0: data = data[:limit] @@ -2998,7 +2998,7 @@ def run_benchmark( cache_path = Path(diary_cache_file) if cache_path.exists(): try: - with open(cache_path) as f: + with open(cache_path, encoding="utf-8") as f: diary_cache = json.load(f) print( f" Diary cache: loaded {len(diary_cache)} sessions from {cache_path.name}" @@ -3031,7 +3031,7 @@ def run_benchmark( # Save progress in case of interruption if cache_path: try: - with open(cache_path, "w") as f: + with open(cache_path, "w", encoding="utf-8") as f: json.dump(diary_cache, f) except Exception: pass @@ -3039,14 +3039,14 @@ def run_benchmark( # Final cache save if cache_path: try: - with open(cache_path, "w") as f: + with open(cache_path, "w", encoding="utf-8") as f: json.dump(diary_cache, f) - print(f" Diary cache saved → {cache_path.name}") + print(f" Diary cache saved -> {cache_path.name}") except Exception: pass print(f"\n{'=' * 60}") - print(" MemPal × LongMemEval Benchmark") + print(" MemPal x LongMemEval Benchmark") print(f"{'=' * 60}") print(f" Data: {Path(data_file).name}") print(f" Questions: {len(data)}") @@ -3055,7 +3055,7 @@ def run_benchmark( rerank_label = f" + LLM re-rank ({model_short})" if llm_rerank_enabled else "" diary_label = f" [diary ingest: {model_short}]" if mode == "diary" else "" print(f" Mode: {mode}{diary_label}{rerank_label}") - print(f"{'─' * 60}\n") + print(f"{'-' * 60}\n") # Collect metrics ks = [1, 3, 5, 10, 30, 50] @@ -3216,7 +3216,7 @@ def run_benchmark( # Print results print(f"\n{'=' * 60}") - print(f" RESULTS — MemPal ({mode} mode, {granularity} granularity)") + print(f" RESULTS - MemPal ({mode} mode, {granularity} granularity)") print(f"{'=' * 60}") print(f" Time: {elapsed:.1f}s ({elapsed / len(data):.2f}s per question)\n") @@ -3245,15 +3245,15 @@ def run_benchmark( if mode == "diary" and diary_cache and diary_cache_file: try: real_cache = {k: v for k, v in diary_cache.items() if v is not None} - with open(diary_cache_file, "w") as f: + with open(diary_cache_file, "w", encoding="utf-8") as f: json.dump(real_cache, f) - print(f" Diary cache saved: {len(real_cache)} sessions → {diary_cache_file}") + print(f" Diary cache saved: {len(real_cache)} sessions -> {diary_cache_file}") except Exception as e: print(f" Warning: could not save diary cache: {e}") # Save results if out_file: - with open(out_file, "w") as f: + with open(out_file, "w", encoding="utf-8") as f: for entry in results_log: f.write(json.dumps(entry) + "\n") print(f" Results saved to: {out_file}") @@ -3393,7 +3393,7 @@ def run_benchmark( if args.create_split: if not args.split_file: args.split_file = "benchmarks/lme_split_50_450.json" - with open(args.data_file) as f: + with open(args.data_file, encoding="utf-8") as f: _all_data = json.load(f) _load_or_create_split(args.split_file, _all_data) sys.exit(0) diff --git a/tests/benchmarks/test_longmemeval_bench.py b/tests/benchmarks/test_longmemeval_bench.py new file mode 100644 index 0000000000..5bb439021b --- /dev/null +++ b/tests/benchmarks/test_longmemeval_bench.py @@ -0,0 +1,89 @@ +import importlib.util +import json +import sys +import types +from pathlib import Path + + +def _load_longmemeval_bench(monkeypatch): + module_name = "tests._longmemeval_bench_under_test" + module_path = Path(__file__).resolve().parents[2] / "benchmarks" / "longmemeval_bench.py" + + class _DummyEphemeralClient: + def delete_collection(self, name): + return None + + def create_collection(self, name, embedding_function=None): + return object() + + fake_chromadb = types.ModuleType("chromadb") + fake_chromadb.EphemeralClient = lambda: _DummyEphemeralClient() + monkeypatch.setitem(sys.modules, "chromadb", fake_chromadb) + + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _force_cp1252_default(monkeypatch, target_path): + original_open = open + target_path = Path(target_path) + + def fake_open(file, mode="r", *args, **kwargs): + if Path(file) == target_path and "b" not in mode and "encoding" not in kwargs: + kwargs["encoding"] = "cp1252" + return original_open(file, mode, *args, **kwargs) + + monkeypatch.setattr("builtins.open", fake_open) + + +def test_load_or_create_split_reads_utf8_when_windows_default_is_cp1252(tmp_path, monkeypatch): + bench = _load_longmemeval_bench(monkeypatch) + split_path = tmp_path / "lme_split.json" + expected = {"dev": ["frage-ä"], "held_out": ["sess-😀"], "seed": 42, "dev_size": 1} + split_path.write_text(json.dumps(expected, ensure_ascii=False), encoding="utf-8") + + _force_cp1252_default(monkeypatch, split_path) + + split = bench._load_or_create_split(str(split_path), data=[]) + + assert split == expected + + +def test_run_benchmark_reads_utf8_questions_when_windows_default_is_cp1252( + tmp_path, monkeypatch +): + bench = _load_longmemeval_bench(monkeypatch) + question = "Wann hat Zoë das Café 😀 erwähnt?" + answer = "Beim Frühstück im Café 😀." + data = [ + { + "question_id": "q1", + "question_type": "single-session-user", + "question": question, + "answer": answer, + "answer_session_ids": ["sess_1"], + "haystack_sessions": [[{"role": "user", "content": "stub memory"}]], + "haystack_session_ids": ["sess_1"], + "haystack_dates": ["2024-01-01"], + } + ] + data_path = tmp_path / "longmemeval.json" + data_path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + + captured = {} + + def fake_build(entry, granularity="session", n_results=50): + captured["question"] = entry["question"] + captured["answer"] = entry["answer"] + return [0], ["stub memory"], ["sess_1"], ["2024-01-01"] + + monkeypatch.setattr(bench, "build_palace_and_retrieve", fake_build) + _force_cp1252_default(monkeypatch, data_path) + + bench.run_benchmark(str(data_path), limit=1, out_file=None) + + assert captured["question"] == question + assert captured["answer"] == answer