diff --git a/docs/requirements.md b/docs/requirements.md index 82555770..4dcab6a0 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -47,7 +47,7 @@ 4. 公开仓库不得包含凭据、真实关注地区、真实 RSS URL、经纬度、Webhook、地理编码缓存、历史简报或运行状态。仓库只维护无效或公开地点的结构示例。 5. 提交前运行通用秘密扫描，并对项目特有的隐私配置进行检查。 6. 需求变更应同步更新本文档。 -7. 配置项应经过类型、范围和必填校验；provider、source 和状态存储保持明确扩展边界，避免厂商逻辑进入核心编排。 +7. 配置项应经过类型、范围和必填校验；非法布尔字面量、缺少必填字段、把 JSON 数组字段配置为其他类型，或在 RSS 可选数组中配置非字符串、空字符串及无效 selector/regex 时，必须给出包含配置名或字段路径的配置错误。RSS 可选数组字段的 `null` 保持为空数组语义。provider、source 和状态存储保持明确扩展边界，避免厂商逻辑进入核心编排。 8. `LLM_PROVIDER` 使用 any-llm provider ID，`LLM_MODEL` 使用对应模型 ID，凭据与 API Base 使用 SDK 为该 provider 定义的环境变量。应用仅兼容已投入使用的 `DEEPSEEK_MODEL` 与 `DEEPSEEK_BASE_URL`。开发依赖安装 `any-llm-sdk[all]`；运行依赖只安装 SDK 核心包，由部署者显式、可复现地安装实际 provider extras。RSS 只从命名 JSON 文件读取。SQLite 没有原生日期时间类型，应用仅在持久化边界将时区感知时间转换为固定宽度 UTC 文本，使文本字典序等同绝对时间顺序。 9. 地理范围、空气质量分级与健康提示、正文清洗默认规则、provider 默认顺序及厂商指数代码等纯领域数据应存放在独立数据文件中，由实现代码加载并校验。 10. INFO 日志必须记录每个地点的有效天气 provider 顺序，并为每次天气 API 逻辑调用记录 provider、成功或失败、耗时、成功时的实际来源与观测时间，以及失败时不包含请求内容的阶段、HTTP 状态或异常类型，使自动降级可追溯。所有实际外部 HTTP 请求，包括天气、空气质量、地理编码、LLM、RSS、辅助上下文和 Telegram 投递，还必须统一记录静态 provider、operation、方法、成功或失败、耗时，以及 HTTP 状态或异常类型；重试和分片分别按实际请求次数记录。常规 INFO 日志及仅由 DEBUG 级别启用的非敏感诊断不得包含坐标、凭据、标题、正文、URL、接收方标识、私有 endpoint、响应正文或异常消息。DEBUG 非敏感诊断应以元数据覆盖 RSS 清洗、权威预报转发、平台渲染和分片投递边界，使运维可以根据来源、发布时间、字符数、分片数和平台接受状态定位内容丢失阶段。完整渲染正文属于敏感诊断数据，默认不得记录；运行时可以通过 CLI 临时启用、查询或关闭该行为，无需重启 daemon。启用时长必须为正且不超过 24 小时，到期自动失效。只有 DEBUG 日志级别与临时开关同时有效时才输出投递 provider 生成的完整消息及平台分片；这些文本可能包含来源内容、来源 URL、坐标和其他位置上下文，但仍不得包含投递凭据、接收方标识或请求 endpoint。诊断状态初始化或读取失败不得阻断正常投递。 diff --git a/tests/test_config.py b/tests/test_config.py index de5d4f4c..9f3adac1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +import json from pathlib import Path import pytest @@ -142,6 +143,112 @@ def test_rss_source_treats_null_optional_arrays_as_empty(monkeypatch, tmp_path: assert feed.location_ids == () +@pytest.mark.parametrize( + "field", + ( + "verbatim_title_patterns", + "forecast_title_patterns", + "content_remove_selectors", + "content_remove_patterns", + "location_ids", + ), +) +def test_rss_source_optional_arrays_reject_non_arrays(monkeypatch, tmp_path: Path, field: str) -> None: + _required_environment(monkeypatch) + source_file = tmp_path / "rss-sources.json" + source_file.write_text( + json.dumps( + [ + { + "id": "test", + "name": "Test", + "url": "https://example.invalid/feed", + field: "not-an-array", + } + ] + ), + encoding="utf-8", + ) + monkeypatch.setenv("RSS_SOURCES_FILE", str(source_file)) + + with pytest.raises(ConfigurationError, match=rf"RSS source test field {field} must be a JSON array"): + Settings.from_env() + + +@pytest.mark.parametrize( + ("field", "entry"), + ( + ("verbatim_title_patterns", ""), + ("forecast_title_patterns", " "), + ("content_remove_selectors", 1), + ("content_remove_patterns", {}), + ("location_ids", None), + ), +) +def test_rss_source_optional_arrays_require_non_empty_strings( + monkeypatch, + tmp_path: Path, + field: str, + entry: object, +) -> None: + _required_environment(monkeypatch) + source_file = tmp_path / "rss-sources.json" + source_file.write_text( + json.dumps( + [ + { + "id": "test", + "name": "Test", + "url": "https://example.invalid/feed", + field: [entry], + } + ] + ), + encoding="utf-8", + ) + monkeypatch.setenv("RSS_SOURCES_FILE", str(source_file)) + + with pytest.raises( + ConfigurationError, + match=rf"RSS source test field {field}\[0\] must be a non-empty string", + ): + Settings.from_env() + + +@pytest.mark.parametrize( + ("field", "entry"), + ( + ("content_remove_selectors", "["), + ("content_remove_patterns", "["), + ), +) +def test_rss_source_cleaning_rules_reject_invalid_syntax( + monkeypatch, + tmp_path: Path, + field: str, + entry: str, +) -> None: + _required_environment(monkeypatch) + source_file = tmp_path / "rss-sources.json" + source_file.write_text( + json.dumps( + [ + { + "id": "test", + "name": "Test", + "url": "https://example.invalid/feed", + field: [entry], + } + ] + ), + encoding="utf-8", + ) + monkeypatch.setenv("RSS_SOURCES_FILE", str(source_file)) + + with pytest.raises(ConfigurationError, match=rf"RSS source test field {field}\[0\] is invalid"): + Settings.from_env() + + def test_location_file_supports_name_coordinates_or_both(monkeypatch, tmp_path: Path) -> None: _required_environment(monkeypatch) location_file = tmp_path / "locations.json" @@ -275,7 +382,7 @@ def test_env_value_with_unmatched_quotes_is_unchanged(monkeypatch, value: str) - assert settings.api_key == value -@pytest.mark.parametrize("value", ("1", "true", "yes", "'true'", '"yes"')) +@pytest.mark.parametrize("value", ("1", "true", "yes", "'true'", '"yes"', "' true '", '" yes "')) def test_debug_accepts_truthy_values_with_optional_outer_quotes(monkeypatch, value: str) -> None: _required_environment(monkeypatch) monkeypatch.setenv("DEBUG", value) @@ -283,6 +390,23 @@ def test_debug_accepts_truthy_values_with_optional_outer_quotes(monkeypatch, val assert Settings.from_env().debug +@pytest.mark.parametrize("value", ("", "0", "false", "no", "'false'", '"no"', "' false '", '" no "')) +def test_debug_accepts_false_values_with_optional_outer_quotes(monkeypatch, value: str) -> None: + _required_environment(monkeypatch) + monkeypatch.setenv("DEBUG", value) + + assert not Settings.from_env().debug + + +@pytest.mark.parametrize("value", ("tru", "enabled", "2")) +def test_debug_rejects_unknown_values(monkeypatch, value: str) -> None: + _required_environment(monkeypatch) + monkeypatch.setenv("DEBUG", value) + + with pytest.raises(ConfigurationError, match="DEBUG must be one of"): + Settings.from_env() + + @pytest.mark.parametrize( "name", ( @@ -532,6 +656,36 @@ def test_context_sources_not_array_raises_error(self, monkeypatch) -> None: with pytest.raises(ConfigurationError, match="CONTEXT_SOURCES_JSON must be a JSON array"): Settings.from_env() + def test_context_source_not_object_includes_index_in_error(self, monkeypatch) -> None: + _required_environment(monkeypatch) + monkeypatch.setenv("CONTEXT_SOURCES_JSON", "[1]") + + with pytest.raises(ConfigurationError, match=r"CONTEXT_SOURCES_JSON\[0\] must be a JSON object"): + Settings.from_env() + + @pytest.mark.parametrize("field", ("id", "name", "url")) + def test_context_source_requires_named_fields(self, monkeypatch, field: str) -> None: + _required_environment(monkeypatch) + source = {"id": "context", "name": "Context", "url": "https://example.invalid/context"} + del source[field] + monkeypatch.setenv("CONTEXT_SOURCES_JSON", json.dumps([source])) + + with pytest.raises(ConfigurationError, match=rf"CONTEXT_SOURCES_JSON\[0\]\.{field}"): + Settings.from_env() + + def test_context_source_accepts_and_strips_required_strings(self, monkeypatch) -> None: + _required_environment(monkeypatch) + monkeypatch.setenv( + "CONTEXT_SOURCES_JSON", + '[{"id":" context ","name":" Context ","url":" https://example.invalid/context "}]', + ) + + source = Settings.from_env().context_sources[0] + + assert source.id == "context" + assert source.name == "Context" + assert source.url == "https://example.invalid/context" + def test_invalid_timezone_raises_error(self, monkeypatch) -> None: _required_environment(monkeypatch) monkeypatch.setenv("BRIEFING_TIMEZONE", "Invalid/Timezone") diff --git a/weather_briefing/config.py b/weather_briefing/config.py index a5fb1467..412c2bf8 100644 --- a/weather_briefing/config.py +++ b/weather_briefing/config.py @@ -4,6 +4,8 @@ import json import os +import re +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from typing import Any, overload @@ -11,6 +13,8 @@ import pendulum from any_llm import AnyLLM from apscheduler.triggers.cron import CronTrigger +from soupsieve import SelectorSyntaxError +from soupsieve import compile as compile_selector from .models import ContextSourceConfig, FeedConfig, LocationSpec, ResolvedLocation from .reference_data import reference_string_tuple @@ -97,6 +101,15 @@ def _positive_float(name: str, default: float) -> float: return value +def _boolean(name: str, default: bool) -> bool: + value = _clean_env(os.getenv(name, str(default))).strip().casefold() + if value in {"1", "true", "yes"}: + return True + if value in {"0", "false", "no", ""}: + return False + raise ConfigurationError(f"{name} must be one of: true, false, 1, 0, yes, no") + + def _json_file(path: Path) -> list[dict[str, Any]]: if not path.exists(): return [] @@ -109,6 +122,45 @@ def _json_file(path: Path) -> list[dict[str, Any]]: return value +def _optional_string_array( + item: dict[str, Any], + source_id: str, + field: str, + *, + validator: Callable[[str], object] | None = None, +) -> tuple[str, ...]: + value = item.get(field) + if value is None: + return () + if not isinstance(value, list): + raise ConfigurationError(f"RSS source {source_id} field {field} must be a JSON array") + entries: list[str] = [] + for index, entry in enumerate(value): + path = f"RSS source {source_id} field {field}[{index}]" + if not isinstance(entry, str) or not entry.strip(): + raise ConfigurationError(f"{path} must be a non-empty string") + if validator is not None: + try: + validator(entry) + except (re.error, SelectorSyntaxError) as exc: + raise ConfigurationError(f"{path} is invalid") from exc + entries.append(entry) + return tuple(entries) + + +def _context_source(item: object, index: int) -> ContextSourceConfig: + if not isinstance(item, dict): + raise ConfigurationError(f"CONTEXT_SOURCES_JSON[{index}] must be a JSON object") + + values: dict[str, str] = {} + for field in ("id", "name", "url"): + value = item.get(field) + if not isinstance(value, str) or not value.strip(): + raise ConfigurationError(f"CONTEXT_SOURCES_JSON[{index}].{field} must be a non-empty string") + values[field] = value.strip() + return ContextSourceConfig(id=values["id"], name=values["name"], url=values["url"]) + + def _configured_weather_providers() -> tuple[str, ...] | None: configured = _clean_env(os.getenv("WEATHER_PROVIDERS")) if configured is None: @@ -184,13 +236,21 @@ def _feeds(path: Path) -> tuple[FeedConfig, ...]: id=source_id, name=source_name, url=source_url, - verbatim_title_patterns=tuple(str(pattern) for pattern in item.get("verbatim_title_patterns") or []), - forecast_title_patterns=tuple(str(pattern) for pattern in item.get("forecast_title_patterns") or []), - content_remove_selectors=tuple( - str(selector) for selector in item.get("content_remove_selectors") or [] + verbatim_title_patterns=_optional_string_array(item, source_id, "verbatim_title_patterns"), + forecast_title_patterns=_optional_string_array(item, source_id, "forecast_title_patterns"), + content_remove_selectors=_optional_string_array( + item, + source_id, + "content_remove_selectors", + validator=compile_selector, + ), + content_remove_patterns=_optional_string_array( + item, + source_id, + "content_remove_patterns", + validator=re.compile, ), - content_remove_patterns=tuple(str(pattern) for pattern in item.get("content_remove_patterns") or []), - location_ids=tuple(str(location_id) for location_id in item.get("location_ids") or []), + location_ids=_optional_string_array(item, source_id, "location_ids"), ) ) return tuple(feeds) @@ -258,12 +318,9 @@ def from_env(cls) -> Settings: context_items = json.loads(context_raw) except json.JSONDecodeError as exc: raise ConfigurationError("CONTEXT_SOURCES_JSON must contain valid JSON") from exc - if not isinstance(context_items, list) or not all(isinstance(item, dict) for item in context_items): + if not isinstance(context_items, list): raise ConfigurationError("CONTEXT_SOURCES_JSON must be a JSON array of objects") - context_sources = tuple( - ContextSourceConfig(id=str(item["id"]), name=str(item["name"]), url=str(item["url"])) - for item in context_items - ) + context_sources = tuple(_context_source(item, index) for index, item in enumerate(context_items)) try: timezone = pendulum.timezone(_clean_env(os.getenv("BRIEFING_TIMEZONE", "Asia/Shanghai"))) except (ValueError, KeyError) as exc: @@ -371,5 +428,5 @@ def from_env(cls) -> Settings: greeting_hour=daily_cron_hour, greeting_minute=daily_cron_minute, hourly_cron=hourly_cron, - debug=_clean_env(os.getenv("DEBUG", "")).lower() in ("1", "true", "yes"), + debug=_boolean("DEBUG", False), )