From 70a73b253e04af66ec5175bd0cab7dddc529c925 Mon Sep 17 00:00:00 2001 From: Adarsh0427 <124577434+Adarsh0427@users.noreply.github.com> Date: Fri, 22 Aug 2025 03:57:24 +0530 Subject: [PATCH 01/17] Implement Initial Query Parser for Search Interfaces [#2038] --- backend/apps/common/search/__init__.py | 0 backend/apps/common/search/query_parser.py | 469 ++++++++++++++++++ backend/poetry.lock | 77 ++- backend/pyproject.toml | 1 + backend/tests/apps/common/search/__init__.py | 0 .../apps/common/search/query_parser_test.py | 187 +++++++ 6 files changed, 731 insertions(+), 3 deletions(-) create mode 100644 backend/apps/common/search/__init__.py create mode 100644 backend/apps/common/search/query_parser.py create mode 100644 backend/tests/apps/common/search/__init__.py create mode 100644 backend/tests/apps/common/search/query_parser_test.py diff --git a/backend/apps/common/search/__init__.py b/backend/apps/common/search/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/apps/common/search/query_parser.py b/backend/apps/common/search/query_parser.py new file mode 100644 index 0000000000..9ec47dabff --- /dev/null +++ b/backend/apps/common/search/query_parser.py @@ -0,0 +1,469 @@ +"""Query parser module for parsing search queries into structured conditions.""" + +from enum import Enum + +from pyparsing import ( + Combine, + Group, + ParseException, + QuotedString, + Regex, + Suppress, + Word, + ZeroOrMore, + alphanums, + alphas, +) + + +class FieldType(Enum): + """Supported field types for query parsing.""" + + STRING = "string" + NUMBER = "number" + DATE = "date" + BOOLEAN = "boolean" + + +class QueryParserError(Exception): + """Exception for query parsing errors with context.""" + + def __init__( + self, + message: str, + error_type: str = "PARSE_ERROR", + query: str = "", + field: str = "", + ): + """Initialize QueryParserError. + + Args: + message: Human-readable error message + error_type: Category of error (PARSE_ERROR, FIELD_ERROR, VALUE_ERROR, etc.) + query: The original query string that caused the error + field: Field name related to the error (if applicable) + + """ + super().__init__(message) + self.message = message + self.error_type = error_type + self.query = query + self.field = field + + def __str__(self) -> str: + """Return formatted error message with context.""" + parts = [f"[{self.error_type}] {self.message}"] + + if self.query: + parts.append(f"Query: '{self.query}'") + + if self.field: + parts.append(f"Field: '{self.field}'") + + return "\n".join(parts) + + def to_dict(self) -> dict: + """Return error information as dictionary for API responses.""" + return { + "error": True, + "error_type": self.error_type, + "message": self.message, + "query": self.query, + "field": self.field, + } + + +class QueryParser: + """QueryParser class for parsing search queries into structured conditions. + + Features: + - Equality checks (field:value) + - Numeric comparisons (>, <, >=, <=, =) + - Quoted strings for multi-word values + - Boolean field support + - Date field parsing with comparisons + - Only support implicit AND logic between conditions + - Handle case sensitivity/insensitivity values + """ + + # Grammar definitions (class-level constants) + _FIELD_NAME = Word(alphas, alphanums + "_") + _QUOTED_VALUE = QuotedString('"', escChar="\\") + _UNQUOTED_VALUE = Word(alphanums + "+-.<>=/_") + _FIELD_VALUE = _QUOTED_VALUE | _UNQUOTED_VALUE + _TOKEN_PATTERN = Group(_FIELD_NAME + Suppress(":") + _FIELD_VALUE) + _COMP_OPERATOR = Combine( + Regex(r"^(>=|<=|>|<|=)?") + ) # valid operator {"=", ">", "<", ">=", "<="} + _COMP_PATTERN = Group(_COMP_OPERATOR + _UNQUOTED_VALUE) + _UNQUOTED_STRING = Word(alphanums + "+-.<>=/ ") + _DATE_PATTERN = Regex(r"\d{4}-\d{2}-\d{2}") # YYYY-MM-DD format + + _BOOLEAN_TRUE_VALUES = {"true", "1", "yes", "on"} + _BOOLEAN_FALSE_VALUES = {"false", "0", "no", "off"} + + def __init__( + self, + allowed_fields: dict[str, str], + default_field: str = "query", + *, + case_sensitive: bool = False, + strict: bool = False, + ): + """Initialize the query parser. + + Args: + allowed_fields: Dictionary mapping field names to their types + (e.g., {"stars": "number", "language": "string"}) + default_field: Default field name for free text search + case_sensitive: If True, field names are case-sensitive + strict: If True, raises exception for unknown fields; if False, issues warnings + + Raises: + QueryParserError: If allowed_fields contains invalid field types + + """ + try: + self.allowed_fields = self._validate_allowed_fields(allowed_fields, default_field) + self.default_field = default_field + self.case_sensitive = case_sensitive + self.strict = strict + except QueryParserError: + raise + except Exception as e: + raise QueryParserError( + message=f"Invalid query parser configuration: {e!s}", + error_type="CONFIGURATION_ERROR", + ) from e + + @staticmethod + def _raise_invalid_field_name(field: str) -> None: + msg = f"Field name '{field}' must be alphanumeric (letters, numbers, underscores only)" + raise QueryParserError(message=msg, error_type="FIELD_NAME_ERROR", field=field) + + @staticmethod + def _raise_invalid_field_type(field: str, field_type: str) -> None: + msg = f"Invalid field type '{field_type}' for field '{field}'" + raise QueryParserError(message=msg, error_type="FIELD_TYPE_ERROR", field=field) + + @staticmethod + def _validate_allowed_fields( + allowed_fields: dict[str, str], default_field: str + ) -> dict[str, FieldType]: + """Validate and convert allowed_fields to use FieldType enum. + + Args: + allowed_fields: Raw field type mapping + default_field: Default field name for free text search + + Returns: + Validated field type mapping using FieldType enum + + Raises: + QueryParserError: If field type is not supported + + """ + if not QueryParser._FIELD_NAME.parseString(default_field, parseAll=True): + QueryParser._raise_invalid_field_name(default_field) + validated_fields = {} + + for field, field_type in allowed_fields.items(): + if not QueryParser._FIELD_NAME.parseString(field, parseAll=True): + QueryParser._raise_invalid_field_name(field) + if field_type not in FieldType._value2member_map_: + QueryParser._raise_invalid_field_type(field, field_type) + validated_fields[field] = FieldType(field_type) + + validated_fields[default_field] = FieldType.STRING + return validated_fields + + @staticmethod + def _split_tokens(query: str) -> list[str]: + """Split query string into individual tokens, respecting quoted strings. + + Args: + query: Raw query string + + Returns: + List of individual tokens + + """ + reg_components = { + "key_value": r'\S+:"[^"]*"', + "quoted_string": r'"[^"]*"', + "unquoted_word": r"\S+", + } + + reg = [ + reg_components["key_value"], + reg_components["quoted_string"], + reg_components["unquoted_word"], + ] + + token_regex = "|".join(reg) + parser = ZeroOrMore(Regex(token_regex)) + + try: + result = parser.parseString(query, parseAll=True) + return result.asList() + except ParseException as e: + raise QueryParserError( + message="Failed to tokenize query", error_type="TOKENIZATION_ERROR", query=query + ) from e + + @staticmethod + def _parse_token(token: str) -> tuple[str | None, str]: + """Parse a single token into field name and value. + + Args: + token: Individual token string (e.g., "field:value") raise exception if null + + Returns: + Tuple of (field_name, raw_value) or (None, token) if parsing fails + + """ + try: + result = QueryParser._TOKEN_PATTERN.parseString(token, parseAll=True) + return result[0][0], result[0][1] + except ParseException: + if token.startswith('"') and token.endswith('"'): + token = token[1:-1] + return None, token + + @staticmethod + def _parse_comparison_pattern(value: str) -> tuple[str, str]: + """Parse comparison operator and value from a string. + + Args: + value: Value string potentially containing comparison operator + + Returns: + Tuple of (operator, clean_value), if operator is none then (None, clean_value) + + Raises: + QueryParserError: If the comparison pattern is invalid + + """ + try: + match = QueryParser._COMP_PATTERN.parseString(value, parseAll=True) + if match[0][0] is None: + return "=", match[0][1] + return match[0][0], match[0][1] + except ParseException as e: + raise QueryParserError( + message="Invalid comparison pattern in value", error_type="COMPARISON_ERROR" + ) from e + + @staticmethod + def _parse_string_value(field: str, value: str) -> str: + """Parse and validate string value. + + Args: + field: Field name + value: Raw string value + + Returns: + Cleaned string value + + Raises: + QueryParserError: If string parsing fails + + """ + try: + result = QueryParser._UNQUOTED_STRING.parseString(value) + return result[0] + except ParseException as e: + raise QueryParserError( + message="Invalid string value", error_type="STRING_VALUE_ERROR", field=field + ) from e + + @staticmethod + def _parse_boolean_value(field: str, value: str) -> str: + """Parse boolean value from string. + + Args: + field: Field name + value: String value to parse + + Returns: + Boolean value + + Raises: + QueryParserError: If value is not a valid boolean + + """ + if value.lower() in QueryParser._BOOLEAN_TRUE_VALUES: + return "True" + if value.lower() in QueryParser._BOOLEAN_FALSE_VALUES: + return "False" + + raise QueryParserError( + message=f"Invalid boolean value '{value}'", + error_type="BOOLEAN_VALUE_ERROR", + field=field, + ) + + @staticmethod + def _parse_number_value(field: str, value: str) -> tuple[str, int]: + """Parse numeric value with optional comparison operator. + + Args: + field: Field name (for error messages) + value: String value to parse + + Returns: + Tuple of (operator, numeric_value) + + Raises: + QueryParserError: If value is not a valid number + + """ + try: + operator, clean_value = QueryParser._parse_comparison_pattern(value) + numeric_value = int(float(clean_value)) + + except ValueError as e: + raise QueryParserError( + message=f"Invalid numeric value '{value}'", + error_type="NUMBER_VALUE_ERROR", + field=field, + ) from e + + return operator, numeric_value + + @staticmethod + def _parse_date_value(field: str, value: str) -> tuple[str, str]: + """Parse date value with optional comparison operator. + + Args: + field: Field name + value: String value to parse + + Returns: + Tuple of (operator, date_string) + + Raises: + QueryParserError: If value is not a valid date format + + """ + try: + operator, clean_value = QueryParser._parse_comparison_pattern(value) + result = QueryParser._DATE_PATTERN.parseString(clean_value, parseAll=True) + return operator, result[0] + + except (ValueError, ParseException, QueryParserError) as e: + raise QueryParserError( + message=f"Invalid date value '{value}'", error_type="DATE_VALUE_ERROR", field=field + ) from e + + def to_dict(self, field: str, field_type: FieldType, raw_value: str) -> dict[str, str]: + """Convert condition to dictionary format. + + Args: + field: Field name + field_type: FieldType of the condition + raw_value: Raw string value to parse + + Returns: + Dictionary representation of the condition + + Raises: + QueryParserError: If parsing fails + + """ + condition_dict = { + "type": field_type.value, + "field": field, + } + if field_type == FieldType.BOOLEAN: + condition_dict["boolean"] = self._parse_boolean_value(field, raw_value) + elif field_type == FieldType.NUMBER: + operator, numeric_value = self._parse_number_value(field, raw_value) + condition_dict["op"] = operator + condition_dict["number"] = str(numeric_value) + elif field_type == FieldType.DATE: + operator, date_value = self._parse_date_value(field, raw_value) + condition_dict["op"] = operator + condition_dict["date"] = date_value + else: # STRING + condition_dict["string"] = self._parse_string_value(field, raw_value) + + return condition_dict + + def _create_text_search_condition(self, token: str) -> dict[str, str]: + """Create a text search condition for tokens that aren't field:value pairs. + + Args: + token: Raw token string + + Returns: + QueryCondition for text search + + """ + return {"type": FieldType.STRING.value, "field": self.default_field, "string": token} + + def _handle_unknown_field(self, field: str) -> None: + """Handle unknown field based on strict mode setting. + + Args: + field: Unknown field name + + Raises: + QueryParserError: If in strict mode + + """ + if self.strict: + raise QueryParserError( + message=f"Unknown field '{field}'", + error_type="UNKNOWN_FIELD_ERROR", + field=field, + ) + + def parse(self, query: str) -> list[dict[str, str]]: + """Parse query string into structured conditions. + + Args: + query: Query string to parse + + Returns: + List of dictionaries representing parsed conditions + + Raises: + QueryParserError: If parsing fails + + """ + conditions = [] + try: + if not self.case_sensitive: + query = query.lower().strip() + tokens = self._split_tokens(query) + for token in tokens: + if not token or token.strip() == "": + continue + field, raw_value = self._parse_token(token) + if raw_value is None or raw_value.strip() == "": + continue + + if field is None: + conditions.append(self._create_text_search_condition(raw_value)) + continue + + field = field.lower() # field name normalization + + if field not in self.allowed_fields: + self._handle_unknown_field(field) + continue + + field_type = self.allowed_fields[field] + condition_dict = self.to_dict(field, field_type, raw_value) + conditions.append(condition_dict) + except QueryParserError: + raise + except Exception as e: + raise QueryParserError( + message=f"Unexpected error during parsing: {e!s}", + error_type="INTERNAL_ERROR", + query=query, + ) from e + + return conditions diff --git a/backend/poetry.lock b/backend/poetry.lock index 6ace034c39..92f0b66421 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -1120,7 +1120,7 @@ description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_version == \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")" +markers = "python_version < \"3.14\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")" files = [ {file = "greenlet-3.2.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:8c68325b0d0acf8d91dde4e6f930967dd52a5302cd4062932a6b2e7c2969f47c"}, {file = "greenlet-3.2.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:94385f101946790ae13da500603491f04a76b6e4c059dab271b3ce2e283b2590"}, @@ -2927,6 +2927,21 @@ cffi = ">=1.4.1" docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] +[[package]] +name = "pyparsing" +version = "3.2.3" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf"}, + {file = "pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "pytest" version = "8.4.1" @@ -3749,6 +3764,62 @@ optional = false python-versions = ">=3.7" groups = ["main"] files = [ + {file = "SQLAlchemy-2.0.43-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:21ba7a08a4253c5825d1db389d4299f64a100ef9800e4624c8bf70d8f136e6ed"}, + {file = "SQLAlchemy-2.0.43-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11b9503fa6f8721bef9b8567730f664c5a5153d25e247aadc69247c4bc605227"}, + {file = "SQLAlchemy-2.0.43-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07097c0a1886c150ef2adba2ff7437e84d40c0f7dcb44a2c2b9c905ccfc6361c"}, + {file = "SQLAlchemy-2.0.43-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:cdeff998cb294896a34e5b2f00e383e7c5c4ef3b4bfa375d9104723f15186443"}, + {file = "SQLAlchemy-2.0.43-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:bcf0724a62a5670e5718957e05c56ec2d6850267ea859f8ad2481838f889b42c"}, + {file = "SQLAlchemy-2.0.43-cp37-cp37m-win32.whl", hash = "sha256:c697575d0e2b0a5f0433f679bda22f63873821d991e95a90e9e52aae517b2e32"}, + {file = "SQLAlchemy-2.0.43-cp37-cp37m-win_amd64.whl", hash = "sha256:d34c0f6dbefd2e816e8f341d0df7d4763d382e3f452423e752ffd1e213da2512"}, + {file = "sqlalchemy-2.0.43-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:70322986c0c699dca241418fcf18e637a4369e0ec50540a2b907b184c8bca069"}, + {file = "sqlalchemy-2.0.43-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:87accdbba88f33efa7b592dc2e8b2a9c2cdbca73db2f9d5c510790428c09c154"}, + {file = "sqlalchemy-2.0.43-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c00e7845d2f692ebfc7d5e4ec1a3fd87698e4337d09e58d6749a16aedfdf8612"}, + {file = "sqlalchemy-2.0.43-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:022e436a1cb39b13756cf93b48ecce7aa95382b9cfacceb80a7d263129dfd019"}, + {file = "sqlalchemy-2.0.43-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c5e73ba0d76eefc82ec0219d2301cb33bfe5205ed7a2602523111e2e56ccbd20"}, + {file = "sqlalchemy-2.0.43-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9c2e02f06c68092b875d5cbe4824238ab93a7fa35d9c38052c033f7ca45daa18"}, + {file = "sqlalchemy-2.0.43-cp310-cp310-win32.whl", hash = "sha256:e7a903b5b45b0d9fa03ac6a331e1c1d6b7e0ab41c63b6217b3d10357b83c8b00"}, + {file = "sqlalchemy-2.0.43-cp310-cp310-win_amd64.whl", hash = "sha256:4bf0edb24c128b7be0c61cd17eef432e4bef507013292415f3fb7023f02b7d4b"}, + {file = "sqlalchemy-2.0.43-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:52d9b73b8fb3e9da34c2b31e6d99d60f5f99fd8c1225c9dad24aeb74a91e1d29"}, + {file = "sqlalchemy-2.0.43-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f42f23e152e4545157fa367b2435a1ace7571cab016ca26038867eb7df2c3631"}, + {file = "sqlalchemy-2.0.43-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fb1a8c5438e0c5ea51afe9c6564f951525795cf432bed0c028c1cb081276685"}, + {file = "sqlalchemy-2.0.43-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db691fa174e8f7036afefe3061bc40ac2b770718be2862bfb03aabae09051aca"}, + {file = "sqlalchemy-2.0.43-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2b3b4927d0bc03d02ad883f402d5de201dbc8894ac87d2e981e7d87430e60d"}, + {file = "sqlalchemy-2.0.43-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4d3d9b904ad4a6b175a2de0738248822f5ac410f52c2fd389ada0b5262d6a1e3"}, + {file = "sqlalchemy-2.0.43-cp311-cp311-win32.whl", hash = "sha256:5cda6b51faff2639296e276591808c1726c4a77929cfaa0f514f30a5f6156921"}, + {file = "sqlalchemy-2.0.43-cp311-cp311-win_amd64.whl", hash = "sha256:c5d1730b25d9a07727d20ad74bc1039bbbb0a6ca24e6769861c1aa5bf2c4c4a8"}, + {file = "sqlalchemy-2.0.43-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:20d81fc2736509d7a2bd33292e489b056cbae543661bb7de7ce9f1c0cd6e7f24"}, + {file = "sqlalchemy-2.0.43-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25b9fc27650ff5a2c9d490c13c14906b918b0de1f8fcbb4c992712d8caf40e83"}, + {file = "sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6772e3ca8a43a65a37c88e2f3e2adfd511b0b1da37ef11ed78dea16aeae85bd9"}, + {file = "sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a113da919c25f7f641ffbd07fbc9077abd4b3b75097c888ab818f962707eb48"}, + {file = "sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4286a1139f14b7d70141c67a8ae1582fc2b69105f1b09d9573494eb4bb4b2687"}, + {file = "sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:529064085be2f4d8a6e5fab12d36ad44f1909a18848fcfbdb59cc6d4bbe48efe"}, + {file = "sqlalchemy-2.0.43-cp312-cp312-win32.whl", hash = "sha256:b535d35dea8bbb8195e7e2b40059e2253acb2b7579b73c1b432a35363694641d"}, + {file = "sqlalchemy-2.0.43-cp312-cp312-win_amd64.whl", hash = "sha256:1c6d85327ca688dbae7e2b06d7d84cfe4f3fffa5b5f9e21bb6ce9d0e1a0e0e0a"}, + {file = "sqlalchemy-2.0.43-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e7c08f57f75a2bb62d7ee80a89686a5e5669f199235c6d1dac75cd59374091c3"}, + {file = "sqlalchemy-2.0.43-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14111d22c29efad445cd5021a70a8b42f7d9152d8ba7f73304c4d82460946aaa"}, + {file = "sqlalchemy-2.0.43-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21b27b56eb2f82653168cefe6cb8e970cdaf4f3a6cb2c5e3c3c1cf3158968ff9"}, + {file = "sqlalchemy-2.0.43-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c5a9da957c56e43d72126a3f5845603da00e0293720b03bde0aacffcf2dc04f"}, + {file = "sqlalchemy-2.0.43-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d79f9fdc9584ec83d1b3c75e9f4595c49017f5594fee1a2217117647225d738"}, + {file = "sqlalchemy-2.0.43-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9df7126fd9db49e3a5a3999442cc67e9ee8971f3cb9644250107d7296cb2a164"}, + {file = "sqlalchemy-2.0.43-cp313-cp313-win32.whl", hash = "sha256:7f1ac7828857fcedb0361b48b9ac4821469f7694089d15550bbcf9ab22564a1d"}, + {file = "sqlalchemy-2.0.43-cp313-cp313-win_amd64.whl", hash = "sha256:971ba928fcde01869361f504fcff3b7143b47d30de188b11c6357c0505824197"}, + {file = "sqlalchemy-2.0.43-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4e6aeb2e0932f32950cf56a8b4813cb15ff792fc0c9b3752eaf067cfe298496a"}, + {file = "sqlalchemy-2.0.43-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:61f964a05356f4bca4112e6334ed7c208174511bd56e6b8fc86dad4d024d4185"}, + {file = "sqlalchemy-2.0.43-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46293c39252f93ea0910aababa8752ad628bcce3a10d3f260648dd472256983f"}, + {file = "sqlalchemy-2.0.43-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:136063a68644eca9339d02e6693932116f6a8591ac013b0014479a1de664e40a"}, + {file = "sqlalchemy-2.0.43-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:6e2bf13d9256398d037fef09fd8bf9b0bf77876e22647d10761d35593b9ac547"}, + {file = "sqlalchemy-2.0.43-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:44337823462291f17f994d64282a71c51d738fc9ef561bf265f1d0fd9116a782"}, + {file = "sqlalchemy-2.0.43-cp38-cp38-win32.whl", hash = "sha256:13194276e69bb2af56198fef7909d48fd34820de01d9c92711a5fa45497cc7ed"}, + {file = "sqlalchemy-2.0.43-cp38-cp38-win_amd64.whl", hash = "sha256:334f41fa28de9f9be4b78445e68530da3c5fa054c907176460c81494f4ae1f5e"}, + {file = "sqlalchemy-2.0.43-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ceb5c832cc30663aeaf5e39657712f4c4241ad1f638d487ef7216258f6d41fe7"}, + {file = "sqlalchemy-2.0.43-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:11f43c39b4b2ec755573952bbcc58d976779d482f6f832d7f33a8d869ae891bf"}, + {file = "sqlalchemy-2.0.43-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:413391b2239db55be14fa4223034d7e13325a1812c8396ecd4f2c08696d5ccad"}, + {file = "sqlalchemy-2.0.43-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c379e37b08c6c527181a397212346be39319fb64323741d23e46abd97a400d34"}, + {file = "sqlalchemy-2.0.43-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:03d73ab2a37d9e40dec4984d1813d7878e01dbdc742448d44a7341b7a9f408c7"}, + {file = "sqlalchemy-2.0.43-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:8cee08f15d9e238ede42e9bbc1d6e7158d0ca4f176e4eab21f88ac819ae3bd7b"}, + {file = "sqlalchemy-2.0.43-cp39-cp39-win32.whl", hash = "sha256:b3edaec7e8b6dc5cd94523c6df4f294014df67097c8217a89929c99975811414"}, + {file = "sqlalchemy-2.0.43-cp39-cp39-win_amd64.whl", hash = "sha256:227119ce0a89e762ecd882dc661e0aa677a690c914e358f0dd8932a2e8b2765b"}, + {file = "sqlalchemy-2.0.43-py3-none-any.whl", hash = "sha256:1681c21dd2ccee222c2fe0bef671d1aef7c504087c9c4e800371cfcc8ac966fc"}, {file = "sqlalchemy-2.0.43.tar.gz", hash = "sha256:788bfcef6787a7764169cfe9859fe425bf44559619e1d9f56f5bddf2ebf6f417"}, ] @@ -4254,4 +4325,4 @@ cffi = ["cffi (>=1.17) ; python_version >= \"3.13\" and platform_python_implemen [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "52c97a78f5d6a7d2367d6bb92032038024f09b8dc3a32e156fdb0c793fff63ec" +content-hash = "769ef14e6f445b23be7e6b0f3bbbbdccedc3639f832513188ebd699aa449b11d" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 770bbe8c16..5d4dcff3ec 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -47,6 +47,7 @@ slack-sdk = "^3.35.0" strawberry-graphql = { extras = [ "django" ], version = "^0.280.0" } strawberry-graphql-django = "^0.65.1" thefuzz = "^0.22.1" +pyparsing = "^3.2.3" [tool.poetry.group.dev.dependencies] djlint = "^1.36.4" diff --git a/backend/tests/apps/common/search/__init__.py b/backend/tests/apps/common/search/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/tests/apps/common/search/query_parser_test.py b/backend/tests/apps/common/search/query_parser_test.py new file mode 100644 index 0000000000..07477ebc79 --- /dev/null +++ b/backend/tests/apps/common/search/query_parser_test.py @@ -0,0 +1,187 @@ +import pytest + +from apps.common.search.query_parser import QueryParser, QueryParserError + + +class TestQueryParser: + def setup_method(self): + self.allowed_fields = { + "author": "string", + "stars": "number", + "issues": "number", + "project": "string", + "archived": "boolean", + "language": "string", + "created": "date", + } + + self.parser = QueryParser(allowed_fields=self.allowed_fields) + self.case_sensitive_parser = QueryParser( + allowed_fields=self.allowed_fields, default_field="test", case_sensitive=True + ) + self.strict_parser = QueryParser(allowed_fields=self.allowed_fields, strict=True) + self.case_sensitive_strict_parser = QueryParser( + allowed_fields=self.allowed_fields, strict=True, case_sensitive=True + ) + + def test_invalid_parser_field_validation(self): + with pytest.raises(QueryParserError, match="Invalid field type"): + QueryParser(allowed_fields={"field": "invalid_type"}) + + with pytest.raises(Exception, match="Expected end of text"): + QueryParser(allowed_fields={"field-with-dash": "string"}) + + @pytest.mark.parametrize( + ("parser_type", "expected_string"), + [ + ("parser", "john doe"), + ("case_sensitive_parser", "John Doe"), + ("strict_parser", "john doe"), + ("case_sensitive_strict_parser", "John Doe"), + ], + ) + def test_basic_field_parsing_all_types(self, parser_type, expected_string): + parser = getattr(self, parser_type) + result = parser.parse('Author:"John Doe"') + assert len(result) == 1 + assert result[0]["field"] == "author" + assert result[0]["type"] == "string" + assert result[0]["string"] == expected_string + + def test_parser_configuration_differences(self): + query = "AUTHOR:test" + result = self.case_sensitive_parser.parse(query) + assert len(result) == 1 + assert result[0]["field"] == "author" + + def test_comparison_operators_and_boolean_variations(self): + @pytest.mark.parametrize("op", ["<", "<=", ">", ">=", "="]) + def test_operator_variations(self, op): + result = self.parser.parse(f"stars:{op}100") + assert len(result) == 1 + assert result[0]["field"] == "stars" + assert result[0]["type"] == "number" + assert result[0]["op"] == op + assert result[0]["number"] == "100" + + boolean_values = ["true", "false", "True", "False", "TRUE", "FALSE"] + + @pytest.mark.parametrize("bool_val", boolean_values) + def test_boolean_variations(self, bool_val): + result = self.parser.parse(f"archived:{bool_val}") + assert len(result) == 1 + assert result[0]["field"] == "archived" + assert result[0]["type"] == "boolean" + assert result[0]["boolean"] == bool_val + + @pytest.mark.parametrize( + ("query", "expected"), + [ + ( + 'author:"John Doe" stars:>100 archived:false some "free text"', + [ + {"type": "string", "field": "author", "string": "john doe"}, + {"type": "number", "field": "stars", "op": ">", "number": "100"}, + {"type": "boolean", "field": "archived", "boolean": "False"}, + {"type": "string", "field": "query", "string": "some"}, + {"type": "string", "field": "query", "string": "free text"}, + ], + ), + ( + 'project:"my-awesome-project" language:"C++"', + [ + {"type": "string", "field": "project", "string": "my-awesome-project"}, + {"type": "string", "field": "language", "string": "c++"}, + ], + ), + ], + ) + def test_complex_queries_and_mixed_content(self, query, expected): + result = self.parser.parse(query) + assert result == expected + + def test_error_handling_and_malformed_queries(self): + malformed_queries = ["field_without_colon", ":value_without_field", "field:", ""] + + @pytest.mark.parametrize("query", malformed_queries) + def test_malformed_queries(self, query): + result = self.parser.parse(query) + if query == "": + assert not result + else: + assert result[0]["field"] == "query" + assert result[0]["type"] == "string" + assert result[0]["string"] == query + + @pytest.mark.parametrize("query", malformed_queries) + def test_invalid_queries(self, query): + with pytest.raises(QueryParserError): + self.strict_parser.parse(query) + + def test_date_format_validation(self): + @pytest.mark.parametrize("date_str", ["2023-01-01", "2023-12-31"]) + def test_valid_dates(self, date_str): + result = self.parser.parse(f"created:{date_str}") + assert len(result) == 1 + assert result[0]["date"] == date_str + + @pytest.mark.parametrize("inv_date_str", ["invalid-date", "2023-2213-01", "2023-01-322"]) + def test_invalid_dates(self, inv_date_str): + result = self.parser.parse(f"created:{inv_date_str}") + assert not result + + def test_initialization(self): + def test_invalid_initialization(): + with pytest.raises(QueryParserError): + QueryParser(allowed_fields=None) + + @pytest.mark.parametrize( + ("allowed_fields", "default_field", "query", "expected"), + [ + ( + {"query": "string"}, + "query", + "query:test free_text", + [ + {"type": "string", "field": "query", "string": "test"}, + {"type": "string", "field": "query", "string": "free_text"}, + ], + ) + ], + ) + def test_valid_initialization(allowed_fields, default_field, query, expected): + parser_with_default = QueryParser( + allowed_fields=allowed_fields, default_field=default_field + ) + result = parser_with_default.parse(query) + assert result == expected + + def test_boundary_conditions_and_edge_cases(self): + @pytest.mark.parametrize( + ("query", "expected"), + [ + ("author:", {"field": "query", "type": "string", "string": "author:"}), + (":123", {"field": "query", "type": "string", "string": ":123"}), + ('"field: true"', {"field": "query", "type": "string", "boolean": True}), + ("2341", {"field": "query", "type": "string", "string": "2341"}), + ], + ) + def test_boundary_conditions_and_edge_cases(self, query, expected): + result = self.parser.parse(query) + assert result == [expected] + + def test_long_field_names(): + long_field_parser = QueryParser(allowed_fields={"verylongfieldname": "string"}) + long_value = "a" * 1000 + result = long_field_parser.parse(f'verylongfieldname:"{long_value}"') + assert isinstance(result, list) + assert result[0]["string"] == long_value + + @pytest.mark.parametrize("val", ["0", "-1", "999999", "3.14159"]) + def test_numeric_values(self, val): + result = self.parser.parse(f"stars:{val}") + assert len(result) == 1 + assert result[0]["type"] == "number" + assert result[0]["field"] == "stars" + assert result[0]["number"] == str(int(float(val))) + assert result[0]["op"] == "=" From a430a0e768a64931ca3f689fc157273534244dac Mon Sep 17 00:00:00 2001 From: Adarsh0427 <124577434+Adarsh0427@users.noreply.github.com> Date: Fri, 22 Aug 2025 14:07:33 +0530 Subject: [PATCH 02/17] fixed suggestions addressed bots --- backend/apps/common/search/query_parser.py | 146 ++++++++------ .../apps/common/search/query_parser_test.py | 185 ++++++++---------- 2 files changed, 168 insertions(+), 163 deletions(-) diff --git a/backend/apps/common/search/query_parser.py b/backend/apps/common/search/query_parser.py index 9ec47dabff..98b4d3b0f9 100644 --- a/backend/apps/common/search/query_parser.py +++ b/backend/apps/common/search/query_parser.py @@ -3,8 +3,8 @@ from enum import Enum from pyparsing import ( - Combine, Group, + Optional, ParseException, QuotedString, Regex, @@ -13,10 +13,11 @@ ZeroOrMore, alphanums, alphas, + oneOf, ) -class FieldType(Enum): +class FieldType(str, Enum): """Supported field types for query parsing.""" STRING = "string" @@ -25,6 +26,28 @@ class FieldType(Enum): BOOLEAN = "boolean" +def is_in_enum(enum_class, value): + """Check if a value exists in the given enum class. + + Args: + enum_class: The enum class to check against. + value: The value to check for existence in the enum. + + Returns: + bool: True if the value exists in the enum, False otherwise. + + Raises: + None: This function catches ValueError internally and returns False. + + """ + try: + enum_class(value) + except ValueError: + return False + else: + return True + + class QueryParserError(Exception): """Exception for query parsing errors with context.""" @@ -32,8 +55,8 @@ def __init__( self, message: str, error_type: str = "PARSE_ERROR", - query: str = "", - field: str = "", + query: str = "", + field: str = "", ): """Initialize QueryParserError. @@ -82,26 +105,33 @@ class QueryParser: - Quoted strings for multi-word values - Boolean field support - Date field parsing with comparisons - - Only support implicit AND logic between conditions - Handle case sensitivity/insensitivity values + - Only supports implicit AND logic between conditions """ # Grammar definitions (class-level constants) _FIELD_NAME = Word(alphas, alphanums + "_") - _QUOTED_VALUE = QuotedString('"', escChar="\\") + _QUOTED_VALUE = QuotedString(quoteChar='"', escChar="\\", unquoteResults=False) _UNQUOTED_VALUE = Word(alphanums + "+-.<>=/_") _FIELD_VALUE = _QUOTED_VALUE | _UNQUOTED_VALUE _TOKEN_PATTERN = Group(_FIELD_NAME + Suppress(":") + _FIELD_VALUE) - _COMP_OPERATOR = Combine( - Regex(r"^(>=|<=|>|<|=)?") - ) # valid operator {"=", ">", "<", ">=", "<="} + _COMP_OPERATOR = Optional(oneOf(">= <= > < ="), default="=") _COMP_PATTERN = Group(_COMP_OPERATOR + _UNQUOTED_VALUE) - _UNQUOTED_STRING = Word(alphanums + "+-.<>=/ ") _DATE_PATTERN = Regex(r"\d{4}-\d{2}-\d{2}") # YYYY-MM-DD format _BOOLEAN_TRUE_VALUES = {"true", "1", "yes", "on"} _BOOLEAN_FALSE_VALUES = {"false", "0", "no", "off"} + @staticmethod + def _is_field_name(field_name: str) -> bool: + """Check if a field name is valid.""" + try: + QueryParser._FIELD_NAME.parseString(field_name, parseAll=True) + except ParseException: + return False + else: + return bool(field_name.islower()) + def __init__( self, allowed_fields: dict[str, str], @@ -114,13 +144,14 @@ def __init__( Args: allowed_fields: Dictionary mapping field names to their types - (e.g., {"stars": "number", "language": "string"}) + (e.g., {"stars": "number", "language": "string"}). + Field names must be lowercase. default_field: Default field name for free text search case_sensitive: If True, field names are case-sensitive strict: If True, raises exception for unknown fields; if False, issues warnings Raises: - QueryParserError: If allowed_fields contains invalid field types + QueryParserError: If allowed_fields contains invalid field names or types """ try: @@ -163,15 +194,15 @@ def _validate_allowed_fields( QueryParserError: If field type is not supported """ - if not QueryParser._FIELD_NAME.parseString(default_field, parseAll=True): + if not QueryParser._is_field_name(default_field): QueryParser._raise_invalid_field_name(default_field) validated_fields = {} for field, field_type in allowed_fields.items(): - if not QueryParser._FIELD_NAME.parseString(field, parseAll=True): - QueryParser._raise_invalid_field_name(field) - if field_type not in FieldType._value2member_map_: + if not is_in_enum(FieldType, field_type): QueryParser._raise_invalid_field_type(field, field_type) + if not QueryParser._is_field_name(field): + QueryParser._raise_invalid_field_name(field) validated_fields[field] = FieldType(field_type) validated_fields[default_field] = FieldType.STRING @@ -224,11 +255,9 @@ def _parse_token(token: str) -> tuple[str | None, str]: """ try: result = QueryParser._TOKEN_PATTERN.parseString(token, parseAll=True) - return result[0][0], result[0][1] + return (result[0][0], result[0][1]) except ParseException: - if token.startswith('"') and token.endswith('"'): - token = token[1:-1] - return None, token + return (None, token) @staticmethod def _parse_comparison_pattern(value: str) -> tuple[str, str]: @@ -238,7 +267,7 @@ def _parse_comparison_pattern(value: str) -> tuple[str, str]: value: Value string potentially containing comparison operator Returns: - Tuple of (operator, clean_value), if operator is none then (None, clean_value) + Tuple of (operator, clean_value), if operator is none then ("=", clean_value) Raises: QueryParserError: If the comparison pattern is invalid @@ -246,9 +275,7 @@ def _parse_comparison_pattern(value: str) -> tuple[str, str]: """ try: match = QueryParser._COMP_PATTERN.parseString(value, parseAll=True) - if match[0][0] is None: - return "=", match[0][1] - return match[0][0], match[0][1] + return (match[0][0], match[0][1]) except ParseException as e: raise QueryParserError( message="Invalid comparison pattern in value", error_type="COMPARISON_ERROR" @@ -256,7 +283,7 @@ def _parse_comparison_pattern(value: str) -> tuple[str, str]: @staticmethod def _parse_string_value(field: str, value: str) -> str: - """Parse and validate string value. + """Parse and validate string value(handle quotes and unquoted string). Args: field: Field name @@ -269,24 +296,25 @@ def _parse_string_value(field: str, value: str) -> str: QueryParserError: If string parsing fails """ + if value.startswith('"') and value.endswith('"'): + return value try: - result = QueryParser._UNQUOTED_STRING.parseString(value) + result = QueryParser._UNQUOTED_VALUE.parseString(value) return result[0] except ParseException as e: - raise QueryParserError( - message="Invalid string value", error_type="STRING_VALUE_ERROR", field=field - ) from e + msg = f"Invalid string value for {field}:{value}" + raise QueryParserError(msg) from e @staticmethod def _parse_boolean_value(field: str, value: str) -> str: - """Parse boolean value from string. + """Parse boolean value from string and return its string representation. Args: field: Field name value: String value to parse Returns: - Boolean value + String representation of boolean value ("True" or "False") Raises: QueryParserError: If value is not a valid boolean @@ -433,37 +461,29 @@ def parse(self, query: str) -> list[dict[str, str]]: """ conditions = [] - try: - if not self.case_sensitive: - query = query.lower().strip() - tokens = self._split_tokens(query) - for token in tokens: - if not token or token.strip() == "": - continue - field, raw_value = self._parse_token(token) - if raw_value is None or raw_value.strip() == "": - continue - - if field is None: - conditions.append(self._create_text_search_condition(raw_value)) - continue - - field = field.lower() # field name normalization - - if field not in self.allowed_fields: - self._handle_unknown_field(field) - continue - - field_type = self.allowed_fields[field] - condition_dict = self.to_dict(field, field_type, raw_value) - conditions.append(condition_dict) - except QueryParserError: - raise - except Exception as e: - raise QueryParserError( - message=f"Unexpected error during parsing: {e!s}", - error_type="INTERNAL_ERROR", - query=query, - ) from e + query = query.strip() + if not self.case_sensitive: + query = query.lower() + tokens = self._split_tokens(query) + for token in tokens: + field, raw_value = self._parse_token(token) + + if field is None: + conditions.append(self._create_text_search_condition(raw_value)) + continue + field = field.lower() # field name normalization + + if field not in self.allowed_fields: + self._handle_unknown_field(field) + continue + + field_type = self.allowed_fields[field] + try: + condition_dict = self.to_dict(field, field_type, raw_value) + except QueryParserError: + if self.strict: + raise + continue + conditions.append(condition_dict) return conditions diff --git a/backend/tests/apps/common/search/query_parser_test.py b/backend/tests/apps/common/search/query_parser_test.py index 07477ebc79..bad60c0abd 100644 --- a/backend/tests/apps/common/search/query_parser_test.py +++ b/backend/tests/apps/common/search/query_parser_test.py @@ -25,19 +25,21 @@ def setup_method(self): ) def test_invalid_parser_field_validation(self): - with pytest.raises(QueryParserError, match="Invalid field type"): + with pytest.raises(QueryParserError) as ei: QueryParser(allowed_fields={"field": "invalid_type"}) + assert ei.value.error_type == "FIELD_TYPE_ERROR" - with pytest.raises(Exception, match="Expected end of text"): + with pytest.raises(QueryParserError) as ei: QueryParser(allowed_fields={"field-with-dash": "string"}) + assert ei.value.error_type == "FIELD_NAME_ERROR" @pytest.mark.parametrize( ("parser_type", "expected_string"), [ - ("parser", "john doe"), - ("case_sensitive_parser", "John Doe"), - ("strict_parser", "john doe"), - ("case_sensitive_strict_parser", "John Doe"), + ("parser", '"john doe"'), + ("case_sensitive_parser", '"John Doe"'), + ("strict_parser", '"john doe"'), + ("case_sensitive_strict_parser", '"John Doe"'), ], ) def test_basic_field_parsing_all_types(self, parser_type, expected_string): @@ -54,25 +56,25 @@ def test_parser_configuration_differences(self): assert len(result) == 1 assert result[0]["field"] == "author" - def test_comparison_operators_and_boolean_variations(self): - @pytest.mark.parametrize("op", ["<", "<=", ">", ">=", "="]) - def test_operator_variations(self, op): - result = self.parser.parse(f"stars:{op}100") - assert len(result) == 1 - assert result[0]["field"] == "stars" - assert result[0]["type"] == "number" - assert result[0]["op"] == op - assert result[0]["number"] == "100" + @pytest.mark.parametrize("op", ["<", "<=", ">", ">=", "="]) + def test_comparison_operators(self, op): + result = self.parser.parse(f"stars:{op}100") + assert len(result) == 1 + assert result[0]["field"] == "stars" + assert result[0]["type"] == "number" + assert result[0]["op"] == op + assert result[0]["number"] == "100" - boolean_values = ["true", "false", "True", "False", "TRUE", "FALSE"] + boolean_values = ["true", "false", "True", "False", "TRUE", "FALSE"] - @pytest.mark.parametrize("bool_val", boolean_values) - def test_boolean_variations(self, bool_val): - result = self.parser.parse(f"archived:{bool_val}") - assert len(result) == 1 - assert result[0]["field"] == "archived" - assert result[0]["type"] == "boolean" - assert result[0]["boolean"] == bool_val + @pytest.mark.parametrize("bool_val", boolean_values) + def test_boolean_variations(self, bool_val): + result = self.parser.parse(f"archived:{bool_val}") + assert len(result) == 1 + assert result[0]["field"] == "archived" + assert result[0]["type"] == "boolean" + expected = "True" if bool_val.lower() == "true" else "False" + assert result[0]["boolean"] == expected @pytest.mark.parametrize( ("query", "expected"), @@ -80,18 +82,18 @@ def test_boolean_variations(self, bool_val): ( 'author:"John Doe" stars:>100 archived:false some "free text"', [ - {"type": "string", "field": "author", "string": "john doe"}, + {"type": "string", "field": "author", "string": '"john doe"'}, {"type": "number", "field": "stars", "op": ">", "number": "100"}, {"type": "boolean", "field": "archived", "boolean": "False"}, {"type": "string", "field": "query", "string": "some"}, - {"type": "string", "field": "query", "string": "free text"}, + {"type": "string", "field": "query", "string": '"free text"'}, ], ), ( 'project:"my-awesome-project" language:"C++"', [ - {"type": "string", "field": "project", "string": "my-awesome-project"}, - {"type": "string", "field": "language", "string": "c++"}, + {"type": "string", "field": "project", "string": '"my-awesome-project"'}, + {"type": "string", "field": "language", "string": '"c++"'}, ], ), ], @@ -100,88 +102,71 @@ def test_complex_queries_and_mixed_content(self, query, expected): result = self.parser.parse(query) assert result == expected - def test_error_handling_and_malformed_queries(self): - malformed_queries = ["field_without_colon", ":value_without_field", "field:", ""] + malformed_queries = ['not_field:"operation"', "not_field:operation"] - @pytest.mark.parametrize("query", malformed_queries) - def test_malformed_queries(self, query): - result = self.parser.parse(query) - if query == "": - assert not result - else: - assert result[0]["field"] == "query" - assert result[0]["type"] == "string" - assert result[0]["string"] == query + @pytest.mark.parametrize("query", malformed_queries) + def test_malformed_queries(self, query): + result = self.parser.parse(query) + assert not result - @pytest.mark.parametrize("query", malformed_queries) - def test_invalid_queries(self, query): - with pytest.raises(QueryParserError): - self.strict_parser.parse(query) + @pytest.mark.parametrize("query", malformed_queries) + def test_invalid_queries(self, query): + with pytest.raises(QueryParserError): + self.strict_parser.parse(query) def test_date_format_validation(self): - @pytest.mark.parametrize("date_str", ["2023-01-01", "2023-12-31"]) - def test_valid_dates(self, date_str): + valid_dates = ["2023-01-01", "2023-12-31"] + for date_str in valid_dates: result = self.parser.parse(f"created:{date_str}") assert len(result) == 1 assert result[0]["date"] == date_str - @pytest.mark.parametrize("inv_date_str", ["invalid-date", "2023-2213-01", "2023-01-322"]) - def test_invalid_dates(self, inv_date_str): + invalid_dates = ["invalid-date", "2023-163-01", "2023-01-362"] + for inv_date_str in invalid_dates: + with pytest.raises(QueryParserError, match="Invalid date value"): + self.strict_parser.parse(f"created:{inv_date_str}") + + for inv_date_str in invalid_dates: result = self.parser.parse(f"created:{inv_date_str}") assert not result - def test_initialization(self): - def test_invalid_initialization(): - with pytest.raises(QueryParserError): - QueryParser(allowed_fields=None) - - @pytest.mark.parametrize( - ("allowed_fields", "default_field", "query", "expected"), - [ - ( - {"query": "string"}, - "query", - "query:test free_text", - [ - {"type": "string", "field": "query", "string": "test"}, - {"type": "string", "field": "query", "string": "free_text"}, - ], - ) - ], - ) - def test_valid_initialization(allowed_fields, default_field, query, expected): - parser_with_default = QueryParser( - allowed_fields=allowed_fields, default_field=default_field - ) - result = parser_with_default.parse(query) - assert result == expected - - def test_boundary_conditions_and_edge_cases(self): - @pytest.mark.parametrize( - ("query", "expected"), - [ - ("author:", {"field": "query", "type": "string", "string": "author:"}), - (":123", {"field": "query", "type": "string", "string": ":123"}), - ('"field: true"', {"field": "query", "type": "string", "boolean": True}), - ("2341", {"field": "query", "type": "string", "string": "2341"}), - ], + def test_invalid_initialization(self): + with pytest.raises(QueryParserError): + QueryParser(allowed_fields={"field": "invalid_type"}) + + def test_custom_default_field(self): + parser_with_default = QueryParser( + allowed_fields={"test": "string"}, default_field="default" ) - def test_boundary_conditions_and_edge_cases(self, query, expected): - result = self.parser.parse(query) - assert result == [expected] - - def test_long_field_names(): - long_field_parser = QueryParser(allowed_fields={"verylongfieldname": "string"}) - long_value = "a" * 1000 - result = long_field_parser.parse(f'verylongfieldname:"{long_value}"') - assert isinstance(result, list) - assert result[0]["string"] == long_value - - @pytest.mark.parametrize("val", ["0", "-1", "999999", "3.14159"]) - def test_numeric_values(self, val): - result = self.parser.parse(f"stars:{val}") - assert len(result) == 1 - assert result[0]["type"] == "number" - assert result[0]["field"] == "stars" - assert result[0]["number"] == str(int(float(val))) - assert result[0]["op"] == "=" + result = parser_with_default.parse("query:test free_text") + expected = [ + {"type": "string", "field": "default", "string": "free_text"}, + ] + assert result == expected + + @pytest.mark.parametrize( + ("query"), + ["author:", ":123", '"field: true"', "2341"], + ) + def test_boundary_conditions(self, query): + result = self.parser.parse(query) + assert len(result) == 1 + assert result[0]["field"] == "query" + assert result[0]["type"] == "string" + assert result[0]["string"] == query + + def test_long_field_names_and_values(self): + long_field_parser = QueryParser(allowed_fields={"verylongfieldname": "string"}) + long_value = "a" * 1000 + result = long_field_parser.parse(f'verylongfieldname:"{long_value}"') + assert isinstance(result, list) + assert result[0]["string"] == '"' + long_value + '"' + + @pytest.mark.parametrize("val", ["0", "-1", "999999", "3.14159"]) + def test_numeric_values(self, val): + result = self.parser.parse(f"stars:{val}") + assert len(result) == 1 + assert result[0]["type"] == "number" + assert result[0]["field"] == "stars" + assert result[0]["number"] == str(int(float(val))) + assert result[0]["op"] == "=" From fbb779630aba9300db874b2326eb4abb54fdcff6 Mon Sep 17 00:00:00 2001 From: Adarsh0427 <124577434+Adarsh0427@users.noreply.github.com> Date: Fri, 22 Aug 2025 18:47:53 +0530 Subject: [PATCH 03/17] fixed edge cases for token parsing and enhaced test cases --- backend/apps/common/search/query_parser.py | 286 ++++++++++-------- .../apps/common/search/query_parser_test.py | 39 +-- 2 files changed, 181 insertions(+), 144 deletions(-) diff --git a/backend/apps/common/search/query_parser.py b/backend/apps/common/search/query_parser.py index 98b4d3b0f9..fe9cf9d932 100644 --- a/backend/apps/common/search/query_parser.py +++ b/backend/apps/common/search/query_parser.py @@ -8,7 +8,6 @@ ParseException, QuotedString, Regex, - Suppress, Word, ZeroOrMore, alphanums, @@ -112,26 +111,15 @@ class QueryParser: # Grammar definitions (class-level constants) _FIELD_NAME = Word(alphas, alphanums + "_") _QUOTED_VALUE = QuotedString(quoteChar='"', escChar="\\", unquoteResults=False) - _UNQUOTED_VALUE = Word(alphanums + "+-.<>=/_") + _UNQUOTED_VALUE = Word(alphanums + '+-.<>=/_"') _FIELD_VALUE = _QUOTED_VALUE | _UNQUOTED_VALUE - _TOKEN_PATTERN = Group(_FIELD_NAME + Suppress(":") + _FIELD_VALUE) - _COMP_OPERATOR = Optional(oneOf(">= <= > < ="), default="=") + _COMPARISON_OPERATORS = {">", "<", ">=", "<=", "="} + _COMP_OPERATOR = Optional(oneOf(_COMPARISON_OPERATORS), default="=") _COMP_PATTERN = Group(_COMP_OPERATOR + _UNQUOTED_VALUE) _DATE_PATTERN = Regex(r"\d{4}-\d{2}-\d{2}") # YYYY-MM-DD format - _BOOLEAN_TRUE_VALUES = {"true", "1", "yes", "on"} _BOOLEAN_FALSE_VALUES = {"false", "0", "no", "off"} - @staticmethod - def _is_field_name(field_name: str) -> bool: - """Check if a field name is valid.""" - try: - QueryParser._FIELD_NAME.parseString(field_name, parseAll=True) - except ParseException: - return False - else: - return bool(field_name.islower()) - def __init__( self, allowed_fields: dict[str, str], @@ -167,13 +155,139 @@ def __init__( error_type="CONFIGURATION_ERROR", ) from e + def parse(self, query: str) -> list[dict[str, str]]: + """Parse query string into structured conditions. + + Args: + query: Query string to parse + + Returns: + List of dictionaries representing parsed conditions + + Raises: + QueryParserError: If parsing fails + + """ + conditions = [] + query = query.strip() + if not self.case_sensitive: + query = query.lower() + tokens = self._split_tokens(query) + for token in tokens: + field, raw_value = self._parse_token(token) + + if field is None: + conditions.append(self._create_text_search_condition(raw_value)) + continue + + field = field.lower() # field name normalization + + if field not in self.allowed_fields: + self._handle_unknown_field(field) + continue + + field_type = self.allowed_fields[field] + try: + condition_dict = self.to_dict(field, field_type, raw_value) + except QueryParserError: + if self.strict: + raise + continue + conditions.append(condition_dict) + return conditions + + def to_dict(self, field: str, field_type: FieldType, raw_value: str) -> dict[str, str]: + """Convert condition to dictionary format. + + Args: + field: Field name + field_type: FieldType of the condition + raw_value: Raw string value to parse + + Returns: + Dictionary representation of the condition + + Raises: + QueryParserError: If parsing fails + + """ + condition_dict = { + "type": field_type.value, + "field": field, + } + if field_type == FieldType.BOOLEAN: + condition_dict["boolean"] = self._parse_boolean_value(field, raw_value) + elif field_type == FieldType.NUMBER: + operator, numeric_value = self._parse_number_value(field, raw_value) + condition_dict["op"] = operator + condition_dict["number"] = str(numeric_value) + elif field_type == FieldType.DATE: + operator, date_value = self._parse_date_value(field, raw_value) + condition_dict["op"] = operator + condition_dict["date"] = date_value + else: # STRING + condition_dict["string"] = self._parse_string_value(field, raw_value) + + return condition_dict + + def _create_text_search_condition(self, token: str) -> dict[str, str]: + """Create a text search condition for tokens that aren't field:value pairs. + + Args: + token: Raw token string + + Returns: + QueryCondition for text search + + """ + return {"type": FieldType.STRING.value, "field": self.default_field, "string": token} + + def _handle_unknown_field(self, field: str) -> None: + """Handle unknown field based on strict mode setting. + + Args: + field: Unknown field name + + Raises: + QueryParserError: If in strict mode + + """ + if self.strict: + raise QueryParserError( + message=f"Unknown field '{field}'", + error_type="UNKNOWN_FIELD_ERROR", + field=field, + ) + + @staticmethod + def _is_field_name(field_name: str) -> bool: + """Check if a field name is valid.""" + try: + QueryParser._FIELD_NAME.parseString(field_name, parseAll=True) + except ParseException: + return False + else: + return bool(field_name.islower()) + + @staticmethod + def _is_field_value(field_value: str) -> bool: + """Check if a field value is valid.""" + try: + QueryParser._FIELD_VALUE.parseString(field_value, parseAll=True) + except ParseException: + return False + else: + return True + @staticmethod def _raise_invalid_field_name(field: str) -> None: + """Raise an error for invalid field names.""" msg = f"Field name '{field}' must be alphanumeric (letters, numbers, underscores only)" raise QueryParserError(message=msg, error_type="FIELD_NAME_ERROR", field=field) @staticmethod def _raise_invalid_field_type(field: str, field_type: str) -> None: + """Raise an error for invalid field types.""" msg = f"Invalid field type '{field_type}' for field '{field}'" raise QueryParserError(message=msg, error_type="FIELD_TYPE_ERROR", field=field) @@ -253,11 +367,32 @@ def _parse_token(token: str) -> tuple[str | None, str]: Tuple of (field_name, raw_value) or (None, token) if parsing fails """ - try: - result = QueryParser._TOKEN_PATTERN.parseString(token, parseAll=True) - return (result[0][0], result[0][1]) - except ParseException: + token = token.strip() + if token[0] == '"': + return (None, token) + if token.count(":") == 0: + return (None, token) + field, value = token.split(":", 1) + field = field.strip().lower() + value = value.strip() + if not QueryParser._is_field_name(field): + return (None, token) + if not QueryParser._is_field_value(value): return (None, token) + return (field, value) + + @staticmethod + def _remove_quotes(value: str) -> str: + """Remove surrounding quotes from a string. + + Args: + value: Input string + + Returns: + String without surrounding quotes + + """ + return value.strip().strip('"') @staticmethod def _parse_comparison_pattern(value: str) -> tuple[str, str]: @@ -274,6 +409,7 @@ def _parse_comparison_pattern(value: str) -> tuple[str, str]: """ try: + value = QueryParser._remove_quotes(value) match = QueryParser._COMP_PATTERN.parseString(value, parseAll=True) return (match[0][0], match[0][1]) except ParseException as e: @@ -320,9 +456,11 @@ def _parse_boolean_value(field: str, value: str) -> str: QueryParserError: If value is not a valid boolean """ - if value.lower() in QueryParser._BOOLEAN_TRUE_VALUES: + value = QueryParser._remove_quotes(value) + value = value.lower() + if value in QueryParser._BOOLEAN_TRUE_VALUES: return "True" - if value.lower() in QueryParser._BOOLEAN_FALSE_VALUES: + if value in QueryParser._BOOLEAN_FALSE_VALUES: return "False" raise QueryParserError( @@ -347,6 +485,7 @@ def _parse_number_value(field: str, value: str) -> tuple[str, int]: """ try: + value = QueryParser._remove_quotes(value) operator, clean_value = QueryParser._parse_comparison_pattern(value) numeric_value = int(float(clean_value)) @@ -375,6 +514,7 @@ def _parse_date_value(field: str, value: str) -> tuple[str, str]: """ try: + value = QueryParser._remove_quotes(value) operator, clean_value = QueryParser._parse_comparison_pattern(value) result = QueryParser._DATE_PATTERN.parseString(clean_value, parseAll=True) return operator, result[0] @@ -383,107 +523,3 @@ def _parse_date_value(field: str, value: str) -> tuple[str, str]: raise QueryParserError( message=f"Invalid date value '{value}'", error_type="DATE_VALUE_ERROR", field=field ) from e - - def to_dict(self, field: str, field_type: FieldType, raw_value: str) -> dict[str, str]: - """Convert condition to dictionary format. - - Args: - field: Field name - field_type: FieldType of the condition - raw_value: Raw string value to parse - - Returns: - Dictionary representation of the condition - - Raises: - QueryParserError: If parsing fails - - """ - condition_dict = { - "type": field_type.value, - "field": field, - } - if field_type == FieldType.BOOLEAN: - condition_dict["boolean"] = self._parse_boolean_value(field, raw_value) - elif field_type == FieldType.NUMBER: - operator, numeric_value = self._parse_number_value(field, raw_value) - condition_dict["op"] = operator - condition_dict["number"] = str(numeric_value) - elif field_type == FieldType.DATE: - operator, date_value = self._parse_date_value(field, raw_value) - condition_dict["op"] = operator - condition_dict["date"] = date_value - else: # STRING - condition_dict["string"] = self._parse_string_value(field, raw_value) - - return condition_dict - - def _create_text_search_condition(self, token: str) -> dict[str, str]: - """Create a text search condition for tokens that aren't field:value pairs. - - Args: - token: Raw token string - - Returns: - QueryCondition for text search - - """ - return {"type": FieldType.STRING.value, "field": self.default_field, "string": token} - - def _handle_unknown_field(self, field: str) -> None: - """Handle unknown field based on strict mode setting. - - Args: - field: Unknown field name - - Raises: - QueryParserError: If in strict mode - - """ - if self.strict: - raise QueryParserError( - message=f"Unknown field '{field}'", - error_type="UNKNOWN_FIELD_ERROR", - field=field, - ) - - def parse(self, query: str) -> list[dict[str, str]]: - """Parse query string into structured conditions. - - Args: - query: Query string to parse - - Returns: - List of dictionaries representing parsed conditions - - Raises: - QueryParserError: If parsing fails - - """ - conditions = [] - query = query.strip() - if not self.case_sensitive: - query = query.lower() - tokens = self._split_tokens(query) - for token in tokens: - field, raw_value = self._parse_token(token) - - if field is None: - conditions.append(self._create_text_search_condition(raw_value)) - continue - - field = field.lower() # field name normalization - - if field not in self.allowed_fields: - self._handle_unknown_field(field) - continue - - field_type = self.allowed_fields[field] - try: - condition_dict = self.to_dict(field, field_type, raw_value) - except QueryParserError: - if self.strict: - raise - continue - conditions.append(condition_dict) - return conditions diff --git a/backend/tests/apps/common/search/query_parser_test.py b/backend/tests/apps/common/search/query_parser_test.py index bad60c0abd..ca639cfe72 100644 --- a/backend/tests/apps/common/search/query_parser_test.py +++ b/backend/tests/apps/common/search/query_parser_test.py @@ -23,6 +23,9 @@ def setup_method(self): self.case_sensitive_strict_parser = QueryParser( allowed_fields=self.allowed_fields, strict=True, case_sensitive=True ) + self.boolean_values = self.parser._BOOLEAN_TRUE_VALUES | self.parser._BOOLEAN_FALSE_VALUES + self.true_boolean_values = self.parser._BOOLEAN_TRUE_VALUES + self.comparison_operators = self.parser._COMPARISON_OPERATORS def test_invalid_parser_field_validation(self): with pytest.raises(QueryParserError) as ei: @@ -56,25 +59,23 @@ def test_parser_configuration_differences(self): assert len(result) == 1 assert result[0]["field"] == "author" - @pytest.mark.parametrize("op", ["<", "<=", ">", ">=", "="]) - def test_comparison_operators(self, op): - result = self.parser.parse(f"stars:{op}100") - assert len(result) == 1 - assert result[0]["field"] == "stars" - assert result[0]["type"] == "number" - assert result[0]["op"] == op - assert result[0]["number"] == "100" - - boolean_values = ["true", "false", "True", "False", "TRUE", "FALSE"] - - @pytest.mark.parametrize("bool_val", boolean_values) - def test_boolean_variations(self, bool_val): - result = self.parser.parse(f"archived:{bool_val}") - assert len(result) == 1 - assert result[0]["field"] == "archived" - assert result[0]["type"] == "boolean" - expected = "True" if bool_val.lower() == "true" else "False" - assert result[0]["boolean"] == expected + def test_comparison_operators(self): + for op in self.comparison_operators: + result = self.parser.parse(f"stars:{op}100") + assert len(result) == 1 + assert result[0]["field"] == "stars" + assert result[0]["type"] == "number" + assert result[0]["op"] == op + assert result[0]["number"] == "100" + + def test_boolean_variations(self): + for bool_val in self.boolean_values: + result = self.parser.parse(f"archived:{bool_val}") + assert len(result) == 1 + assert result[0]["field"] == "archived" + assert result[0]["type"] == "boolean" + expected = "True" if bool_val in self.true_boolean_values else "False" + assert result[0]["boolean"] == expected @pytest.mark.parametrize( ("query", "expected"), From b8331ff3467d73a6c27ed28045456b00550a283b Mon Sep 17 00:00:00 2001 From: Adarsh0427 <124577434+Adarsh0427@users.noreply.github.com> Date: Fri, 22 Aug 2025 19:37:26 +0530 Subject: [PATCH 04/17] fixed suggestions addressed by bot in query_parser_test.py --- .../apps/common/search/query_parser_test.py | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/backend/tests/apps/common/search/query_parser_test.py b/backend/tests/apps/common/search/query_parser_test.py index ca639cfe72..a2e702e893 100644 --- a/backend/tests/apps/common/search/query_parser_test.py +++ b/backend/tests/apps/common/search/query_parser_test.py @@ -23,9 +23,9 @@ def setup_method(self): self.case_sensitive_strict_parser = QueryParser( allowed_fields=self.allowed_fields, strict=True, case_sensitive=True ) - self.boolean_values = self.parser._BOOLEAN_TRUE_VALUES | self.parser._BOOLEAN_FALSE_VALUES - self.true_boolean_values = self.parser._BOOLEAN_TRUE_VALUES - self.comparison_operators = self.parser._COMPARISON_OPERATORS + self.boolean_values = {"true", "1", "yes", "on", "false", "0", "no", "off"} + self.true_boolean_values = {"true", "1", "yes", "on"} + self.comparison_operators = ["<", "<=", ">", ">=", "="] def test_invalid_parser_field_validation(self): with pytest.raises(QueryParserError) as ei: @@ -103,37 +103,43 @@ def test_complex_queries_and_mixed_content(self, query, expected): result = self.parser.parse(query) assert result == expected - malformed_queries = ['not_field:"operation"', "not_field:operation"] + unknown_field_queries = ['not_field:"operation"', "not_field:operation"] - @pytest.mark.parametrize("query", malformed_queries) - def test_malformed_queries(self, query): + @pytest.mark.parametrize("query", unknown_field_queries) + def test_unknown_field_non_strict_drops_token(self, query): result = self.parser.parse(query) assert not result - @pytest.mark.parametrize("query", malformed_queries) - def test_invalid_queries(self, query): - with pytest.raises(QueryParserError): + @pytest.mark.parametrize("query", unknown_field_queries) + def test_unknown_field_strict_raises(self, query): + with pytest.raises(QueryParserError) as ei: self.strict_parser.parse(query) + assert ei.value.error_type == "UNKNOWN_FIELD_ERROR" def test_date_format_validation(self): - valid_dates = ["2023-01-01", "2023-12-31"] + valid_dates = ["2023-12-31", '"2023-12-31"', '"2023-12-31'] for date_str in valid_dates: result = self.parser.parse(f"created:{date_str}") assert len(result) == 1 - assert result[0]["date"] == date_str + assert result[0]["date"] == '2023-12-31' + assert result[0]["field"] == "created" + assert result[0]["type"] == "date" + assert result[0]["op"] == "=" invalid_dates = ["invalid-date", "2023-163-01", "2023-01-362"] for inv_date_str in invalid_dates: - with pytest.raises(QueryParserError, match="Invalid date value"): + with pytest.raises(QueryParserError, match="Invalid date value") as ei: self.strict_parser.parse(f"created:{inv_date_str}") + assert ei.value.error_type == "DATE_VALUE_ERROR" for inv_date_str in invalid_dates: result = self.parser.parse(f"created:{inv_date_str}") assert not result - def test_invalid_initialization(self): - with pytest.raises(QueryParserError): - QueryParser(allowed_fields={"field": "invalid_type"}) + def test_invalid_default_field_name(self): + with pytest.raises(QueryParserError) as ei: + QueryParser(allowed_fields={"test": "string"}, default_field="Default") + assert ei.value.error_type == "FIELD_NAME_ERROR" def test_custom_default_field(self): parser_with_default = QueryParser( @@ -162,6 +168,8 @@ def test_long_field_names_and_values(self): result = long_field_parser.parse(f'verylongfieldname:"{long_value}"') assert isinstance(result, list) assert result[0]["string"] == '"' + long_value + '"' + assert result[0]["field"] == "verylongfieldname" + assert result[0]["type"] == "string" @pytest.mark.parametrize("val", ["0", "-1", "999999", "3.14159"]) def test_numeric_values(self, val): From 919d872dfbe48fcdc4cde47cac22ecec43f62707 Mon Sep 17 00:00:00 2001 From: Adarsh0427 <124577434+Adarsh0427@users.noreply.github.com> Date: Thu, 28 Aug 2025 19:23:50 +0530 Subject: [PATCH 05/17] addressed the suggestions and fixed them --- backend/apps/common/search/query_parser.py | 268 ++++++++++-------- .../apps/common/search/query_parser_test.py | 99 +++---- 2 files changed, 184 insertions(+), 183 deletions(-) diff --git a/backend/apps/common/search/query_parser.py b/backend/apps/common/search/query_parser.py index fe9cf9d932..cc6beb86c4 100644 --- a/backend/apps/common/search/query_parser.py +++ b/backend/apps/common/search/query_parser.py @@ -25,7 +25,7 @@ class FieldType(str, Enum): BOOLEAN = "boolean" -def is_in_enum(enum_class, value): +def has_enum_value(enum_class, value): """Check if a value exists in the given enum class. Args: @@ -60,9 +60,9 @@ def __init__( """Initialize QueryParserError. Args: - message: Human-readable error message + message: Human-readable error message by the parser error_type: Category of error (PARSE_ERROR, FIELD_ERROR, VALUE_ERROR, etc.) - query: The original query string that caused the error + query: The original query string that caused the error (if applicable) field: Field name related to the error (if applicable) """ @@ -84,14 +84,14 @@ def __str__(self) -> str: return "\n".join(parts) - def to_dict(self) -> dict: + def to_response(self) -> dict: """Return error information as dictionary for API responses.""" return { "error": True, "error_type": self.error_type, "message": self.message, - "query": self.query, - "field": self.field, + "query": self.query or "", + "field": self.field or "", } @@ -100,7 +100,7 @@ class QueryParser: Features: - Equality checks (field:value) - - Numeric comparisons (>, <, >=, <=, =) + - Numeric and date comparisons (>, <, >=, <=, =) - Quoted strings for multi-word values - Boolean field support - Date field parsing with comparisons @@ -114,15 +114,15 @@ class QueryParser: _UNQUOTED_VALUE = Word(alphanums + '+-.<>=/_"') _FIELD_VALUE = _QUOTED_VALUE | _UNQUOTED_VALUE _COMPARISON_OPERATORS = {">", "<", ">=", "<=", "="} - _COMP_OPERATOR = Optional(oneOf(_COMPARISON_OPERATORS), default="=") - _COMP_PATTERN = Group(_COMP_OPERATOR + _UNQUOTED_VALUE) - _DATE_PATTERN = Regex(r"\d{4}-\d{2}-\d{2}") # YYYY-MM-DD format + _COMPARISON_OPERATOR = Optional(oneOf(_COMPARISON_OPERATORS), default="=") + _COMPARISON_PATTERN = Group(_COMPARISON_OPERATOR + _UNQUOTED_VALUE) + _DATE_PATTERN = Regex(r"\d{4}-\d{2}-\d{2}") | Regex(r"\d{8}") # YYYY-MM-DD or YYYYMMDD format _BOOLEAN_TRUE_VALUES = {"true", "1", "yes", "on"} _BOOLEAN_FALSE_VALUES = {"false", "0", "no", "off"} def __init__( self, - allowed_fields: dict[str, str], + field_schema: dict[str, str], default_field: str = "query", *, case_sensitive: bool = False, @@ -131,19 +131,19 @@ def __init__( """Initialize the query parser. Args: - allowed_fields: Dictionary mapping field names to their types - (e.g., {"stars": "number", "language": "string"}). - Field names must be lowercase. + field_schema: Dictionary mapping field names to their types + (e.g., {"stars": "number", "language": "string"}). + Field names must be lowercase. default_field: Default field name for free text search case_sensitive: If True, field names are case-sensitive strict: If True, raises exception for unknown fields; if False, issues warnings Raises: - QueryParserError: If allowed_fields contains invalid field names or types + QueryParserError: If field_schema contains invalid field names or types """ try: - self.allowed_fields = self._validate_allowed_fields(allowed_fields, default_field) + self.field_schema = self._validate_field_schema(field_schema, default_field) self.default_field = default_field self.case_sensitive = case_sensitive self.strict = strict @@ -173,31 +173,34 @@ def parse(self, query: str) -> list[dict[str, str]]: if not self.case_sensitive: query = query.lower() tokens = self._split_tokens(query) - for token in tokens: - field, raw_value = self._parse_token(token) - - if field is None: - conditions.append(self._create_text_search_condition(raw_value)) - continue - - field = field.lower() # field name normalization - - if field not in self.allowed_fields: - self._handle_unknown_field(field) - continue - - field_type = self.allowed_fields[field] - try: - condition_dict = self.to_dict(field, field_type, raw_value) - except QueryParserError: - if self.strict: - raise - continue - conditions.append(condition_dict) - return conditions + try: + for token in tokens: + if QueryParser._is_empty_string(token): + continue + field, raw_value = self._parse_token(token) + + if field is None: + conditions.append(self._create_text_search_condition(raw_value)) + continue + + if field not in self.field_schema: + self._handle_unknown_field(field) + continue + + field_type = self.field_schema[field] + condition = self.create_condition(field, field_type, raw_value) + if condition: + conditions.append(condition) + except QueryParserError as e: + e.query = query + raise + else: + return conditions - def to_dict(self, field: str, field_type: FieldType, raw_value: str) -> dict[str, str]: - """Convert condition to dictionary format. + def create_condition( + self, field: str, field_type: FieldType, raw_value: str + ) -> dict[str, str] | None: + """Create a structured condition dictionary from field and raw value. Args: field: Field name @@ -211,24 +214,33 @@ def to_dict(self, field: str, field_type: FieldType, raw_value: str) -> dict[str QueryParserError: If parsing fails """ - condition_dict = { + condition = { "type": field_type.value, "field": field, } - if field_type == FieldType.BOOLEAN: - condition_dict["boolean"] = self._parse_boolean_value(field, raw_value) - elif field_type == FieldType.NUMBER: - operator, numeric_value = self._parse_number_value(field, raw_value) - condition_dict["op"] = operator - condition_dict["number"] = str(numeric_value) - elif field_type == FieldType.DATE: - operator, date_value = self._parse_date_value(field, raw_value) - condition_dict["op"] = operator - condition_dict["date"] = date_value - else: # STRING - condition_dict["string"] = self._parse_string_value(field, raw_value) - - return condition_dict + try: + match field_type: + case FieldType.BOOLEAN: + condition["boolean"] = self._parse_boolean_value( + raw_value + ) # "True" or "False" + case FieldType.NUMBER: + operator, numeric_value = self._parse_number_value(raw_value) + condition["op"] = operator + condition["number"] = numeric_value + case FieldType.DATE: + operator, date_value = self._parse_date_value(raw_value) + condition["op"] = operator + condition["date"] = date_value + case FieldType.STRING: + condition["string"] = self._parse_string_value(raw_value) + except QueryParserError as e: + if self.strict: + e.field = field + raise + return None + else: + return condition def _create_text_search_condition(self, token: str) -> dict[str, str]: """Create a text search condition for tokens that aren't field:value pairs. @@ -254,7 +266,7 @@ def _handle_unknown_field(self, field: str) -> None: """ if self.strict: raise QueryParserError( - message=f"Unknown field '{field}'", + message="Unknown field", error_type="UNKNOWN_FIELD_ERROR", field=field, ) @@ -266,54 +278,48 @@ def _is_field_name(field_name: str) -> bool: QueryParser._FIELD_NAME.parseString(field_name, parseAll=True) except ParseException: return False - else: - return bool(field_name.islower()) + return field_name.islower() @staticmethod - def _is_field_value(field_value: str) -> bool: - """Check if a field value is valid.""" - try: - QueryParser._FIELD_VALUE.parseString(field_value, parseAll=True) - except ParseException: - return False - else: - return True + def _is_empty_string(token: str) -> bool: + """Check if a token is empty or contains only whitespace/quotes.""" + return token.strip().strip('"').strip() == "" @staticmethod def _raise_invalid_field_name(field: str) -> None: """Raise an error for invalid field names.""" - msg = f"Field name '{field}' must be alphanumeric (letters, numbers, underscores only)" + msg = "Field name must be lowercase alphanumeric (letters, numbers, underscores only)" raise QueryParserError(message=msg, error_type="FIELD_NAME_ERROR", field=field) @staticmethod def _raise_invalid_field_type(field: str, field_type: str) -> None: """Raise an error for invalid field types.""" - msg = f"Invalid field type '{field_type}' for field '{field}'" + msg = f"Invalid field type: '{field_type}'" raise QueryParserError(message=msg, error_type="FIELD_TYPE_ERROR", field=field) @staticmethod - def _validate_allowed_fields( - allowed_fields: dict[str, str], default_field: str + def _validate_field_schema( + field_schema: dict[str, str], default_field: str ) -> dict[str, FieldType]: - """Validate and convert allowed_fields to use FieldType enum. + """Validate field schema and convert string field types to FieldType enums. Args: - allowed_fields: Raw field type mapping + field_schema: Raw field type mapping with string values default_field: Default field name for free text search Returns: - Validated field type mapping using FieldType enum + Validated field type mapping with string types converted to FieldType enum values Raises: - QueryParserError: If field type is not supported + QueryParserError: If field names are invalid or field types are not supported """ if not QueryParser._is_field_name(default_field): QueryParser._raise_invalid_field_name(default_field) validated_fields = {} - for field, field_type in allowed_fields.items(): - if not is_in_enum(FieldType, field_type): + for field, field_type in field_schema.items(): + if not has_enum_value(FieldType, field_type): QueryParser._raise_invalid_field_type(field, field_type) if not QueryParser._is_field_name(field): QueryParser._raise_invalid_field_name(field) @@ -353,7 +359,9 @@ def _split_tokens(query: str) -> list[str]: return result.asList() except ParseException as e: raise QueryParserError( - message="Failed to tokenize query", error_type="TOKENIZATION_ERROR", query=query + message=f"Failed to tokenize query: {e!s}", + error_type="TOKENIZATION_ERROR", + query=query, ) from e @staticmethod @@ -361,25 +369,24 @@ def _parse_token(token: str) -> tuple[str | None, str]: """Parse a single token into field name and value. Args: - token: Individual token string (e.g., "field:value") raise exception if null + token: Individual token string (e.g., "field:value"). Returns: - Tuple of (field_name, raw_value) or (None, token) if parsing fails + Tuple of (field_name, raw_value) or returns (None, token) + if token is not a field:value pattern. """ token = token.strip() if token[0] == '"': - return (None, token) + return None, token if token.count(":") == 0: - return (None, token) + return None, token field, value = token.split(":", 1) field = field.strip().lower() value = value.strip() - if not QueryParser._is_field_name(field): - return (None, token) - if not QueryParser._is_field_value(value): - return (None, token) - return (field, value) + if QueryParser._is_empty_string(field) or QueryParser._is_empty_string(value): + return None, token + return field, value @staticmethod def _remove_quotes(value: str) -> str: @@ -394,6 +401,21 @@ def _remove_quotes(value: str) -> str: """ return value.strip().strip('"') + @staticmethod + def _normalize_date(date: str) -> str: + """Normalize the date string to YYYY-MM-DD format. + + Args: + date: Raw date string + + Returns: + Normalized date string + + """ + if "-" not in date: + return f"{date[:4]}-{date[4:6]}-{date[6:]}" + return date + @staticmethod def _parse_comparison_pattern(value: str) -> tuple[str, str]: """Parse comparison operator and value from a string. @@ -402,7 +424,7 @@ def _parse_comparison_pattern(value: str) -> tuple[str, str]: value: Value string potentially containing comparison operator Returns: - Tuple of (operator, clean_value), if operator is none then ("=", clean_value) + Tuple of (operator, clean_value), "=" is the default operator when none is specified Raises: QueryParserError: If the comparison pattern is invalid @@ -410,75 +432,69 @@ def _parse_comparison_pattern(value: str) -> tuple[str, str]: """ try: value = QueryParser._remove_quotes(value) - match = QueryParser._COMP_PATTERN.parseString(value, parseAll=True) - return (match[0][0], match[0][1]) - except ParseException as e: + match = QueryParser._COMPARISON_PATTERN.parseString(value, parseAll=True) + return match[0][0], match[0][1] + except (ValueError, ParseException) as e: raise QueryParserError( - message="Invalid comparison pattern in value", error_type="COMPARISON_ERROR" + message=f"Invalid comparison pattern in value: {e!s}", + error_type="COMPARISON_ERROR", ) from e @staticmethod - def _parse_string_value(field: str, value: str) -> str: - """Parse and validate string value(handle quotes and unquoted string). + def _parse_string_value(value: str) -> str: + """Process string values by validating their format. Args: - field: Field name value: Raw string value Returns: - Cleaned string value + Cleaned and validated string value Raises: QueryParserError: If string parsing fails """ - if value.startswith('"') and value.endswith('"'): - return value try: - result = QueryParser._UNQUOTED_VALUE.parseString(value) + result = QueryParser._FIELD_VALUE.parseString(value, parseAll=True) return result[0] - except ParseException as e: - msg = f"Invalid string value for {field}:{value}" - raise QueryParserError(msg) from e + except (ValueError, ParseException) as e: + raise QueryParserError( + message=f"Invalid string value: {e!s}", error_type="STRING_VALUE_ERROR" + ) from e @staticmethod - def _parse_boolean_value(field: str, value: str) -> str: + def _parse_boolean_value(value: str) -> str: """Parse boolean value from string and return its string representation. Args: - field: Field name value: String value to parse Returns: - String representation of boolean value ("True" or "False") + String ("True" or "False") based on the boolean input value. Raises: QueryParserError: If value is not a valid boolean """ - value = QueryParser._remove_quotes(value) - value = value.lower() + value = QueryParser._remove_quotes(value).lower() if value in QueryParser._BOOLEAN_TRUE_VALUES: return "True" if value in QueryParser._BOOLEAN_FALSE_VALUES: return "False" - raise QueryParserError( - message=f"Invalid boolean value '{value}'", - error_type="BOOLEAN_VALUE_ERROR", - field=field, + message=f"Invalid boolean value: {value}", error_type="BOOLEAN_VALUE_ERROR" ) @staticmethod - def _parse_number_value(field: str, value: str) -> tuple[str, int]: + def _parse_number_value(value: str) -> tuple[str, str]: """Parse numeric value with optional comparison operator. Args: - field: Field name (for error messages) value: String value to parse Returns: - Tuple of (operator, numeric_value) + Tuple of (operator, numeric_value) where numeric_value is string + of integer representation and "=" is the default operator. Raises: QueryParserError: If value is not a valid number @@ -488,26 +504,24 @@ def _parse_number_value(field: str, value: str) -> tuple[str, int]: value = QueryParser._remove_quotes(value) operator, clean_value = QueryParser._parse_comparison_pattern(value) numeric_value = int(float(clean_value)) - - except ValueError as e: + return operator, str(numeric_value) + except QueryParserError: + raise + except (ValueError, ParseException) as e: raise QueryParserError( - message=f"Invalid numeric value '{value}'", - error_type="NUMBER_VALUE_ERROR", - field=field, + message=f"Invalid numeric value: {e!s}", error_type="NUMBER_VALUE_ERROR" ) from e - return operator, numeric_value - @staticmethod - def _parse_date_value(field: str, value: str) -> tuple[str, str]: + def _parse_date_value(value: str) -> tuple[str, str]: """Parse date value with optional comparison operator. Args: - field: Field name value: String value to parse Returns: - Tuple of (operator, date_string) + Tuple of (operator, date_string) where date_string + format is "YYYY-MM-DD" and "=" is the default operator. Raises: QueryParserError: If value is not a valid date format @@ -517,9 +531,11 @@ def _parse_date_value(field: str, value: str) -> tuple[str, str]: value = QueryParser._remove_quotes(value) operator, clean_value = QueryParser._parse_comparison_pattern(value) result = QueryParser._DATE_PATTERN.parseString(clean_value, parseAll=True) + result[0] = QueryParser._normalize_date(result[0]) return operator, result[0] - - except (ValueError, ParseException, QueryParserError) as e: + except QueryParserError: + raise + except (ValueError, ParseException) as e: raise QueryParserError( - message=f"Invalid date value '{value}'", error_type="DATE_VALUE_ERROR", field=field + message=f"Invalid date value: {e!s}", error_type="DATE_VALUE_ERROR" ) from e diff --git a/backend/tests/apps/common/search/query_parser_test.py b/backend/tests/apps/common/search/query_parser_test.py index a2e702e893..a0aa2b4a86 100644 --- a/backend/tests/apps/common/search/query_parser_test.py +++ b/backend/tests/apps/common/search/query_parser_test.py @@ -5,7 +5,7 @@ class TestQueryParser: def setup_method(self): - self.allowed_fields = { + self.field_schema = { "author": "string", "stars": "number", "issues": "number", @@ -15,26 +15,26 @@ def setup_method(self): "created": "date", } - self.parser = QueryParser(allowed_fields=self.allowed_fields) + self.parser = QueryParser(field_schema=self.field_schema) self.case_sensitive_parser = QueryParser( - allowed_fields=self.allowed_fields, default_field="test", case_sensitive=True + field_schema=self.field_schema, default_field="test", case_sensitive=True ) - self.strict_parser = QueryParser(allowed_fields=self.allowed_fields, strict=True) + self.strict_parser = QueryParser(field_schema=self.field_schema, strict=True) self.case_sensitive_strict_parser = QueryParser( - allowed_fields=self.allowed_fields, strict=True, case_sensitive=True + field_schema=self.field_schema, strict=True, case_sensitive=True ) self.boolean_values = {"true", "1", "yes", "on", "false", "0", "no", "off"} self.true_boolean_values = {"true", "1", "yes", "on"} self.comparison_operators = ["<", "<=", ">", ">=", "="] def test_invalid_parser_field_validation(self): - with pytest.raises(QueryParserError) as ei: - QueryParser(allowed_fields={"field": "invalid_type"}) - assert ei.value.error_type == "FIELD_TYPE_ERROR" + with pytest.raises(QueryParserError) as exception_info: + QueryParser(field_schema={"field": "invalid_type"}) + assert exception_info.value.error_type == "FIELD_TYPE_ERROR" - with pytest.raises(QueryParserError) as ei: - QueryParser(allowed_fields={"field-with-dash": "string"}) - assert ei.value.error_type == "FIELD_NAME_ERROR" + with pytest.raises(QueryParserError) as exception_info: + QueryParser(field_schema={"field-with-dash": "string"}) + assert exception_info.value.error_type == "FIELD_NAME_ERROR" @pytest.mark.parametrize( ("parser_type", "expected_string"), @@ -48,34 +48,27 @@ def test_invalid_parser_field_validation(self): def test_basic_field_parsing_all_types(self, parser_type, expected_string): parser = getattr(self, parser_type) result = parser.parse('Author:"John Doe"') - assert len(result) == 1 - assert result[0]["field"] == "author" - assert result[0]["type"] == "string" - assert result[0]["string"] == expected_string + expected = [{"field": "author", "type": "string", "string": expected_string}] + assert result == expected def test_parser_configuration_differences(self): query = "AUTHOR:test" result = self.case_sensitive_parser.parse(query) - assert len(result) == 1 - assert result[0]["field"] == "author" + expected = [{"field": "author", "type": "string", "string": "test"}] + assert result == expected def test_comparison_operators(self): for op in self.comparison_operators: result = self.parser.parse(f"stars:{op}100") - assert len(result) == 1 - assert result[0]["field"] == "stars" - assert result[0]["type"] == "number" - assert result[0]["op"] == op - assert result[0]["number"] == "100" + expected = [{"field": "stars", "type": "number", "op": op, "number": "100"}] + assert result == expected def test_boolean_variations(self): for bool_val in self.boolean_values: result = self.parser.parse(f"archived:{bool_val}") - assert len(result) == 1 - assert result[0]["field"] == "archived" - assert result[0]["type"] == "boolean" - expected = "True" if bool_val in self.true_boolean_values else "False" - assert result[0]["boolean"] == expected + expected_bool = "True" if bool_val in self.true_boolean_values else "False" + expected = [{"field": "archived", "type": "boolean", "boolean": expected_bool}] + assert result == expected @pytest.mark.parametrize( ("query", "expected"), @@ -112,39 +105,34 @@ def test_unknown_field_non_strict_drops_token(self, query): @pytest.mark.parametrize("query", unknown_field_queries) def test_unknown_field_strict_raises(self, query): - with pytest.raises(QueryParserError) as ei: + with pytest.raises(QueryParserError) as exception_info: self.strict_parser.parse(query) - assert ei.value.error_type == "UNKNOWN_FIELD_ERROR" + assert exception_info.value.error_type == "UNKNOWN_FIELD_ERROR" def test_date_format_validation(self): - valid_dates = ["2023-12-31", '"2023-12-31"', '"2023-12-31'] + valid_dates = ["2023-12-31", '"2023-12-31"', '"2023-12-31', "20231231"] for date_str in valid_dates: result = self.parser.parse(f"created:{date_str}") - assert len(result) == 1 - assert result[0]["date"] == '2023-12-31' - assert result[0]["field"] == "created" - assert result[0]["type"] == "date" - assert result[0]["op"] == "=" + expected = [{"date": "2023-12-31", "field": "created", "type": "date", "op": "="}] + assert result == expected invalid_dates = ["invalid-date", "2023-163-01", "2023-01-362"] for inv_date_str in invalid_dates: - with pytest.raises(QueryParserError, match="Invalid date value") as ei: + with pytest.raises(QueryParserError) as exception_info: self.strict_parser.parse(f"created:{inv_date_str}") - assert ei.value.error_type == "DATE_VALUE_ERROR" + assert exception_info.value.error_type == "DATE_VALUE_ERROR" for inv_date_str in invalid_dates: result = self.parser.parse(f"created:{inv_date_str}") assert not result def test_invalid_default_field_name(self): - with pytest.raises(QueryParserError) as ei: - QueryParser(allowed_fields={"test": "string"}, default_field="Default") - assert ei.value.error_type == "FIELD_NAME_ERROR" + with pytest.raises(QueryParserError) as exception_info: + QueryParser(field_schema={"test": "string"}, default_field="Default") + assert exception_info.value.error_type == "FIELD_NAME_ERROR" def test_custom_default_field(self): - parser_with_default = QueryParser( - allowed_fields={"test": "string"}, default_field="default" - ) + parser_with_default = QueryParser(field_schema={"test": "string"}, default_field="default") result = parser_with_default.parse("query:test free_text") expected = [ {"type": "string", "field": "default", "string": "free_text"}, @@ -157,25 +145,22 @@ def test_custom_default_field(self): ) def test_boundary_conditions(self, query): result = self.parser.parse(query) - assert len(result) == 1 - assert result[0]["field"] == "query" - assert result[0]["type"] == "string" - assert result[0]["string"] == query + expected = [{"field": "query", "type": "string", "string": query}] + assert result == expected def test_long_field_names_and_values(self): - long_field_parser = QueryParser(allowed_fields={"verylongfieldname": "string"}) + long_field_parser = QueryParser(field_schema={"verylongfieldname": "string"}) long_value = "a" * 1000 result = long_field_parser.parse(f'verylongfieldname:"{long_value}"') - assert isinstance(result, list) - assert result[0]["string"] == '"' + long_value + '"' - assert result[0]["field"] == "verylongfieldname" - assert result[0]["type"] == "string" + expected = [ + {"string": '"' + long_value + '"', "field": "verylongfieldname", "type": "string"} + ] + assert result == expected @pytest.mark.parametrize("val", ["0", "-1", "999999", "3.14159"]) def test_numeric_values(self, val): result = self.parser.parse(f"stars:{val}") - assert len(result) == 1 - assert result[0]["type"] == "number" - assert result[0]["field"] == "stars" - assert result[0]["number"] == str(int(float(val))) - assert result[0]["op"] == "=" + expected = [ + {"type": "number", "field": "stars", "number": str(int(float(val))), "op": "="} + ] + assert result == expected From 656dc4c9933a63c83f54faa4955de68c8403d25e Mon Sep 17 00:00:00 2001 From: Adarsh Singh <124577434+Adarsh0427@users.noreply.github.com> Date: Thu, 28 Aug 2025 20:05:19 +0530 Subject: [PATCH 06/17] Refine field name regex and validation logic Updated the field name regex pattern and modified the return logic in the field name validation method. --- backend/apps/common/search/query_parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/apps/common/search/query_parser.py b/backend/apps/common/search/query_parser.py index cc6beb86c4..a5a48b1ee9 100644 --- a/backend/apps/common/search/query_parser.py +++ b/backend/apps/common/search/query_parser.py @@ -109,7 +109,7 @@ class QueryParser: """ # Grammar definitions (class-level constants) - _FIELD_NAME = Word(alphas, alphanums + "_") + _FIELD_NAME = Regex(r"[a-z0-9_]+") _QUOTED_VALUE = QuotedString(quoteChar='"', escChar="\\", unquoteResults=False) _UNQUOTED_VALUE = Word(alphanums + '+-.<>=/_"') _FIELD_VALUE = _QUOTED_VALUE | _UNQUOTED_VALUE @@ -278,7 +278,8 @@ def _is_field_name(field_name: str) -> bool: QueryParser._FIELD_NAME.parseString(field_name, parseAll=True) except ParseException: return False - return field_name.islower() + else: + return True @staticmethod def _is_empty_string(token: str) -> bool: From f1a0e0176be31bba647c40e6b6ac1d8cbef65c2f Mon Sep 17 00:00:00 2001 From: Adarsh0427 <124577434+Adarsh0427@users.noreply.github.com> Date: Thu, 28 Aug 2025 22:35:16 +0530 Subject: [PATCH 07/17] fixed suggestions addressed by bot --- backend/apps/common/search/query_parser.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/apps/common/search/query_parser.py b/backend/apps/common/search/query_parser.py index a5a48b1ee9..764ec775ad 100644 --- a/backend/apps/common/search/query_parser.py +++ b/backend/apps/common/search/query_parser.py @@ -11,7 +11,6 @@ Word, ZeroOrMore, alphanums, - alphas, oneOf, ) @@ -104,7 +103,7 @@ class QueryParser: - Quoted strings for multi-word values - Boolean field support - Date field parsing with comparisons - - Handle case sensitivity/insensitivity values + - Case sensitivity: normalize field names only, preserve values - Only supports implicit AND logic between conditions """ @@ -113,8 +112,7 @@ class QueryParser: _QUOTED_VALUE = QuotedString(quoteChar='"', escChar="\\", unquoteResults=False) _UNQUOTED_VALUE = Word(alphanums + '+-.<>=/_"') _FIELD_VALUE = _QUOTED_VALUE | _UNQUOTED_VALUE - _COMPARISON_OPERATORS = {">", "<", ">=", "<=", "="} - _COMPARISON_OPERATOR = Optional(oneOf(_COMPARISON_OPERATORS), default="=") + _COMPARISON_OPERATOR = Optional(oneOf(">= <= > < ="), default="=") _COMPARISON_PATTERN = Group(_COMPARISON_OPERATOR + _UNQUOTED_VALUE) _DATE_PATTERN = Regex(r"\d{4}-\d{2}-\d{2}") | Regex(r"\d{8}") # YYYY-MM-DD or YYYYMMDD format _BOOLEAN_TRUE_VALUES = {"true", "1", "yes", "on"} From 84df75766553b3ca646dae55ee125fe32796797e Mon Sep 17 00:00:00 2001 From: Adarsh0427 <124577434+Adarsh0427@users.noreply.github.com> Date: Thu, 28 Aug 2025 23:16:57 +0530 Subject: [PATCH 08/17] Fix Docs case-sensitivity behavior explanation --- backend/apps/common/search/query_parser.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/apps/common/search/query_parser.py b/backend/apps/common/search/query_parser.py index 764ec775ad..c72f3a08bd 100644 --- a/backend/apps/common/search/query_parser.py +++ b/backend/apps/common/search/query_parser.py @@ -103,7 +103,7 @@ class QueryParser: - Quoted strings for multi-word values - Boolean field support - Date field parsing with comparisons - - Case sensitivity: normalize field names only, preserve values + - Case sensitivity: normalize field names only, preserve original value casing - Only supports implicit AND logic between conditions """ @@ -130,10 +130,11 @@ def __init__( Args: field_schema: Dictionary mapping field names to their types - (e.g., {"stars": "number", "language": "string"}). - Field names must be lowercase. + (e.g., {"stars": "number", "language": "string"}). + Field names must be lowercase. default_field: Default field name for free text search - case_sensitive: If True, field names are case-sensitive + case_sensitive: Controls value-case behavior for downstream search, + field names are always normalized to lowercase. strict: If True, raises exception for unknown fields; if False, issues warnings Raises: From 91b01b22dffb9db1b3a13f288cdb6088deda40ad Mon Sep 17 00:00:00 2001 From: Adarsh0427 <124577434+Adarsh0427@users.noreply.github.com> Date: Sun, 31 Aug 2025 05:33:17 +0530 Subject: [PATCH 09/17] Resolved pnpm run format issue in some frontend files --- frontend/src/app/about/page.tsx | 2 +- frontend/src/app/global-error.tsx | 2 +- frontend/src/app/globals.css | 2 +- frontend/src/app/page.tsx | 6 +++--- frontend/src/app/snapshots/[id]/page.tsx | 4 ++-- frontend/src/app/snapshots/page.tsx | 2 +- frontend/src/components/Card.tsx | 4 ++-- frontend/src/components/CardDetailsPage.tsx | 2 +- frontend/src/components/Footer.tsx | 6 +++--- .../components/GeneralCompliantComponent.tsx | 2 +- frontend/src/components/Header.tsx | 8 ++++---- frontend/src/components/InfoBlock.tsx | 2 +- frontend/src/components/ItemCardList.tsx | 2 +- frontend/src/components/LoginPageContent.tsx | 2 +- frontend/src/components/LogoCarousel.tsx | 8 ++++---- frontend/src/components/MetricsCard.tsx | 2 +- .../src/components/MetricsScoreCircle.tsx | 8 ++++---- frontend/src/components/Modal.tsx | 4 ++-- frontend/src/components/ModeToggle.tsx | 2 +- frontend/src/components/ModuleCard.tsx | 2 +- frontend/src/components/ModuleForm.tsx | 20 +++++++++---------- frontend/src/components/MultiSearch.tsx | 6 +++--- frontend/src/components/NavButton.tsx | 2 +- frontend/src/components/NavDropDown.tsx | 2 +- frontend/src/components/ProgramForm.tsx | 18 ++++++++--------- frontend/src/components/RecentReleases.tsx | 2 +- frontend/src/components/RepositoriesCard.tsx | 2 +- frontend/src/components/ScrollToTop.tsx | 2 +- frontend/src/components/Search.tsx | 6 +++--- frontend/src/components/SearchPageLayout.tsx | 2 +- frontend/src/components/SingleModuleCard.tsx | 2 +- frontend/src/components/SnapshotCard.tsx | 2 +- frontend/src/components/SortBy.tsx | 2 +- .../src/components/TopContributorsList.tsx | 2 +- frontend/src/components/TruncatedText.tsx | 2 +- frontend/src/components/UserCard.tsx | 10 +++++----- frontend/src/components/UserMenu.tsx | 4 ++-- .../components/skeletons/ApiKeySkelton.tsx | 4 ++-- frontend/src/components/skeletons/Card.tsx | 4 ++-- 39 files changed, 83 insertions(+), 83 deletions(-) diff --git a/frontend/src/app/about/page.tsx b/frontend/src/app/about/page.tsx index 690a7c3819..db80862e09 100644 --- a/frontend/src/app/about/page.tsx +++ b/frontend/src/app/about/page.tsx @@ -106,7 +106,7 @@ const About = () => { return (
-

About

+

About

}> {aboutText.map((text) => (
diff --git a/frontend/src/app/global-error.tsx b/frontend/src/app/global-error.tsx index e984ccf804..518e0f9538 100644 --- a/frontend/src/app/global-error.tsx +++ b/frontend/src/app/global-error.tsx @@ -40,7 +40,7 @@ export const ErrorDisplay: React.FC = ({ statusCode, title, m

{message}

diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css index 7b3b311eb0..f7cf9df912 100644 --- a/frontend/src/app/globals.css +++ b/frontend/src/app/globals.css @@ -333,7 +333,7 @@ } .dropdown-menu { - @apply absolute top-full left-0 mt-2 w-48 rounded-lg bg-white p-3 shadow-lg dark:bg-gray-800; + @apply absolute left-0 top-full mt-2 w-48 rounded-lg bg-white p-3 shadow-lg dark:bg-gray-800; @apply invisible opacity-0 transition-all duration-200 ease-in-out; @apply flex flex-col gap-2; /* Stack items vertically */ diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index d194b77a0b..6a0228fbcf 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -141,7 +141,7 @@ export default function Home() {

Welcome to OWASP Nest

-

+

Your gateway to OWASP. Discover, engage, and help shape the future!

@@ -255,7 +255,7 @@ export default function Home() { {data.recentProjects?.map((project) => (
-

+

@@ -369,7 +369,7 @@ export default function Home() { ))}
-
+

Ready to Make a Difference?

diff --git a/frontend/src/app/snapshots/[id]/page.tsx b/frontend/src/app/snapshots/[id]/page.tsx index 3f1ebb0507..a5a2e30f4e 100644 --- a/frontend/src/app/snapshots/[id]/page.tsx +++ b/frontend/src/app/snapshots/[id]/page.tsx @@ -110,7 +110,7 @@ const SnapshotDetailsPage: React.FC = () => { return (

-
+

@@ -164,7 +164,7 @@ const SnapshotDetailsPage: React.FC = () => { {snapshot.newReleases.map((release, index) => (
diff --git a/frontend/src/app/snapshots/page.tsx b/frontend/src/app/snapshots/page.tsx index cf0312c0a8..d9d4bfc548 100644 --- a/frontend/src/app/snapshots/page.tsx +++ b/frontend/src/app/snapshots/page.tsx @@ -63,7 +63,7 @@ const SnapshotsPage: React.FC = () => { return (
-
+
{!snapshots?.length ? (
No Snapshots found
diff --git a/frontend/src/components/Card.tsx b/frontend/src/components/Card.tsx index 960881d8e6..a042af4604 100644 --- a/frontend/src/components/Card.tsx +++ b/frontend/src/components/Card.tsx @@ -29,7 +29,7 @@ const Card = ({ timeline, }: CardProps) => { return ( -
+
{/* Card Header with Badge and Title */}
@@ -54,7 +54,7 @@ const Card = ({ {/* Project title and link */}

{ router.push(`${window.location.pathname}/edit`) }} diff --git a/frontend/src/components/Footer.tsx b/frontend/src/components/Footer.tsx index cc0feb2821..a4e6504c5b 100644 --- a/frontend/src/components/Footer.tsx +++ b/frontend/src/components/Footer.tsx @@ -20,8 +20,8 @@ export default function Footer() { }, []) return ( -