diff --git a/services/search/query.py b/services/search/query.py index 3bb398446..194610f38 100644 --- a/services/search/query.py +++ b/services/search/query.py @@ -34,8 +34,14 @@ def _extract_entities(query: str) -> Dict[str, List[str]]: cleaned = query if qtype: cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip() - for token in re.findall(r"\b[A-Z][a-zA-Z]+\b", cleaned): - entities["names"].append(token) + # Unicode-aware capitalized-word (name) detection. The old [A-Z][a-zA-Z]+ + # class missed non-ASCII names like "İstanbul"/"Zürich" (dropped) and + # "São" (shredded). Keep the ASCII behaviour — the word boundary already + # excludes camelCase mid-word capitals — by requiring an all-alphabetic + # token of length > 1 whose first character is uppercase. + for token in re.findall(r"\b\w+\b", cleaned): + if len(token) > 1 and token[0].isupper() and token.isalpha(): + entities["names"].append(token) for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned): entities["dates"].append(year) month_day_year = re.findall( diff --git a/tests/test_search_query_unicode_names.py b/tests/test_search_query_unicode_names.py new file mode 100644 index 000000000..104ba310f --- /dev/null +++ b/tests/test_search_query_unicode_names.py @@ -0,0 +1,31 @@ +"""Regression: _extract_entities must find non-ASCII capitalized names. + +The name extractor used the ASCII-only class [A-Z][a-zA-Z]+, so a query like +"İstanbul weather" or "Zürich hotels" yielded no name entities at all, and +"São Paulo" lost "São" — non-English/accented place and proper names were +silently dropped from query enhancement. Detection is now Unicode-aware; +ASCII behaviour (including camelCase mid-word capitals not counting as names) +is preserved. +""" +from services.search.query import _extract_entities + + +def _names(q): + return _extract_entities(q)["names"] + + +def test_non_ascii_names_are_extracted(): + assert "İstanbul" in _names("İstanbul weather") + assert "Zürich" in _names("Zürich hotels") + assert set(_names("trip to São Paulo")) >= {"São", "Paulo"} + + +def test_ascii_names_unchanged(): + assert _names("What did Alice do in 2024") == ["Alice"] + assert _names("news about OpenAI and Google") == ["OpenAI", "Google"] + + +def test_lowercase_camelcase_and_numbers_are_not_names(): + assert _names("the iphone price") == [] + assert _names("iPhone price") == [] # mid-word capital is not a name + assert _names("top 50 albums") == []