mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-07-08 11:56:59 +00:00
fix(search): extract non-ASCII capitalized names in _extract_entities
_extract_entities used the ASCII-only class [A-Z][a-zA-Z]+ to pull name
entities from a query, so non-ASCII names were dropped ("İstanbul",
"Zürich" yielded nothing) or shredded ("São Paulo" -> only "Paulo"),
degrading query enhancement for non-English/accented searches. Match
Unicode words and keep the alphabetic, uppercase-initial ones; ASCII
behaviour (the word boundary already excludes camelCase mid-word
capitals) is unchanged.
This commit is contained in:
@@ -34,8 +34,14 @@ def _extract_entities(query: str) -> Dict[str, List[str]]:
|
|||||||
cleaned = query
|
cleaned = query
|
||||||
if qtype:
|
if qtype:
|
||||||
cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip()
|
cleaned = re.sub(rf"^{qtype}\b", "", cleaned, flags=re.I).strip()
|
||||||
for token in re.findall(r"\b[A-Z][a-zA-Z]+\b", cleaned):
|
# Unicode-aware capitalized-word (name) detection. The old [A-Z][a-zA-Z]+
|
||||||
entities["names"].append(token)
|
# class missed non-ASCII names like "İstanbul"/"Zürich" (dropped) and
|
||||||
|
# "São" (shredded). Keep the ASCII behaviour — the word boundary already
|
||||||
|
# excludes camelCase mid-word capitals — by requiring an all-alphabetic
|
||||||
|
# token of length > 1 whose first character is uppercase.
|
||||||
|
for token in re.findall(r"\b\w+\b", cleaned):
|
||||||
|
if len(token) > 1 and token[0].isupper() and token.isalpha():
|
||||||
|
entities["names"].append(token)
|
||||||
for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned):
|
for year in re.findall(r"\b(?:19|20)\d{2}\b", cleaned):
|
||||||
entities["dates"].append(year)
|
entities["dates"].append(year)
|
||||||
month_day_year = re.findall(
|
month_day_year = re.findall(
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
"""Regression: _extract_entities must find non-ASCII capitalized names.
|
||||||
|
|
||||||
|
The name extractor used the ASCII-only class [A-Z][a-zA-Z]+, so a query like
|
||||||
|
"İstanbul weather" or "Zürich hotels" yielded no name entities at all, and
|
||||||
|
"São Paulo" lost "São" — non-English/accented place and proper names were
|
||||||
|
silently dropped from query enhancement. Detection is now Unicode-aware;
|
||||||
|
ASCII behaviour (including camelCase mid-word capitals not counting as names)
|
||||||
|
is preserved.
|
||||||
|
"""
|
||||||
|
from services.search.query import _extract_entities
|
||||||
|
|
||||||
|
|
||||||
|
def _names(q):
|
||||||
|
return _extract_entities(q)["names"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_ascii_names_are_extracted():
|
||||||
|
assert "İstanbul" in _names("İstanbul weather")
|
||||||
|
assert "Zürich" in _names("Zürich hotels")
|
||||||
|
assert set(_names("trip to São Paulo")) >= {"São", "Paulo"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_ascii_names_unchanged():
|
||||||
|
assert _names("What did Alice do in 2024") == ["Alice"]
|
||||||
|
assert _names("news about OpenAI and Google") == ["OpenAI", "Google"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_lowercase_camelcase_and_numbers_are_not_names():
|
||||||
|
assert _names("the iphone price") == []
|
||||||
|
assert _names("iPhone price") == [] # mid-word capital is not a name
|
||||||
|
assert _names("top 50 albums") == []
|
||||||
Reference in New Issue
Block a user