Merge pull request #3658 from lonvia/minor-query-parsing-optimisations

Minor query parsing optimisations

Author: lonvia

docs/develop/ICU-Tokenizer-Modules.md

@@ -60,13 +60,19 @@ The order of phrases matters to Nominatim when doing further processing.
 Thus, while you may split or join phrases, you should not reorder them
 unless you really know what you are doing.
 
-Phrase types (`nominatim_api.search.PhraseType`) can further help narrowing
-down how the tokens in the phrase are interpreted. The following phrase types
-are known:
-
-::: nominatim_api.search.PhraseType
-    options:
-        heading_level: 6
+Phrase types can further help narrowing down how the tokens in the phrase
+are interpreted. The following phrase types are known:
+
+| Name           | Description |
+|----------------|-------------|
+| PHRASE_ANY     | No specific designation (i.e. source is free-form query) |
+| PHRASE_AMENITY | Contains name or type of a POI |
+| PHRASE_STREET  | Contains a street name optionally with a housenumber |
+| PHRASE_CITY    | Contains the postal city |
+| PHRASE_COUNTY  | Contains the equivalent of a county |
+| PHRASE_STATE   | Contains a state or province |
+| PHRASE_POSTCODE| Contains a postal code |
+| PHRASE_COUNTRY | Contains the country name or code |
 
 
 ## Custom sanitizer modules

src/nominatim_api/core.py

@@ -26,7 +26,7 @@
 from .status import get_status, StatusResult
 from .lookup import get_places, get_detailed_place
 from .reverse import ReverseGeocoder
-from .search import ForwardGeocoder, Phrase, PhraseType, make_query_analyzer
+from . import search as nsearch
 from . import types as ntyp
 from .results import DetailedResult, ReverseResult, SearchResults
 
@@ -207,7 +207,7 @@ async def details(self, place: ntyp.PlaceRef, **params: Any) -> Optional[Detaile
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             if details.keywords:
-                await make_query_analyzer(conn)
+                await nsearch.make_query_analyzer(conn)
             return await get_detailed_place(conn, place, details)
 
     async def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults:
@@ -219,7 +219,7 @@ async def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> Search
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             if details.keywords:
-                await make_query_analyzer(conn)
+                await nsearch.make_query_analyzer(conn)
             return await get_places(conn, places, details)
 
     async def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]:
@@ -237,7 +237,7 @@ async def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[Reverse
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             if details.keywords:
-                await make_query_analyzer(conn)
+                await nsearch.make_query_analyzer(conn)
             geocoder = ReverseGeocoder(conn, details,
                                        self.reverse_restrict_to_country_area)
             return await geocoder.lookup(coord)
@@ -251,10 +251,10 @@ async def search(self, query: str, **params: Any) -> SearchResults:
 
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
-            geocoder = ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params),
-                                       self.config.get_int('REQUEST_TIMEOUT')
-                                       if self.config.REQUEST_TIMEOUT else None)
-            phrases = [Phrase(PhraseType.NONE, p.strip()) for p in query.split(',')]
+            geocoder = nsearch.ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params),
+                                               self.config.get_int('REQUEST_TIMEOUT')
+                                               if self.config.REQUEST_TIMEOUT else None)
+            phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p.strip()) for p in query.split(',')]
             return await geocoder.lookup(phrases)
 
     async def search_address(self, amenity: Optional[str] = None,
@@ -271,22 +271,22 @@ async def search_address(self, amenity: Optional[str] = None,
             conn.set_query_timeout(self.query_timeout)
             details = ntyp.SearchDetails.from_kwargs(params)
 
-            phrases: List[Phrase] = []
+            phrases: List[nsearch.Phrase] = []
 
             if amenity:
-                phrases.append(Phrase(PhraseType.AMENITY, amenity))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_AMENITY, amenity))
             if street:
-                phrases.append(Phrase(PhraseType.STREET, street))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_STREET, street))
             if city:
-                phrases.append(Phrase(PhraseType.CITY, city))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_CITY, city))
             if county:
-                phrases.append(Phrase(PhraseType.COUNTY, county))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_COUNTY, county))
             if state:
-                phrases.append(Phrase(PhraseType.STATE, state))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_STATE, state))
             if postalcode:
-                phrases.append(Phrase(PhraseType.POSTCODE, postalcode))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_POSTCODE, postalcode))
             if country:
-                phrases.append(Phrase(PhraseType.COUNTRY, country))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_COUNTRY, country))
 
             if not phrases:
                 raise UsageError('Nothing to search for.')
@@ -309,9 +309,9 @@ async def search_address(self, amenity: Optional[str] = None,
                 if amenity:
                     details.layers |= ntyp.DataLayer.POI
 
-            geocoder = ForwardGeocoder(conn, details,
-                                       self.config.get_int('REQUEST_TIMEOUT')
-                                       if self.config.REQUEST_TIMEOUT else None)
+            geocoder = nsearch.ForwardGeocoder(conn, details,
+                                               self.config.get_int('REQUEST_TIMEOUT')
+                                               if self.config.REQUEST_TIMEOUT else None)
             return await geocoder.lookup(phrases)
 
     async def search_category(self, categories: List[Tuple[str, str]],
@@ -328,15 +328,15 @@ async def search_category(self, categories: List[Tuple[str, str]],
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             if near_query:
-                phrases = [Phrase(PhraseType.NONE, p) for p in near_query.split(',')]
+                phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p) for p in near_query.split(',')]
             else:
                 phrases = []
                 if details.keywords:
-                    await make_query_analyzer(conn)
+                    await nsearch.make_query_analyzer(conn)
 
-            geocoder = ForwardGeocoder(conn, details,
-                                       self.config.get_int('REQUEST_TIMEOUT')
-                                       if self.config.REQUEST_TIMEOUT else None)
+            geocoder = nsearch.ForwardGeocoder(conn, details,
+                                               self.config.get_int('REQUEST_TIMEOUT')
+                                               if self.config.REQUEST_TIMEOUT else None)
             return await geocoder.lookup_pois(categories, phrases)
 
 

src/nominatim_api/search/__init__.py

@@ -9,5 +9,12 @@
 """
 from .geocoder import (ForwardGeocoder as ForwardGeocoder)
 from .query import (Phrase as Phrase,
-                    PhraseType as PhraseType)
+                    PHRASE_ANY as PHRASE_ANY,
+                    PHRASE_AMENITY as PHRASE_AMENITY,
+                    PHRASE_STREET as PHRASE_STREET,
+                    PHRASE_CITY as PHRASE_CITY,
+                    PHRASE_COUNTY as PHRASE_COUNTY,
+                    PHRASE_STATE as PHRASE_STATE,
+                    PHRASE_POSTCODE as PHRASE_POSTCODE,
+                    PHRASE_COUNTRY as PHRASE_COUNTRY)
 from .query_analyzer_factory import (make_query_analyzer as make_query_analyzer)

src/nominatim_api/search/db_search_builder.py

@@ -11,7 +11,7 @@
 import heapq
 
 from ..types import SearchDetails, DataLayer
-from .query import QueryStruct, Token, TokenType, TokenRange, BreakType
+from . import query as qmod
 from .token_assignment import TokenAssignment
 from . import db_search_fields as dbf
 from . import db_searches as dbs
@@ -51,7 +51,7 @@ class SearchBuilder:
     """ Build the abstract search queries from token assignments.
     """
 
-    def __init__(self, query: QueryStruct, details: SearchDetails) -> None:
+    def __init__(self, query: qmod.QueryStruct, details: SearchDetails) -> None:
         self.query = query
         self.details = details
 
@@ -97,7 +97,7 @@ def build(self, assignment: TokenAssignment) -> Iterator[dbs.AbstractSearch]:
                 builder = self.build_poi_search(sdata)
             elif assignment.housenumber:
                 hnr_tokens = self.query.get_tokens(assignment.housenumber,
-                                                   TokenType.HOUSENUMBER)
+                                                   qmod.TOKEN_HOUSENUMBER)
                 builder = self.build_housenumber_search(sdata, hnr_tokens, assignment.address)
             else:
                 builder = self.build_special_search(sdata, assignment.address,
@@ -128,7 +128,7 @@ def build_poi_search(self, sdata: dbf.SearchData) -> Iterator[dbs.AbstractSearch
             yield dbs.PoiSearch(sdata)
 
     def build_special_search(self, sdata: dbf.SearchData,
-                             address: List[TokenRange],
+                             address: List[qmod.TokenRange],
                              is_category: bool) -> Iterator[dbs.AbstractSearch]:
         """ Build abstract search queries for searches that do not involve
             a named place.
@@ -150,8 +150,8 @@ def build_special_search(self, sdata: dbf.SearchData,
                                                  lookups.Restrict)]
             yield dbs.PostcodeSearch(penalty, sdata)
 
-    def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[Token],
-                                 address: List[TokenRange]) -> Iterator[dbs.AbstractSearch]:
+    def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[qmod.Token],
+                                 address: List[qmod.TokenRange]) -> Iterator[dbs.AbstractSearch]:
         """ Build a simple address search for special entries where the
             housenumber is the main name token.
         """
@@ -173,7 +173,7 @@ def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[Token],
                                                  list(partials), lookups.LookupAll))
         else:
             addr_fulls = [t.token for t
-                          in self.query.get_tokens(address[0], TokenType.WORD)]
+                          in self.query.get_tokens(address[0], qmod.TOKEN_WORD)]
             if len(addr_fulls) > 5:
                 return
             sdata.lookups.append(
@@ -183,7 +183,7 @@ def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[Token],
         yield dbs.PlaceSearch(0.05, sdata, expected_count)
 
     def build_name_search(self, sdata: dbf.SearchData,
-                          name: TokenRange, address: List[TokenRange],
+                          name: qmod.TokenRange, address: List[qmod.TokenRange],
                           is_category: bool) -> Iterator[dbs.AbstractSearch]:
         """ Build abstract search queries for simple name or address searches.
         """
@@ -196,7 +196,7 @@ def build_name_search(self, sdata: dbf.SearchData,
                 sdata.lookups = lookup
                 yield dbs.PlaceSearch(penalty + name_penalty, sdata, count)
 
-    def yield_lookups(self, name: TokenRange, address: List[TokenRange]
+    def yield_lookups(self, name: qmod.TokenRange, address: List[qmod.TokenRange]
                       ) -> Iterator[Tuple[float, int, List[dbf.FieldLookup]]]:
         """ Yield all variants how the given name and address should best
             be searched for. This takes into account how frequent the terms
@@ -216,7 +216,7 @@ def yield_lookups(self, name: TokenRange, address: List[TokenRange]
 
         addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 30000
         # Partial term to frequent. Try looking up by rare full names first.
-        name_fulls = self.query.get_tokens(name, TokenType.WORD)
+        name_fulls = self.query.get_tokens(name, qmod.TOKEN_WORD)
         if name_fulls:
             fulls_count = sum(t.count for t in name_fulls)
 
@@ -235,7 +235,7 @@ def yield_lookups(self, name: TokenRange, address: List[TokenRange]
                 self.get_name_address_ranking(list(name_partials.keys()), addr_partials)
 
     def get_name_address_ranking(self, name_tokens: List[int],
-                                 addr_partials: List[Token]) -> List[dbf.FieldLookup]:
+                                 addr_partials: List[qmod.Token]) -> List[dbf.FieldLookup]:
         """ Create a ranking expression looking up by name and address.
         """
         lookup = [dbf.FieldLookup('name_vector', name_tokens, lookups.LookupAll)]
@@ -257,7 +257,7 @@ def get_name_address_ranking(self, name_tokens: List[int],
 
         return lookup
 
-    def get_full_name_ranking(self, name_fulls: List[Token], addr_partials: List[Token],
+    def get_full_name_ranking(self, name_fulls: List[qmod.Token], addr_partials: List[qmod.Token],
                               use_lookup: bool) -> List[dbf.FieldLookup]:
         """ Create a ranking expression with full name terms and
             additional address lookup. When 'use_lookup' is true, then
@@ -281,19 +281,19 @@ def get_full_name_ranking(self, name_fulls: List[Token], addr_partials: List[Tok
         return dbf.lookup_by_any_name([t.token for t in name_fulls],
                                       addr_restrict_tokens, addr_lookup_tokens)
 
-    def get_name_ranking(self, trange: TokenRange,
+    def get_name_ranking(self, trange: qmod.TokenRange,
                          db_field: str = 'name_vector') -> dbf.FieldRanking:
         """ Create a ranking expression for a name term in the given range.
         """
-        name_fulls = self.query.get_tokens(trange, TokenType.WORD)
+        name_fulls = self.query.get_tokens(trange, qmod.TOKEN_WORD)
         ranks = [dbf.RankedTokens(t.penalty, [t.token]) for t in name_fulls]
         ranks.sort(key=lambda r: r.penalty)
         # Fallback, sum of penalty for partials
         name_partials = self.query.get_partials_list(trange)
         default = sum(t.penalty for t in name_partials) + 0.2
         return dbf.FieldRanking(db_field, default, ranks)
 
-    def get_addr_ranking(self, trange: TokenRange) -> dbf.FieldRanking:
+    def get_addr_ranking(self, trange: qmod.TokenRange) -> dbf.FieldRanking:
         """ Create a list of ranking expressions for an address term
             for the given ranges.
         """
@@ -304,10 +304,10 @@ def get_addr_ranking(self, trange: TokenRange) -> dbf.FieldRanking:
         while todo:
             neglen, pos, rank = heapq.heappop(todo)
             for tlist in self.query.nodes[pos].starting:
-                if tlist.ttype in (TokenType.PARTIAL, TokenType.WORD):
+                if tlist.ttype in (qmod.TOKEN_PARTIAL, qmod.TOKEN_WORD):
                     if tlist.end < trange.end:
                         chgpenalty = PENALTY_WORDCHANGE[self.query.nodes[tlist.end].btype]
-                        if tlist.ttype == TokenType.PARTIAL:
+                        if tlist.ttype == qmod.TOKEN_PARTIAL:
                             penalty = rank.penalty + chgpenalty \
                                       + max(t.penalty for t in tlist.tokens)
                             heapq.heappush(todo, (neglen - 1, tlist.end,
@@ -317,7 +317,7 @@ def get_addr_ranking(self, trange: TokenRange) -> dbf.FieldRanking:
                                 heapq.heappush(todo, (neglen - 1, tlist.end,
                                                       rank.with_token(t, chgpenalty)))
                     elif tlist.end == trange.end:
-                        if tlist.ttype == TokenType.PARTIAL:
+                        if tlist.ttype == qmod.TOKEN_PARTIAL:
                             ranks.append(dbf.RankedTokens(rank.penalty
                                                           + max(t.penalty for t in tlist.tokens),
                                                           rank.tokens))
@@ -357,11 +357,11 @@ def get_search_data(self, assignment: TokenAssignment) -> Optional[dbf.SearchDat
         if assignment.housenumber:
             sdata.set_strings('housenumbers',
                               self.query.get_tokens(assignment.housenumber,
-                                                    TokenType.HOUSENUMBER))
+                                                    qmod.TOKEN_HOUSENUMBER))
         if assignment.postcode:
             sdata.set_strings('postcodes',
                               self.query.get_tokens(assignment.postcode,
-                                                    TokenType.POSTCODE))
+                                                    qmod.TOKEN_POSTCODE))
         if assignment.qualifier:
             tokens = self.get_qualifier_tokens(assignment.qualifier)
             if not tokens:
@@ -386,23 +386,23 @@ def get_search_data(self, assignment: TokenAssignment) -> Optional[dbf.SearchDat
 
         return sdata
 
-    def get_country_tokens(self, trange: TokenRange) -> List[Token]:
+    def get_country_tokens(self, trange: qmod.TokenRange) -> List[qmod.Token]:
         """ Return the list of country tokens for the given range,
             optionally filtered by the country list from the details
             parameters.
         """
-        tokens = self.query.get_tokens(trange, TokenType.COUNTRY)
+        tokens = self.query.get_tokens(trange, qmod.TOKEN_COUNTRY)
         if self.details.countries:
             tokens = [t for t in tokens if t.lookup_word in self.details.countries]
 
         return tokens
 
-    def get_qualifier_tokens(self, trange: TokenRange) -> List[Token]:
+    def get_qualifier_tokens(self, trange: qmod.TokenRange) -> List[qmod.Token]:
         """ Return the list of qualifier tokens for the given range,
             optionally filtered by the qualifier list from the details
             parameters.
         """
-        tokens = self.query.get_tokens(trange, TokenType.QUALIFIER)
+        tokens = self.query.get_tokens(trange, qmod.TOKEN_QUALIFIER)
         if self.details.categories:
             tokens = [t for t in tokens if t.get_category() in self.details.categories]
 
@@ -415,7 +415,7 @@ def get_near_items(self, assignment: TokenAssignment) -> Optional[dbf.WeightedCa
         """
         if assignment.near_item:
             tokens: Dict[Tuple[str, str], float] = {}
-            for t in self.query.get_tokens(assignment.near_item, TokenType.NEAR_ITEM):
+            for t in self.query.get_tokens(assignment.near_item, qmod.TOKEN_NEAR_ITEM):
                 cat = t.get_category()
                 # The category of a near search will be that of near_item.
                 # Thus, if search is restricted to a category parameter,
@@ -429,11 +429,11 @@ def get_near_items(self, assignment: TokenAssignment) -> Optional[dbf.WeightedCa
 
 
 PENALTY_WORDCHANGE = {
-    BreakType.START: 0.0,
-    BreakType.END: 0.0,
-    BreakType.PHRASE: 0.0,
-    BreakType.SOFT_PHRASE: 0.0,
-    BreakType.WORD: 0.1,
-    BreakType.PART: 0.2,
-    BreakType.TOKEN: 0.4
+    qmod.BREAK_START: 0.0,
+    qmod.BREAK_END: 0.0,
+    qmod.BREAK_PHRASE: 0.0,
+    qmod.BREAK_SOFT_PHRASE: 0.0,
+    qmod.BREAK_WORD: 0.1,
+    qmod.BREAK_PART: 0.2,
+    qmod.BREAK_TOKEN: 0.4
 }