Merge pull request #3714 from lonvia/postcode-update-without-project-dir

Change postcode update function to work without a project directory

Author: lonvia

src/nominatim_db/tools/postcodes.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Functions for importing, updating and otherwise maintaining the table
@@ -64,11 +64,15 @@ def add(self, postcode: str, x: float, y: float) -> None:
             if normalized:
                 self.collected[normalized] += (x, y)
 
-    def commit(self, conn: Connection, analyzer: AbstractAnalyzer, project_dir: Path) -> None:
-        """ Update postcodes for the country from the postcodes selected so far
-            as well as any externally supplied postcodes.
+    def commit(self, conn: Connection, analyzer: AbstractAnalyzer,
+               project_dir: Optional[Path]) -> None:
+        """ Update postcodes for the country from the postcodes selected so far.
+
+            When 'project_dir' is set, then any postcode files found in this
+            directory are taken into account as well.
         """
-        self._update_from_external(analyzer, project_dir)
+        if project_dir is not None:
+            self._update_from_external(analyzer, project_dir)
         to_add, to_delete, to_update = self._compute_changes(conn)
 
         LOG.info("Processing country '%s' (%s added, %s deleted, %s updated).",
@@ -170,7 +174,7 @@ def _open_external(self, project_dir: Path) -> Optional[TextIO]:
         return None
 
 
-def update_postcodes(dsn: str, project_dir: Path, tokenizer: AbstractTokenizer) -> None:
+def update_postcodes(dsn: str, project_dir: Optional[Path], tokenizer: AbstractTokenizer) -> None:
     """ Update the table of artificial postcodes.
 
         Computes artificial postcode centroids from the placex table,

test/bdd/test_db.py

@@ -11,7 +11,6 @@
 """
 import asyncio
 import re
-from pathlib import Path
 
 import psycopg
 
@@ -130,7 +129,7 @@ def do_import(db_conn, def_config):
     create_table_triggers(db_conn, def_config)
     asyncio.run(load_data(def_config.get_libpq_dsn(), 1))
     tokenizer = tokenizer_factory.get_tokenizer_for_db(def_config)
-    update_postcodes(def_config.get_libpq_dsn(), Path('/xxxx'), tokenizer)
+    update_postcodes(def_config.get_libpq_dsn(), None, tokenizer)
     cli.nominatim(['index', '-q'], def_config.environ)
 
     return _collect_place_ids(db_conn)

test/python/tools/test_postcodes.py

@@ -85,19 +85,17 @@ def _insert_implicit_postcode(osm_id, country, geometry, address):
     return _insert_implicit_postcode
 
 
-def test_postcodes_empty(dsn, postcode_table, place_table,
-                         tmp_path, tokenizer):
-    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+def test_postcodes_empty(dsn, postcode_table, place_table, tokenizer):
+    postcodes.update_postcodes(dsn, None, tokenizer)
 
     assert not postcode_table.row_set
 
 
-def test_postcodes_add_new(dsn, postcode_table, tmp_path,
-                           insert_implicit_postcode, tokenizer):
+def test_postcodes_add_new(dsn, postcode_table, insert_implicit_postcode, tokenizer):
     insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='9486'))
     postcode_table.add('yy', '9486', 99, 34)
 
-    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+    postcodes.update_postcodes(dsn, None, tokenizer)
 
     assert postcode_table.row_set == {('xx', '9486', 10, 12), }
 
@@ -112,49 +110,48 @@ def test_postcodes_replace_coordinates(dsn, postcode_table, tmp_path,
     assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
 
 
-def test_postcodes_replace_coordinates_close(dsn, postcode_table, tmp_path,
+def test_postcodes_replace_coordinates_close(dsn, postcode_table,
                                              insert_implicit_postcode, tokenizer):
     insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
     postcode_table.add('xx', 'AB 4511', 10, 11.99999)
 
-    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+    postcodes.update_postcodes(dsn, None, tokenizer)
 
     assert postcode_table.row_set == {('xx', 'AB 4511', 10, 11.99999)}
 
 
-def test_postcodes_remove(dsn, postcode_table, tmp_path,
+def test_postcodes_remove(dsn, postcode_table,
                           insert_implicit_postcode, tokenizer):
     insert_implicit_postcode(1, 'xx', 'POINT(10 12)', dict(postcode='AB 4511'))
     postcode_table.add('xx', 'badname', 10, 12)
 
-    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+    postcodes.update_postcodes(dsn, None, tokenizer)
 
     assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
 
 
-def test_postcodes_ignore_empty_country(dsn, postcode_table, tmp_path,
+def test_postcodes_ignore_empty_country(dsn, postcode_table,
                                         insert_implicit_postcode, tokenizer):
     insert_implicit_postcode(1, None, 'POINT(10 12)', dict(postcode='AB 4511'))
-    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+    postcodes.update_postcodes(dsn, None, tokenizer)
     assert not postcode_table.row_set
 
 
-def test_postcodes_remove_all(dsn, postcode_table, place_table,
-                              tmp_path, tokenizer):
+def test_postcodes_remove_all(dsn, postcode_table, place_table, tokenizer):
     postcode_table.add('ch', '5613', 10, 12)
-    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+    postcodes.update_postcodes(dsn, None, tokenizer)
 
     assert not postcode_table.row_set
 
 
-def test_postcodes_multi_country(dsn, postcode_table, tmp_path,
+def test_postcodes_multi_country(dsn, postcode_table,
                                  insert_implicit_postcode, tokenizer):
     insert_implicit_postcode(1, 'de', 'POINT(10 12)', dict(postcode='54451'))
     insert_implicit_postcode(2, 'cc', 'POINT(100 56)', dict(postcode='DD23 T'))
     insert_implicit_postcode(3, 'de', 'POINT(10.3 11.0)', dict(postcode='54452'))
     insert_implicit_postcode(4, 'cc', 'POINT(10.3 11.0)', dict(postcode='54452'))
 
-    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+    postcodes.update_postcodes(dsn, None, tokenizer)
 
     assert postcode_table.row_set == {('de', '54451', 10, 12),
                                       ('de', '54452', 10.3, 11.0),
@@ -211,7 +208,7 @@ def test_can_compute(dsn, table_factory):
     assert postcodes.can_compute(dsn)
 
 
-def test_no_placex_entry(dsn, tmp_path, temp_db_cursor, place_row, postcode_table, tokenizer):
+def test_no_placex_entry(dsn, temp_db_cursor, place_row, postcode_table, tokenizer):
     # Rewrite the get_country_code function to verify its execution.
     temp_db_cursor.execute("""
         CREATE OR REPLACE FUNCTION get_country_code(place geometry)
@@ -220,12 +217,12 @@ def test_no_placex_entry(dsn, tmp_path, temp_db_cursor, place_row, postcode_tabl
         END; $$ LANGUAGE plpgsql;
     """)
     place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
-    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+    postcodes.update_postcodes(dsn, None, tokenizer)
 
     assert postcode_table.row_set == {('yy', 'AB 4511', 10, 12)}
 
 
-def test_discard_badly_formatted_postcodes(dsn, tmp_path, temp_db_cursor, place_row,
+def test_discard_badly_formatted_postcodes(dsn, temp_db_cursor, place_row,
                                            postcode_table, tokenizer):
     # Rewrite the get_country_code function to verify its execution.
     temp_db_cursor.execute("""
@@ -235,6 +232,6 @@ def test_discard_badly_formatted_postcodes(dsn, tmp_path, temp_db_cursor, place_
         END; $$ LANGUAGE plpgsql;
     """)
     place_row(geom='SRID=4326;POINT(10 12)', address=dict(postcode='AB 4511'))
-    postcodes.update_postcodes(dsn, tmp_path, tokenizer)
+    postcodes.update_postcodes(dsn, None, tokenizer)
 
     assert not postcode_table.row_set