feat: download fixed files list instead of scraping

Author: takb

osm/dl_pbf.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# read the pbf list and download each file
+pbf_list="pbf_list.txt"
+if [[ ! -f "$pbf_list" ]]; then
+  echo "$(date +'%Y-%m-%d %H:%M:%S') - pbf list not found!" >> download.log
+  exit 1
+fi
+while IFS= read -r file; do
+  if [[ -z "$file" || "$file" =~ ^# ]]; then
+    continue
+  fi
+  mkdir -p "$(dirname "$file")"
+  echo "$(date +'%Y-%m-%d %H:%M:%S') - Downloading https://download.geofabrik.de/$file" >> download.log
+  wget -q "https://download.geofabrik.de/$file" -O "$file"
+  if [[ $? -ne 0 ]]; then
+    echo "$(date +'%Y-%m-%d %H:%M:%S') - Failed to download $file"  >> download.log
+  else
+    # output timestamp
+    echo "$(date +'%Y-%m-%d %H:%M:%S') - Downloaded $file" >> download.log
+  fi
+done < "$pbf_list"
+echo "$(date +'%Y-%m-%d %H:%M:%S') - All downloads completed." >> download.log

osm/pbf_list.txt

@@ -0,0 +1,115 @@
+africa-latest.osm.pbf
+antarctica-latest.osm.pbf
+asia/afghanistan-latest.osm.pbf
+asia/armenia-latest.osm.pbf
+asia/azerbaijan-latest.osm.pbf
+asia/bangladesh-latest.osm.pbf
+asia/bhutan-latest.osm.pbf
+asia/cambodia-latest.osm.pbf
+asia/china-latest.osm.pbf
+asia/east-timor-latest.osm.pbf
+asia/gcc-states-latest.osm.pbf
+asia/india-latest.osm.pbf
+asia/indonesia-latest.osm.pbf
+asia/iran-latest.osm.pbf
+asia/iraq-latest.osm.pbf
+asia/israel-and-palestine-latest.osm.pbf
+asia/japan-latest.osm.pbf
+asia/jordan-latest.osm.pbf
+asia/kazakhstan-latest.osm.pbf
+asia/kyrgyzstan-latest.osm.pbf
+asia/laos-latest.osm.pbf
+asia/lebanon-latest.osm.pbf
+asia/malaysia-singapore-brunei-latest.osm.pbf
+asia/maldives-latest.osm.pbf
+asia/mongolia-latest.osm.pbf
+asia/myanmar-latest.osm.pbf
+asia/nepal-latest.osm.pbf
+asia/north-korea-latest.osm.pbf
+asia/pakistan-latest.osm.pbf
+asia/philippines-latest.osm.pbf
+asia/south-korea-latest.osm.pbf
+asia/sri-lanka-latest.osm.pbf
+asia/syria-latest.osm.pbf
+asia/taiwan-latest.osm.pbf
+asia/tajikistan-latest.osm.pbf
+asia/thailand-latest.osm.pbf
+asia/turkmenistan-latest.osm.pbf
+asia/uzbekistan-latest.osm.pbf
+asia/vietnam-latest.osm.pbf
+asia/yemen-latest.osm.pbf
+australia-oceania-latest.osm.pbf
+central-america-latest.osm.pbf
+europe/albania-latest.osm.pbf
+europe/andorra-latest.osm.pbf
+europe/austria-latest.osm.pbf
+europe/azores-latest.osm.pbf
+europe/belarus-latest.osm.pbf
+europe/belgium-latest.osm.pbf
+europe/bosnia-herzegovina-latest.osm.pbf
+europe/bulgaria-latest.osm.pbf
+europe/croatia-latest.osm.pbf
+europe/cyprus-latest.osm.pbf
+europe/czech-republic-latest.osm.pbf
+europe/denmark-latest.osm.pbf
+europe/estonia-latest.osm.pbf
+europe/faroe-islands-latest.osm.pbf
+europe/finland-latest.osm.pbf
+europe/france-latest.osm.pbf
+europe/georgia-latest.osm.pbf
+europe/germany-latest.osm.pbf
+europe/great-britain-latest.osm.pbf
+europe/greece-latest.osm.pbf
+europe/guernsey-jersey-latest.osm.pbf
+europe/hungary-latest.osm.pbf
+europe/iceland-latest.osm.pbf
+europe/ireland-and-northern-ireland-latest.osm.pbf
+europe/isle-of-man-latest.osm.pbf
+europe/italy-latest.osm.pbf
+europe/kosovo-latest.osm.pbf
+europe/latvia-latest.osm.pbf
+europe/liechtenstein-latest.osm.pbf
+europe/lithuania-latest.osm.pbf
+europe/luxembourg-latest.osm.pbf
+europe/macedonia-latest.osm.pbf
+europe/malta-latest.osm.pbf
+europe/moldova-latest.osm.pbf
+europe/monaco-latest.osm.pbf
+europe/montenegro-latest.osm.pbf
+europe/netherlands-latest.osm.pbf
+europe/norway-latest.osm.pbf
+europe/poland-latest.osm.pbf
+europe/portugal-latest.osm.pbf
+europe/romania-latest.osm.pbf
+europe/serbia-latest.osm.pbf
+europe/slovakia-latest.osm.pbf
+europe/slovenia-latest.osm.pbf
+europe/spain-latest.osm.pbf
+europe/sweden-latest.osm.pbf
+europe/switzerland-latest.osm.pbf
+europe/turkey-latest.osm.pbf
+europe/ukraine-latest.osm.pbf
+europe/united-kingdom-latest.osm.pbf
+north-america/canada/alberta-latest.osm.pbf
+north-america/canada/british-columbia-latest.osm.pbf
+north-america/canada/manitoba-latest.osm.pbf
+north-america/canada/new-brunswick-latest.osm.pbf
+north-america/canada/newfoundland-and-labrador-latest.osm.pbf
+north-america/canada/northwest-territories-latest.osm.pbf
+north-america/canada/nova-scotia-latest.osm.pbf
+north-america/canada/nunavut-latest.osm.pbf
+north-america/canada/ontario-latest.osm.pbf
+north-america/canada/prince-edward-island-latest.osm.pbf
+north-america/canada/quebec-latest.osm.pbf
+north-america/canada/saskatchewan-latest.osm.pbf
+north-america/canada/yukon-latest.osm.pbf
+north-america/greenland-latest.osm.pbf
+north-america/mexico-latest.osm.pbf
+north-america/us-midwest-latest.osm.pbf
+north-america/us-northeast-latest.osm.pbf
+north-america/us-pacific-latest.osm.pbf
+north-america/us-south-latest.osm.pbf
+north-america/us-west-latest.osm.pbf
+russia-latest.osm.pbf
+south-america-latest.osm.pbf
+# keep last line empty