Skip to content

Commit d2f522f

Browse files
gitmur444Timurenzet
authored
Fix OSM API gzip response handling.
Co-authored-by: Timur <timur@User1s-MacBook-Air.local> Co-authored-by: Sergey Vartanov <me@enzet.ru>
1 parent 8ef6c63 commit d2f522f

File tree

1 file changed

+16
-1
lines changed

1 file changed

+16
-1
lines changed

map_machine/osm/osm_getter.py

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Getting OpenStreetMap data from the web."""
22
import logging
33
import time
4+
import gzip
45
from dataclasses import dataclass
56
from pathlib import Path
67

@@ -40,6 +41,13 @@ def get_osm(
4041
{"bbox": boundary_box.get_format()},
4142
)
4243

44+
# Try to decompress gzip content if needed.
45+
try:
46+
if content.startswith(b"\x1f\x8b"): # gzip magic header.
47+
content = gzip.decompress(content)
48+
except Exception:
49+
pass # If decompression fails, continue with original content.
50+
4351
if not content.startswith(b"<"):
4452
if content == (
4553
b"You requested too many nodes (limit is 50000). Either request a "
@@ -67,11 +75,18 @@ def get_data(address: str, parameters: dict[str, str]) -> bytes:
6775
:return: connection descriptor
6876
"""
6977
logging.info(f"Getting {address}...")
78+
headers = {
79+
"User-Agent": "map-machine/1.0",
80+
# Disable compression to avoid gzip issues.
81+
"Accept-Encoding": "identity",
82+
}
7083
pool_manager: urllib3.PoolManager = urllib3.PoolManager()
7184
urllib3.disable_warnings()
7285

7386
try:
74-
result = pool_manager.request("GET", address, fields=parameters)
87+
result = pool_manager.request(
88+
"GET", address, fields=parameters, headers=headers
89+
)
7590
except urllib3.exceptions.MaxRetryError:
7691
raise NetworkError("Cannot download data: too many attempts.")
7792

0 commit comments

Comments
 (0)