|
1 | 1 | """Getting OpenStreetMap data from the web.""" |
2 | 2 | import logging |
3 | 3 | import time |
| 4 | +import gzip |
4 | 5 | from dataclasses import dataclass |
5 | 6 | from pathlib import Path |
6 | 7 |
|
@@ -40,6 +41,13 @@ def get_osm( |
40 | 41 | {"bbox": boundary_box.get_format()}, |
41 | 42 | ) |
42 | 43 |
|
| 44 | + # Try to decompress gzip content if needed. |
| 45 | + try: |
| 46 | + if content.startswith(b"\x1f\x8b"): # gzip magic header. |
| 47 | + content = gzip.decompress(content) |
| 48 | + except Exception: |
| 49 | + pass # If decompression fails, continue with original content. |
| 50 | + |
43 | 51 | if not content.startswith(b"<"): |
44 | 52 | if content == ( |
45 | 53 | b"You requested too many nodes (limit is 50000). Either request a " |
@@ -67,11 +75,18 @@ def get_data(address: str, parameters: dict[str, str]) -> bytes: |
67 | 75 | :return: connection descriptor |
68 | 76 | """ |
69 | 77 | logging.info(f"Getting {address}...") |
| 78 | + headers = { |
| 79 | + "User-Agent": "map-machine/1.0", |
| 80 | + # Disable compression to avoid gzip issues. |
| 81 | + "Accept-Encoding": "identity", |
| 82 | + } |
70 | 83 | pool_manager: urllib3.PoolManager = urllib3.PoolManager() |
71 | 84 | urllib3.disable_warnings() |
72 | 85 |
|
73 | 86 | try: |
74 | | - result = pool_manager.request("GET", address, fields=parameters) |
| 87 | + result = pool_manager.request( |
| 88 | + "GET", address, fields=parameters, headers=headers |
| 89 | + ) |
75 | 90 | except urllib3.exceptions.MaxRetryError: |
76 | 91 | raise NetworkError("Cannot download data: too many attempts.") |
77 | 92 |
|
|
0 commit comments