From 77f11b93b6e8eec3ef0efe681584f9b150982274 Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Tue, 27 May 2025 19:39:16 -0700 Subject: [PATCH 1/4] Add retries to get_external.py downloads as well --- PCbuild/get_external.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 4ecc8925349c93..d0cd94aa99a393 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -1,14 +1,39 @@ #!/usr/bin/env python3 import argparse +import html import os import pathlib +import random import sys import time import zipfile from urllib.request import urlretrieve +def retrieve_with_retries(download_location, + output_path, + reporthook, + max_retries = 7, + base_delay = 2.25, + max_jitter = 1.0): + """Download a file with exponential backoff retry and save to disk.""" + for attempt in range(max_retries): + try: + resp = urlretrieve( + download_location, + output_path, + reporthook=reporthook, + ) + except ConnectionError as ex: + if attempt == max_retries: + msg = f"Download from {download_location} failed." + raise OSError(msg) from ex + time.sleep(base_delay**attempt + random.uniform(0, max_jitter)) + else: + return resp + + def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): repo = f'cpython-{"bin" if binary else "source"}-deps' url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' @@ -16,10 +41,10 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): if verbose: reporthook = print zip_dir.mkdir(parents=True, exist_ok=True) - filename, headers = urlretrieve( + filename, _headers = retrieve_with_retries( url, zip_dir / f'{commit_hash}.zip', - reporthook=reporthook, + reporthook ) return filename From ef5731bf8da812e674f8eafb772e808cd2bfdf8d Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Tue, 27 May 2025 19:39:47 -0700 Subject: [PATCH 2/4] Remove unused import --- PCbuild/get_external.py | 1 - 1 file changed, 1 deletion(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index d0cd94aa99a393..98e547f6542433 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import argparse -import html import os import pathlib import random From 209ecccaa319b9e4fc972f8e11ea9617f88c8a0b Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Wed, 28 May 2025 11:57:36 -0700 Subject: [PATCH 3/4] Wrap function signature tighter Co-authored-by: Semyon Moroz --- PCbuild/get_external.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 98e547f6542433..d337af563b1217 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -10,12 +10,8 @@ from urllib.request import urlretrieve -def retrieve_with_retries(download_location, - output_path, - reporthook, - max_retries = 7, - base_delay = 2.25, - max_jitter = 1.0): +def retrieve_with_retries(download_location, output_path, reporthook, + max_retries=7, base_delay=2.25, max_jitter=1.0): """Download a file with exponential backoff retry and save to disk.""" for attempt in range(max_retries): try: From c893998072f6c49835d1a7fbd3dc22f0a582a7a7 Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Wed, 28 May 2025 12:07:36 -0700 Subject: [PATCH 4/4] Simplify to exponential backoff --- PCbuild/get_external.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index d337af563b1217..99aff63882f5ba 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -3,7 +3,6 @@ import argparse import os import pathlib -import random import sys import time import zipfile @@ -11,7 +10,7 @@ def retrieve_with_retries(download_location, output_path, reporthook, - max_retries=7, base_delay=2.25, max_jitter=1.0): + max_retries=7): """Download a file with exponential backoff retry and save to disk.""" for attempt in range(max_retries): try: @@ -24,7 +23,7 @@ def retrieve_with_retries(download_location, output_path, reporthook, if attempt == max_retries: msg = f"Download from {download_location} failed." raise OSError(msg) from ex - time.sleep(base_delay**attempt + random.uniform(0, max_jitter)) + time.sleep(2.25**attempt) else: return resp